nightly.yml

name: Nightly Compatibility

on:
  schedule:
    - cron: "0 3 * * *"  # 03:00 UTC daily
  workflow_dispatch:  # Manual trigger

# No workflow-level permissions — each job declares the minimal set it needs.

# Set FORGELM_OPERATOR so future operator-identity checks (Phase 3 hardening)
# do not raise on distroless runners where USER is unset.
env:
  FORGELM_OPERATOR: ci-smoke

jobs:
  test-latest-deps:
    name: Test with latest dependencies
    runs-on: ubuntu-latest
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11", "3.12"]
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - uses: actions/setup-python@v7
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install with latest deps (no upper bounds)
        # NOTE: intentionally unconstrained — designed to surface upstream-API
        # breakage early. The notify-failure job dedups against existing
        # 'nightly-failure' issues to bound noise (see also: testing.md
        # CI gates discussion). Do NOT add `|| true` here.
        #
        # This list MUST mirror ``pyproject.toml`` ``[project].dependencies``
        # because the next step (``pip install -e . --no-deps``) does not
        # install transitive deps.  When a new runtime dependency lands in
        # pyproject, add it here too or the matrix job will crash in
        # ``import forgelm`` before any test runs.  Each entry below maps
        # 1:1 to a pyproject line; the comment after each call out the
        # source-of-truth.
        run: |
          pip install --upgrade pip
          # Core: torch / transformers / peft / datasets / accelerate /
          # trl / pydantic / pyyaml / tensorboard / huggingface_hub /
          # requests / requests-toolbelt (the last needed by
          # forgelm._http for HostHeaderSSLAdapter; SSRF DNS-rebinding
          # hardening, issue #14, v0.6.0).
          pip install torch transformers peft datasets accelerate trl \
            pydantic pyyaml tensorboard huggingface_hub requests requests-toolbelt
          pip install pytest pytest-cov ruff

      - name: Install ForgeLM (editable, no deps — already installed above)
        run: pip install -e . --no-deps

      - name: Show dependency versions
        run: |
          python -c "
          import importlib
          for pkg in ['torch', 'transformers', 'peft', 'datasets', 'accelerate', 'trl', 'pydantic']:
              mod = importlib.import_module(pkg)
              print(f'{pkg}: {getattr(mod, \"__version__\", \"unknown\")}')
          "

      - name: Lint check
        run: ruff check .

      - name: Run tests
        run: pytest tests/ -q --tb=short

      - name: CLI smoke test
        run: |
          forgelm --version
          forgelm --config config_template.yaml --dry-run
          forgelm --config config_template.yaml --dry-run --output-format json
          # `python -m` is the form the contributor gauntlet runs; keep it
          # covered against upstream dep drift alongside the console script.
          python -m forgelm --version
          python -m forgelm --config config_template.yaml --dry-run

      - name: Quickstart templates smoke test
        run: |
          # Every bundled template must render to a valid YAML that
          # passes pydantic validation. Catches template drift the
          # moment a config schema changes.
          #
          # Note: smoke-tests 4 of 5 templates. `domain-expert` is BYOD
          # (no bundled dataset), so `quickstart --dry-run` rejects it
          # without `--dataset PATH`. It's covered separately by the
          # pytest unit `test_domain_expert_intentionally_has_no_bundled_data`.
          forgelm quickstart --list
          for tpl in customer-support code-assistant medical-qa-tr grpo-math; do
            echo "=== Quickstart dry-run: $tpl ==="
            forgelm quickstart "$tpl" --dry-run --output "/tmp/qs-$tpl.yaml"
            forgelm --config "/tmp/qs-$tpl.yaml" --dry-run
          done

      - name: Forbid git+https in notebooks
        run: |
          # Notebooks must install ForgeLM from PyPI with a pinned version,
          # never from a git+https URL. Closure plan Faz 5 — git+https makes
          # Colab installs depend on a specific branch HEAD, breaking
          # reproducibility and confusing users when main moves ahead of
          # the latest released wheel.
          if grep -rn 'git+https://github.com/HodeTech/ForgeLM' notebooks/; then
            echo "::error::Notebooks must install from PyPI, not git+https"
            exit 1
          fi

      - name: Ingestion + audit smoke test (Phase 11)
        run: |
          # Minimal end-to-end: TXT in → JSONL out → audit report out.
          # Plain TXT path doesn't need the [ingestion] extra; the audit
          # module is pure stdlib. Catches CLI wiring drift without paying
          # for pypdf / langdetect installs.
          mkdir -p /tmp/p11
          echo "Article 10 governs data quality." > /tmp/p11/sample.txt
          echo "Section two: representativeness, traceability, bias review." > /tmp/p11/sample2.txt
          forgelm ingest /tmp/p11/ --recursive --output /tmp/p11/out.jsonl
          test -s /tmp/p11/out.jsonl
          forgelm audit /tmp/p11/out.jsonl --output /tmp/p11/audit/
          test -s /tmp/p11/audit/data_audit_report.json

  wheel-install-smoke:
    # ------------------------------------------------------------------
    # This is the only test that catches package_data globs being broken
    # — editable installs always hide this. `pip install -e .` resolves
    # forgelm/templates/* via Path(__file__).parent regardless of what
    # setuptools would actually copy into the wheel, so a broken
    # [tool.setuptools.package-data] entry is invisible until a real user
    # runs `pip install forgelm` from PyPI and gets a missing-asset
    # FileNotFoundError on `forgelm quickstart`.
    #
    # We split this into a dedicated job (rather than appending to
    # test-latest-deps) because:
    #   1. Building a wheel + spawning a fresh venv is unrelated to the
    #      "latest deps still resolve" axis and adds ~1 minute the matrix
    #      doesn't need to pay twice.
    #   2. A failure here points unambiguously at packaging, not at a
    #      transitive dep bump that broke training.
    #   3. We deliberately install from the wheel WITHOUT --no-deps so
    #      the install path matches what end-users hit.
    # ------------------------------------------------------------------
    name: Wheel install smoke test
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - uses: actions/setup-python@v7
        with:
          python-version: "3.11"

      - name: Build wheel
        run: |
          set -euo pipefail
          pip install --upgrade pip build
          python -m build --wheel

      - name: Install wheel into a fresh venv and run quickstart from /tmp
        run: |
          set -euo pipefail
          # Build a clean venv so nothing from the checkout's cwd leaks in.
          python -m venv /tmp/wheel-test
          /tmp/wheel-test/bin/pip install --upgrade pip

          # Resolve the wheel glob explicitly: there must be exactly one
          # forgelm-*.whl, otherwise the test environment is ambiguous.
          shopt -s nullglob
          wheels=(dist/forgelm-*.whl)
          if [ "${#wheels[@]}" -ne 1 ]; then
            echo "Expected exactly one forgelm wheel in dist/; found ${#wheels[@]}: ${wheels[*]:-(none)}" >&2
            exit 1
          fi
          wheel="${wheels[0]}"
          echo "Installing: $wheel"
          /tmp/wheel-test/bin/pip install "$wheel"

          # Run from /tmp specifically so the source tree is NOT on
          # sys.path — any template asset must come from the wheel's
          # site-packages copy, not from the checkout.
          cd /tmp

          # `python -m forgelm` must work from the installed wheel too —
          # this is what proves forgelm/__main__.py is actually packaged
          # rather than merely present in the source tree.
          echo "=== python -m forgelm --version (from the wheel) ==="
          /tmp/wheel-test/bin/python -m forgelm --version

          echo "=== forgelm quickstart --list ==="
          /tmp/wheel-test/bin/forgelm quickstart --list | tee /tmp/qs-list.txt

          # Every registered template must appear in the listing.
          for tpl in customer-support code-assistant domain-expert medical-qa-tr grpo-math; do
            if ! grep -q "$tpl" /tmp/qs-list.txt; then
              echo "MISSING template '$tpl' from quickstart --list output" >&2
              exit 1
            fi
          done

          echo "=== forgelm quickstart customer-support --dry-run ==="
          /tmp/wheel-test/bin/forgelm quickstart customer-support \
            --dry-run --output /tmp/wheel-qs.yaml

          if [ ! -f /tmp/wheel-qs.yaml ]; then
            echo "Quickstart did not materialize /tmp/wheel-qs.yaml" >&2
            exit 1
          fi
          echo "Wheel-install smoke test passed."

  test-min-deps:
    name: Test with minimum supported versions
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - uses: actions/setup-python@v7
        with:
          python-version: "3.10"

      - name: Install minimum dependency versions
        # This list MUST mirror the documented minimums in
        # ``pyproject.toml`` ``[project].dependencies`` because the
        # next step (``pip install -e . --no-deps``) does not install
        # transitive deps.  When a new runtime dependency lands in
        # pyproject, add it here too or the job will crash in ``import
        # forgelm`` before any test runs.
        run: |
          pip install --upgrade pip
          # transformers-5 migration floor (v0.9.0). Every gate is pinned
          # EXPLICITLY so this job tests the true documented minimum — trl puts
          # no upper bound on transformers, so an unpinned resolve would silently
          # pull the newest 5.x and stop exercising the floor. transformers
          # 5.10.1 is the first installable release past CVE-2026-4372,
          # CVE-2026-5241 and CVE-2026-9856 (5.10.0, which carries the last of
          # them, was yanked); it requires huggingface_hub>=1.5 /
          # tokenizers>=0.22 and imports torch.float8_e8m0fnu, which first
          # exists in torch 2.7.0 (this job failed on torch 2.4.0 with
          # AttributeError at `import transformers`); trl 1.0.0
          # pulls the accelerate>=1.4 and datasets>=4.7 floor. Keep 1:1 with the
          # pyproject [project].dependencies minimums (tests/test_packaging.py
          # compares them).
          pip install \
            "torch==2.7.0" \
            "transformers==5.10.1" \
            "huggingface_hub==1.5.0" \
            "peft==0.19.0" \
            "accelerate==1.4.0" \
            "datasets==4.7.0" \
            "trl==1.0.0" \
            "pydantic==2.0.0" \
            "pyyaml==6.0.1" \
            "tensorboard==2.15.0" \
            "requests-toolbelt==1.0.0"
          pip install pytest pytest-cov

      - name: Install ForgeLM
        run: pip install -e . --no-deps

      - name: Run tests
        run: pytest tests/ -q --tb=short

  supply-chain-security:
    # ------------------------------------------------------------------
    # Wave 4 / Faz 23: ISO 27001 / SOC 2 Type II alignment supply-chain
    # security gate.
    #
    # Two scans, both run on the latest published deps so we catch CVEs
    # the moment a dependency advisory drops, even before the team
    # re-bumps versions:
    #
    # - pip-audit (transitive CVEs against PyPI's vulnerability data, its default service)
    # - bandit (static-analysis on forgelm/ production code only)
    #
    # Severity policy (per tools/check_pip_audit.py + tools/check_bandit.py):
    # HIGH → fail; MEDIUM → ::warning::; LOW → silent.
    # (CRITICAL findings are reported by pip-audit at HIGH tier — same gate.)
    # Operators install the same tooling locally via
    # `pip install forgelm[security]`.
    # ------------------------------------------------------------------
    name: Supply-chain security (pip-audit + bandit)
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - uses: actions/setup-python@v7
        with:
          python-version: "3.11"
          cache: pip

      - name: Install ForgeLM with [security] extra
        run: |
          pip install --upgrade pip
          # setuptools ships in the runner image and is what pip-audit scans
          # (pyproject's build-system floor only governs the isolated build
          # env, not this scanned one). Nightly's charter is to run against
          # the latest published deps, so upgrade unconditionally rather than
          # re-pinning a floor here that would go stale the moment the real
          # floor moves. The authoritative minimum is pyproject.toml's
          # [build-system].requires ("setuptools>=83.0.0", fixing
          # PYSEC-2026-3447); tests/test_packaging.py machine-checks that
          # floor never regresses below 83.0.0.
          pip install --upgrade setuptools
          pip install -e ".[security]"

      - name: pip-audit (transitive CVE scan)
        run: |
          # pip-audit captures every finding; tools/check_pip_audit.py
          # applies ForgeLM's severity policy (HIGH → fail,
          # MEDIUM → ::warning::, UNKNOWN → fail closed) on the JSON.
          #
          # Project-internal CVE suppressions live in
          # tools/pip_audit_ignores.yaml; --ignores is opt-in so
          # standalone deployer invocations of check_pip_audit.py
          # inherit nothing (per docs/reference/supply_chain_security.md).
          # Adding an entry to that file is a security-policy change;
          # see the file header for the required schema (id, package,
          # reason, threat_model, verified_at, reevaluate_after).
          pip-audit \
              --format json \
              --output /tmp/pip-audit.json \
              || true
          python3 tools/check_pip_audit.py /tmp/pip-audit.json \
              --ignores tools/pip_audit_ignores.yaml

      - name: bandit (static security analysis)
        run: |
          # Same scope/severity policy as ci.yml; nightly cadence catches
          # newly-introduced HIGH issues even on long-lived branches.
          bandit -c pyproject.toml -r forgelm/ -f json -o /tmp/bandit.json || true
          python3 tools/check_bandit.py /tmp/bandit.json

      - name: Upload security scan artifacts
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: supply-chain-scans
          path: |
            /tmp/pip-audit.json
            /tmp/bandit.json

  notify-failure:
    name: Notify on failure
    needs: [test-latest-deps, test-min-deps, wheel-install-smoke, supply-chain-security]
    if: failure()
    runs-on: ubuntu-latest
    permissions:
      contents: read  # github-script needs to read the repo
      issues: write   # required for creating failure notification issues
    steps:
      - name: Create issue on failure
        uses: actions/github-script@v9
        with:
          script: |
            const title = `Nightly CI failure — ${new Date().toISOString().split('T')[0]}`;
            const existing = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              state: 'open',
              labels: 'nightly-failure',
              per_page: 1,
            });
            if (existing.data.length > 0) {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: existing.data[0].number,
                body: `Nightly CI failed again on ${new Date().toISOString().split('T')[0]}.\n\n[View run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`,
              });
            } else {
              await github.rest.issues.create({
                owner: context.repo.owner,
                repo: context.repo.repo,
                title: title,
                // Events caused by GITHUB_TOKEN start no other workflow, so the triage gate cannot label this issue.
                labels: ['nightly-failure', 'bug', 'needs-triage'],
                body: `## Nightly CI Failure\n\nThe nightly compatibility test failed.\n\n**Run:** [View details](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})\n\nThis may indicate a breaking change in a dependency. Check the test logs for details.`,
              });
            }