ci: run the tools against a released ix and generate no-backend fixtures

.github/workflows/ci.yml

@@ -55,17 +55,98 @@ jobs:
       - name: Test
         run: bun test
 
+  # ── Real ix ──────────────────────────────────────────────────────────────
+  # The `test` job only ever meets a fake `ix` (tests/fake-ix.ts), which refuses
+  # just the flags someone taught it to. This job installs a released CLI --
+  # pinned, checksummed -- and runs the tools against it with no backend:
+  # IX_ENDPOINT is a port nothing answers on and IX_HOME is empty, so graph
+  # reads return ix's real `workspace_not_mapped` record and `ix text`,
+  # `ix --version` and the argv parser are fully real. tests/real-ix.test.ts
+  # fails if ix rejects any argv a tool builds (unknown option/command), so a
+  # pin to a CLI that lacks a flag the plugin uses turns this job red.
+  #
+  # It also regenerates the fixtures that need no backend
+  # (tests/fixtures/regen-no-backend.ts) and fails if they differ from the
+  # committed ones. Fixtures that need a populated graph stay hand-captured;
+  # see tests/fixtures/ix-v0.12.0/README.md.
+  real-ix:
+    name: Real ix (v0.12.0, no backend)
+    runs-on: ubuntu-latest
+    env:
+      IX_VERSION: "0.12.0"
+      # sha256 of ix-<IX_VERSION>-linux-amd64.tar.gz, pinned here so a release
+      # asset replaced after the fact (tarball and .sha256 together) fails the
+      # job. Bump it with IX_VERSION.
+      IX_SHA256: "b9606253c5e300dbf4b933c82c6e0dc1d40a028aa344b5dc90dfe6dc34d89817"
+      IX_ENDPOINT: "http://127.0.0.1:1"
+      IX_NO_UPDATE_CHECK: "1"
+    steps:
+      - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
+        with:
+          egress-policy: audit
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      # The ix launcher is a Node script and needs Node 22+.
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+        with:
+          bun-version: "1.4.2"
+      # `ix text` shells out to ripgrep.
+      - name: Install ripgrep
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq ripgrep
+          rg --version
+      - name: Install ix ${{ env.IX_VERSION }}
+        run: |
+          set -euo pipefail
+          tarball="ix-${IX_VERSION}-linux-amd64.tar.gz"
+          base="https://github.com/ix-infrastructure/Ix/releases/download/v${IX_VERSION}"
+          dl="$RUNNER_TEMP/ix-dl"
+          mkdir -p "$dl" "$RUNNER_TEMP/ix"
+          curl -fsSL -o "$dl/$tarball" "$base/$tarball"
+          curl -fsSL -o "$dl/$tarball.sha256" "$base/$tarball.sha256"
+          (cd "$dl" && sha256sum -c "$tarball.sha256")
+          echo "$IX_SHA256  $dl/$tarball" | sha256sum -c -
+          tar -xzf "$dl/$tarball" --strip-components=1 -C "$RUNNER_TEMP/ix"
+          echo "$RUNNER_TEMP/ix" >> "$GITHUB_PATH"
+          echo "IX_HOME=$RUNNER_TEMP/ix-home" >> "$GITHUB_ENV"
+      - name: ix --version
+        run: |
+          set -euo pipefail
+          got="$(ix --version)"
+          echo "ix --version: $got"
+          [ "$got" = "$IX_VERSION" ]
+      - name: Install
+        run: bun install --frozen-lockfile
+      - name: Tools against the real ix
+        env:
+          IX_REAL_TESTS: "1"
+        run: bun test tests/real-ix.test.ts
+      - name: Regenerate no-backend fixtures
+        run: bun tests/fixtures/regen-no-backend.ts
+      - name: Committed fixtures match the real ix
+        run: |
+          git status --porcelain -- tests/fixtures
+          git diff --exit-code -- tests/fixtures
+          [ -z "$(git status --porcelain -- tests/fixtures)" ]
+
   # ── Aggregate gate ───────────────────────────────────────────────────────
   # One stable required-status-check context. Branch protection requires only
   # this job, so the job graph can be reshaped without orphaning required
   # checks. Fails if any upstream job failed OR was skipped/cancelled.
   ci-success:
     name: CI Passed
     if: always()
-    needs: [test]
+    needs: [test, real-ix]
     runs-on: ubuntu-latest
     steps:
       - name: Verify all required jobs succeeded
         run: |
           echo "test=${{ needs.test.result }}"
+          echo "real-ix=${{ needs.real-ix.result }}"
           [ "${{ needs.test.result }}" = "success" ]
+          [ "${{ needs.real-ix.result }}" = "success" ]

tests/fixtures/ix-v0.12.0/README.md

@@ -1,9 +1,9 @@
 # Ix v0.12.0 output fixtures
 
-Captured stdout of the real `ix` CLI, used by `tests/ix-errors.test.ts`,
+Stdout of the real `ix` CLI, used by `tests/ix-errors.test.ts`,
 `tests/success-fixtures.test.ts` and `tests/secrets.test.ts`.
 
-**Source:** Ix `main` at `5ce416c` (v0.12.0 is `8d16f2a`; the one commit on
+**Source of the hand-captured files:** Ix `main` at `5ce416c` (v0.12.0 is `8d16f2a`; the one commit on
 top only touches `ix mcp`), built `ix-cli/dist`, run with
 `node ix-cli/dist/cli/main.js`. That checkout's `package.json` still says
 0.11.1, so `--version` printed 0.11.1; the behaviour is v0.12.0's, including
@@ -13,10 +13,28 @@ backend on :8090. Only read commands were run.
 `manifest.json` maps every captured file to the exact command and its exit code.
 `.json` files are `--format json`, `.txt` files are `--format llm`.
 
-## `unmapped/` (real)
+## Generated vs hand-captured
 
-Run from a fresh `mktemp -d` directory with no Ix config, so no registered
-workspace covers it. Every graph read answers
+Two kinds of file live here:
+
+- **Generated** by `tests/fixtures/regen-no-backend.ts` from the released
+  v0.12.0 CLI with no backend: `unmapped/` and `no-backend/`. Their manifest
+  entries carry `"generatedBy"`. CI's `real-ix` job installs the v0.12.0
+  release tarball, reruns the script and fails if `git diff` shows any change,
+  so do not edit these by hand -- rerun the script
+  (`bun tests/fixtures/regen-no-backend.ts`, real `ix` 0.12.0 first on PATH).
+- **Hand-captured** (or hand-written): `empty-graph/`, `success/`,
+  `empty-repo/` and `synthetic/`. Each needs a backend holding a graph (or a
+  registered workspace the backend answers for), which CI does not have, so
+  the script cannot produce them and leaves them alone. They are described
+  below as they were captured.
+
+## `unmapped/` (generated)
+
+Run from a fixed directory (`/tmp/ix-fixtures-v0.12.0/unmapped`) with a fresh
+`IX_HOME`, so no registered workspace covers it, and `IX_ENDPOINT` pointing at
+a port nothing answers on. Ix decides "not mapped" from `IX_HOME` alone,
+before any network call. Every graph read answers
 `{"error":"workspace_not_mapped",...,"next":"Run \`ix map <dir>\` ..."}` (JSON)
 or `error code=workspace_not_mapped ... hint="..."` (llm), exit 1. Commands:
 impact, callers, trace, explain, locate, `smells --list`, stats, subsystems,
@@ -25,6 +43,24 @@ rank, inventory.
 `smells-list.*` is `ix smells --list`, the only form the ix-smells tool runs
 (bare `ix smells` re-runs detection and writes smell claims).
 
+These were first captured by hand from a `mktemp -d` directory; the only
+change on regeneration was that directory name.
+
+## `no-backend/` (generated)
+
+- `version.*`: `ix --version`. It ignores `--format json` and prints plain
+  `0.12.0` either way.
+- `status.*`: `ix status` with the backend unreachable, exit 1. `--format llm`
+  prints `error code=cli_error message="fetch failed (bad port)"`;
+  `--format json` prints **nothing on stdout** (the error goes to stderr only),
+  so `status.json` is empty on purpose. "bad port" is because the endpoint is
+  port 1, which fetch refuses without connecting.
+- `text*.*`: `ix text` (ripgrep, no graph needed) on a two-file repo the
+  script writes: hits, a `--path`/`--language` scoped search, and no hits.
+  All hits are in one file because `ix text` scores every ripgrep hit the same
+  and keeps ripgrep's unstable cross-file order, so a multi-file result is not
+  reproducible run to run.
+
 ## `empty-graph/` (real)
 
 Run from a fresh `mktemp -d` directory registered in a throwaway `IX_HOME`

tests/fixtures/ix-v0.12.0/manifest.json

@@ -1,83 +1,123 @@
 {
   "unmapped/impact.json": {
     "command": "ix impact Foo --format json",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/impact.txt": {
     "command": "ix impact Foo --format llm",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/callers.json": {
     "command": "ix callers Foo --format json",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/callers.txt": {
     "command": "ix callers Foo --format llm",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/trace.json": {
     "command": "ix trace Foo --format json",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/trace.txt": {
     "command": "ix trace Foo --format llm",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/explain.json": {
     "command": "ix explain Foo --format json",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/explain.txt": {
     "command": "ix explain Foo --format llm",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/locate.json": {
     "command": "ix locate Foo --format json",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/locate.txt": {
     "command": "ix locate Foo --format llm",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/smells-list.json": {
     "command": "ix smells --list --format json",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/smells-list.txt": {
     "command": "ix smells --list --format llm",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/stats.json": {
     "command": "ix stats --format json",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/stats.txt": {
     "command": "ix stats --format llm",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/subsystems.json": {
     "command": "ix subsystems --format json",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/subsystems.txt": {
     "command": "ix subsystems --format llm",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/rank.json": {
     "command": "ix rank --by dependents --kind class --top 5 --format json",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/rank.txt": {
     "command": "ix rank --by dependents --kind class --top 5 --format llm",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/inventory.json": {
     "command": "ix inventory --kind file --path src/ --format json",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "unmapped/inventory.txt": {
     "command": "ix inventory --kind file --path src/ --format llm",
-    "exit": 1
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   },
   "empty-graph/impact.json": {
     "command": "ix impact Foo --format json",
@@ -586,5 +626,68 @@
     "exit": 0,
     "ix": "5ce416c",
     "capturedFrom": "throwaway backend (compose project ix-ocfix, memory-layer 1.0.30)"
+  },
+  "no-backend/version.txt": {
+    "command": "ix --version",
+    "exit": 0,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
+  },
+  "no-backend/version.json": {
+    "command": "ix --version --format json",
+    "exit": 0,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts",
+    "note": "--version ignores --format: plain text, not JSON"
+  },
+  "no-backend/status.json": {
+    "command": "ix status --format json",
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts",
+    "note": "backend unreachable; --format json prints nothing on stdout (the error goes to stderr), llm prints an error record"
+  },
+  "no-backend/status.txt": {
+    "command": "ix status --format llm",
+    "exit": 1,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts",
+    "note": "backend unreachable; --format json prints nothing on stdout (the error goes to stderr), llm prints an error record"
+  },
+  "no-backend/text.json": {
+    "command": "ix text greet --limit 5 --format json",
+    "exit": 0,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
+  },
+  "no-backend/text.txt": {
+    "command": "ix text greet --limit 5 --format llm",
+    "exit": 0,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
+  },
+  "no-backend/text-scoped.json": {
+    "command": "ix text greet --limit 5 --path src --language typescript --format json",
+    "exit": 0,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
+  },
+  "no-backend/text-scoped.txt": {
+    "command": "ix text greet --limit 5 --path src --language typescript --format llm",
+    "exit": 0,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
+  },
+  "no-backend/text-none.json": {
+    "command": "ix text noSuchIdentifierAnywhere --limit 5 --format json",
+    "exit": 0,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
+  },
+  "no-backend/text-none.txt": {
+    "command": "ix text noSuchIdentifierAnywhere --limit 5 --format llm",
+    "exit": 0,
+    "ix": "0.12.0",
+    "generatedBy": "tests/fixtures/regen-no-backend.ts"
   }
 }

tests/fixtures/ix-v0.12.0/no-backend/status.txt

@@ -0,0 +1 @@
+error code=cli_error message="fetch failed (bad port)"

tests/fixtures/ix-v0.12.0/no-backend/text-none.json

@@ -0,0 +1 @@
+[]

tests/fixtures/ix-v0.12.0/no-backend/text-none.txt

@@ -0,0 +1 @@
+text shown=0 scanned=0

tests/fixtures/ix-v0.12.0/no-backend/text-scoped.json

@@ -0,0 +1 @@
+[{"path":"src/main.ts","line_start":1,"line_end":1,"snippet":"export function greet(name: string): string {\n","engine":"ripgrep","score":1,"language":"typescript"},{"path":"src/main.ts","line_start":5,"line_end":5,"snippet":"console.log(greet(\"world\"));\n","engine":"ripgrep","score":1,"language":"typescript"}]

tests/fixtures/ix-v0.12.0/no-backend/text-scoped.txt

@@ -0,0 +1,3 @@
+text shown=2 scanned=2
+match path=src/main.ts line=1 snippet="export function greet(name: string): string {"
+match path=src/main.ts line=5 snippet="console.log(greet(\"world\"));"

tests/fixtures/ix-v0.12.0/no-backend/text.json

@@ -0,0 +1 @@
+[{"path":"src/main.ts","line_start":1,"line_end":1,"snippet":"export function greet(name: string): string {\n","engine":"ripgrep","score":1,"language":"typescript"},{"path":"src/main.ts","line_start":5,"line_end":5,"snippet":"console.log(greet(\"world\"));\n","engine":"ripgrep","score":1,"language":"typescript"}]

tests/fixtures/ix-v0.12.0/no-backend/text.txt

@@ -0,0 +1,3 @@
+text shown=2 scanned=2
+match path=src/main.ts line=1 snippet="export function greet(name: string): string {"
+match path=src/main.ts line=5 snippet="console.log(greet(\"world\"));"