From 4367360ecf65d1034ececd4619e3b63da1cfae3f Mon Sep 17 00:00:00 2001 From: ModelMirror <273825391+modelmirror@users.noreply.github.com> Date: Wed, 23 Sep 2026 19:42:30 +0000 Subject: [PATCH 1/5] docs(security): record the review environment's deployment branches Co-Authored-By: Claude Opus 5.5 (1M context) --- SECURITY.md | 6 +++--- docs/security.md | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index fc243bdca..775f9bb09 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -436,9 +436,9 @@ runbook, [docs/security.md](docs/security.md). role, its own engine keys for the pre-promotion integration runs, the staging read-write role, the staging telemetry App's client id and private key — the Issues-only App the repro leg's rehearsal record is - minted from — and the staging copy of the case-summary lane's key. A third, - `review`, holds no secret, no role, and no branch policy: its entire content - is a required-reviewer rule, and it exists only as the audit-logged hold + minted from. A third, `review`, holds no secret and no role and admits + `main` and `staging`; beyond that its content is a required-reviewer rule, + and it exists only as the audit-logged hold between a plan that would spend and the spend — run-predict, run-evaluate, run-backtest and summarize all bind it; one environment serves every spend hold rather than each minting its own. What each hold covers differs by what diff --git a/docs/security.md b/docs/security.md index 76f00155f..db7f23347 100644 --- a/docs/security.md +++ b/docs/security.md @@ -410,8 +410,11 @@ below; `summarize`'s publish job, which runs on `main` only, is the literal deliberate exceptions, by environment. The `approval` jobs of run-predict, run-evaluate, run-backtest and summarize declare **`review`**, an environment that exists *only* for its required reviewers. -It carries no secrets, no variables, no role, and no deployment-branch -policy; each job it gates runs one echo under `permissions: {}`, so the +It carries no secrets, no variables and no role, and its deployment branches +are `main` and `staging` — `staging` so that summarize's staging rehearsal +(the one held lane whose jobs resolve their environment from the ref) passes +the same hold its production run does; +each job it gates runs one echo under `permissions: {}`, so the environment grants nothing and merely withholds the spend behind it — a fan-out's matrix, the back-test's fortnightly replay, or a case-summary run — until @@ -461,8 +464,9 @@ the ~8 known shapes) would pass. Relatedly, never put anything sensitive in a on the `staging` environment, whose policy is load-bearing twice over: the read-only role's trust names it, and so does the one write-capable role outside `prod`); `review` -deliberately carries no branch policy, since it holds nothing a branch could -take. A job can read the environment's +admits `main` and `staging` — it holds nothing a branch could take, so its +policy only decides which refs may request a spend hold, and a staging +rehearsal of a held lane must be among them. A job can read the environment's secrets only when it runs from `main`, so a workflow authored on a PR branch runs **without** the App key, agent tokens, or S3 role: a malicious or prompt-injected workflow added in a PR cannot exfiltrate secrets on its own PR run; the change From ebc3fe8f052cf86b984eb63de336bf5c29b03923 Mon Sep 17 00:00:00 2001 From: ModelMirror <273825391+modelmirror@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:58:38 -0400 Subject: [PATCH 2/5] ci(gate): measure coverage under sys.monitoring (#1980) Co-authored-by: Claude Opus 5.5 (1M context) --- docs/testing.md | 7 ++++++- scripts/gate.sh | 12 ++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index f1c1b3c25..189e7b946 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -47,7 +47,12 @@ does and additionally honours `@pytest.mark.xdist_group`, so a test that ever do need to run beside its siblings on one worker can say so where it lives instead of needing the gate changed underneath it. Coverage is unaffected: under `GATE_COV=1` each worker measures its own slice and pytest-cov combines them into the single -`.coverage` file the CI job's summary step reads. +`.coverage` file the CI job's summary step reads. The stage measures through +Python's `sys.monitoring` (`COVERAGE_CORE=sysmon`) rather than coverage.py's +default C tracer, which cuts most of coverage's overhead for the same line-coverage +result. What sysmon cannot do on 3.12 — branch coverage, dynamic contexts, +non-thread concurrency — is not configured; configuring one makes coverage warn +and fall back to the C tracer, so the gate slows rather than fails. Set `COVERAGE_CORE` to override the core for a comparison run. Parallel workers are the wrong shape for debugging one failure — a worker has no terminal for `breakpoint()`, and output from several interleaves — so the worker count diff --git a/scripts/gate.sh b/scripts/gate.sh index 9bc185e03..6e88a5f8a 100755 --- a/scripts/gate.sh +++ b/scripts/gate.sh @@ -13,7 +13,9 @@ # scripts/gate.sh lint ruff format --check + ruff check # scripts/gate.sh types mypy # scripts/gate.sh test pytest, fanned across cores (set GATE_COV=1 for -# coverage, as CI does; GATE_TEST_WORKERS=1 for a +# coverage, as CI does, measured under Python's +# sys.monitoring — COVERAGE_CORE overrides the core +# for a comparison run; GATE_TEST_WORKERS=1 for a # serial run when debugging) # scripts/gate.sh data validate data + corpus-status # scripts/gate.sh schemas export-schemas + schema-drift check @@ -68,7 +70,13 @@ test_stage() { # would break the serial path — the one this script promises a debugger — on a # Mac's system bash while leaving the default path working. if [ "${GATE_COV:-0}" = "1" ]; then - uv run pytest ${fanout[@]+"${fanout[@]}"} --cov --cov-report=term-missing + # sys.monitoring rather than coverage.py's default C tracer: about a third of + # the wall time for the same line-coverage result. What it cannot do on 3.12 + # — branch coverage, dynamic contexts, non-thread concurrency — this + # project's coverage config does not use; configuring one makes coverage + # warn and fall back to the C tracer, slower but not failing. + COVERAGE_CORE="${COVERAGE_CORE:-sysmon}" \ + uv run pytest ${fanout[@]+"${fanout[@]}"} --cov --cov-report=term-missing else uv run pytest ${fanout[@]+"${fanout[@]}"} fi From eb73eb3f4f1d63902d42d1fc8159c6a1f4a8ae1e Mon Sep 17 00:00:00 2001 From: ModelMirror <273825391+modelmirror@users.noreply.github.com> Date: Wed, 23 Sep 2026 16:07:59 -0400 Subject: [PATCH 3/5] fix(summaries): hold summaries to the record's own terms (#1981) Co-authored-by: Claude Opus 5.5 (1M context) --- .github/prompts/summarize.md | 19 +++++++++++++++++++ docs/case-summaries.md | 5 +++++ 2 files changed, 24 insertions(+) diff --git a/.github/prompts/summarize.md b/.github/prompts/summarize.md index 11643ef84..1d60b8078 100644 --- a/.github/prompts/summarize.md +++ b/.github/prompts/summarize.md @@ -77,6 +77,25 @@ About **250 words in total**: Make no prediction about what the Court will do, give no view on who is right, and do not characterize the case as important, significant, landmark, closely watched, or the like. +- **Accurate to the record.** Every statement must be something the record + shows. + - *Allegations are attributed.* A fact that only one side's filing + asserts is that side's account: write "the petition says…", "the + respondent answers that…". State as fact only what the lower courts found + or what both sides agree on. + - *Questions keep their direction.* When you restate a question presented, + keep who made the rule, whom it binds, and which way it cuts: a question + about whether a court's rule requiring something is valid must not become + a question about whether a party may do that thing. Check your + restatement against the question's own words before moving on. + - *Procedure in the docket's own terms.* Leave routine docket entries out. + If one matters to where the case stands, name it as the docket does (a + motion to extend a deadline, a waiver of the right to respond) and + explain it in passing; do not guess at an entry's purpose. + - *Dates and counts as the entries show them.* A case distributed in + August for a September conference was distributed in August for a + September conference, not distributed in September; if a case went to two + conferences, say two, or "more than one", not "several". - **People.** Name parties only as the caption and filings name them. Where the filings refer to someone by initials (a minor, for example), use the initials. Add no personal detail — addresses, contact details, health, diff --git a/docs/case-summaries.md b/docs/case-summaries.md index 06051ae81..c8e7192e6 100644 --- a/docs/case-summaries.md +++ b/docs/case-summaries.md @@ -69,6 +69,11 @@ The prompt is `.github/prompts/summarize.md`. Its rules: passing. The repository holds no site glossary, so the prompt defines each term itself, minimally; a site glossary, once it exists, is the definition to align the prompt with. +- **Accurate to the record.** An allegation only one side's filing makes is + attributed to that filing; a question presented is restated without changing + who made the rule, whom it binds, or which way it cuts; routine docket + entries are left out, and one that matters to the posture is named as the + docket names it; dates and counts are as the entries give them. - **People.** Named only as the caption and filings name them; initials stay initials; no personal detail beyond the dispute. From d0bfcbb87e86f08555db7a02b0425d6cf23cb711 Mon Sep 17 00:00:00 2001 From: ModelMirror <273825391+modelmirror@users.noreply.github.com> Date: Wed, 23 Sep 2026 16:29:37 -0400 Subject: [PATCH 4/5] fix(summaries): a filed request is not reported as granted (#1982) Co-authored-by: Claude Opus 5.5 (1M context) --- .github/prompts/summarize.md | 4 ++++ docs/case-summaries.md | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/prompts/summarize.md b/.github/prompts/summarize.md index 1d60b8078..1b8ffbe69 100644 --- a/.github/prompts/summarize.md +++ b/.github/prompts/summarize.md @@ -92,6 +92,10 @@ About **250 words in total**: If one matters to where the case stands, name it as the docket does (a motion to extend a deadline, a waiver of the right to respond) and explain it in passing; do not guess at an entry's purpose. + - *A request is not its outcome.* A motion, application or request the + docket shows only as filed or submitted has not been granted: say what + was asked, not that it happened, unless a later entry records the + Court's action on it. - *Dates and counts as the entries show them.* A case distributed in August for a September conference was distributed in August for a September conference, not distributed in September; if a case went to two diff --git a/docs/case-summaries.md b/docs/case-summaries.md index c8e7192e6..37bd07c19 100644 --- a/docs/case-summaries.md +++ b/docs/case-summaries.md @@ -73,7 +73,8 @@ The prompt is `.github/prompts/summarize.md`. Its rules: attributed to that filing; a question presented is restated without changing who made the rule, whom it binds, or which way it cuts; routine docket entries are left out, and one that matters to the posture is named as the - docket names it; dates and counts are as the entries give them. + docket names it; a request the docket shows only as filed is not reported + as granted; dates and counts are as the entries give them. - **People.** Named only as the caption and filings name them; initials stay initials; no personal detail beyond the dispute. From 0470026163314c2b9859c41cb0fe55f6017da8e1 Mon Sep 17 00:00:00 2001 From: ModelMirror <273825391+modelmirror@users.noreply.github.com> Date: Wed, 23 Sep 2026 20:42:33 +0000 Subject: [PATCH 5/5] fix(summaries): check the staged records before they leave the stage job Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/summarize.yml | 14 ++++ docs/case-summaries.md | 18 +++-- docs/cli.md | 1 + docs/security.md | 7 +- src/fedcourtsai/cli.py | 39 +++++++++ src/fedcourtsai/summaries.py | 123 ++++++++++++++++++++++++++++ tests/test_summaries.py | 133 ++++++++++++++++++++++++++++++- tests/test_workflow_summarize.py | 15 ++++ 8 files changed, 342 insertions(+), 8 deletions(-) diff --git a/.github/workflows/summarize.yml b/.github/workflows/summarize.yml index 5b1425d92..97d0b61ad 100644 --- a/.github/workflows/summarize.yml +++ b/.github/workflows/summarize.yml @@ -301,6 +301,20 @@ jobs: done < "$RUNNER_TEMP/cases.txt" echo "staged ${staged} case record(s), ${failed} failed" | tee -a "$GITHUB_STEP_SUMMARY" + # The staged tree leaves this job as a public run artifact, so it may + # carry only the Court's own docket JSON and filings. The plan screened + # each case's newest snapshot, but staging ran after the review hold and + # took whatever was newest by then; this re-applies the screen to what + # was actually staged, here where the corpus credentials are, and + # removes any case that fails it before the upload below. Unconditional: + # the upload must never run on an unchecked tree. + - name: Check the staged records before they leave this job + run: | + set -euo pipefail + uv run fedcourts summary-stage-check \ + --plan "$RUNNER_TEMP/plan/summary-plan.json" \ + --staged "$RUNNER_TEMP/stage" \ + --summary "$GITHUB_STEP_SUMMARY" - name: Upload the staged records uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/docs/case-summaries.md b/docs/case-summaries.md index 37bd07c19..e45365fc2 100644 --- a/docs/case-summaries.md +++ b/docs/case-summaries.md @@ -249,12 +249,18 @@ shortest GitHub offers, one day. It rides the qp-topic extract's footing — supremecourt.gov content only, since the plan and `summarize` both refuse a CourtListener REST snapshot ([data-sources.md](data-sources.md)) — and is wider than the extract in one way: every stored filing of each planned case -rather than one section of each petition. A race keeps a small residual: a -REST snapshot stored between the plan and the stage for the same day is staged -(and then refused by `summarize`), so it crosses the artifact once. Closing -the channel outright means encrypting the staged tree to a key only the -generate job holds, or staging in the same job as generation with step-scoped -corpus credentials. +rather than one section of each petition. Staging runs after the review hold +and provisions whatever snapshot is newest by then, so the plan's screen alone +would let a REST snapshot stored in between reach the artifact. The stage job +therefore re-applies it to what it actually staged, before the upload and on +the side of the job boundary that holds the corpus credentials +(`summary-stage-check`), as an allowlist: a case crosses only if it was +planned and its tree holds exactly what provisioning writes — the planned day's +snapshot in the Court's own shape, `context.json`, and the documents manifest +with one text file per listed document, each fetched from supremecourt.gov. A +symlink removes the case rather than being followed, and anything else under +the stage root is removed. A removed case is reported skipped and planned again +by the next run. **The written summaries are a public artifact for a week.** The `case-summaries` artifact carries the generated files, after the jail and the diff --git a/docs/cli.md b/docs/cli.md index 84ad771bb..9fb171940 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -146,6 +146,7 @@ committed), plus the spend ledger. | `qp-topics` | Measure a topic labeler's JSONL against the hand reference set and accrue `data/qp-topics/qp-topics.json`: the artifact accumulates, so what is written is the union of the committed artifact and this batch's new rows, with prior entries carried forward unchanged. A row inside the reference set publishes the **hand set's adjudicated label**, never the labeler's — the labeler's call there is measurement input, scored into the agreement rate and discarded, so a flip on a reference case moves this run's rate and can change no published row; every other row is published once, since later batches exclude what is already published, and a second labeling of one stops the run. Each row records the batch that first published it and its `source`, and a per-batch ledger keeps every contributing run's labeler and agreement figures. Every label is validated against the `qp-topic-v0` vocabulary, and the labels joined to the extract and to the reference on `case_id` **and** `docket_number` (a half-matching pair is a mis-join and stops the run; so does a labels file that is not exactly the extract's case set, since a partial run measures a prefix rather than a sample and turns the reported `n` into a membership probe on an outcome-encoding reference set). Reports overall agreement beside the rate a **constant labeler** would score, floor-gated per-label agreement, the constitutional-rights / criminal-law / civil-procedure confusion matrix, and the deterministic shadow rules' disagreement rate — whose *level* is uninterpretable off the reference set, only its movement between runs. What it reports is **agreement with the v0 reference raters, not accuracy** — the pooled rate spans the disclosed two-block frame, and the per-stream split is derived at measurement review ([qp-topic.md](qp-topic.md)). The publication gate takes two conditions, the agreement rate and how much of the reference set the run covered; failing either, the artifact is not written, the measurement prints, and the command exits non-zero. There is no override flag. `--frame-rows` is the QP-bearing frame this batch was cut from, which neither input holds — the extract is one batch of the frame and the artifact is the union of every batch — so the extract job passes its own count in as a plain integer, never the `.batch.json` sidecar (whose Term × fee-class shape stays off the labeling job). It lands on this batch's ledger entry, where the docket pack reads it to measure the labeled share of the frame instead of bounding it; omitted, the entry records none and the cut keeps its bound. A count under the rows the batch labeled is refused, since the batch is drawn from the frame. The `run-analytics` mode that dispatches it is in [pipeline.md](pipeline.md). | `--labels`, `--texts`, `--labeler`, `--out`, `--frame-rows` | | `summarize-plan` | The case-summary lane's dry run ([case-summaries.md](case-summaries.md)): which predicted cases are owed a plain-language summary, and at what cost. Eligible cases carry a committed prediction; one is owed a summary when the `record_digest` of its newest corpus record — sha256 over the snapshot payload minus its generation stamp, plus each stored document's `(kind, sha256(text))` — differs from the one its newest committed `summaries/.md` was written from, or it has none. Prints the `SummaryPlan` JSON (`summarize` consumes it) and, with `--report`, the markdown the `review` hold is judged on: eligible, up-to-date, planned, deferred and snapshot-less counts, and an estimated cost range at `summaries.model`'s rate. `--limit` plans at most that many, cases without any summary first. A case whose newest snapshot is a CourtListener REST docket rather than the Court's own docket JSON is counted (`not_live_shaped`) and not planned. Reads the corpus (the casestore under the corpus-split mode); writes nothing under `data/`; calls no model. Refuses (exit 2) a configured model `pricing.MODEL_RATES` cannot price. | `--limit`, `--out`, `--report`, `--report-run-url`, `--corpus-backend` | | `summarize` | Write the planned case summaries: one Messages API call per case carrying `.github/prompts/summarize.md` and the case's staged record (`provision-snapshot` output under `--staged`; `record/context.json` is not sent) — no tools, thinking off. A response is written to `data/cases///summaries/.md`, under harness-written front matter (`CaseSummaryFrontMatter`), only if it ended normally, has exactly the three contract sections in order, runs 120–450 words, opens no paragraph with "Whether", and passes the secret scan; otherwise, and for a call that still fails after bounded retries on 429/5xx/529, the case is reported skipped with its reason. The report counts written and skipped cases and the cost of every call that returned, from response usage. The API key is the environment's Anthropic key, read from the variable `summaries.API_KEY_ENV` names; missing, exit 2. Refuses a plan priced for another model than the configured one. `--budget-minutes` stops starting new cases once that much time has passed and reports the rest as deferred, so a slow run returns inside its caller's timeout with what it wrote; a deferred case stays owed. A retried throttle honours the server's `retry-after`, capped at 60 seconds. Refuses a staged snapshot that is not the Court's own docket JSON. Exits 1 when the plan held cases and none was written. | `--plan`, `--staged`, `--report`, `--budget-minutes` | +| `summary-stage-check` | Remove every staged case-summary record that may not leave the stage job, before its run-artifact upload. An allowlist: a case is kept only if the plan names it (by its exact docket directory name) and its tree holds exactly what `provision-snapshot` writes — the planned day's snapshot in the Court's own docket-JSON shape, `record/context.json`, and `record/documents/documents.json` with one `.txt` per listed document, every listed `url` on supremecourt.gov over https. An unplanned case, a CourtListener REST snapshot, a snapshot of another day (the corpus moved after the plan), an unreadable file, an unlisted or off-host document, a symlink (never followed), or any stray file under the stage root is removed and printed as a `::warning::`. A removed case is reported skipped by `summarize` and planned again next run. `--summary` appends the kept/removed count line to a file. | `--plan`, `--staged`, `--summary` | | `summary-paths` | The case-summary lane's path filter: print the case summary files (`data/cases///summaries/.md`, added or modified) a `--name-status` change set writes. With `--strict`, its publish jail: exit 1 with an `::error::` per change that is anything else (another path, a deletion, a rename). | `--name-status-file`, `--strict` | | `tool-usage` | Roll every committed `retrieval_log.json` into an **offered-vs-called** report: which configured MCP tools were never called, which are used by some engines and not others, and call counts per tool / engine / actor. Reads `data/` only — no corpus, no network — so it runs offline and in the gate; the `run-analytics` mode that dispatches it is in [pipeline.md](pipeline.md). Counting contract: call names normalize to `.` (engines spell one MCP tool `mcp__x__y` or `mcp_x_y`); engine built-ins (shell, file IO, web search) are counted apart from what the manifest offers; a **code-mode** engine invokes everything from inside a freeform builtin call, and both idioms its program reaches through — the MCP manifest and the engine's own builtins, which is where such a program does most of its work — are lifted out of that source into rows of their own, so the freeform call contributes its own builtin row plus one row per lifted call *site the scan reached* (a site inside a loop is still one row); a lifted manifest row carries the same `mcp____` spelling a direct item would and normalizes into the offered denominator identically, while a lifted builtin row names the builtin and so is counted apart from that denominator, like any other builtin — gate on the MCP normalization wherever the question is manifest use, and read such an engine's raw call volume as counting the wrapper beside everything it wrapped; the offered denominator is each log's `mcp_tools` snapshot, falling back to the current manifest's advertised set for logs predating that field. A zero means **never called**, not useless, and the report says which. The same walk publishes four further cuts: per-engine **result observability** — a captured `result_digest` is the only evidence the answer side was recorded at all, and a null covers both an empty result and an uncaptured one, so the rate is honestly two-state and the per-tool dead-end rate is withheld for any engine that never captured a result rather than printed as 100%; cells and calls by **mode, role, and actor**; **calls beside cost** per cell, joined from each log's sibling `usage.json`, where a missing record degrades to a null cost and never to free; and **call volume against Brier**, joined to the gradings of each predicted cell and segmented by engine, mode, and forecast moment with the `n` beside every mean. That last block prints a grade, so it is scoped like the boards — blessed processes only unless `--all-versions` — and it publishes a correlation only per (mode, moment) population and only above the floor pre-declared in code as `tool_usage.TOOL_USAGE_CORRELATION_MIN_CELLS`; there is deliberately no pooled coefficient. What it may be read for: [metrics/README.md](../metrics/README.md). | `--out`, `--markdown-out`, `--all-versions` | | `ops-report` | Roll pipeline health, **substance**, spend & cost, **agent signals**, data health, and open issues wearing a `run:*` fan-out label into the ops report Markdown (and optional JSON). Each section renders only once its feed exists: `--previous` backs the substance deltas, `--live-frontier` the watchlist readiness, `--corpus-validation` data health, `--trigger-issues` the stale fan-out labels (markers left behind, since nothing keys on a label and each stage derives its own backlog); `--digest-out` renders the weekly performance digest. What each section reports and how `run-ops` publishes it: [pipeline.md](pipeline.md). | `--runs`, `--json`, `--generated-at`, `--corpus-validation`, `--live-frontier`, `--previous`, `--digest-out`, `--data-health-out`, `--trigger-issues`, `--all-versions` | diff --git a/docs/security.md b/docs/security.md index db7f23347..5e6b10127 100644 --- a/docs/security.md +++ b/docs/security.md @@ -1100,7 +1100,12 @@ jobs — the newest snapshot and every stored document's text, after the contact-detail scrub. The lane plans only cases whose newest snapshot is the Court's own docket JSON, so what it carries is supremecourt.gov content, on the footing the qp-topic extract is argued on: the plan refuses a -CourtListener REST snapshot, and so does `summarize` if one is staged. It +CourtListener REST snapshot, and the stage job re-checks what it staged and +removes, before the upload, any case whose tree is not exactly what +provisioning writes — the planned day's Court docket JSON, `context.json`, and +manifest-listed supremecourt.gov documents (`summary-stage-check`) — so a +snapshot that changed after the plan never crosses; `summarize` refuses one as +well. It widens that footing in one way the extract does not: the extract carries one section of each petition, while this carries every stored filing of each planned case. `case-summaries`, seven days, carries the generated summaries diff --git a/src/fedcourtsai/cli.py b/src/fedcourtsai/cli.py index f690d8fb7..05a4cbd25 100644 --- a/src/fedcourtsai/cli.py +++ b/src/fedcourtsai/cli.py @@ -10622,6 +10622,45 @@ def summarize_cmd( raise typer.Exit(code=1) +@app.command("summary-stage-check") +def summary_stage_check_cmd( + plan_file: Annotated[ + Path, typer.Option("--plan", help="The plan JSON `summarize-plan` wrote.") + ], + staged: Annotated[ + Path, + typer.Option(help="The data root `provision-snapshot` staged the planned cases under."), + ], + summary: Annotated[ + Path | None, + typer.Option(help="Append the one-line kept/removed count to this file (a job summary)."), + ] = None, +) -> None: + """Remove every staged record that may not leave the stage job, before it is uploaded. + + The staged tree crosses to the generate job as a run artifact that any + signed-in user can download while it exists, so it may carry only the + Court's own docket JSON and filings. Staging runs after the review hold and + provisions whatever snapshot is newest by then; this keeps a case only if it + was planned and its tree holds exactly what provisioning writes — the + planned day's snapshot in the Court's own shape, ``context.json``, and the + documents manifest with one text file per listed supremecourt.gov document — + and removes everything else under the stage root. Each removal is printed as + a ``::warning::``; the case is then reported skipped by ``summarize`` and + planned again by the next run. + """ + plan = SummaryPlan.model_validate_json(plan_file.read_text()) + removed = summaries.prune_stage(plan, staged) + for what, reason in removed: + typer.echo(f"::warning::removed {what} from the staged records: {reason}") + kept = sum(1 for p in (staged / "cases").glob("*/*") if p.is_dir()) + line = f"stage check: {kept} staged case record(s) kept, {len(removed)} removed" + typer.echo(line) + if summary is not None: + with summary.open("a", encoding="utf-8") as fh: + fh.write(line + "\n") + + @app.command("summary-paths") def summary_paths_cmd( name_status_file: Annotated[ diff --git a/src/fedcourtsai/summaries.py b/src/fedcourtsai/summaries.py index be75cfc8e..751debf4a 100644 --- a/src/fedcourtsai/summaries.py +++ b/src/fedcourtsai/summaries.py @@ -29,11 +29,13 @@ import hashlib import json import re +import shutil from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass, field from datetime import UTC, date, datetime from pathlib import Path from typing import Any +from urllib.parse import urlsplit import httpx import yaml @@ -321,6 +323,127 @@ class StagedDocument: stored_truncated: bool = False +def prune_stage(plan: SummaryPlan, stage_root: Path) -> list[tuple[str, str]]: + """Remove every staged record that may not leave the stage job. + + The staged tree crosses to the generate job as a run artifact, which on a + public repository any signed-in user can download while it exists, so it + may carry the Court's own docket JSON and filings and nothing else. The plan + screened each case's newest snapshot, but staging runs later — after the + review hold — and provisions whatever is newest then, so a CourtListener + REST snapshot stored in between would be staged. This re-applies the screen + to what was actually staged, on the side of the job boundary that holds the + corpus credentials, as an allowlist: a case is kept only if the plan names + it and its tree holds exactly what provisioning writes — the planned day's + snapshot in the Court's shape, ``context.json``, and the documents manifest + with one text file per listed document, each fetched from supremecourt.gov. + Everything else under the stage root is removed, and a symlink anywhere + removes the case rather than being followed. Returns ``(path, reason)`` for + each removal. + """ + planned = {(c.court_id, str(c.docket_id)): c for c in plan.cases} + removed: list[tuple[str, str]] = [] + if not stage_root.is_dir() or stage_root.is_symlink(): + return removed + cases_root = stage_root / "cases" + for entry in sorted(stage_root.iterdir()): + if entry != cases_root or entry.is_symlink() or not entry.is_dir(): + _remove(entry) + removed.append((entry.name, "not a case tree")) + if not cases_root.is_dir(): + return removed + for court_dir in sorted(cases_root.iterdir()): + if court_dir.is_symlink() or not court_dir.is_dir(): + _remove(court_dir) + removed.append((court_dir.name, "not a court directory")) + continue + for case_dir in sorted(court_dir.iterdir()): + label = f"{court_dir.name}/{case_dir.name}" + case = planned.get((court_dir.name, case_dir.name)) + reason = "not planned" if case is None else _stage_problem(case_dir, case) + if reason: + _remove(case_dir) + removed.append((label, reason)) + return removed + + +def _stage_problem(case_dir: Path, case: SummaryPlanCase) -> str: + """Why one planned case's staged tree may not cross the artifact, or ``""``.""" + if case_dir.is_symlink() or not case_dir.is_dir(): + return "not a case directory" + files: set[str] = set() + for path in case_dir.rglob("*"): + if path.is_symlink(): + return f"symlink {path.relative_to(case_dir)}" + if path.is_file(): + files.add(path.relative_to(case_dir).as_posix()) + snapshot = f"record/snapshots/{case.snapshot.isoformat()}.json" + problem = _snapshot_problem(case_dir, snapshot, files) + if problem: + return problem + documents, problem = _staged_documents(case_dir) + if problem: + return problem + extra = sorted(files - {snapshot, "record/context.json"} - documents) + return f"unexpected staged file {extra[0]}" if extra else "" + + +def _snapshot_problem(case_dir: Path, snapshot: str, files: set[str]) -> str: + """Whether the planned day's snapshot is staged, alone, in the Court's shape.""" + if snapshot not in files: + others = sorted(f for f in files if f.startswith("record/snapshots/")) + if others: + return f"staged snapshot {others[0]} is not the planned {Path(snapshot).stem}" + return "no staged snapshot" + try: + payload = json.loads((case_dir / snapshot).read_text()) + except (OSError, ValueError): + return "staged snapshot is unreadable" + if not isinstance(payload, dict) or LIVE_SHAPE_KEY not in payload: + return "staged snapshot is not the Court's own docket JSON" + return "" + + +def _staged_documents(case_dir: Path) -> tuple[set[str], str]: + """The document files the staged manifest accounts for, or why it cannot cross. + + Each listed document must have been fetched from supremecourt.gov (every + ``|``-joined part of its ``url``); a case with no manifest stages no documents. + """ + manifest = "record/documents/documents.json" + if not (case_dir / manifest).is_file(): + return set(), "" + try: + entries = json.loads((case_dir / manifest).read_text()) + except (OSError, ValueError): + return set(), "documents manifest is unreadable" + if not isinstance(entries, list): + return set(), "documents manifest is not a list" + allowed = {manifest} + for entry in entries: + kind = entry.get("kind") if isinstance(entry, dict) else None + if not isinstance(kind, str): + return set(), "documents manifest entry has no kind" + if not all(_is_court_url(url) for url in str(entry.get("url", "")).split("|")): + return set(), f"document {kind!r} was not fetched from supremecourt.gov" + allowed.add(f"record/documents/{kind}.txt") + return allowed, "" + + +def _is_court_url(url: str) -> bool: + host = (urlsplit(url.strip()).hostname or "").lower() + return urlsplit(url.strip()).scheme == "https" and ( + host == "supremecourt.gov" or host.endswith(".supremecourt.gov") + ) + + +def _remove(path: Path) -> None: + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink(missing_ok=True) + + def read_staged_record( stage_root: Path, court_id: str, docket_id: int, day: str ) -> tuple[dict[str, Any], list[StagedDocument]] | None: diff --git a/tests/test_summaries.py b/tests/test_summaries.py index cf5833c6b..f3be6ded4 100644 --- a/tests/test_summaries.py +++ b/tests/test_summaries.py @@ -191,7 +191,16 @@ def _stage(stage_root: Path, court: str = "scotus", docket: int = 1) -> None: paths.documents_dir.mkdir(parents=True) paths.document("petition").write_text("P" * 150_000) paths.documents_manifest.write_text( - json.dumps([{"kind": "petition", "entry_date": "Jan 02 2026", "truncated": False}]) + json.dumps( + [ + { + "kind": "petition", + "url": "https://www.supremecourt.gov/DocketPDF/26/26-1/1/petition.pdf", + "entry_date": "Jan 02 2026", + "truncated": False, + } + ] + ) ) @@ -572,3 +581,125 @@ def test_summary_paths_admits_only_summary_writes(tmp_path: Path) -> None: def test_the_paths_helper_and_the_jail_agree() -> None: path = CasePaths(Path("data"), "scotus", 9026000239).summary("2026-09-20") assert summaries.is_summary_path(path.as_posix()) + + +# --- the stage check ------------------------------------------------------------- + + +def test_stage_check_keeps_a_planned_case_staged_as_planned(tmp_path: Path) -> None: + stage = tmp_path / "stage" + _stage(stage) + plan = _plan(tmp_path / "data", {"scotus/1": _record()}) + assert summaries.prune_stage(plan, stage) == [] + assert CasePaths(stage, "scotus", 1).snapshot("2026-09-22").is_file() + + +def test_stage_check_removes_a_courtlistener_snapshot_staged_after_the_plan( + tmp_path: Path, +) -> None: + stage = tmp_path / "stage" + _stage(stage) + plan = _plan(tmp_path / "data", {"scotus/1": _record()}) + # The corpus moved after the plan: the newest snapshot is now a REST docket, + # on the same day (overwriting) or a later one (beside it). + paths = CasePaths(stage, "scotus", 1) + paths.snapshot("2026-09-22").write_text(json.dumps({"docket_entries": []})) + removed = summaries.prune_stage(plan, stage) + assert removed == [("scotus/1", "staged snapshot is not the Court's own docket JSON")] + assert not paths.base.exists() + + _stage(stage) + paths.snapshot("2026-09-23").write_text(json.dumps({"docket_entries": []})) + removed = summaries.prune_stage(plan, stage) + assert removed == [("scotus/1", "unexpected staged file record/snapshots/2026-09-23.json")] + assert not paths.base.exists() + + +def test_stage_check_removes_what_was_not_planned(tmp_path: Path) -> None: + stage = tmp_path / "stage" + _stage(stage) + _stage(stage, docket=2) + (stage / "stray.txt").write_text("x") + plan = _plan(tmp_path / "data", {"scotus/1": _record()}) + removed = summaries.prune_stage(plan, stage) + assert sorted(removed) == [("scotus/2", "not planned"), ("stray.txt", "not a case tree")] + assert CasePaths(stage, "scotus", 1).snapshot("2026-09-22").is_file() + assert not (stage / "stray.txt").exists() + + +def test_stage_check_removes_an_unreadable_snapshot(tmp_path: Path) -> None: + stage = tmp_path / "stage" + _stage(stage) + CasePaths(stage, "scotus", 1).snapshot("2026-09-22").write_text("{not json") + plan = _plan(tmp_path / "data", {"scotus/1": _record()}) + assert summaries.prune_stage(plan, stage) == [("scotus/1", "staged snapshot is unreadable")] + + +def test_stage_check_does_not_let_a_padded_docket_name_pass_as_a_planned_one( + tmp_path: Path, +) -> None: + stage = tmp_path / "stage" + _stage(stage) + padded = stage / "cases" / "scotus" / "01" / "record" / "snapshots" + padded.mkdir(parents=True) + (padded / "2026-09-22.json").write_text(json.dumps({"docket_entries": []})) + plan = _plan(tmp_path / "data", {"scotus/1": _record()}) + assert summaries.prune_stage(plan, stage) == [("scotus/01", "not planned")] + assert CasePaths(stage, "scotus", 1).snapshot("2026-09-22").is_file() + + +def test_stage_check_holds_a_kept_case_to_what_provisioning_writes(tmp_path: Path) -> None: + plan = _plan(tmp_path / "data", {"scotus/1": _record()}) + stage = tmp_path / "stage" + _stage(stage) + (CasePaths(stage, "scotus", 1).base / "extra.json").write_text("{}") + assert summaries.prune_stage(plan, stage) == [("scotus/1", "unexpected staged file extra.json")] + + _stage(stage) + paths = CasePaths(stage, "scotus", 1) + paths.cell_context.write_text("{}") # provisioning writes it; it is allowed + assert summaries.prune_stage(plan, stage) == [] + + manifest = json.loads(paths.documents_manifest.read_text()) + manifest[0]["url"] = "https://www.courtlistener.com/docket/1/" + paths.documents_manifest.write_text(json.dumps(manifest)) + assert summaries.prune_stage(plan, stage) == [ + ("scotus/1", "document 'petition' was not fetched from supremecourt.gov") + ] + + +def test_stage_check_removes_a_case_holding_a_symlink(tmp_path: Path) -> None: + outside = tmp_path / "outside.json" + outside.write_text("{}") + stage = tmp_path / "stage" + _stage(stage) + (CasePaths(stage, "scotus", 1).record / "link.json").symlink_to(outside) + plan = _plan(tmp_path / "data", {"scotus/1": _record()}) + assert summaries.prune_stage(plan, stage) == [("scotus/1", "symlink record/link.json")] + assert outside.is_file() + + +def test_summary_stage_check_command_warns_and_counts(tmp_path: Path) -> None: + stage = tmp_path / "stage" + _stage(stage) + _stage(stage, docket=2) + plan = _plan(tmp_path / "data", {"scotus/1": _record()}) + plan_file = tmp_path / "plan.json" + plan_file.write_text(plan.model_dump_json()) + summary = tmp_path / "summary.md" + result = CliRunner().invoke( + app, + [ + "summary-stage-check", + "--plan", + str(plan_file), + "--staged", + str(stage), + "--summary", + str(summary), + ], + ) + assert result.exit_code == 0, result.output + assert "::warning::removed scotus/2 from the staged records: not planned" in result.output + assert "stage check: 1 staged case record(s) kept, 1 removed" in result.output + assert summary.read_text() == "stage check: 1 staged case record(s) kept, 1 removed\n" diff --git a/tests/test_workflow_summarize.py b/tests/test_workflow_summarize.py index 9ee225e46..01ee9d7c2 100644 --- a/tests/test_workflow_summarize.py +++ b/tests/test_workflow_summarize.py @@ -147,3 +147,18 @@ def test_its_own_serializing_group_and_a_free_cron_minute() -> None: on = _load(other.name).get(True) or {} for entry in (on.get("schedule") or []) if isinstance(on, dict) else []: assert entry["cron"].split()[0] != minute, f"{other.name} shares minute {minute}" + + +def test_the_staged_records_are_checked_before_they_leave_the_stage_job() -> None: + steps = _jobs()["stage"]["steps"] + names = [str(step.get("name", "")) for step in steps] + stage = names.index("Stage each planned case's record") + check = names.index("Check the staged records before they leave this job") + upload = names.index("Upload the staged records") + assert stage < check < upload + assert "fedcourts summary-stage-check" in steps[check]["run"] + assert "set -euo pipefail" in steps[check]["run"] + assert "if" not in steps[check], "the check must be unconditional" + # A failed check must stop the upload: no step-level override lets it run anyway. + assert "if" not in steps[upload] + assert "continue-on-error" not in steps[check]