answers-recheck: fresh live samples re-checked weekly, drifted answers held back from the pack

.github/workflows/ci.yml

@@ -73,14 +73,46 @@ jobs:
       - run: pip install -r service/requirements-test.txt
       - run: python -m pytest service/tests tests/test_ingest.py -q
   revalidate:
-    name: weekly re-validation (report only)
+    name: weekly re-validation + answers re-check
     if: github.event_name == 'schedule'
     runs-on: ubuntu-latest
+    permissions:
+      contents: write   # pushes the rechecked answers files to a recheck/<date> branch; the PR is opened by hand
     steps:
       - uses: actions/checkout@v4
+      - name: the app's parsers (csv / feed / xml / text samples are parsed exactly as its cards parse them)
+        uses: actions/checkout@v4
+        with:
+          repository: SecureCloudGroup/SmartBrain_3000
+          path: .app
+          sparse-checkout: app/smartbrain_3000
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
-      - run: pip install "httpx==0.28.1" "duckdb==1.5.4"
-      - run: python -m sourcetool validate --only curated
-      - run: git diff --stat
+      - run: pip install "httpx==0.28.1" "duckdb==1.5.4" cryptography
+      - name: re-validate curated records (report only)
+        run: python -m sourcetool validate --only curated || true
+      - name: re-check every answers file against a fresh live sample
+        id: recheck
+        env:
+          SMARTBRAIN_APP: ${{ github.workspace }}/.app/app
+        run: |
+          set +e
+          python -m sourcetool answers-recheck | tee recheck.log
+          echo "rc=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
+      - name: push the changed answers files for a PR
+        if: always()
+        run: |
+          rm -rf .app
+          git add -A answers sources
+          if git diff --cached --quiet; then echo "nothing changed"; exit 0; fi
+          BR="recheck/$(date -u +%F)"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git commit -q -m "answers-recheck $(date -u +%F): $(tail -1 recheck.log)"
+          git push -f origin "HEAD:refs/heads/$BR"
+          { echo "## answers-recheck"; echo; echo '```'; cat recheck.log; echo '```'; echo;
+            echo "Branch \`$BR\` pushed — open the PR: https://github.com/${GITHUB_REPOSITORY}/compare/main...$BR"; } >> "$GITHUB_STEP_SUMMARY"
+      - name: fail the run when any file failed or drifted
+        if: steps.recheck.outputs.rc != '0'
+        run: exit 1

answers/bbc-world.json

@@ -40,5 +40,5 @@
   }
  ],
  "sample_url": "https://feeds.bbci.co.uk/news/world/rss.xml",
- "checked": "2026-09-28"
+ "checked": "2026-10-06"
 }

answers/fred-apu0000702111.json

@@ -69,5 +69,5 @@
   }
  ],
  "sample_url": "https://fred.stlouisfed.org/graph/fredgraph.csv?id=APU0000702111",
- "checked": "2026-09-28"
+ "checked": "2026-10-06"
 }

answers/npr-news.json

@@ -30,5 +30,5 @@
   }
  ],
  "sample_url": "https://feeds.npr.org/1001/rss.xml",
- "checked": "2026-09-28"
+ "checked": "2026-10-06"
 }

answers/nws-alerts-point.json

@@ -67,5 +67,5 @@
     }
   ],
   "sample_url": "https://api.weather.gov/alerts/active?point=63.45,-145.75",
-  "checked": "2026-09-29"
+  "checked": "2026-10-06"
 }

answers/open-meteo-forecast.json

@@ -339,5 +339,5 @@
   }
  ],
  "sample_url": "https://api.open-meteo.com/v1/forecast?latitude=32.7765&longitude=-79.9311&current=temperature_2m,apparent_temperature,relative_humidity_2m,weather_code,wind_speed_10m&hourly=temperature_2m,precipitation_probability,weather_code&daily=weather_code,temperature_2m_max,temperature_2m_min,precipitation_probability_max,snowfall_sum,sunrise,sunset&temperature_unit=fahrenheit&wind_speed_unit=mph&precipitation_unit=inch&timezone=auto",
- "checked": "2026-09-29"
+ "checked": "2026-10-06"
 }

docs/SCHEMA.md

@@ -26,7 +26,7 @@ Each source is one JSON object per line in `sources/**/*.jsonl`. Users' suggesti
 | `votes` | `{yes, no}`, aggregated from users' Yes/No answers through the Library API. |
 | `signals` | Harvest evidence: monthly views, last data update, column names. |
 | `origin` | Who or what created the record, and when. |
-| `answers` | Curated sources: which paths of the response answer which questions, so the app builds cards without guessing paths. Authored in `answers/<id>.json` (`{source_id, answers, sample_url, checked}`) and merged into the record by `sourcetool build`. A source has 1 to 12 answers, each `{name, label, words, primary?, kind: value\|list\|columns, ...}`; a path segment may be a whole `{param}` of the record, a key the plain names can't spell is quoted (`description["#cdata-section"]`), and a list may `filter` its rows by one or by a short fixed value the response uses (`"Final"`). A `time` whose values carry no zone is shown as the source's local clock; `"utc": true` says those zoneless values are UTC. Spec v1.2 adds what an answer delivers (below). The closed keys, types and limits are in `sourcetool/answers.py` (`answer_problems`), and every path must resolve in a real sample (`check_sample`, run live by `sourcetool answers-check`): the first row of a list in full, and in every later row and every column element a present cell must hold its declared type while a cell the row lacks is allowed (the card shows "—" for it). |
+| `answers` | Curated sources: which paths of the response answer which questions, so the app builds cards without guessing paths. Authored in `answers/<id>.json` (`{source_id, answers, sample_url, checked}`, plus `status` and `last_failure` written only by `sourcetool answers-recheck`: a file whose fresh live sample fails the check on two later days is `"drifted"`, `build` holds it back and the app offers the source as a link until a recheck passes and drops the keys again) and merged into the record by `sourcetool build`. A source has 1 to 12 answers, each `{name, label, words, primary?, kind: value\|list\|columns, ...}`; a path segment may be a whole `{param}` of the record, a key the plain names can't spell is quoted (`description["#cdata-section"]`), and a list may `filter` its rows by one or by a short fixed value the response uses (`"Final"`). A `time` whose values carry no zone is shown as the source's local clock; `"utc": true` says those zoneless values are UTC. Spec v1.2 adds what an answer delivers (below). The closed keys, types and limits are in `sourcetool/answers.py` (`answer_problems`), and every path must resolve in a real sample (`check_sample`, run live by `sourcetool answers-check`): the first row of a list in full, and in every later row and every column element a present cell must hold its declared type while a cell the row lacks is allowed (the card shows "—" for it). |
 
 ### Answers spec v1.2: what an answer delivers
 

sourcetool/__main__.py

@@ -18,6 +18,8 @@
   python -m sourcetool answers-check [ID ...]                           fetch samples, check answers/ paths (live)
   python -m sourcetool answers-check --lint                             record + keyword lints (offline, no fetch)
   python -m sourcetool answers-generate                                 write answers for curated feeds + FRED series
+  python -m sourcetool answers-recheck [--since DAYS] [ID ...]          re-fetch live samples; PASS stamps checked,
+                                                                        two failures on later days mark the file drifted
   python -m sourcetool asks-overlap --against FILE [FILE ...]           report asks that overlap any labeled set
                                                                         (exact + jaccard>=0.75); exit nonzero if any
 
@@ -210,6 +212,11 @@ def cmd_answers_generate(args) -> int:
     return answers.cmd_generate(args)
 
 
+def cmd_answers_recheck(args) -> int:
+    from . import recheck
+    return recheck.main(args)
+
+
 def cmd_asks_overlap(args) -> int:
     """`asks-overlap --against FILE [FILE ...]`: report asks that overlap labeled eval sets outside
     the repo. Prints a per-file breakdown (exact / near) and returns 1 if any overlap is found."""
@@ -236,7 +243,7 @@ def cmd_asks_overlap(args) -> int:
 def main(argv: list[str]) -> int:
     if not argv or argv[0] not in {"check", "validate", "harvest", "build", "lookup", "coverage", "resolvers",
                                    "policies", "fills", "evalres", "evallookup", "ingest", "answers-check",
-                                   "answers-generate", "asks-overlap"}:
+                                   "answers-generate", "answers-recheck", "asks-overlap"}:
         print(__doc__)
         return 2
     return globals()["cmd_" + argv[0].replace("-", "_")](argv[1:])

sourcetool/answers.py

@@ -48,6 +48,10 @@
 ROW_KEYS = {"path", "label", "type", "unit", "unit_path", "utc", "tbd_if"}  # utc: zoneless times are UTC
 COLUMN_KEYS = (ROW_KEYS - {"tbd_if"}) | {"codes"}
 FILE_KEYS = {"source_id", "answers", "sample_url", "checked"}
+# spec v1.3 (2026-10-06): written by `answers-recheck`, never by hand — a file whose live sample failed the check
+# twice on later days is "drifted": build holds it back and the app offers the source as a link until it passes again
+FILE_OPTIONAL_KEYS = {"status", "last_failure"}
+FILE_STATUSES = ("drifted",)
 MAX_ANSWERS = 12
 
 _NAME = re.compile(r"[a-z][a-z0-9_]{0,39}")
@@ -585,10 +589,25 @@ def params_of(rec: dict) -> dict:
     return {q["name"]: q.get("example") for q in rec["access"].get("params", [])}
 
 
-def load_answers(params_by_id: dict[str, dict], directory: Path = ANSWERS) -> tuple[dict[str, list], list[str]]:
+def held_back(directory: Path = ANSWERS) -> list[str]:
+    """The source ids whose answers file is marked drifted (served as links until a recheck passes)."""
+    out = []
+    for f in sorted(directory.glob("*.json")):
+        try:
+            d = json.loads(f.read_text())
+        except ValueError:
+            continue
+        if isinstance(d, dict) and d.get("status") == "drifted":
+            out.append(f.stem)
+    return out
+
+
+def load_answers(params_by_id: dict[str, dict], directory: Path = ANSWERS,
+                 include_drifted: bool = False) -> tuple[dict[str, list], list[str]]:
     """Read every answers/<id>.json: ({source_id: answers}, problems). `params_by_id` maps each record id
     to its params (`params_of`). A file whose id has no record, or whose answers fail the schema, is a
-    problem (build refuses it)."""
+    problem (build refuses it). A file marked drifted is validated but left out unless `include_drifted`
+    (the live check and the recheck still want it; build and the lints do not)."""
     out: dict[str, list] = {}
     errs: list[str] = []
     for f in sorted(directory.glob("*.json")):
@@ -597,8 +616,8 @@ def load_answers(params_by_id: dict[str, dict], directory: Path = ANSWERS) -> tu
         except ValueError as e:
             errs.append(f"{f.name}: not JSON ({e})")
             continue
-        if not isinstance(d, dict) or set(d) != FILE_KEYS:
-            errs.append(f"{f.name}: must hold exactly {sorted(FILE_KEYS)}")
+        if not isinstance(d, dict) or not FILE_KEYS <= set(d) or not set(d) <= FILE_KEYS | FILE_OPTIONAL_KEYS:
+            errs.append(f"{f.name}: must hold exactly {sorted(FILE_KEYS)} (plus optional {sorted(FILE_OPTIONAL_KEYS)})")
             continue
         sid = d["source_id"]
         if sid != f.stem:
@@ -612,8 +631,12 @@ def load_answers(params_by_id: dict[str, dict], directory: Path = ANSWERS) -> tu
             bad.append("sample_url must be the https URL fetched")
         if not isinstance(d["checked"], str) or not _DATE.fullmatch(d["checked"]):
             bad.append("checked must be YYYY-MM-DD")
+        if "status" in d and d["status"] not in FILE_STATUSES:
+            bad.append(f"status must be one of {'/'.join(FILE_STATUSES)}")
+        if "last_failure" in d and not (isinstance(d["last_failure"], str) and _DATE.fullmatch(d["last_failure"])):
+            bad.append("last_failure must be YYYY-MM-DD")
         errs += [f"{f.name}: {e}" for e in bad]
-        if not bad:
+        if not bad and (include_drifted or d.get("status") != "drifted"):
             out[sid] = d["answers"]
     return out, errs
 
@@ -627,7 +650,10 @@ def _records(tier: str | None = None) -> dict[str, dict]:
 # --- live samples ----------------------------------------------------------------------------------
 
 class Skip(Exception):
-    """The source cannot be sampled without a user's own value or key."""
+    """The source cannot be sampled HERE: it needs a user's own value or key, or (csv / feed / xml / text) the
+    app's parsers from a SmartBrain_3000 checkout this machine lacks. A skip is never the source's fault, so
+    `answers-recheck` leaves the file alone (Phase 2a, 2026-10-06: before this a missing checkout read as FAIL,
+    and the weekly job would have drifted every non-JSON source within two weeks)."""
 
 
 class Failed(Exception):
@@ -645,8 +671,8 @@ def _formats():
     try:
         from smartbrain_3000 import formats
     except ImportError as e:
-        raise Failed(f"the app's parsers are missing ({APP}/smartbrain_3000/formats.py: {e}); "
-                     "csv/feed/xml/text samples need a SmartBrain_3000 checkout") from None
+        raise Skip(f"the app's parsers are missing ({APP}/smartbrain_3000/formats.py: {e}); "
+                   "csv/feed/xml/text samples need a SmartBrain_3000 checkout") from None
     return formats
 
 
@@ -751,7 +777,7 @@ def cmd_check(ids: list[str]) -> int:
     if "--lint" in ids:  # offline: the lints only, no sample is fetched
         return cmd_lint()
     recs = _records()
-    loaded, errs = load_answers({k: params_of(r) for k, r in recs.items()})
+    loaded, errs = load_answers({k: params_of(r) for k, r in recs.items()}, include_drifted=True)
     for e in errs:
         print(f"INVALID {e}")
     todo = ids or sorted(loaded)

sourcetool/build.py

@@ -159,10 +159,13 @@ def pack_policy(sub: dict) -> dict:
 
 def merge_answers(recs: list[dict], directory=None) -> int:
     """Put each answers/<id>.json file's answers on its record; refuse orphans and malformed answers."""
-    from .answers import ANSWERS, load_answers, params_of
+    from .answers import ANSWERS, held_back, load_answers, params_of
     loaded, errs = load_answers({r["id"]: params_of(r) for r in recs}, directory or ANSWERS)
     if errs:
         raise SystemExit("answers files refused:\n  " + "\n  ".join(errs))
+    held = held_back(directory or ANSWERS)
+    if held:  # drifted since their last pass: the app offers these sources as links until a recheck passes
+        print(f"{len(held)} answers files held back as drifted: {', '.join(held[:12])}{' …' if len(held) > 12 else ''}")
     for r in recs:
         if r["id"] in loaded:
             r["answers"] = loaded[r["id"]]