diff --git a/.github/workflows/current-site.yml b/.github/workflows/current-site.yml index a9a5b19a2..f11db0b71 100644 --- a/.github/workflows/current-site.yml +++ b/.github/workflows/current-site.yml @@ -12,6 +12,9 @@ name: Current research site - 'tools/current_site_history.py' - 'tools/current_site_publication.py' - 'tests/test_current_site*' + - 'tools/node_status_refresh.py' + - 'tests/test_node_status_refresh.py' + - 'tests/fixtures/node_status_refresh/**' - '.github/workflows/current-site.yml' schedule: - cron: '*/30 * * * *' @@ -41,6 +44,26 @@ jobs: python3 -B -O -S -m unittest discover -s tests -p 'test_current_site*.py' -v node --test tests/test_current_site.mjs + node-status-refresh: + # LANES-20261022 Lane 3: report-only freshness of records/node_status. + # Non-blocking: never gates build/deploy, writes nothing, freshness is not scientific validity. + runs-on: ubuntu-24.04 + timeout-minutes: 10 + continue-on-error: true + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Offline freshness tests + run: python3 -B -m unittest tests.test_node_status_refresh + - name: Report-only freshness refresh (read-only GitHub API) + if: github.event_name != 'pull_request' + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 -B -S tools/node_status_refresh.py records/node_status | tee -a "$GITHUB_STEP_SUMMARY" + build: needs: controls if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && github.repository == 'd6g8k5htny-coder/main' diff --git a/tests/fixtures/node_status_refresh/current_revs.json b/tests/fixtures/node_status_refresh/current_revs.json new file mode 100644 index 000000000..8bd3f5eab --- /dev/null +++ b/tests/fixtures/node_status_refresh/current_revs.json @@ -0,0 +1,7 @@ +{ + "https://github.com/d6g8k5htny-coder/main/blob/1111111111111111111111111111111111111111/a.md": "1111111111111111111111111111111111111111", + "https://github.com/d6g8k5htny-coder/main/blob/2222222222222222222222222222222222222222/b.md": "3333333333333333333333333333333333333333", + "https://github.com/d6g8k5htny-coder/main/issues/475": "2026-10-22T18:00:00Z", + "https://github.com/d6g8k5htny-coder/main/pull/490": "2026-10-22T12:00:00Z", + "https://github.com/d6g8k5htny-coder/Math-/pull/463#issuecomment-6198606495": "2026-10-15T17:50:29Z" +} diff --git a/tests/fixtures/node_status_refresh/fresh.json b/tests/fixtures/node_status_refresh/fresh.json new file mode 100644 index 000000000..dab562983 --- /dev/null +++ b/tests/fixtures/node_status_refresh/fresh.json @@ -0,0 +1,18 @@ +{ + "missing_input": [], + "next_action": "review", + "node_id": "fresh", + "observed_at": "2026-10-22T00:00:00Z", + "owner": null, + "schema_version": 0, + "scientific_effect": "NONE", + "source_rev": "1111111111111111111111111111111111111111", + "sources": [ + { + "rev": "1111111111111111111111111111111111111111", + "url": "https://github.com/d6g8k5htny-coder/main/blob/1111111111111111111111111111111111111111/a.md" + } + ], + "status": "fresh", + "stops": [] +} diff --git a/tests/fixtures/node_status_refresh/issue.json b/tests/fixtures/node_status_refresh/issue.json new file mode 100644 index 000000000..3a791bf5f --- /dev/null +++ b/tests/fixtures/node_status_refresh/issue.json @@ -0,0 +1,18 @@ +{ + "missing_input": [], + "next_action": "review", + "node_id": "issue", + "observed_at": "2026-10-22T00:00:00Z", + "owner": null, + "schema_version": 0, + "scientific_effect": "NONE", + "source_rev": "2026-10-22T17:00:00Z", + "sources": [ + { + "rev": "2026-10-22T17:00:00Z", + "url": "https://github.com/d6g8k5htny-coder/main/issues/475" + } + ], + "status": "fresh", + "stops": [] +} diff --git a/tests/fixtures/node_status_refresh/stale.json b/tests/fixtures/node_status_refresh/stale.json new file mode 100644 index 000000000..ad686c991 --- /dev/null +++ b/tests/fixtures/node_status_refresh/stale.json @@ -0,0 +1,18 @@ +{ + "missing_input": [], + "next_action": "review", + "node_id": "stale", + "observed_at": "2026-10-22T00:00:00Z", + "owner": null, + "schema_version": 0, + "scientific_effect": "NONE", + "source_rev": "2222222222222222222222222222222222222222", + "sources": [ + { + "rev": "2222222222222222222222222222222222222222", + "url": "https://github.com/d6g8k5htny-coder/main/blob/2222222222222222222222222222222222222222/b.md" + } + ], + "status": "fresh", + "stops": [] +} diff --git a/tests/fixtures/node_status_refresh/unknown.json b/tests/fixtures/node_status_refresh/unknown.json new file mode 100644 index 000000000..db3da1e6b --- /dev/null +++ b/tests/fixtures/node_status_refresh/unknown.json @@ -0,0 +1,18 @@ +{ + "missing_input": [], + "next_action": "review", + "node_id": "unknown", + "observed_at": "2026-10-22T00:00:00Z", + "owner": null, + "schema_version": 0, + "scientific_effect": "NONE", + "source_rev": "4444444444444444444444444444444444444444", + "sources": [ + { + "rev": "4444444444444444444444444444444444444444", + "url": "https://github.com/d6g8k5htny-coder/Math-/blob/4444444444444444444444444444444444444444/c.md" + } + ], + "status": "fresh", + "stops": [] +} diff --git a/tests/test_node_status_refresh.py b/tests/test_node_status_refresh.py new file mode 100644 index 000000000..4303fa615 --- /dev/null +++ b/tests/test_node_status_refresh.py @@ -0,0 +1,412 @@ +"""Offline tests for tools/node_status_refresh.py (LANES-20261022 Lane 3). No network.""" +import json +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) +import node_status_refresh as nsr # noqa: E402 + +FIX = ROOT / "tests" / "fixtures" / "node_status_refresh" +NOW = "2026-10-22T17:00:00Z" +REVS = json.loads((FIX / "current_revs.json").read_text()) + + +def offline_fetch(url): + if url not in REVS: + raise nsr.SourceError("unreachable (offline fixture)") + return REVS[url] + + +def load(name): + return json.loads((FIX / name).read_text()) + + +class RefreshTests(unittest.TestCase): + def setUp(self): + def no_net(*a, **k): + raise AssertionError("network used in tests") + self._orig = nsr._api_get + nsr._api_get = no_net + + def tearDown(self): + nsr._api_get = self._orig + + def test_unchanged_is_fresh(self): + out = nsr.refresh_record(load("fresh.json"), offline_fetch, NOW) + self.assertEqual(out["status"], "fresh") + self.assertEqual(out["source_rev"], "1111111111111111111111111111111111111111") + self.assertNotIn(",", out["source_rev"]) + self.assertEqual(out["observed_at"], NOW) + self.assertIsNone(out["refresh_error"]) + + def test_changed_rev_is_stale(self): + rec = load("stale.json") + out = nsr.refresh_record(rec, offline_fetch, NOW) + self.assertEqual(out["status"], "stale") + self.assertEqual(out["source_rev"], rec["sources"][0]["rev"]) + self.assertEqual(out["source_rev"], "2222222222222222222222222222222222222222") + self.assertNotIn("3333333333333333333333333333333333333333", json.dumps(out)) + + def test_unreachable_is_unknown_with_error(self): + rec = load("unknown.json") + out = nsr.refresh_record(rec, offline_fetch, NOW) + self.assertEqual(out["status"], "unknown") + self.assertIn("unreachable", out["refresh_error"]) + self.assertEqual(out["source_rev"], rec["source_rev"]) # not advanced on failure + + def test_failure_never_fresh_even_if_other_source_ok(self): + rec = load("fresh.json") + rec["sources"].append({"url": "https://github.com/example/repo/issues/9", "rev": "x"}) + self.assertEqual(nsr.refresh_record(rec, offline_fetch, NOW)["status"], "unknown") + + def test_no_sources_or_bad_schema_is_unknown(self): + rec = load("fresh.json") + rec["sources"] = [] + self.assertEqual(nsr.refresh_record(rec, offline_fetch, NOW)["status"], "unknown") + rec = load("fresh.json") + rec["schema_version"] = 1 + self.assertEqual(nsr.refresh_record(rec, offline_fetch, NOW)["status"], "unknown") + + def test_owner_and_scientific_effect_untouched(self): + rec = load("stale.json") + rec["owner"] = {"id": "agent", "evidence_url": "https://github.com/example/repo/issues/2#issuecomment-1"} + out = nsr.refresh_record(rec, offline_fetch, NOW) + self.assertEqual(out["owner"], rec["owner"]) + self.assertEqual(out["scientific_effect"], "NONE") + + def test_dir_report_only_does_not_write(self): + with tempfile.TemporaryDirectory() as tmp: + for n in ("fresh.json", "stale.json", "unknown.json"): + shutil.copy(FIX / n, tmp) + before = {p.name: p.read_bytes() for p in Path(tmp).glob("*.json")} + report = nsr.refresh_dir(Path(tmp), offline_fetch, write=False, now=NOW) + self.assertEqual({r["file"]: r["status"] for r in report}, + {"fresh.json": "fresh", "stale.json": "stale", "unknown.json": "unknown"}) + self.assertEqual(before, {p.name: p.read_bytes() for p in Path(tmp).glob("*.json")}) + + def test_dir_write_mode(self): + with tempfile.TemporaryDirectory() as tmp: + shutil.copy(FIX / "stale.json", tmp) + nsr.refresh_dir(Path(tmp), offline_fetch, write=True, now=NOW) + self.assertEqual(json.loads((Path(tmp) / "stale.json").read_text())["status"], "stale") + + def test_unsupported_url_raises_source_error(self): + with self.assertRaises(nsr.SourceError): + nsr.github_fetch("https://example.org/x") + + def test_scientific_effect_forced_none_not_passed_through(self): + rec = load("fresh.json") + rec["scientific_effect"] = "ACCEPTED" + out = nsr.refresh_record(rec, offline_fetch, NOW) + self.assertEqual(out["status"], "fresh") + self.assertEqual(out["scientific_effect"], "NONE") + self.assertNotEqual(out["scientific_effect"], "ACCEPTED") + + rec = load("unknown.json") + rec["scientific_effect"] = "ACCEPTED" + out = nsr.refresh_record(rec, offline_fetch, NOW) + self.assertEqual(out["status"], "unknown") + self.assertEqual(out["scientific_effect"], "NONE") + + def test_successful_refresh_clears_stale_refresh_error(self): + rec = load("fresh.json") + rec["refresh_error"] = "leftover from an earlier run" + out = nsr.refresh_record(rec, offline_fetch, NOW) + self.assertEqual(out["status"], "fresh") + self.assertIsNone(out["refresh_error"]) + + rec = load("stale.json") + rec["refresh_error"] = "leftover from an earlier run" + out = nsr.refresh_record(rec, offline_fetch, NOW) + self.assertEqual(out["status"], "stale") + self.assertIsNone(out["refresh_error"]) + + def test_empty_or_missing_source_revision_never_fresh(self): + rec = load("fresh.json") + + def empty_fetch(url): + return "" + + out = nsr.refresh_record(rec, empty_fetch, NOW) + self.assertEqual(out["status"], "unknown") + self.assertIn("refresh_error", out) + self.assertNotEqual(out["status"], "fresh") + + def missing_fetch(url): + return None + + out = nsr.refresh_record(load("fresh.json"), missing_fetch, NOW) + self.assertEqual(out["status"], "unknown") + self.assertNotEqual(out["status"], "fresh") + + missing = load("fresh.json") + del missing["sources"][0]["rev"] + out = nsr.refresh_record(missing, offline_fetch, NOW) + self.assertEqual(out["status"], "unknown") + self.assertIn("refresh_error", out) + self.assertNotEqual(out["status"], "fresh") + + blank = load("fresh.json") + blank["sources"][0]["rev"] = "" + out = nsr.refresh_record(blank, offline_fetch, NOW) + self.assertEqual(out["status"], "unknown") + self.assertNotEqual(out["status"], "fresh") + + def test_source_rev_is_primary_and_moving_ref_is_not_stored(self): + primary = "5" * 40 + other = "6" * 40 + moved = "7" * 40 + rec = load("fresh.json") + rec["source_rev"] = moved + rec["sources"] = [ + {"url": "https://github.com/d6g8k5htny-coder/main/blob/" + primary + "/old.md", "rev": primary}, + {"url": "https://github.com/d6g8k5htny-coder/Math-/tree/" + other + "/new", "rev": other}, + ] + + def fetch(url): + if url.endswith("/old.md"): + return primary + return moved + + out = nsr.refresh_record(rec, fetch, NOW) + self.assertEqual(out["status"], "stale") + self.assertEqual(out["source_rev"], primary) + self.assertEqual(out["sources"][0]["rev"], primary) + self.assertEqual(out["sources"][1]["rev"], other) + self.assertNotIn(moved, json.dumps(out)) + self.assertIsNone(out["refresh_error"]) + + def test_pinned_sha_accepted_and_other_rev_forms_unknown(self): + math_sha = "a" * 40 + rec = load("fresh.json") + rec["sources"] = [{ + "url": "https://github.com/d6g8k5htny-coder/Math-/tree/" + math_sha + "/frontiers/downstream_gate_20260925", + "rev": math_sha, + }] + out = nsr.refresh_record(rec, lambda url: math_sha, NOW) + self.assertEqual(out["status"], "fresh") + self.assertEqual(out["source_rev"], math_sha) + self.assertIsNone(out["refresh_error"]) + + other = load("fresh.json") + other["sources"][0]["url"] = "https://github.com/example/repo/blob/" + ("1" * 40) + "/a.md" + other["sources"][0]["rev"] = "1" * 40 + out = nsr.refresh_record(other, offline_fetch, NOW) + self.assertEqual(out["status"], "unknown") + self.assertIsInstance(out["refresh_error"], str) + + mismatch = load("fresh.json") + mismatch["sources"][0]["rev"] = "2" * 40 + out = nsr.refresh_record(mismatch, offline_fetch, NOW) + self.assertEqual(out["status"], "unknown") + self.assertIn("40-hex sha in its url", out["refresh_error"]) + + upper = load("fresh.json") + upper_sha = "A" * 40 + upper["sources"] = [{ + "url": "https://github.com/d6g8k5htny-coder/main/blob/" + upper_sha + "/a.md", + "rev": upper_sha, + }] + self.assertEqual(nsr.refresh_record(upper, offline_fetch, NOW)["status"], "unknown") + + def test_discussion_updated_at_newer_is_stale_and_unfetched_is_unknown(self): + issue = nsr.refresh_record(load("issue.json"), offline_fetch, NOW) + self.assertEqual(issue["status"], "stale") + self.assertEqual(issue["source_rev"], "2026-10-22T17:00:00Z") + self.assertNotIn("2026-10-22T18:00:00Z", json.dumps(issue)) + self.assertIsNone(issue["refresh_error"]) + + pull = load("fresh.json") + pull["sources"] = [{ + "url": "https://github.com/d6g8k5htny-coder/main/pull/490", + "rev": "2026-10-22T12:00:00Z", + }] + pull["source_rev"] = "9999999999999999999999999999999999999999" + same = nsr.refresh_record(pull, offline_fetch, NOW) + self.assertEqual(same["status"], "fresh") + self.assertEqual(same["source_rev"], "2026-10-22T12:00:00Z") + + comment_url = "https://github.com/d6g8k5htny-coder/Math-/pull/463#issuecomment-6198606495" + comment = load("fresh.json") + comment["sources"] = [{"url": comment_url, "rev": "2026-10-15T18:00:00Z"}] + older = nsr.refresh_record(comment, offline_fetch, NOW) + self.assertEqual(older["status"], "fresh") + self.assertEqual(older["source_rev"], "2026-10-15T18:00:00Z") + + missing = load("fresh.json") + missing["sources"] = [{ + "url": "https://github.com/d6g8k5htny-coder/main/issues/1", + "rev": "2026-10-22T17:00:00Z", + }] + missing["source_rev"] = "2026-10-22T17:00:00Z" + unknown = nsr.refresh_record(missing, offline_fetch, NOW) + self.assertEqual(unknown["status"], "unknown") + self.assertIsInstance(unknown["refresh_error"], str) + self.assertIn("unreachable", unknown["refresh_error"]) + self.assertEqual(unknown["source_rev"], "2026-10-22T17:00:00Z") + + bad_rev = load("issue.json") + bad_rev["sources"][0]["rev"] = "not-a-timestamp" + bad = nsr.refresh_record(bad_rev, offline_fetch, NOW) + self.assertEqual(bad["status"], "unknown") + self.assertIn("UTC Z", bad["refresh_error"]) + + def test_refresh_error_is_string_or_null(self): + fresh = nsr.refresh_record(load("fresh.json"), offline_fetch, NOW) + self.assertIn("refresh_error", fresh) + self.assertIsNone(fresh["refresh_error"]) + unknown = nsr.refresh_record(load("unknown.json"), offline_fetch, NOW) + self.assertIsInstance(unknown["refresh_error"], str) + self.assertTrue(unknown["refresh_error"]) + self.assertIn(type(fresh["refresh_error"]), (str, type(None))) + self.assertIn(type(unknown["refresh_error"]), (str, type(None))) + + def test_release_comment_sets_released_url_only(self): + pickup = "https://github.com/d6g8k5htny-coder/main/issues/475#issuecomment-6346922771" + release = "https://github.com/d6g8k5htny-coder/main/pull/495#issuecomment-6347046676" + body = "**RELEASE: nonauthor review of #495**\n\n- Releases PICKUP 6346922771.\n" + rec = load("fresh.json") + rec["owner"] = {"id": "agent-9", "evidence_url": pickup, "lane": "2"} + + def comments(url): + self.assertEqual(url, pickup) + return [ + {"url": "https://github.com/d6g8k5htny-coder/main/issues/475#issuecomment-6346922770", + "body": "**RELEASE: too early**\n\nPICKUP 6346922771\n"}, + {"url": "https://github.com/d6g8k5htny-coder/main/issues/475#issuecomment-6347000000", + "body": "**RELEASE: other**\n\nPICKUP 111\n"}, + {"url": release, "body": body}, + ] + + out = nsr.refresh_record(rec, offline_fetch, NOW, comments) + self.assertEqual(out["status"], "fresh") + self.assertEqual(out["owner"]["id"], "agent-9") + self.assertEqual(out["owner"]["evidence_url"], pickup) + self.assertEqual(out["owner"]["lane"], "2") + self.assertEqual(out["owner"]["released_url"], release) + self.assertEqual(set(out["owner"]), {"id", "evidence_url", "lane", "released_url"}) + + quiet = nsr.refresh_record(rec, offline_fetch, NOW, lambda url: []) + self.assertEqual(quiet["owner"], rec["owner"]) + self.assertNotIn("released_url", quiet["owner"]) + + def boom(url): + raise nsr.SourceError("comment listing incomplete") + + failed = nsr.refresh_record(rec, offline_fetch, NOW, boom) + self.assertEqual(failed["status"], "unknown") + self.assertIsInstance(failed["refresh_error"], str) + self.assertEqual(failed["owner"], rec["owner"]) + + def test_release_sets_released_url_only_for_matching_pickup_id(self): + pickup = "https://github.com/d6g8k5htny-coder/main/issues/475#issuecomment-6346922771" + other = "https://github.com/d6g8k5htny-coder/main/pull/495#issuecomment-6347042124" + match = "https://github.com/d6g8k5htny-coder/main/pull/495#issuecomment-6347046676" + rec = load("fresh.json") + rec["owner"] = {"id": "agent-9", "evidence_url": pickup} + + def refresh(body, url): + return nsr.refresh_record(rec, offline_fetch, NOW, lambda evidence: [{"url": url, "body": body}]) + + different = refresh("**RELEASE: other pickup**\n\n- Releases PICKUP 111.\n", other) + self.assertNotIn("released_url", different["owner"]) + self.assertEqual(different["owner"], rec["owner"]) + + unnamed = refresh("**RELEASE: no pickup id**\n\nThe review is finished.\n", other) + self.assertNotIn("released_url", unnamed["owner"]) + self.assertEqual(unnamed["owner"], rec["owner"]) + + matched = refresh("**RELEASE: this pickup**\n\n- Releases PICKUP 6346922771.\n", match) + self.assertEqual(matched["owner"]["released_url"], match) + self.assertEqual(matched["owner"]["id"], "agent-9") + self.assertEqual(matched["owner"]["evidence_url"], pickup) + + def test_report_only_includes_discovered_released_url(self): + pickup = "https://github.com/d6g8k5htny-coder/main/issues/475#issuecomment-6346922771" + release = "https://github.com/d6g8k5htny-coder/main/pull/495#issuecomment-6347046676" + body = "**RELEASE: nonauthor review**\n\n- Releases PICKUP 6346922771.\n" + rec = load("fresh.json") + rec["owner"] = {"id": "agent-9", "evidence_url": pickup} + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "fresh.json" + path.write_text(json.dumps(rec), encoding="utf-8") + before = path.read_bytes() + report = nsr.refresh_dir( + Path(tmp), offline_fetch, write=False, now=NOW, + comments=lambda url: [{"url": release, "body": body}]) + self.assertEqual(path.read_bytes(), before) + self.assertEqual(report[0]["status"], "fresh") + self.assertEqual(report[0].get("released_url"), release) + quiet = nsr.refresh_dir( + Path(tmp), offline_fetch, write=False, now=NOW, comments=lambda url: []) + self.assertNotIn("released_url", quiet[0]) + self.assertEqual(path.read_bytes(), before) + + def test_duplicate_json_key_is_unknown_and_not_rewritten(self): + text = json.dumps(load("fresh.json")) + owner = text.replace('"owner": null', '"owner": {"id": "unbacked"}', 1) + raw_owner = owner[:-1] + ',"owner": null}' + item = ('{"id": "hold-1", "text": "Unresolved synthetic item", "status": "open", ' + '"source_url": "https://github.com/example/repo/issues/1"}') + raw_carried = text[:-1] + ',"carried_forward": [' + item + '], "carried_forward": []}' + calls = [] + + def fetch(url): + calls.append(url) + return offline_fetch(url) + + for label, raw, key in ( + ("owner", raw_owner, "owner"), + ("carried_forward", raw_carried, "carried_forward"), + ): + with self.subTest(label=label): + calls.clear() + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "fresh.json" + path.write_text(raw, encoding="utf-8") + before = path.read_bytes() + for write in (False, True): + report = nsr.refresh_dir(Path(tmp), fetch, write=write, now=NOW) + self.assertEqual(len(report), 1) + self.assertEqual(report[0]["status"], "unknown") + self.assertIn("duplicate JSON key: " + key, report[0]["refresh_error"]) + self.assertEqual(path.read_bytes(), before) + self.assertEqual(calls, []) + + def test_github_fetch_uses_moving_ref_for_pinned_url(self): + seen = [] + + def fake(path): + seen.append(path) + if "/commits?" in path: + return [{"sha": "ab" * 20, "commit": {"committer": {"date": "2026-10-22T18:00:00+00:00"}}}] + return {"updated_at": "2026-10-22T17:30:25+00:00"} + + nsr._api_get = fake + pin = "1" * 40 + url = "https://github.com/d6g8k5htny-coder/Math-/blob/" + pin + "/frontiers/downstream_gate_20260925/GRAPH.json" + self.assertEqual(nsr.github_fetch(url), "ab" * 20) + self.assertEqual(len(seen), 1) + self.assertIn("/repos/d6g8k5htny-coder/Math-/commits?", seen[0]) + self.assertNotIn("sha=", seen[0]) + self.assertNotIn(pin, seen[0]) + self.assertEqual(nsr.github_fetch("https://github.com/d6g8k5htny-coder/main/issues/475"), "2026-10-22T17:30:25Z") + self.assertIn("/repos/d6g8k5htny-coder/main/issues/475", seen[-1]) + self.assertEqual(nsr.github_fetch("https://github.com/d6g8k5htny-coder/main/pull/495"), "2026-10-22T17:30:25Z") + self.assertIn("/repos/d6g8k5htny-coder/main/pulls/495", seen[-1]) + self.assertEqual( + nsr.github_fetch("https://github.com/d6g8k5htny-coder/main/pull/495#issuecomment-6347046676"), + "2026-10-22T17:30:25Z") + self.assertIn("/repos/d6g8k5htny-coder/main/issues/comments/6347046676", seen[-1]) + with self.assertRaises(nsr.SourceError): + nsr.github_fetch("https://example.org/x") + with self.assertRaises(nsr.SourceError): + nsr.github_fetch("https://github.com/example/repo/blob/" + pin + "/a.md") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/node_status_refresh.py b/tools/node_status_refresh.py new file mode 100644 index 000000000..4fa5945b1 --- /dev/null +++ b/tools/node_status_refresh.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python3 +"""Recompute freshness for node-status records (LANES-20261022 Lane 3). + +Settled rule: source_rev is sources[0].rev, the primary source. For a pinned +blob or tree on d6g8k5htny-coder/main or Math-, that rev is the 40-hex sha in +the URL. The moving-ref sha is never written into the record. If that moving +ref has moved past source_rev, status is stale. If it cannot be resolved, +status is unknown and refresh_error says why. + +#495 is adding the other three forms this tool already accepts: an updated_at +rev (UTC Z string) for an issue, pull, or comment URL on main or Math-; an +optional refresh_error (string or null); and an optional owner.released_url. +An issue, pull, or comment source is stale when the object's updated_at is +newer than its rev, and unknown when that updated_at cannot be fetched. + +Only status, observed_at, source_rev (set to the primary rev, not a fetched +tip), refresh_error, and that one owner field are written. status is freshness +only, never scientific validity. scientific_effect stays "NONE". Owner identity +is never inferred. + +Rules: + - no source moved past its recorded rev -> "fresh" + - any source moved past its recorded rev -> "stale" + - any source unreachable / not a valid rev -> "unknown" (never "fresh") +Precedence: unknown > stale > fresh. + +The default fetcher reads the public GitHub REST API (read-only, optional +GITHUB_TOKEN from the workflow). Tests inject an offline fetcher. +Report-only: without --write, records are not modified. +""" +from __future__ import annotations + +import argparse +import datetime as _dt +import json +import os +import re +import sys +import urllib.request +from pathlib import Path +from typing import Callable +from urllib.parse import quote + +SCHEMA_VERSION = 0 +STATUSES = ("fresh", "stale", "unknown") +API = "https://api.github.com" +_COMMENT_PAGE = 100 +_COMMENT_PAGE_LIMIT = 10 + +_SHA = re.compile(r"^[0-9a-f]{40}$") +_UTC = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") +_PINNED = re.compile( + r"^https://github\.com/d6g8k5htny-coder/(main|Math-)/(blob|tree)/" + r"([0-9a-f]{40})/([A-Za-z0-9_./-]+)$") +_DISCUSSION = re.compile( + r"^https://github\.com/d6g8k5htny-coder/(main|Math-)/(issues|pull)/" + r"([1-9][0-9]*)(?:#issuecomment-([1-9][0-9]*))?$") +_COMMENT = re.compile( + r"^https://github\.com/d6g8k5htny-coder/(main|Math-)/(issues|pull)/" + r"([1-9][0-9]*)#issuecomment-([1-9][0-9]*)$") +_RELEASE = re.compile(r"(?im)^\s*\**\s*RELEASE\b") + +Fetcher = Callable[[str], object] +Comments = Callable[[str], list] + + +class SourceError(Exception): + """A source could not be resolved to a current revision.""" + + +def _utc_now() -> str: + return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _as_z(value: object) -> str: + if not isinstance(value, str): + return "" + if value.endswith("+00:00"): + value = value[:-6] + "Z" + return value + + +def _api_get(path: str) -> object: + req = urllib.request.Request(API + path, headers={"Accept": "application/vnd.github+json", + "User-Agent": "node-status-refresh"}) + token = os.environ.get("GITHUB_TOKEN") + if token: + req.add_header("Authorization", "Bearer " + token) + try: + with urllib.request.urlopen(req, timeout=20) as resp: + return json.load(resp) + except Exception as exc: # network, HTTP, JSON + raise SourceError(f"{type(exc).__name__}: {exc}") from exc + + +def _classify(url: object) -> tuple[str, re.Match[str]] | tuple[None, None]: + if not isinstance(url, str): + return None, None + pinned = _PINNED.match(url) + if pinned: + return "sha", pinned + discussion = _DISCUSSION.fullmatch(url) + if discussion: + return "time", discussion + return None, None + + +def _recorded_ok(kind: str | None, match: re.Match[str] | None, recorded: object) -> bool: + if kind == "sha" and match is not None: + return isinstance(recorded, str) and bool(_SHA.fullmatch(recorded)) and recorded == match.group(3) + if kind == "time": + return isinstance(recorded, str) and bool(_UTC.fullmatch(recorded)) + return False + + +def github_fetch(url: str) -> str: + """Return the current moving-ref sha, or a discussion object's updated_at. + + Neither value is written into the record. Callers compare it with the + recorded rev and keep sources[0].rev as source_rev. + """ + kind, match = _classify(url) + if kind == "sha" and match is not None: + repo, path = match.group(1), match.group(4) + query = f"/repos/d6g8k5htny-coder/{repo}/commits?path={quote(path, safe='/')}&per_page=1" + commits = _api_get(query) + if not isinstance(commits, list) or not commits or not isinstance(commits[0], dict): + raise SourceError(f"no commits for path: {url}") + sha = commits[0].get("sha") + if not isinstance(sha, str) or not _SHA.fullmatch(sha): + raise SourceError(f"{url}: moving ref is not a 40-hex sha") + return sha + if kind == "time" and match is not None: + repo, disc_kind, number, comment_id = match.group(1), match.group(2), match.group(3), match.group(4) + if comment_id: + path = f"/repos/d6g8k5htny-coder/{repo}/issues/comments/{comment_id}" + elif disc_kind == "pull": + path = f"/repos/d6g8k5htny-coder/{repo}/pulls/{number}" + else: + path = f"/repos/d6g8k5htny-coder/{repo}/issues/{number}" + data = _api_get(path) + if not isinstance(data, dict): + raise SourceError(f"{url}: updated_at could not be read") + updated = _as_z(data.get("updated_at")) + if not _UTC.fullmatch(updated): + raise SourceError(f"{url}: updated_at is not a UTC Z timestamp") + return updated + raise SourceError(f"unsupported source url: {url}") + + +def github_comments(evidence_url: str) -> list[dict]: + """Read issue comments for a PICKUP URL. One short page means the read ended.""" + m = _COMMENT.fullmatch(evidence_url) + if not m: + raise SourceError(f"unsupported owner evidence url: {evidence_url}") + repo, number = m.group(1), m.group(3) + found: list[dict] = [] + page = 1 + while True: + batch = _api_get( + f"/repos/d6g8k5htny-coder/{repo}/issues/{number}/comments?per_page={_COMMENT_PAGE}&page={page}") + if not isinstance(batch, list): + raise SourceError("comment listing is not a list") + for item in batch: + if not isinstance(item, dict): + raise SourceError("comment is not an object") + found.append({"url": item.get("html_url"), "body": item.get("body") or ""}) + if len(batch) < _COMMENT_PAGE: + return found + page += 1 + if page > _COMMENT_PAGE_LIMIT: + raise SourceError("comment listing incomplete") + + +def _release_url(evidence_url: str, thread: object) -> str | None: + """URL of the latest RELEASE comment that names this PICKUP and follows it.""" + pickup = _COMMENT.fullmatch(evidence_url) + if not pickup or not isinstance(thread, list): + return None + pickup_id = int(pickup.group(4)) + best_id = pickup_id + best_url = None + for item in thread: + if not isinstance(item, dict): + continue + url = item.get("url") + body = item.get("body") if isinstance(item.get("body"), str) else "" + if not isinstance(url, str): + continue + comment = _COMMENT.fullmatch(url) + if not comment: + continue + comment_id = int(comment.group(4)) + if comment_id <= pickup_id or comment_id <= best_id: + continue + if not _RELEASE.search(body): + continue + if not re.search(rf"(?i)\bPICKUP\b[\s:#-]*{pickup_id}\b", body): + continue + best_id = comment_id + best_url = url + return best_url + + +def _current(fetched: object) -> object: + if isinstance(fetched, tuple) and fetched: + return fetched[0] + return fetched + + +def _apply_release(out: dict, record: dict, comments: Comments | None, errors: list[str]) -> None: + owner = record.get("owner") + if not isinstance(owner, dict): + return + evidence = owner.get("evidence_url") + if not isinstance(evidence, str) or not _COMMENT.fullmatch(evidence): + return + try: + thread = github_comments(evidence) if comments is None else comments(evidence) + except Exception as exc: # incomplete or failed read stays unknown + errors.append(f"{evidence}: {exc}") + return + if not isinstance(thread, list): + errors.append(f"{evidence}: comment listing is not a list") + return + released = _release_url(evidence, thread) + if not released: + return + updated = dict(owner) + updated["released_url"] = released + out["owner"] = updated + + +def _primary_rev(sources: object) -> str | None: + if not isinstance(sources, list) or not sources or not isinstance(sources[0], dict): + return None + kind, match = _classify(sources[0].get("url")) + recorded = sources[0].get("rev") + if _recorded_ok(kind, match, recorded) and isinstance(recorded, str): + return recorded + return None + + +def refresh_record(record: dict, fetch: Fetcher, now: str | None = None, + comments: Comments | None = None) -> dict: + """Return a new record with freshness recomputed. Never raises for source errors.""" + out = dict(record) + out.pop("refresh_error", None) + out["observed_at"] = now or _utc_now() + out["scientific_effect"] = "NONE" + errors: list[str] = [] + changed = False + sources = record.get("sources") + primary = _primary_rev(sources) + if record.get("schema_version") != SCHEMA_VERSION: + errors.append(f"unsupported schema_version: {record.get('schema_version')!r}") + if not isinstance(sources, list) or not sources: + errors.append("no sources to check") + sources = [] + for src in sources: + url = src.get("url") if isinstance(src, dict) else None + kind, match = _classify(url) + recorded = src.get("rev") if isinstance(src, dict) else None + if kind is None: + errors.append(f"{url}: source url is not a pinned blob or tree, or a main/Math- issue, pull, or comment") + continue + if not isinstance(recorded, str) or not recorded: + errors.append(f"{url}: empty or missing source revision") + continue + if not _recorded_ok(kind, match, recorded): + if kind == "sha": + errors.append(f"{url}: source revision must be the 40-hex sha in its url") + else: + errors.append(f"{url}: source revision must be a UTC Z timestamp") + continue + try: + fetched = fetch(url) + except Exception as exc: # any fetch failure is unknown, never fresh + errors.append(f"{url}: {exc}") + continue + current = _current(fetched) + if kind == "sha": + if not isinstance(current, str) or not _SHA.fullmatch(current): + errors.append(f"{url}: moving ref is not a 40-hex sha") + continue + if current != recorded: + changed = True + else: + current_z = _as_z(current) + if not _UTC.fullmatch(current_z): + errors.append(f"{url}: updated_at is not a UTC Z timestamp") + continue + if current_z > recorded: + changed = True + _apply_release(out, record, comments, errors) + if primary is not None: + out["source_rev"] = primary + if errors: + out["status"] = "unknown" + out["refresh_error"] = "; ".join(errors) + else: + out["status"] = "stale" if changed else "fresh" + out["refresh_error"] = None + return out + + +def _unique_pairs(pairs): + """Reject duplicate JSON names before observing or rewriting a record.""" + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def refresh_dir(directory: Path, fetch: Fetcher, write: bool, now: str | None = None, + comments: Comments | None = None) -> list[dict]: + report = [] + for path in sorted(directory.glob("*.json")): + try: + record = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=_unique_pairs) + if not isinstance(record, dict): + raise ValueError("record is not an object") + except Exception as exc: + report.append({"file": path.name, "status": "unknown", "refresh_error": f"unreadable: {exc}"}) + continue + new = refresh_record(record, fetch, now, comments) + row = {"file": path.name, "node_id": new.get("node_id"), "status": new["status"], + "previous_status": record.get("status"), + "refresh_error": new.get("refresh_error")} + owner = new.get("owner") + if isinstance(owner, dict) and isinstance(owner.get("released_url"), str) and owner["released_url"]: + row["released_url"] = owner["released_url"] + report.append(row) + if write: + path.write_text(json.dumps(new, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("records", nargs="?", default="records/node_status") + ap.add_argument("--write", action="store_true", help="rewrite records (default: report only)") + args = ap.parse_args(argv) + directory = Path(args.records) + if not directory.is_dir(): + print(json.dumps({"records": str(directory), "report": [], "note": "no records directory; nothing to refresh", + "scientific_effect": "NONE"}, indent=2)) + return 0 + report = refresh_dir(directory, github_fetch, args.write) + print(json.dumps({"records": str(directory), "report": report, "scientific_effect": "NONE"}, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main())