From a2d9e45d3044f396a2b05c3a69631e73db2a705b Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Fri, 18 Sep 2026 03:13:13 +0300 Subject: [PATCH 1/2] =?UTF-8?q?manual-generate-release:=20=D7=A0=D7=9B?= =?UTF-8?q?=D7=A1=20library=5Fstats.json=20(=D7=A1=D7=A4=D7=A8=D7=99=D7=9D?= =?UTF-8?q?/=D7=A7=D7=99=D7=A9=D7=95=D7=A8=D7=99=D7=9D/=D7=A9=D7=95=D7=A8?= =?UTF-8?q?=D7=95=D7=AA)=20=D7=91=D7=9B=D7=9C=20=D7=A8=D7=9C=D7=99=D7=A1?= =?UTF-8?q?=20DB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit צעד חדש "Compute library stats (advisory)" אחרי דחיסת ה-DB: מריץ שאילתת json_object על build/seforim.db ‏(sqlite3 -init /dev/null -readonly) וכותב build/library_stats.json בשורה אחת. "Stage release assets" מעתיק אותו ל-release-staging לפני חישוב build_provenance.json, כך שהוא נכס רגיל: רשום עם sha256 ב-provenance, עולה באותה לולאת draft → verify → publish, ובדיקת ה-reuse ‏(provenance assets + 1) ממשיכה להתקיים. הצעד מייעץ בלבד: sqlite3 חסר, שאילתה שנכשלה, תוצאה בצורה לא מדויקת או db_version שאינו של הבנייה הזו — כולם ::warning:: ו-exit 0 בלי להשאיר קובץ, והרליס יוצא בדיוק כמו קודם. טסט חוזה חדש ב-test_manual_release_workflow. Co-Authored-By: Claude Opus 5 --- .../scripts/test_manual_release_workflow.py | 75 +++++++++++++++++++ .github/workflows/manual-generate-release.yml | 58 ++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/.github/scripts/test_manual_release_workflow.py b/.github/scripts/test_manual_release_workflow.py index d6183d47..fefae9cf 100644 --- a/.github/scripts/test_manual_release_workflow.py +++ b/.github/scripts/test_manual_release_workflow.py @@ -2270,6 +2270,81 @@ def test_reuse_scan_summarises_instead_of_one_ok_line_per_release(self): self.assertIn("validate_build_provenance.py \"$STAGE/build_provenance.json\"", stage) self.assertNotIn("--quiet", stage) + def test_library_stats_are_an_advisory_staged_asset(self): + # library_stats.json is display-only (the website's stats banner), so it + # must never cost the release — and it must still be a STAGED asset: + # verify_remote demands remote == staged, and the reuse scan counts + # `provenance assets + 1`, so an asset uploaded after publish breaks both. + stats = self.step("Compute library stats (advisory)") + stage = self.step("Stage release assets") + + # Reads build/seforim.db, which the compress step keeps (-o, no --rm), + # and must finish before staging hashes the directory. + self.assertLess( + self.workflow.index(" - name: Compress Seforim Database (zstd)\n"), + self.workflow.index(" - name: Compute library stats (advisory)\n"), + ) + self.assertLess( + self.workflow.index(" - name: Compute library stats (advisory)\n"), + self.workflow.index(" - name: Stage release assets\n"), + ) + self.assertIn("EXPECTED_DB_VERSION: ${{ steps.discover.outputs.db_version }}", stats) + + # Advisory: GitHub runs the step with `bash -e`, so the script itself + # never adds -e and every command that can fail is guarded by + # `|| skip` — each failure path is skip() → warning + exit 0, and skip() + # leaves no partial file behind for staging to pick up. + self.assertIn("set -uo pipefail", stats) + self.assertNotIn("set -euo pipefail", stats) + self.assertNotIn("exit 1", stats) + self.assertNotIn("::error::", stats) + self.assertIn('rm -f "$OUT" "$OUT.tmp"\n', stats) + self.assertIn('echo "::warning::library stats: $1', stats) + # A ~/.sqliterc on the runner must not change the CLI's output format. + self.assertIn("sqlite3 -init /dev/null -readonly -bail build/seforim.db", stats) + self.assertIn("exit 0", stats) + + # Staged conditionally, and BEFORE the provenance hashes the stage. + copy = 'if [ -s build/library_stats.json ]; then\n cp build/library_stats.json "$STAGE/"' + self.assertIn(copy, stage) + self.assertLess(stage.index(copy), stage.index('python3 - "$STAGE"')) + + # The query produces the exact published bytes, and the step's own + # validator accepts them — and rejects a foreign db_version. + query = re.search(r'build/seforim\.db "(select json_object\(.*?\);)"', stats).group(1) + validator = textwrap.dedent( + stats.split("<<'PY' || skip", 1)[1].split("\n", 1)[1].split("\n PY\n", 1)[0] + ) + import sqlite3 # noqa: PLC0415 - stdlib, only this test needs it + + with tempfile.TemporaryDirectory() as tmp: + db = Path(tmp) / "seforim.db" + with sqlite3.connect(db) as conn: + conn.executescript( + "create table schema_meta(key text primary key, value text);" + "insert into schema_meta values ('db_version', '28');" + "create table book(id); insert into book values (1), (2);" + "create table link(id); insert into link values (1);" + "create table line(id); insert into line values (1), (2), (3);" + ) + row = conn.execute(query).fetchone()[0] + conn.close() + out = Path(tmp) / "library_stats.json" + out.write_bytes(row.encode() + b"\n") + self.assertEqual( + out.read_bytes(), + b'{"schema_version":1,"db_version":28,"books":2,"links":1,"lines":3}\n', + ) + script = Path(tmp) / "validate.py" + script.write_text(validator, encoding="utf-8") + ok = subprocess.run([sys.executable, str(script), str(out), "28"]) + self.assertEqual(ok.returncode, 0) + wrong = subprocess.run([sys.executable, str(script), str(out), "29"]) + self.assertNotEqual(wrong.returncode, 0) + out.write_bytes(row.encode() + b"\r\n") + crlf = subprocess.run([sys.executable, str(script), str(out), "28"]) + self.assertNotEqual(crlf.returncode, 0) + def test_build_provenance_quiet_only_silences_the_positive_line(self): sys.path.insert(0, str(Path(__file__).parent)) import test_build_provenance # noqa: PLC0415 - sibling fixture, not a package diff --git a/.github/workflows/manual-generate-release.yml b/.github/workflows/manual-generate-release.yml index ef4e3e94..395467ff 100644 --- a/.github/workflows/manual-generate-release.yml +++ b/.github/workflows/manual-generate-release.yml @@ -2108,6 +2108,59 @@ jobs: . .pipeline-control/.github/scripts/zstd_workers.sh zstd -T"$(zstd_workers)" -19 -f -o build/seforim.db.zst build/seforim.db + # ─── Library stats (advisory) ────────────────────────────────────────── + # One line of JSON — {"schema_version":1,"db_version":N,"books":…, + # "links":…,"lines":…} — so a consumer (the website's stats banner) can + # show the size of the library without downloading the 1.4 GB DB. + # + # It is written to build/ and copied by "Stage release assets" BEFORE the + # provenance is computed, so it is an ordinary staged asset: listed with + # its sha256 in build_provenance.json, uploaded by the same + # draft → verify → publish loop (verify_remote demands remote == staged), + # and the reuse scan's `provenance assets + 1` count still holds. An asset + # uploaded to the release afterwards would break both of those. + # + # Advisory by design: the numbers are display-only, so no failure here may + # cost the weekly release. sqlite3 missing, a failing query, or a result + # that does not have the exact expected shape (or names a db_version other + # than this build's) each emit a ::warning:: and leave NO file behind — the + # release then ships exactly the asset set it shipped before this step. + - name: Compute library stats (advisory) + env: + EXPECTED_DB_VERSION: ${{ steps.discover.outputs.db_version }} + run: | + set -uo pipefail + OUT=build/library_stats.json + rm -f "$OUT" "$OUT.tmp" + skip() { + echo "::warning::library stats: $1 — library_stats.json is not published on this release" + rm -f "$OUT" "$OUT.tmp" + exit 0 + } + command -v sqlite3 >/dev/null 2>&1 || skip "sqlite3 not found" + [ -s build/seforim.db ] || skip "build/seforim.db is missing" + sqlite3 -init /dev/null -readonly -bail build/seforim.db "select json_object('schema_version',1,'db_version',(select cast(value as integer) from schema_meta where key='db_version'),'books',(select count(*) from book),'links',(select count(*) from link),'lines',(select count(*) from line));" \ + > "$OUT.tmp" || skip "the stats query failed" + [[ "$EXPECTED_DB_VERSION" =~ ^[1-9][0-9]*$ ]] || skip "no db_version for this build" + # Exact shape AND exact bytes: key order, integer counts > 0, compact + # separators, one trailing LF — what sqlite3's json_object prints. + python3 - "$OUT.tmp" "$EXPECTED_DB_VERSION" <<'PY' || skip "unexpected result $(head -c 300 "$OUT.tmp")" + import json, sys + raw = open(sys.argv[1], "rb").read() + value = json.loads(raw) + ok = ( + isinstance(value, dict) + and list(value) == ["schema_version", "db_version", "books", "links", "lines"] + and value["schema_version"] == 1 + and type(value["db_version"]) is int and value["db_version"] == int(sys.argv[2]) + and all(type(value[k]) is int and value[k] > 0 for k in ("books", "links", "lines")) + and raw == json.dumps(value, separators=(",", ":")).encode() + b"\n" + ) + sys.exit(0 if ok else 1) + PY + mv "$OUT.tmp" "$OUT" || skip "could not write $OUT" + echo "library_stats.json: $(cat "$OUT")" + # ─── Stage release assets ────────────────────────────────────────────── - name: Stage release assets env: @@ -2145,6 +2198,11 @@ jobs: cp patches/patch-*.db.zst "$STAGE/" cp patches/patch-*.db.zst.manifest.json "$STAGE/" fi + # Advisory display stats — present only when "Compute library stats + # (advisory)" produced a valid file; its absence never fails staging. + if [ -s build/library_stats.json ]; then + cp build/library_stats.json "$STAGE/" + fi echo "Staged assets:" ls -lh "$STAGE/" # Sanity: every asset must fit GitHub's 2 GiB per-asset cap. From baeab11aa878ebccdc5daddc8bea145b34b6e82a Mon Sep 17 00:00:00 2001 From: ypl <7353755@gmail.com> Date: Wed, 23 Sep 2026 01:19:13 +0300 Subject: [PATCH 2/2] fix(release): keep advisory stats staging non-blocking --- .github/scripts/test_manual_release_workflow.py | 10 ++++++++-- .github/workflows/manual-generate-release.yml | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/scripts/test_manual_release_workflow.py b/.github/scripts/test_manual_release_workflow.py index fefae9cf..1b2cc28c 100644 --- a/.github/scripts/test_manual_release_workflow.py +++ b/.github/scripts/test_manual_release_workflow.py @@ -2305,8 +2305,14 @@ def test_library_stats_are_an_advisory_staged_asset(self): self.assertIn("exit 0", stats) # Staged conditionally, and BEFORE the provenance hashes the stage. - copy = 'if [ -s build/library_stats.json ]; then\n cp build/library_stats.json "$STAGE/"' - self.assertIn(copy, stage) + copy = 'if [ -s build/library_stats.json ]; then' + self.assertIn( + copy + "\n" + ' if ! cp build/library_stats.json "$STAGE/"; then\n' + ' echo "::warning::library stats: could not stage library_stats.json', + stage, + ) + self.assertIn('rm -f "$STAGE/library_stats.json" || true', stage) self.assertLess(stage.index(copy), stage.index('python3 - "$STAGE"')) # The query produces the exact published bytes, and the step's own diff --git a/.github/workflows/manual-generate-release.yml b/.github/workflows/manual-generate-release.yml index 395467ff..2ee6e568 100644 --- a/.github/workflows/manual-generate-release.yml +++ b/.github/workflows/manual-generate-release.yml @@ -2201,7 +2201,10 @@ jobs: # Advisory display stats — present only when "Compute library stats # (advisory)" produced a valid file; its absence never fails staging. if [ -s build/library_stats.json ]; then - cp build/library_stats.json "$STAGE/" + if ! cp build/library_stats.json "$STAGE/"; then + echo "::warning::library stats: could not stage library_stats.json — omitting it from this release" + rm -f "$STAGE/library_stats.json" || true + fi fi echo "Staged assets:" ls -lh "$STAGE/"