manual-generate-release: נכס library_stats.json (ספרים/קישורים/שורות) בכל רליס DB

.github/scripts/test_manual_release_workflow.py

@@ -2270,6 +2270,87 @@ def test_reuse_scan_summarises_instead_of_one_ok_line_per_release(self):
         self.assertIn("validate_build_provenance.py \"$STAGE/build_provenance.json\"", stage)
         self.assertNotIn("--quiet", stage)
 
+    def test_library_stats_are_an_advisory_staged_asset(self):
+        # library_stats.json is display-only (the website's stats banner), so it
+        # must never cost the release — and it must still be a STAGED asset:
+        # verify_remote demands remote == staged, and the reuse scan counts
+        # `provenance assets + 1`, so an asset uploaded after publish breaks both.
+        stats = self.step("Compute library stats (advisory)")
+        stage = self.step("Stage release assets")
+
+        # Reads build/seforim.db, which the compress step keeps (-o, no --rm),
+        # and must finish before staging hashes the directory.
+        self.assertLess(
+            self.workflow.index("      - name: Compress Seforim Database (zstd)\n"),
+            self.workflow.index("      - name: Compute library stats (advisory)\n"),
+        )
+        self.assertLess(
+            self.workflow.index("      - name: Compute library stats (advisory)\n"),
+            self.workflow.index("      - name: Stage release assets\n"),
+        )
+        self.assertIn("EXPECTED_DB_VERSION: ${{ steps.discover.outputs.db_version }}", stats)
+
+        # Advisory: GitHub runs the step with `bash -e`, so the script itself
+        # never adds -e and every command that can fail is guarded by
+        # `|| skip` — each failure path is skip() → warning + exit 0, and skip()
+        # leaves no partial file behind for staging to pick up.
+        self.assertIn("set -uo pipefail", stats)
+        self.assertNotIn("set -euo pipefail", stats)
+        self.assertNotIn("exit 1", stats)
+        self.assertNotIn("::error::", stats)
+        self.assertIn('rm -f "$OUT" "$OUT.tmp"\n', stats)
+        self.assertIn('echo "::warning::library stats: $1', stats)
+        # A ~/.sqliterc on the runner must not change the CLI's output format.
+        self.assertIn("sqlite3 -init /dev/null -readonly -bail build/seforim.db", stats)
+        self.assertIn("exit 0", stats)
+
+        # Staged conditionally, and BEFORE the provenance hashes the stage.
+        copy = 'if [ -s build/library_stats.json ]; then'
+        self.assertIn(
+            copy + "\n"
+            '            if ! cp build/library_stats.json "$STAGE/"; then\n'
+            '              echo "::warning::library stats: could not stage library_stats.json',
+            stage,
+        )
+        self.assertIn('rm -f "$STAGE/library_stats.json" || true', stage)
+        self.assertLess(stage.index(copy), stage.index('python3 - "$STAGE"'))
+
+        # The query produces the exact published bytes, and the step's own
+        # validator accepts them — and rejects a foreign db_version.
+        query = re.search(r'build/seforim\.db "(select json_object\(.*?\);)"', stats).group(1)
+        validator = textwrap.dedent(
+            stats.split("<<'PY' || skip", 1)[1].split("\n", 1)[1].split("\n          PY\n", 1)[0]
+        )
+        import sqlite3  # noqa: PLC0415 - stdlib, only this test needs it
+
+        with tempfile.TemporaryDirectory() as tmp:
+            db = Path(tmp) / "seforim.db"
+            with sqlite3.connect(db) as conn:
+                conn.executescript(
+                    "create table schema_meta(key text primary key, value text);"
+                    "insert into schema_meta values ('db_version', '28');"
+                    "create table book(id); insert into book values (1), (2);"
+                    "create table link(id); insert into link values (1);"
+                    "create table line(id); insert into line values (1), (2), (3);"
+                )
+                row = conn.execute(query).fetchone()[0]
+            conn.close()
+            out = Path(tmp) / "library_stats.json"
+            out.write_bytes(row.encode() + b"\n")
+            self.assertEqual(
+                out.read_bytes(),
+                b'{"schema_version":1,"db_version":28,"books":2,"links":1,"lines":3}\n',
+            )
+            script = Path(tmp) / "validate.py"
+            script.write_text(validator, encoding="utf-8")
+            ok = subprocess.run([sys.executable, str(script), str(out), "28"])
+            self.assertEqual(ok.returncode, 0)
+            wrong = subprocess.run([sys.executable, str(script), str(out), "29"])
+            self.assertNotEqual(wrong.returncode, 0)
+            out.write_bytes(row.encode() + b"\r\n")
+            crlf = subprocess.run([sys.executable, str(script), str(out), "28"])
+            self.assertNotEqual(crlf.returncode, 0)
+
     def test_build_provenance_quiet_only_silences_the_positive_line(self):
         sys.path.insert(0, str(Path(__file__).parent))
         import test_build_provenance  # noqa: PLC0415 - sibling fixture, not a package

.github/workflows/manual-generate-release.yml

@@ -2108,6 +2108,59 @@ jobs:
           . .pipeline-control/.github/scripts/zstd_workers.sh
           zstd -T"$(zstd_workers)" -19 -f -o build/seforim.db.zst build/seforim.db
 
+      # ─── Library stats (advisory) ──────────────────────────────────────────
+      # One line of JSON — {"schema_version":1,"db_version":N,"books":…,
+      # "links":…,"lines":…} — so a consumer (the website's stats banner) can
+      # show the size of the library without downloading the 1.4 GB DB.
+      #
+      # It is written to build/ and copied by "Stage release assets" BEFORE the
+      # provenance is computed, so it is an ordinary staged asset: listed with
+      # its sha256 in build_provenance.json, uploaded by the same
+      # draft → verify → publish loop (verify_remote demands remote == staged),
+      # and the reuse scan's `provenance assets + 1` count still holds. An asset
+      # uploaded to the release afterwards would break both of those.
+      #
+      # Advisory by design: the numbers are display-only, so no failure here may
+      # cost the weekly release. sqlite3 missing, a failing query, or a result
+      # that does not have the exact expected shape (or names a db_version other
+      # than this build's) each emit a ::warning:: and leave NO file behind — the
+      # release then ships exactly the asset set it shipped before this step.
+      - name: Compute library stats (advisory)
+        env:
+          EXPECTED_DB_VERSION: ${{ steps.discover.outputs.db_version }}
+        run: |
+          set -uo pipefail
+          OUT=build/library_stats.json
+          rm -f "$OUT" "$OUT.tmp"
+          skip() {
+            echo "::warning::library stats: $1 — library_stats.json is not published on this release"
+            rm -f "$OUT" "$OUT.tmp"
+            exit 0
+          }
+          command -v sqlite3 >/dev/null 2>&1 || skip "sqlite3 not found"
+          [ -s build/seforim.db ] || skip "build/seforim.db is missing"
+          sqlite3 -init /dev/null -readonly -bail build/seforim.db "select json_object('schema_version',1,'db_version',(select cast(value as integer) from schema_meta where key='db_version'),'books',(select count(*) from book),'links',(select count(*) from link),'lines',(select count(*) from line));" \
+            > "$OUT.tmp" || skip "the stats query failed"
+          [[ "$EXPECTED_DB_VERSION" =~ ^[1-9][0-9]*$ ]] || skip "no db_version for this build"
+          # Exact shape AND exact bytes: key order, integer counts > 0, compact
+          # separators, one trailing LF — what sqlite3's json_object prints.
+          python3 - "$OUT.tmp" "$EXPECTED_DB_VERSION" <<'PY' || skip "unexpected result $(head -c 300 "$OUT.tmp")"
+          import json, sys
+          raw = open(sys.argv[1], "rb").read()
+          value = json.loads(raw)
+          ok = (
+              isinstance(value, dict)
+              and list(value) == ["schema_version", "db_version", "books", "links", "lines"]
+              and value["schema_version"] == 1
+              and type(value["db_version"]) is int and value["db_version"] == int(sys.argv[2])
+              and all(type(value[k]) is int and value[k] > 0 for k in ("books", "links", "lines"))
+              and raw == json.dumps(value, separators=(",", ":")).encode() + b"\n"
+          )
+          sys.exit(0 if ok else 1)
+          PY
+          mv "$OUT.tmp" "$OUT" || skip "could not write $OUT"
+          echo "library_stats.json: $(cat "$OUT")"
+
       # ─── Stage release assets ──────────────────────────────────────────────
       - name: Stage release assets
         env:
@@ -2145,6 +2198,14 @@ jobs:
             cp patches/patch-*.db.zst "$STAGE/"
             cp patches/patch-*.db.zst.manifest.json "$STAGE/"
           fi
+          # Advisory display stats — present only when "Compute library stats
+          # (advisory)" produced a valid file; its absence never fails staging.
+          if [ -s build/library_stats.json ]; then
+            if ! cp build/library_stats.json "$STAGE/"; then
+              echo "::warning::library stats: could not stage library_stats.json — omitting it from this release"
+              rm -f "$STAGE/library_stats.json" || true
+            fi
+          fi
           echo "Staged assets:"
           ls -lh "$STAGE/"
           # Sanity: every asset must fit GitHub's 2 GiB per-asset cap.