diff --git a/docs/02-understanding-fuel.md b/docs/02-understanding-fuel.md index 09b2e95..07de454 100644 --- a/docs/02-understanding-fuel.md +++ b/docs/02-understanding-fuel.md @@ -54,6 +54,11 @@ Deleting a ref or expiring metadata does not reclaim immutable Git history; those bytes remain in file storage until the relay is deleted. See [GRASP-01 Git hosting](22-grasp-01-git-hosting.md) for the full limits. +Git file usage includes transaction receipts and retained bookkeeping as +well as packed files. It is not a fixed reservation per repository: the +total grows with its history, including ref-only updates. The Health tab +shows the combined file total, not a breakdown of these Git components. + ## Prices past the allowance | Measured | Price | diff --git a/docs/15-costs-and-margins.md b/docs/15-costs-and-margins.md index 09ecc41..1d84c18 100644 --- a/docs/15-costs-and-margins.md +++ b/docs/15-costs-and-margins.md @@ -158,6 +158,46 @@ lower CPU cost or a complete request-memory bound. Bulk initial checkpoint construction avoids writing intermediate receipt-index nodes; it removes no stored data and does not start migration automatically. +The isolated checkpoint fixture separates physical storage from the objects +referenced by the current format. Both workloads use one 214-byte Git pack, +260 transactions and an explicit checkpoint after transaction 128. One keeps +updating the same ref to the same commit; the other retains 259 new tags. +These are local measurements with ntig 0.2.1, not production usage or a +per-repository minimum. + +| Retained Git payload | One fixed ref | Growing tags | +|---|---:|---:| +| Pack | 214 bytes | 214 bytes | +| Current root and manifest | 430 bytes | 16,888 bytes | +| All 260 retry records and current receipt index | 136,165 bytes | 126,982 bytes | +| Current-format total | 136,809 bytes | 144,084 bytes | +| Superseded manifests | 42,900 bytes | 1,671,120 bytes | +| Superseded receipt-index nodes | 332,722 bytes | 332,722 bytes | +| Physical objects and reserved bytes | 512,431 bytes | 2,147,926 bytes | + +The ledger equals the paginated R2 inventory for every key and size in these +quiet fixtures: there is no reservation slack. Each untouched fixture +has no Git R2 keys; its first commit uses 631 bytes in three keys. SQLite +allocation, provider operations and execution are separate quantities. +The larger growing-tag total comes mainly from retained copies of earlier +ref maps. Its current manifest is only 16,783 bytes. Fixed-ref transaction +records are larger because their updates include an old object ID. + +All historical retry records remain required by the current retention +contract. Current-format membership includes every pack in the manifest; +it does not establish that every object inside those packs is reachable +from today's refs. Superseded metadata is outside the latest root's +dependency set, but older readers and in-flight publishers can still need +it. This inventory supplies no online deletion authority. No collector or +storage saving is deployed, and these bytes still count toward tenant file +usage. Coordinated metadata collection is the larger target in this fixture; +changing receipt retention would change retry semantics. + +`npm test -- test/object/grasp-checkpoint.test.ts --reporter=verbose --disableConsoleIntercept` +reproduces the inventory, current-format traversal, all historical receipt +lookups and unchanged root checks. The measurements apply to healthy, +quiescent fixtures; ambiguous writes can still leave reservation slack. + `graspBusy`, `graspControls` and the job round's `working` flag fence work within a live instance. Async handlers can interleave across awaits. Those flags are admission controls, not persisted crash recovery or authority to diff --git a/test/helpers/grasp-inventory.ts b/test/helpers/grasp-inventory.ts new file mode 100644 index 0000000..7ac3a61 --- /dev/null +++ b/test/helpers/grasp-inventory.ts @@ -0,0 +1,97 @@ +// Quiescent test fixtures reconcile the quota ledger against R2 and classify +// current-format dependencies. This walk is not an online deletion policy. +import { expect } from "vitest"; +import { npubEncode } from "nostr-tools/nip19"; +import { repositoryAddress, repositoryStoragePrefix, type WalRepository } from "ntig"; +import type { Relay } from "../../src/relay.ts"; +import type { RepositoryAnnouncement } from "../../src/grasp-policy.ts"; + +const digest = async (bytes: Uint8Array) => Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)), (b) => b.toString(16).padStart(2, "0")).join(""); + +// inventory reconciles an isolated repository after its writer has stopped. +export async function inventory(relay: Relay, repo: RepositoryAnnouncement, wal: WalRepository) { + const prefix = `${relay.slug}/git/${await repositoryStoragePrefix(repositoryAddress(npubEncode(repo.owner), repo.identifier))}data/`; + const root = await relay.media.get(`${prefix}root.json`); + const rootText = root ? await root.text() : null; + const snapshot = await wal.load(); + const physical = new Map(); + let cursor: string | undefined; + do { + const page = await relay.media.list({ prefix, cursor, limit: 100 }); + for (const item of page.objects) physical.set(item.key, item.size); + cursor = page.truncated ? page.cursor : undefined; + } while (cursor); + const ledger = relay.sql.exec<{ key: string; size: number }>("SELECT key, size FROM grasp_objects WHERE owner=?", repo.owner).toArray().filter(({ key }) => key.startsWith(prefix)); + expect(new Map(ledger.map(({ key, size }) => [key, size]))).toEqual(physical); + const live = new Set(); + const records = new Map(); + const sequences = new Set(); + const read = async (family: string, hash: string) => { + expect(hash).toMatch(/^[0-9a-f]{64}$/); + const key = `${prefix}${family}/${hash}`; + const item = await relay.media.get(key); + expect(item).not.toBeNull(); + const bytes = new Uint8Array(await item!.arrayBuffer()); + expect(bytes.length).toBe(physical.get(key)); + expect(await digest(bytes)).toBe(hash); + live.add(key); + return JSON.parse(new TextDecoder().decode(bytes)); + }; + if (root) live.add(root.key); + if (snapshot.checkpoint) { + await read("manifests", snapshot.checkpoint.manifestHash); + for (const id of snapshot.checkpoint.packIds) live.add(`${prefix}packs/${id}`); + const walk = async (hash: string, route = ""): Promise => { + expect(route.length).toBeLessThanOrEqual(64); + expect(live.has(`${prefix}receipt-index/${hash}`)).toBe(false); + const node = await read("receipt-index", hash); + expect(node.v).toBe(1); + if (node.t === "l") { + for (const [key, recordHash] of node.e as [string, string][]) { + expect(key.startsWith(route)).toBe(true); + const record = await read("records", recordHash); + expect(await digest(new TextEncoder().encode(record.id))).toBe(key); + expect(records.has(record.id)).toBe(false); + records.set(record.id, recordHash); + sequences.add(record.sequence); + expect(await wal.lookupRecord(record.id)).toEqual(record); + } + } else { + expect(node.t).toBe("b"); + expect(node.c).toHaveLength(16); + for (const [i, child] of (node.c as (string | null)[]).entries()) { + if (child !== null) await walk(child, route + i.toString(16)); + } + } + }; + if (snapshot.checkpoint.receiptRoot) await walk(snapshot.checkpoint.receiptRoot); + expect(records.size).toBe(snapshot.sequence); + expect([...sequences].sort((a, b) => a - b)).toEqual(Array.from({ length: snapshot.sequence }, (_, i) => i + 1)); + } else { + let tip = snapshot.tip; + while (tip) { + expect(live.has(`${prefix}records/${tip}`)).toBe(false); + const record = await read("records", tip); + if (record.pack) live.add(`${prefix}packs/${record.pack}`); + tip = record.parent; + } + } + for (const key of live) expect(physical.has(key)).toBe(true); + const categories: Record = {}; + for (const [key, bytes] of physical) { + const family = key.slice(prefix.length).split("/")[0]; + const category = `${live.has(key) ? "current" : "unreferenced"}/${family}`; + const row = categories[category] ??= { objects: 0, bytes: 0 }; + row.objects++; row.bytes += bytes; + } + const after = await relay.media.get(`${prefix}root.json`); + expect(after?.etag).toBe(root?.etag); + expect(after ? await after.text() : null).toBe(rootText); + return { + sequence: snapshot.sequence, refs: Object.keys(snapshot.refs).length, + physicalBytes: [...physical.values()].reduce((a, b) => a + b, 0), + reservedBytes: ledger.reduce((n, row) => n + row.size, 0), + currentBytes: [...live].reduce((n, key) => n + physical.get(key)!, 0), + objects: physical.size, categories, + }; +} diff --git a/test/object/grasp-checkpoint.test.ts b/test/object/grasp-checkpoint.test.ts index cd94ee7..bbea8b2 100644 --- a/test/object/grasp-checkpoint.test.ts +++ b/test/object/grasp-checkpoint.test.ts @@ -11,6 +11,7 @@ import { gitRepository } from "../../src/grasp.ts"; import type { RepositoryAnnouncement } from "../../src/grasp-policy.ts"; import { ev, pk, rpc } from "../helpers/relay.ts"; import { WS } from "../helpers/ws.ts"; +import { inventory } from "../helpers/grasp-inventory.ts"; const encoder = new TextEncoder(); const oid = async (type: string, data: Uint8Array) => { @@ -38,8 +39,8 @@ const announcement = (sk: Uint8Array, host: string, identifier: string) => ev(sk ]); describe("GRASP checkpoint integration", () => { - it("migrates an isolated legacy repository and retains indexed receipts", async () => { - const host = "grasp-checkpoint.bind.ws"; + it.each(["growing", "fixed"])("migrates an isolated %s-ref repository and reconciles retained storage", async (workload) => { + const host = `grasp-checkpoint-${workload}.bind.ws`; const owner = generateSecretKey(); const identifier = "checkpoint"; await rpc(host, owner, "claim"); @@ -54,12 +55,14 @@ describe("GRASP checkpoint integration", () => { const repo = repository(relay, pk(owner), identifier) as RepositoryAnnouncement | null; expect(repo).not.toBeNull(); const wal = await gitRepository(relay, repo!); + const empty = await inventory(relay, repo!, wal); const first = await wal.commit({ id: "checkpoint-initial", updates: [{ name: "refs/heads/main", old: null, new: fixture.commitID }], pack: fixture.pack }); + const oneCommit = await inventory(relay, repo!, wal); const legacy: WalRecord[] = []; legacy.push((await wal.lookupRecord!(first.id))!); let prior = fixture.commitID; for (let i = 1; i < 128; i++) { - const receipt = await wal.commit({ id: `checkpoint-legacy-${i}`, updates: [{ name: `refs/tags/legacy-${i}`, old: null, new: prior }] }); + const receipt = await wal.commit({ id: `checkpoint-legacy-${i}`, updates: [{ name: workload === "growing" ? `refs/tags/legacy-${i}` : "refs/heads/main", old: workload === "growing" ? null : prior, new: prior }] }); legacy.push((await wal.lookupRecord!(receipt.id))!); } expect((await wal.load()).sequence).toBe(128); @@ -77,14 +80,14 @@ describe("GRASP checkpoint integration", () => { // Format 2 allows more than the legacy 128-record ceiling. Ref-only // commits exercise manifest/index growth without creating more packs. for (let i = 128; i < 260; i++) { - await wal.commit({ id: `checkpoint-v2-${i}`, updates: [{ name: `refs/tags/v2-${i}`, old: null, new: prior }] }); + await wal.commit({ id: `checkpoint-v2-${i}`, updates: [{ name: workload === "growing" ? `refs/tags/v2-${i}` : "refs/heads/main", old: workload === "growing" ? null : prior, new: prior }] }); } const replay = await wal.commit({ id: first.id, updates: [{ name: "refs/heads/main", old: null, new: fixture.commitID }], pack: fixture.pack }); expect(replay).toMatchObject({ id: first.id, sequence: 1, replayed: true }); const rebuilt = await gitRepository(relay, repo!); const cold = await rebuilt.load(); expect(cold.sequence).toBe(260); - expect(cold.refs["refs/tags/v2-259"]).toBe(fixture.commitID); + expect(cold.refs[workload === "growing" ? "refs/tags/v2-259" : "refs/heads/main"]).toBe(fixture.commitID); const advertised = await rebuilt.loadRefs!(); expect(advertised.sequence).toBe(260); expect(advertised.refs["refs/heads/main"]).toBe(fixture.commitID); @@ -98,10 +101,13 @@ describe("GRASP checkpoint integration", () => { counts[category] = (counts[category] ?? 0) + 1; return counts; }, {}); - return { before, after, keyCount: keys.length, categories, sequence: cold.sequence }; + const retained = await inventory(relay, repo!, rebuilt); + return { before, after, keyCount: keys.length, categories, sequence: cold.sequence, empty, oneCommit, retained }; }); + console.info("checkpoint inventory", JSON.stringify({ workload, empty: measurements.empty, oneCommit: measurements.oneCommit, retained: measurements.retained })); console.info("checkpoint storage measurement", { + workload, beforeObjects: measurements.before.n, afterObjects: measurements.after.n, beforeBytes: measurements.before.bytes, @@ -117,6 +123,11 @@ describe("GRASP checkpoint integration", () => { expect(measurements.after.bytes).toBeGreaterThan(measurements.before.bytes); expect(measurements.keyCount).toBeGreaterThan(128); expect(measurements.sequence).toBe(260); + expect(measurements.empty.physicalBytes).toBe(0); + expect(measurements.empty.reservedBytes).toBe(0); + expect(measurements.retained.currentBytes).toBeLessThan(measurements.retained.physicalBytes); + expect(measurements.retained.categories["unreferenced/records"]).toBeUndefined(); + expect(measurements.retained.categories["unreferenced/packs"]).toBeUndefined(); }); it("retains a reservation when an immutable checkpoint write is ambiguous", async () => { diff --git a/test/object/grasp.test.ts b/test/object/grasp.test.ts index f962eef..598457a 100644 --- a/test/object/grasp.test.ts +++ b/test/object/grasp.test.ts @@ -422,6 +422,12 @@ describe("GRASP", () => { const replay = await runInDurableObject(stub, async (relay) => { const bucket = relay.media; let roots = 0; + const published: string[] = []; + const broadcast = relay.broadcast.bind(relay); + Object.defineProperty(relay, "broadcast", { configurable: true, value: (event: Parameters[0]) => { + published.push(event.id); + broadcast(event); + } }); const observed = new Proxy(bucket, { get(target, name) { if (name === "get") return (key: string, options?: R2GetOptions) => { // The replay loads the WAL, then promotion loads it again. Change @@ -443,15 +449,23 @@ describe("GRASP", () => { body: concat(packet(`${"0".repeat(40)} ${pack.commitID} refs/heads/main\0report-status\n`), flush(), pack.pack), })); expect(roots).toBeGreaterThanOrEqual(2); + expect(published).toContain(announcement.id); + expect(published).toContain(state.id); + expect(published).not.toContain(pr.id); return await response.text(); - } finally { Reflect.deleteProperty(relay, "media"); } + } finally { + Reflect.deleteProperty(relay, "media"); + Reflect.deleteProperty(relay, "broadcast"); + } }); expect(replay).toContain("ok refs/heads/main"); await runInDurableObject(stub, async (relay) => { const wal = await gitRepository(relay, storedRepository(relay, pk(owner), "promotion")!); expect((await wal.load()).sequence).toBe(1); expect(relay.sql.exec("SELECT id FROM grasp_pending WHERE id IN (?,?)", announcement.id, state.id).toArray()).toEqual([]); - expect(relay.sql.exec("SELECT id FROM grasp_pending WHERE id=?", pr.id).toArray()).toHaveLength(1); + // Expiry cleanup can remove the event and its pending row between + // requests. If the event survives, it must still be held from readers. + expect(relay.sql.exec("SELECT id FROM events WHERE id=? AND NOT EXISTS (SELECT 1 FROM grasp_pending WHERE grasp_pending.id=events.id)", pr.id).toArray()).toEqual([]); }); const reader = await WS.connect(host); expect((await reader.query({ ids: [announcement.id, state.id] })).events).toHaveLength(2);