diff --git a/docs/features/content-storage.md b/docs/features/content-storage.md index a025f283d..71cba7a0d 100644 --- a/docs/features/content-storage.md +++ b/docs/features/content-storage.md @@ -232,7 +232,7 @@ handlePublishRoute / publishDataRow └─→ emit publish.after hook ``` -The published `SiteDocument` is stored ONCE per full publish in `site_snapshots` (with a content hash used by the publish-status check); each page's `data_row_versions` row references it via `site_snapshot_id` and carries only its page-scoped `runtime_assets_json`. Readers reassemble the `PublishedPageSnapshot` from the join — that snapshot is the canonical audit record. After it's written, the publisher routes through the three-layer pipeline: +The published `SiteDocument` is stored ONCE per full publish in `site_snapshots` (with a content hash used by the publish-status check); each page's `data_row_versions` row references it via `site_snapshot_id` and carries only its page-scoped `runtime_assets_json`. Readers reassemble the `PublishedPageSnapshot` from the join — that snapshot is the canonical audit record. Each full publish prunes the snapshots no page row's active version references (`pruneUnreferencedSiteSnapshots`); superseded version rows keep their history with `site_snapshot_id` set to null. After it's written, the publisher routes through the three-layer pipeline: - **Layer A** — the publisher renders **every** page (for postType rows, the matched entry template), runs the full `applyPublishedHtmlPipeline`, and writes the final HTML to `uploads/published//.html` — a fully-static page bakes a complete document, a page with dynamic nodes bakes a static shell with `` placeholders. The CSS bundles + runtime JS are baked into the same slot. After all pages are written the symlink `current` atomic-flips to the new slot. Served entirely from disk — no DB for HTML/CSS/JS. - **Layer B** — the in-memory render cache evicts lazily via `bumpPublishVersion()`. diff --git a/docs/features/publisher.md b/docs/features/publisher.md index c31f99cf6..6e3bb59cf 100644 --- a/docs/features/publisher.md +++ b/docs/features/publisher.md @@ -535,7 +535,7 @@ The visitor-facing artefacts are: 2. **In-memory LRU entries** — for dynamic routes (loops, request-dependent bindings). Filled lazily, evicted on every publish. 3. **`` fragment responses** at `/_instatic/hole/?v=&u=` — for dynamic nodes inside otherwise-cacheable pages. Fetched lazily by the IntersectionObserver runtime; shared responses are cached in Layer B, while per-visitor holes bypass it. -The published `SiteDocument` is stored once per publish in `site_snapshots` and referenced by `data_row_versions.site_snapshot_id`; the `PublishedPageSnapshot` reassembled from that join remains the canonical audit record — all three layers derive from it. At request time it is memoised per publish version (`server/publish/publishedSnapshotCache.ts`, shared by the public router's row resolution, the hole endpoint, and the loop endpoint), and `renderPublicResolution` serves a warm Layer B entry *before* any route resolution — a cache hit does zero DB work. +The published `SiteDocument` is stored once per publish in `site_snapshots` and referenced by `data_row_versions.site_snapshot_id`; the `PublishedPageSnapshot` reassembled from that join remains the canonical audit record — all three layers derive from it. Snapshots no active page version references are pruned in the same publish transaction, so the table holds one site document per distinct active publish. At request time it is memoised per publish version (`server/publish/publishedSnapshotCache.ts`, shared by the public router's row resolution, the hole endpoint, and the loop endpoint), and `renderPublicResolution` serves a warm Layer B entry *before* any route resolution — a cache hit does zero DB work. --- diff --git a/server/db/migrations-pg.ts b/server/db/migrations-pg.ts index 89d028114..b546679a2 100644 --- a/server/db/migrations-pg.ts +++ b/server/db/migrations-pg.ts @@ -1370,4 +1370,15 @@ export const pgMigrations: Migration[] = [ alter table data_tables add column created_by_plugin_id text; `, }, + { + // Each full publish prunes the site snapshots no active page version + // references. Deleting one sets `site_snapshot_id` to null on the older + // version rows that pointed at it (`on delete set null`); without this + // index every deleted snapshot scanned the whole versions table. + id: '032_data_row_versions_site_snapshot_idx', + sql: ` + create index if not exists data_row_versions_site_snapshot_idx + on data_row_versions (site_snapshot_id); + `, + }, ] diff --git a/server/db/migrations-sqlite.ts b/server/db/migrations-sqlite.ts index bb30694b2..c3168f069 100644 --- a/server/db/migrations-sqlite.ts +++ b/server/db/migrations-sqlite.ts @@ -1526,4 +1526,15 @@ export const sqliteMigrations: Migration[] = [ alter table data_tables add column created_by_plugin_id text; `, }, + { + // Each full publish prunes the site snapshots no active page version + // references. Deleting one sets `site_snapshot_id` to null on the older + // version rows that pointed at it (`on delete set null`); without this + // index every deleted snapshot scanned the whole versions table. + id: '032_data_row_versions_site_snapshot_idx', + sql: ` + create index if not exists data_row_versions_site_snapshot_idx + on data_row_versions (site_snapshot_id); + `, + }, ] diff --git a/server/repositories/publish.ts b/server/repositories/publish.ts index a8a4cd31d..372863bbe 100644 --- a/server/repositories/publish.ts +++ b/server/repositories/publish.ts @@ -217,6 +217,28 @@ export async function getDraftPublishStatus(db: DbClient): Promise { + await tx` + delete from site_snapshots + where id <> ${keepSnapshotId} + and id not in ( + select data_row_versions.site_snapshot_id + from data_rows + join data_row_versions on data_row_versions.id = data_rows.active_version_id + where data_row_versions.site_snapshot_id is not null + ) + ` +} + /** * Transactional write of one full publish: the site snapshot row plus one * `data_row_versions` row (and its runtime asset files) per page, flipping @@ -278,6 +300,7 @@ export async function persistSitePublish( await tx`delete from data_row_versions where id = ${page.versionId}` } } + await pruneUnreferencedSiteSnapshots(tx, input.siteSnapshotId) }) } diff --git a/src/__tests__/server/cmsPublish.test.ts b/src/__tests__/server/cmsPublish.test.ts index 3a94b3a89..c5ed51ba4 100644 --- a/src/__tests__/server/cmsPublish.test.ts +++ b/src/__tests__/server/cmsPublish.test.ts @@ -146,6 +146,20 @@ function createPublishFakeDb() { }) return { rows: [], rowCount: 1 } } + // pruneUnreferencedSiteSnapshots — params: $1=keepSnapshotId + if (sql.startsWith('delete from site_snapshots')) { + const referenced = new Set( + state.dataRows + .map((r) => state.dataRowVersions.find((v) => v.id === r.active_version_id)?.site_snapshot_id) + .filter(Boolean), + ) + const before = state.siteSnapshots.length + state.siteSnapshots = state.siteSnapshots.filter((s) => s.id === params[0] || referenced.has(s.id)) + for (const v of state.dataRowVersions) { + if (!state.siteSnapshots.some((s) => s.id === v.site_snapshot_id)) v.site_snapshot_id = null + } + return { rows: [], rowCount: before - state.siteSnapshots.length } + } // savePublishedRuntimeAssets — insert into published_runtime_assets if (sql.startsWith('insert into published_runtime_assets')) { state.runtimeAssets.push({ diff --git a/src/__tests__/server/publishSiteSnapshotRetention.test.ts b/src/__tests__/server/publishSiteSnapshotRetention.test.ts new file mode 100644 index 000000000..e8e6e49f9 --- /dev/null +++ b/src/__tests__/server/publishSiteSnapshotRetention.test.ts @@ -0,0 +1,101 @@ +/** + * Every full publish stores the whole site document once in `site_snapshots`. + * Readers only reach a snapshot through a page row's active version, so a + * publish prunes the snapshots no active version references. Without this the + * table grew by one full site copy per publish (439 MB on one production + * site, 7 of 293 snapshots still in use). + */ +import { describe, expect, it } from 'bun:test' +import { createTestDb } from '../helpers/createTestDb' +import { getPublishedPageBySlug, persistSitePublish } from '../../../server/repositories/publish' +import { createDataRow } from '../../../server/repositories/data/rows/mutations' +import { createUser } from '../../../server/repositories/users' +import { hashPassword } from '../../../server/auth/tokens' +import { MAIN_SCOPE } from '../../../server/branches/scope' +import type { DbClient } from '../../../server/db' +import { makePage, makeSite } from '../fixtures' + +async function publishOnce(db: DbClient, pageIds: string[], n: number): Promise { + const site = makeSite({ pages: pageIds.map((id) => makePage({ id, slug: id })) }) + await persistSitePublish(db, { + siteSnapshotId: `snap-${n}`, + site, + serializedImportmap: null, + pages: pageIds.map((pageId) => ({ + pageId, + title: pageId, + slug: pageId, + versionId: `${pageId}-v${n}`, + versionNumber: n, + runtimeAssets: null, + runtimeFiles: [], + })), + publishedByUserId: 'owner_1', + }) +} + +async function storedSnapshots(db: DbClient): Promise { + const { rows } = await db<{ id: string }>`select id from site_snapshots order by id` + return rows.map((row) => row.id) +} + +async function setup(): Promise<{ db: DbClient; cleanup: () => Promise }> { + const { db, cleanup } = await createTestDb() + await createUser(db, { + id: 'owner_1', + email: 'owner@example.com', + displayName: 'Owner', + passwordHash: await hashPassword('long-enough-password'), + roleId: 'owner', + allowOwnerRole: true, + }) + for (const id of ['home', 'about']) { + await createDataRow(db, MAIN_SCOPE, { id, tableId: 'pages', cells: { title: id }, slug: id }) + } + return { db, cleanup } +} + +describe('site snapshot retention', () => { + it('keeps only the snapshot the active versions reference', async () => { + const { db, cleanup } = await setup() + try { + await publishOnce(db, ['home', 'about'], 1) + await publishOnce(db, ['home', 'about'], 2) + await publishOnce(db, ['home', 'about'], 3) + expect(await storedSnapshots(db)).toEqual(['snap-3']) + + // Version history is kept; superseded rows just lose their snapshot link. + const { rows } = await db<{ id: string; site_snapshot_id: string | null }>` + select id, site_snapshot_id from data_row_versions order by id + ` + expect(rows).toEqual([ + { id: 'about-v1', site_snapshot_id: null }, + { id: 'about-v2', site_snapshot_id: null }, + { id: 'about-v3', site_snapshot_id: 'snap-3' }, + { id: 'home-v1', site_snapshot_id: null }, + { id: 'home-v2', site_snapshot_id: null }, + { id: 'home-v3', site_snapshot_id: 'snap-3' }, + ]) + + expect((await getPublishedPageBySlug(db, 'about'))?.pageRowId).toBe('about') + } finally { + await cleanup() + } + }) + + it('keeps an older snapshot while a page is still served from it', async () => { + const { db, cleanup } = await setup() + try { + await publishOnce(db, ['home', 'about'], 1) + // A publish that does not cover `about` leaves its active version on snap-1. + await publishOnce(db, ['home'], 2) + expect(await storedSnapshots(db)).toEqual(['snap-1', 'snap-2']) + expect((await getPublishedPageBySlug(db, 'about'))?.pageRowId).toBe('about') + + await publishOnce(db, ['home', 'about'], 3) + expect(await storedSnapshots(db)).toEqual(['snap-3']) + } finally { + await cleanup() + } + }) +})