From d1a74a55c9584cda59c4a838c680a9df9e42f501 Mon Sep 17 00:00:00 2001 From: Ben Jacobson Date: Sun, 27 Sep 2026 16:19:43 +0000 Subject: [PATCH] fix: index numeric Datalog ranges across live and snapshot reads --- .changeset/numeric-due-work-index.md | 10 ++ docs/current-state.md | 4 +- docs/custom-runtimes.md | 2 +- docs/datalog-performance.md | 79 +++++++++++ docs/host-integration.md | 29 ++++- packages/cloudflare/src/migrations.ts | 15 ++- packages/cloudflare/src/schema.ts | 7 +- packages/cloudflare/test/runtime.test.ts | 7 +- .../test/integration/postgresql.test.ts | 44 +++++++ packages/sql/README.md | 11 +- packages/sql/src/index.ts | 1 + packages/sql/src/migrations.ts | 15 ++- packages/sql/src/schema.ts | 14 +- packages/sql/test/sql-contract.test.ts | 6 +- packages/sqlite/test/migrations.test.ts | 21 ++- packages/sqlite/test/numeric-index.test.ts | 10 ++ packages/testkit/src/index.ts | 123 ++++++++++++++++++ test/fixtures/numeric-index.ts | 91 +++++++++++++ 18 files changed, 465 insertions(+), 24 deletions(-) create mode 100644 .changeset/numeric-due-work-index.md create mode 100644 packages/sqlite/test/numeric-index.test.ts create mode 100644 test/fixtures/numeric-index.ts diff --git a/.changeset/numeric-due-work-index.md b/.changeset/numeric-due-work-index.md new file mode 100644 index 0000000..f8e1e42 --- /dev/null +++ b/.changeset/numeric-due-work-index.md @@ -0,0 +1,10 @@ +--- +"@triplex-build/triplex-sql": patch +"@triplex-build/triplex-testkit": patch +--- + +Provision a shared numeric/datetime expression index through an additive SQL migration, enabling +Datalog range scans for live and snapshot-pinned actor due-work queries without application DDL. +Preserve numeric comparison semantics and add shared backend conformance coverage for due-work +ordering, limits, temporal visibility, and continuation after retraction. Existing SQL databases +must apply migration v2; creating the index scans existing numeric history and can block writes. diff --git a/docs/current-state.md b/docs/current-state.md index 986727c..e467c70 100644 --- a/docs/current-state.md +++ b/docs/current-state.md @@ -43,10 +43,10 @@ work that is not complete. - A backend-neutral configuration-derived HTTP package with exact-keyword runtime schemas, REST CRUD, OpenAPI, immutable version resolution, host authorization, bounded exact-cut pages, and atomic constraint-enforced writes over both in-memory KV and SQLite. -- One greenfield SQL v1 migration, host-owned migration entrypoints, Changesets configuration and +- A baseline SQL v1 migration, additive v2 numeric range index, host-owned migration entrypoints, Changesets configuration and release automation, dist-only exports, package tarball checks (including the installed CLI), and Effect dependencies aligned through the root pnpm catalog. -- An additive Cloudflare v2 migration for snapshot tables, and a Cloudflare runtime implementation +- Additive Cloudflare v2 snapshot and v3 numeric index migrations, and a Cloudflare runtime implementation using only public core and SQL exports. Existing facts are preserved; snapshot backfill is explicit, and pre-upgrade Cloudflare pagination cursors must be restarted. - PostgreSQL layers for standalone pools, an ambient host-owned `SqlClient`, and validated diff --git a/docs/custom-runtimes.md b/docs/custom-runtimes.md index 3156c9a..2a836da 100644 --- a/docs/custom-runtimes.md +++ b/docs/custom-runtimes.md @@ -256,7 +256,7 @@ and change events. Test those boundaries for each new adapter. ## Existing backends and release sequencing `makeCloudflareRuntime` is the executable example: the Cloudflare package now uses public core -and SQL exports. Its additive v2 migration creates snapshot tables for existing v1 databases. +and SQL exports. Its additive v2 migration creates snapshot tables for existing v1 databases; v3 adds the numeric range index. It does not backfill existing entities; use `SnapshotWriter.backfill()` if those projections are needed. `CloudflareTriples.layer` remains a convenience API with an explicit `Capabilities.none`; its old string scope is wrapped in the new versioned identity format. Previously issued Cloudflare cursors diff --git a/docs/datalog-performance.md b/docs/datalog-performance.md index 070fbc7..0e12edd 100644 --- a/docs/datalog-performance.md +++ b/docs/datalog-performance.md @@ -94,3 +94,82 @@ EXPLAIN ANALYZE with buffer statistics. Backend suites can also use an explicitl ```sh PG_TEST_URL=postgresql://localhost/triplex_test pnpm test:postgres:integration ``` + +## Numeric ranges and actor due-work + +Triplex provisions `idx_attr_numeric` on `(attribute, COALESCE(value_number, value_datetime))` +for facts whose `value_type` is `number` or `datetime`. It matches the compiler's existing scalar +expression: both storage types compare numerically, and equal numeric values still collapse in +Datalog projections. The existing number-only index remains available for typed storage reads. + +This covers [Runfold PR #4](https://github.com/bjacobso/runfold/pull/4)'s `src/actor.ts` due-work +query without an application index declaration or per-actor DDL: + +```ts check +import { Effect } from "effect"; +import { Triples } from "@triplex-build/triplex"; + +const dueRows = (attribute: string, marker: string, now: number) => + Effect.gen(function* () { + const triples = yield* Triples; + return yield* triples.query( + { + find: ["?item", "?actor", "?due"], + where: [ + ["?item", attribute, "?due"], + ["<=", "?due", now], + ["?item", marker, "?actor"], + ], + orderBy: [{ variable: "?due", direction: "asc" }], + limit: 128, + }, + { pageSize: 128 }, + ); + }); +``` + +`limit` is the logical result bound; `pageSize` controls the public page. Without the explicit +`pageSize`, current Triplex returns up to 100 rows and a continuation even with `limit: 128`. + +Unlike Runfold's `runfold_actor_due` workaround, this index includes retracted numeric facts. +`Triples.query` pins a recorded position and time, and can see facts retracted after that snapshot. +A partial index restricted to `retracted_at IS NULL` cannot serve those reads. Valid-time and +recorded-time predicates remain residual filters; the index does not change visibility. + +The regression fixture has 10,000 pending items: 160 due, 9,840 future, alternating numeric and +datetime storage, plus 80 retracted numeric facts and 10,000 pending markers. After `ANALYZE`, +SQLite and PostgreSQL select the expression index with **both attribute equality and a numeric +range bound**, for live reads and the actual SQL emitted by `Triples.query`. PostgreSQL's bitmap +index scan visits 240 numeric candidates (160 live plus 80 historical), excluding the 9,840 future +values. Removing the new index restores an attribute/temporal scan: PostgreSQL's snapshot +query reads 10,080 due-attribute candidates and filters out 9,920 of them. SQLite likewise switches +from the expression range to `idx_attribute_history` for snapshot pages. The regression tests +assert a numeric range in the index condition, not merely that an index exists. + +These are local plan observations, not a promise of a particular plan on every dataset. +The PostgreSQL planner may still scan pending markers for its join. Both backends still sort and +deduplicate; a 128-row limit does not guarantee only 128 facts are examined. Dense due queues and +large numeric histories need their own measurements. KV retains equivalent results using its +existing executor and does not gain a SQL-style range-plan optimization. + +To inspect the plans, run these tests with `TRIPLEX_EXPLAIN=1` (add `--disableConsoleIntercept` +if the test reporter suppresses output): + +```sh +TRIPLEX_EXPLAIN=1 pnpm --filter @triplex-build/triplex-sqlite exec vitest run test/numeric-index.test.ts +PG_TEST_URL=postgresql://localhost/disposable_test TRIPLEX_EXPLAIN=1 pnpm --filter @triplex-build/triplex-postgres exec vitest run test/integration/postgresql.test.ts -t 'expression range' --disableConsoleIntercept +``` + +### Scope of consumer indexing + +This change takes the core-index route rather than adding a new portable declaration API: the +concrete due-work requirement is shared by numeric Datalog queries across attributes. Consumers +express the query through the same backend-portable `Triples` API and provision its index once +through the [migration API](/host-integration#host-controlled-migrations). + +Triplex does **not** yet expose arbitrary consumer index declarations. Workloads needing narrower +attribute-specific indexes, compound application projections, custom uniqueness, or text search +still need a separate design. Host-owned SQL DDL remains a backend-specific escape hatch at +provisioning/migration time; it is not a portable indexing contract. A future declaration API +must define backend capabilities, unsupported requirements, naming, changes/removal, and migration +ownership, including KV behavior. Do not create indexes per actor, request, or query. diff --git a/docs/host-integration.md b/docs/host-integration.md index 43a8b8a..b298e20 100644 --- a/docs/host-integration.md +++ b/docs/host-integration.md @@ -146,8 +146,33 @@ from the host's deployment process against the same database or scoped schema be an unmigrated runtime. Triplex records its state in `triplex_schema_migrations`, avoiding the host's migration table. -Triplex currently publishes one complete greenfield v1 schema. It does not upgrade databases -created by unpublished development builds. A host adopting an existing EAV store should rehearse +SQL migration v1 is the baseline; additive v2 creates the shared numeric/datetime expression +index. Cloudflare has a separate sequence: v2 adds snapshots and v3 adds the same numeric index. +Existing baseline databases upgrade in place; applied versions are skipped on subsequent runs. +The baseline definitions are unchanged. `INDEX_DDLS` and `INDEX_NAMES` describe all current indexes, +including the new index, so SQLite bulk-load index rebuilding preserves it. + +Use the exported `runMigrations` Effect with the host's `SqlClient` at deployment/provisioning +time, or execute the ordered `migrations` with the host's migration tooling. Convenience layers +apply pending migrations on startup; unmigrated layers and `layerFromSqlClient` leave DDL to the +host. KV needs no schema migration for this change. + +Index creation scans existing numeric facts and adds storage/write cost, including retained +history. The default migration uses ordinary `CREATE INDEX`, which can block writes; serialize +migration execution and schedule it appropriately for large databases. PostgreSQL hosts that +need an online build can pre-create the exact index with `CREATE INDEX CONCURRENTLY` outside a +transaction, verify that the index is valid, then run the migration to record its version. +`IF NOT EXISTS` checks the name, not the definition or validity: reserve `idx_attr_numeric` for +Triplex and resolve any conflicting or invalid index before migrating. + +Runfold can remove its `runfold_actor_due` DDL after deploying this Triplex version and applying +the migration to every database. Triplex does not drop that host-owned index; dropping an existing +redundant index is a separate host migration. Older binaries can read the additive schema, but +old SQLite bulk-loading code that drops/rebuilds only its known indexes cannot suspend maintenance +of the new index. Deploy the migration and updated runtime together when using bulk loading. +See [numeric query plans and indexing scope](/datalog-performance#numeric-ranges-and-actor-due-work). + +Triplex does not upgrade databases created by unpublished development builds. A host adopting an existing EAV store should rehearse the copy against production-shaped data, preserve every assertion and retraction, rebuild journal positions deterministically, and compare live and historical reads before cutover. diff --git a/packages/cloudflare/src/migrations.ts b/packages/cloudflare/src/migrations.ts index 9bd4707..9ff32f7 100644 --- a/packages/cloudflare/src/migrations.ts +++ b/packages/cloudflare/src/migrations.ts @@ -1,10 +1,11 @@ import { COMMAND_RECEIPTS_TABLE_DDL, COMMIT_POSITION_TABLE_DDL, - INDEX_DDLS, + BASELINE_INDEX_DDLS, TRIPLES_TABLE_DDL, } from "./schema.js"; import { + NUMERIC_VALUE_INDEX_DDL, ENTITY_BLOBS_TABLE_DDL, ENTITY_SNAPSHOTS_TABLE_DDL, SNAPSHOT_INDEX_DDLS, @@ -20,11 +21,21 @@ export const migrations: readonly Migration[] = [ { version: 1, name: "triplex_baseline", - up: [TRIPLES_TABLE_DDL, ...INDEX_DDLS, COMMIT_POSITION_TABLE_DDL, COMMAND_RECEIPTS_TABLE_DDL], + up: [ + TRIPLES_TABLE_DDL, + ...BASELINE_INDEX_DDLS, + COMMIT_POSITION_TABLE_DDL, + COMMAND_RECEIPTS_TABLE_DDL, + ], }, { version: 2, name: "triplex_entity_snapshots", up: [ENTITY_BLOBS_TABLE_DDL, ENTITY_SNAPSHOTS_TABLE_DDL, ...SNAPSHOT_INDEX_DDLS], }, + { + version: 3, + name: "triplex_numeric_value_index", + up: [NUMERIC_VALUE_INDEX_DDL], + }, ]; diff --git a/packages/cloudflare/src/schema.ts b/packages/cloudflare/src/schema.ts index 36cba56..2d95834 100644 --- a/packages/cloudflare/src/schema.ts +++ b/packages/cloudflare/src/schema.ts @@ -1,3 +1,5 @@ +import { NUMERIC_VALUE_INDEX_DDL } from "@triplex-build/triplex-sql"; + /** * Cloudflare database schema support. * @@ -53,7 +55,7 @@ export const COMMAND_RECEIPTS_TABLE_DDL = ` ) `; -export const INDEX_DDLS = [ +export const BASELINE_INDEX_DDLS = [ "CREATE INDEX IF NOT EXISTS idx_entity ON triples(entity_id) WHERE retracted_at IS NULL", "CREATE INDEX IF NOT EXISTS idx_attribute ON triples(attribute) WHERE retracted_at IS NULL", "CREATE INDEX IF NOT EXISTS idx_attribute_history ON triples(attribute, recorded_position, retracted_position)", @@ -68,6 +70,8 @@ export const INDEX_DDLS = [ "CREATE INDEX IF NOT EXISTS idx_tx_id ON triples(tx_id) WHERE retracted_at IS NULL", ] as const; +export const INDEX_DDLS = [...BASELINE_INDEX_DDLS, NUMERIC_VALUE_INDEX_DDL] as const; + export const INDEX_NAMES = [ "idx_entity", "idx_attribute", @@ -76,6 +80,7 @@ export const INDEX_NAMES = [ "idx_type", "idx_attr_string", "idx_attr_number", + "idx_attr_numeric", "idx_ref_target", "idx_temporal", "idx_recorded_position", diff --git a/packages/cloudflare/test/runtime.test.ts b/packages/cloudflare/test/runtime.test.ts index e1960e7..6187485 100644 --- a/packages/cloudflare/test/runtime.test.ts +++ b/packages/cloudflare/test/runtime.test.ts @@ -18,7 +18,7 @@ import { MIGRATIONS_TABLE_DDL } from "../src/schema.js"; import { makeMockDOState } from "./fixtures/MockDOState.js"; describe("public Cloudflare runtime", () => { - it("upgrades a v1 database without losing facts and applies v2 once", async () => { + it("upgrades a v1 database without losing facts and applies upgrades once", async () => { const state = makeMockDOState(); const sql = state.storage.sql; sql.exec(MIGRATIONS_TABLE_DDL); @@ -45,7 +45,10 @@ describe("public Cloudflare runtime", () => { expect(await Effect.runPromise(adapter.getByEntity("existing"))).toHaveLength(1); expect( sql.exec("SELECT version FROM triplex_schema_migrations ORDER BY version").toArray(), - ).toEqual([{ version: 1 }, { version: 2 }]); + ).toEqual([{ version: 1 }, { version: 2 }, { version: 3 }]); + expect( + sql.exec("SELECT name FROM sqlite_master WHERE name = 'idx_attr_numeric'").toArray(), + ).toHaveLength(1); expect(sql.exec("SELECT * FROM entity_snapshots").toArray()).toEqual([]); }); it("passes shared conformance through the public builder", async () => { diff --git a/packages/postgres/test/integration/postgresql.test.ts b/packages/postgres/test/integration/postgresql.test.ts index d0e2e56..6269c01 100644 --- a/packages/postgres/test/integration/postgresql.test.ts +++ b/packages/postgres/test/integration/postgresql.test.ts @@ -11,6 +11,7 @@ * pnpm test --filter @triplex-build/triplex-postgres -- test/integration/postgresql.test.ts */ +import { numericIndexPlan } from "../../../../test/fixtures/numeric-index.js"; import { describe, it, expect } from "vitest"; import { Context, Effect, Layer, Redacted } from "effect"; import { SqlClient } from "effect/unstable/sql"; @@ -36,6 +37,7 @@ import { DatabaseManagerLive, DatabaseRegistryLive, runMigrations, + migrations, } from "@triplex-build/triplex-sql"; import { databaseToSchema, @@ -95,6 +97,48 @@ const prepareHostTables = Effect.gen(function* () { }); describe("PostgreSQL Integration", () => { + it.skipIf(!DOCKER_AVAILABLE)( + "uses an expression range index for live and snapshot actor due-work", + { timeout: 120_000 }, + async () => { + await runWithHostSql(numericIndexPlan("postgres")); + }, + ); + it.skipIf(!DOCKER_AVAILABLE)( + "upgrades an existing v1 schema once without losing data", + { timeout: 120_000 }, + async () => { + await runWithHostSql( + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql.withTransaction( + Effect.gen(function* () { + yield* sql`CREATE SCHEMA numeric_index_upgrade`; + yield* sql`SET LOCAL search_path TO numeric_index_upgrade`; + for (const statement of migrations[0]!.up) yield* sql.unsafe(statement); + yield* sql`CREATE TABLE triplex_schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at BIGINT NOT NULL)`; + yield* sql`INSERT INTO triplex_schema_migrations VALUES (1, 'triplex_baseline', 0)`; + yield* sql`INSERT INTO triples (id, entity_id, attribute, value_type, value_number, recorded_at, recorded_position, valid_from) VALUES ('old', 'old', ':due', 'number', 42, 1, 1, 1)`; + expect( + yield* sql`SELECT indexname FROM pg_indexes WHERE schemaname = 'numeric_index_upgrade' AND indexname = 'idx_attr_numeric'`, + ).toHaveLength(0); + yield* runMigrations; + yield* runMigrations; + expect( + yield* sql`SELECT version FROM triplex_schema_migrations ORDER BY version`, + ).toEqual([{ version: 1 }, { version: 2 }]); + expect(yield* sql`SELECT value_number FROM triples`).toEqual([{ value_number: 42 }]); + expect( + yield* sql`SELECT indexname FROM pg_indexes WHERE schemaname = 'numeric_index_upgrade' AND indexname = 'idx_attr_numeric'`, + ).toHaveLength(1); + yield* sql`DROP SCHEMA numeric_index_upgrade CASCADE`; + }), + ); + }), + ); + }, + ); + // Helper to run effects with PostgreSQL via testcontainers const runWithPostgres = (effect: Effect.Effect) => Effect.runPromise(Effect.provide(effect, PgTestLayer)); diff --git a/packages/sql/README.md b/packages/sql/README.md index c857f08..d67f869 100644 --- a/packages/sql/README.md +++ b/packages/sql/README.md @@ -6,7 +6,7 @@ install this transitively through a concrete backend package. Install `@triplex-build/triplex-sql` from npm when building a custom SQL runtime. Use Node.js 22+ and the compatible Effect 4 release candidate. -The public surface includes the ordered greenfield `migrations`, explicit `runMigrations`, SQL +The public surface includes the ordered `migrations`, explicit `runMigrations`, SQL query executors, and SQL-backed `DatabaseManager`/registry layers. Use `@triplex-build/triplex-sqlite` or `@triplex-build/triplex-postgres` for a concrete client and adapter. @@ -20,4 +20,13 @@ refinement and have snapshot tables provisioned. The portable core storage contr require raw SQL. Install this layer through `entitySnapshots(SqlSnapshotsLive)`; projection failures occur after the source transaction commits and cannot roll it back. +Numeric Datalog ranges share a number/datetime expression index, installed by additive migration +v2 (including existing baseline v1 databases). Run `runMigrations` with the host's `SqlClient` +during provisioning/deployment, or apply the ordered definitions with host-owned tooling. +`NUMERIC_VALUE_INDEX_DDL` exposes the same definition used by migrations and index rebuilding. +This index includes history so snapshot-pinned pages can use it. Creation scans existing facts +and can block writes; see [migration rollout](../../docs/host-integration.md#host-controlled-migrations). +No application-specific index declaration is needed for actor due-work queries. Arbitrary portable +consumer index declarations remain outside this API; see [scope and query plans](../../docs/datalog-performance.md#numeric-ranges-and-actor-due-work). + MIT © 2026 Ben Jacobson. diff --git a/packages/sql/src/index.ts b/packages/sql/src/index.ts index 21d0c04..281bb75 100644 --- a/packages/sql/src/index.ts +++ b/packages/sql/src/index.ts @@ -12,6 +12,7 @@ export { COMMIT_POSITION_TABLE_DDL, COMMAND_RECEIPTS_TABLE_DDL, INDEX_DDLS, + NUMERIC_VALUE_INDEX_DDL, INDEX_NAMES, ENTITY_BLOBS_TABLE_DDL, ENTITY_SNAPSHOTS_TABLE_DDL, diff --git a/packages/sql/src/migrations.ts b/packages/sql/src/migrations.ts index 28cfbf8..8c18233 100644 --- a/packages/sql/src/migrations.ts +++ b/packages/sql/src/migrations.ts @@ -3,7 +3,8 @@ import { SqlClient } from "effect/unstable/sql"; import { MigrationError } from "@triplex-build/triplex/internal"; import { TRIPLES_TABLE_DDL, - INDEX_DDLS, + BASELINE_INDEX_DDLS, + NUMERIC_VALUE_INDEX_DDL, ENTITY_BLOBS_TABLE_DDL, ENTITY_SNAPSHOTS_TABLE_DDL, SNAPSHOT_INDEX_DDLS, @@ -18,10 +19,11 @@ export interface Migration { } /** - * The complete greenfield Triplex schema. + * The baseline Triplex schema and ordered additive upgrades. * * DDL is imported from schema.ts so explicit host-owned migration execution and - * convenience auto-migration use the same single v1 definition. + * convenience auto-migration use the same definitions. The baseline stays + * unchanged for existing v1 databases. */ export const migrations: readonly Migration[] = [ { @@ -29,7 +31,7 @@ export const migrations: readonly Migration[] = [ name: "triplex_baseline", up: [ TRIPLES_TABLE_DDL, - ...INDEX_DDLS, + ...BASELINE_INDEX_DDLS, ENTITY_BLOBS_TABLE_DDL, ENTITY_SNAPSHOTS_TABLE_DDL, ...SNAPSHOT_INDEX_DDLS, @@ -37,6 +39,11 @@ export const migrations: readonly Migration[] = [ COMMAND_RECEIPTS_TABLE_DDL, ], }, + { + version: 2, + name: "triplex_numeric_value_index", + up: [NUMERIC_VALUE_INDEX_DDL], + }, ]; export const runMigrations = Effect.gen(function* () { diff --git a/packages/sql/src/schema.ts b/packages/sql/src/schema.ts index 40110f2..4edb249 100644 --- a/packages/sql/src/schema.ts +++ b/packages/sql/src/schema.ts @@ -94,7 +94,7 @@ export const COMMAND_RECEIPTS_TABLE_DDL = ` // ============================================================================= /** - * All index definitions for the triples table. + * Frozen v1 index definitions for the triples table. * * These indexes optimize common query patterns: * - Entity lookups (idx_entity) @@ -106,7 +106,7 @@ export const COMMAND_RECEIPTS_TABLE_DDL = ` * - Entity+attribute lookups (idx_entity_attr) * - Transaction queries (idx_tx_id) */ -export const INDEX_DDLS = [ +export const BASELINE_INDEX_DDLS = [ "CREATE INDEX IF NOT EXISTS idx_entity ON triples(entity_id) WHERE retracted_at IS NULL", "CREATE INDEX IF NOT EXISTS idx_attribute ON triples(attribute) WHERE retracted_at IS NULL", "CREATE INDEX IF NOT EXISTS idx_attribute_history ON triples(attribute, recorded_position, retracted_position)", @@ -121,6 +121,15 @@ export const INDEX_DDLS = [ "CREATE INDEX IF NOT EXISTS idx_tx_id ON triples(tx_id) WHERE retracted_at IS NULL", ] as const; +/** + * Matches Datalog's shared number/datetime scalar expression. Include retracted + * facts: snapshot-pinned pages can still see them at an earlier commit position. + */ +export const NUMERIC_VALUE_INDEX_DDL = + "CREATE INDEX IF NOT EXISTS idx_attr_numeric ON triples(attribute, COALESCE(value_number, value_datetime)) WHERE value_type IN ('number', 'datetime')"; + +export const INDEX_DDLS = [...BASELINE_INDEX_DDLS, NUMERIC_VALUE_INDEX_DDL] as const; + /** * Index names for drop/recreate operations during bulk loading */ @@ -132,6 +141,7 @@ export const INDEX_NAMES = [ "idx_type", "idx_attr_string", "idx_attr_number", + "idx_attr_numeric", "idx_ref_target", "idx_temporal", "idx_recorded_position", diff --git a/packages/sql/test/sql-contract.test.ts b/packages/sql/test/sql-contract.test.ts index 93b1bc2..25c7b44 100644 --- a/packages/sql/test/sql-contract.test.ts +++ b/packages/sql/test/sql-contract.test.ts @@ -5,12 +5,14 @@ import { INDEX_NAMES, migrations, packValue } from "../src/index.js"; describe("SQL package contract", () => { it("keeps migration versions ordered and index declarations complete", () => { - expect(migrations.map((migration) => migration.version)).toEqual([1]); + expect(migrations.map((migration) => migration.version)).toEqual([1, 2]); expect(new Set(migrations.flatMap((migration) => migration.up)).size).toBe( migrations.flatMap((migration) => migration.up).length, ); for (const name of INDEX_NAMES) { - expect(migrations[0]?.up.some((statement) => statement.includes(name))).toBe(true); + expect(migrations.flatMap(({ up }) => up).some((statement) => statement.includes(name))).toBe( + true, + ); } }); diff --git a/packages/sqlite/test/migrations.test.ts b/packages/sqlite/test/migrations.test.ts index 427f5bf..e050460 100644 --- a/packages/sqlite/test/migrations.test.ts +++ b/packages/sqlite/test/migrations.test.ts @@ -38,8 +38,8 @@ describe("host-owned SQLite migrations", () => { ); expect(result.before).toHaveLength(0); - expect(migrations.map(({ version }) => version)).toEqual([1]); - expect(result.applied.map(({ version }) => version)).toEqual([1]); + expect(migrations.map(({ version }) => version)).toEqual([1, 2]); + expect(result.applied.map(({ version }) => version)).toEqual([1, 2]); expect(result.columns.map(({ name }) => name)).toEqual( expect.arrayContaining([ "recorded_at", @@ -58,15 +58,21 @@ describe("host-owned SQLite migrations", () => { expect.arrayContaining(["command_id", "transaction_id", "recorded_at"]), ); expect(result.indexes.map(({ name }) => name)).toEqual( - expect.arrayContaining(["idx_attribute_history", "idx_attribute_temporal"]), + expect.arrayContaining([ + "idx_attribute_history", + "idx_attribute_temporal", + "idx_attr_numeric", + ]), ); }); - it("treats an existing v1 database as current and preserves its data", async () => { + it("upgrades an existing v1 database once and preserves its data", async () => { const result = await runUnmigrated( Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; - yield* runMigrations; + for (const statement of migrations[0]!.up) yield* sql.unsafe(statement); + yield* sql`CREATE TABLE triplex_schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at BIGINT NOT NULL)`; + yield* sql`INSERT INTO triplex_schema_migrations VALUES (1, 'triplex_baseline', 0)`; yield* sql` INSERT INTO triples ( id, entity_id, attribute, value_type, value_string, @@ -79,6 +85,11 @@ describe("host-owned SQLite migrations", () => { yield* runMigrations; + yield* runMigrations; + const indexes = yield* sql`SELECT name FROM sqlite_master WHERE name = 'idx_attr_numeric'`; + expect(indexes).toHaveLength(1); + const applied = yield* sql`SELECT version FROM triplex_schema_migrations ORDER BY version`; + expect(applied).toEqual([{ version: 1 }, { version: 2 }]); return yield* sql<{ value_string: string }>` SELECT value_string FROM triples WHERE entity_id = 'migration:entity' `; diff --git a/packages/sqlite/test/numeric-index.test.ts b/packages/sqlite/test/numeric-index.test.ts new file mode 100644 index 0000000..998bf39 --- /dev/null +++ b/packages/sqlite/test/numeric-index.test.ts @@ -0,0 +1,10 @@ +import { it } from "vitest"; +import { Effect } from "effect"; +import { SqliteTriples } from "../src/index.js"; +import { numericIndexPlan } from "../../../test/fixtures/numeric-index.js"; + +it("uses an expression range index for live and snapshot actor due-work", async () => { + await Effect.runPromise( + numericIndexPlan("sqlite").pipe(Effect.provide(SqliteTriples.layerMemory)), + ); +}); diff --git a/packages/testkit/src/index.ts b/packages/testkit/src/index.ts index 969fef9..d410781 100644 --- a/packages/testkit/src/index.ts +++ b/packages/testkit/src/index.ts @@ -80,6 +80,129 @@ export interface ConformanceCase { * The behavioral cases every `Triples` backend must pass. */ export const triplesConformanceCases: readonly ConformanceCase[] = [ + { + name: "actor due-work ranges preserve numeric aliases, visibility, ordering, and limits", + run: Effect.gen(function* () { + const t = yield* Triples; + const due = ":conf/actor-due"; + const pending = ":conf/actor-pending"; + const facts = yield* t.assertBatch( + Array.from({ length: 180 }, (_, index) => index + 1).flatMap((value) => [ + { + entityId: eid(`conf:due:${value}`), + attribute: due, + value: value % 2 === 0 ? datetime(value) : number(value), + validFrom: 1, + }, + { + entityId: eid(`conf:due:${value}`), + attribute: pending, + value: ref("conf:actor"), + validFrom: 1, + }, + ]), + ); + // Scalar aliases on the same item must collapse under DISTINCT. + yield* t.assert({ + entityId: eid("conf:due:80"), + attribute: due, + value: number(80), + validFrom: 1, + }); + for (const [suffix, value, validFrom, validTo] of [ + ["text", string("0"), 1, undefined], + ["boolean", boolean(false), 1, undefined], + ["future-valid", number(0), 3_000, undefined], + ["expired", datetime(0), 1, 2_000], + ] as const) { + yield* t.assertBatch([ + { + entityId: eid(`conf:due:${suffix}`), + attribute: due, + value, + validFrom, + ...(validTo === undefined ? {} : { validTo }), + }, + { + entityId: eid(`conf:due:${suffix}`), + attribute: pending, + value: ref("conf:actor"), + validFrom: 1, + }, + ]); + } + const query = { + find: ["?item", "?actor", "?due"], + where: [ + ["?item", due, "?due"], + ["<=", "?due", 160], + ["?item", pending, "?actor"], + ], + orderBy: [{ variable: "?due", direction: "asc" }], + limit: 128, + } as const; + const options = { basis: { validAt: 2_000 }, pageSize: 128 }; + const first = yield* t.query(query, options); + yield* check(first.results.length === 128, "due-work must honor the explicit 128-row page"); + yield* check( + first.results.every( + (row, index) => row["?due"] === index + 1 && row["?actor"] === "conf:actor", + ), + "numbers and datetimes must sort together and exclude nonnumeric/invisible facts", + ); + const page = yield* t.query(query, { basis: options.basis }); + yield* check( + page.results.length === 100 && page.nextCursor !== undefined, + "logical limit must retain the default 100-row page size", + ); + // Retract a due fact and a pending marker after capturing the first page. + yield* t.retract( + facts.find((fact) => fact.entityId === "conf:due:110" && fact.attribute === due)!.id, + ); + yield* t.retract( + facts.find((fact) => fact.entityId === "conf:due:111" && fact.attribute === pending)!.id, + ); + const continuation = yield* t.query(query, { cursor: page.nextCursor! }); + yield* check( + continuation.results.length === 28 && + continuation.results.every((row, index) => row["?due"] === index + 101), + "snapshot continuation must preserve subsequently retracted due and marker facts", + ); + const current = yield* t.query(query, options); + const expected = Array.from({ length: 130 }, (_, index) => index + 1).filter( + (value) => value !== 110 && value !== 111, + ); + yield* check( + JSON.stringify(current.results.map((row) => row["?due"])) === JSON.stringify(expected), + "fresh due-work reads must exclude retractions before applying the limit", + ); + for (const [operator, cutoff, expectedValues] of [ + ["<", 2, [1]], + ["<=", 2, [1, 2]], + [">", 179, [180]], + [">=", 179, [180, 179]], + ] as const) { + const result = yield* t.query( + { + ...query, + where: [ + ["?item", due, "?due"], + [operator, "?due", cutoff], + ["?item", pending, "?actor"], + ], + orderBy: [{ variable: "?due", direction: operator.startsWith(">") ? "desc" : "asc" }], + limit: 2, + }, + options, + ); + yield* check( + JSON.stringify(result.results.map((row) => row["?due"])) === + JSON.stringify(expectedValues), + `numeric ${operator} must preserve boundaries and direction`, + ); + } + }), + }, { name: "Datalog reads are bounded by default and cursors traverse the complete snapshot", run: Effect.gen(function* () { diff --git a/test/fixtures/numeric-index.ts b/test/fixtures/numeric-index.ts new file mode 100644 index 0000000..d16702e --- /dev/null +++ b/test/fixtures/numeric-index.ts @@ -0,0 +1,91 @@ +import { Effect } from "effect"; +import { SqlClient } from "effect/unstable/sql"; +import { Triples } from "@triplex-build/triplex"; +import { expect } from "vitest"; + +/** Exercise the public page SQL, including its recorded-position snapshot. */ +export const numericIndexPlan = (backend: "sqlite" | "postgres") => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const triples = yield* Triples; + yield* sql`DELETE FROM triples WHERE attribute IN (':plan/due', ':plan/pending')`; + // 160 due items, 9,840 sleeping items, plus their pending markers. Numeric + // timestamps alternate storage types; physical rows avoid journal overhead. + yield* sql.unsafe(` + WITH RECURSIVE items(n) AS ( + SELECT 1 UNION ALL SELECT n + 1 FROM items WHERE n < 10000 + ) + INSERT INTO triples (id, entity_id, attribute, value_type, value_number, + value_datetime, recorded_at, recorded_position, valid_from) + SELECT 'plan:due:' || n, 'plan:item:' || n, ':plan/due', + CASE WHEN n % 2 = 0 THEN 'datetime' ELSE 'number' END, + CASE WHEN n % 2 = 1 THEN CASE WHEN n <= 160 THEN n ELSE 1000000 + n END END, + CASE WHEN n % 2 = 0 THEN CASE WHEN n <= 160 THEN n ELSE 1000000 + n END END, + 1, 0, 1 FROM items + `); + yield* sql.unsafe(` + INSERT INTO triples (id, entity_id, attribute, value_type, value_string, + recorded_at, recorded_position, valid_from) + SELECT 'marker:' || id, entity_id, ':plan/pending', 'ref', 'plan:actor', 1, 0, 1 + FROM triples WHERE attribute = ':plan/due' + `); + // Include history: the numeric index must remain eligible for snapshot reads. + yield* sql.unsafe(` + INSERT INTO triples (id, entity_id, attribute, value_type, value_number, + recorded_at, recorded_position, valid_from, retracted_at, retracted_position) + SELECT 'history:' || id, entity_id, attribute, 'number', 0, + 0, 0, 0, 1, 0 FROM triples WHERE attribute = ':plan/due' AND value_number <= 160 + `); + yield* sql.unsafe("ANALYZE triples"); + const query = { + find: ["?item", "?actor", "?due"], + where: [ + ["?item", ":plan/due", "?due"], + ["<=", "?due", 160], + ["?item", ":plan/pending", "?actor"], + ], + orderBy: [{ variable: "?due", direction: "asc" }], + limit: 128, + } as const; + // queryAll also checks the live-read predicate, while query checks the real + // Runfold call path (with its default outer page size) and explicit 128 pages. + for (const mode of ["live", "default-page", "128-page"] as const) { + const result = yield* mode === "live" + ? triples.queryAll(query, { debug: true }) + : triples.query(query, { debug: true, ...(mode === "128-page" ? { pageSize: 128 } : {}) }); + expect(result.results.map((row) => row["?due"])).toEqual( + Array.from({ length: mode === "default-page" ? 100 : 128 }, (_, index) => index + 1), + ); + const debug = result.debug!; + const plan = yield* sql.unsafe( + `${backend === "sqlite" ? "EXPLAIN QUERY PLAN" : "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)"} ${debug.generatedSql}`, + debug.params, + ); + if (process.env["TRIPLEX_EXPLAIN"]) { + console.info( + JSON.stringify({ backend, mode, sql: debug.generatedSql, params: debug.params, plan }), + ); + } + if (backend === "sqlite") { + const details = plan.map((row) => String(row["detail"])); + expect( + details.some((detail) => /idx_attr_numeric.*attribute=\?.*[] = []; + const visit = (value: unknown): void => { + if (typeof value !== "object" || value === null) return; + if ("Index Name" in value) nodes.push(value as Record); + for (const child of Object.values(value)) visit(child); + }; + visit(plan); + expect( + nodes.some( + (node) => + node["Index Name"] === "idx_attr_numeric" && + /attribute.*COALESCE.*<=/s.test(String(node["Index Cond"])), + ), + ).toBe(true); + } + } + });