From 86b604069553d628bc00cb73c87260781f7953af Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:37:42 -0700 Subject: [PATCH] Use built-in pagination instead of convex-helpers paginator Migrations now paginate with ctx.db.query(table).paginate() by default, which components can call once per transaction. The convex-helpers paginator is only used to resume in-progress migrations whose stored cursor came from the paginator (versions 0.3.5-0.3.7). The schema option is no longer required for customRange, except to resume such a legacy cursor with a custom range. Adds isPaginatorCursor (isNewFormatCursor is kept, deprecated), installs convex-test from pkg.pr.new for the one-paginate-per-transaction check, and runs runToCompletion in tests from an action context. --- README.md | 12 +-- example/convex/example.test.ts | 40 ++++++++- example/convex/localComponent/example.ts | 5 +- package-lock.json | 6 +- package.json | 2 +- src/client/index.test.ts | 60 +++++++++---- src/client/index.ts | 103 ++++++++++++++--------- 7 files changed, 158 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 410acf3..e96ec63 100644 --- a/README.md +++ b/README.md @@ -98,11 +98,9 @@ import schema from "./_generated/schema.js"; export const migrations = new Migrations(components.migrations, { schema }); ``` -The `schema` provides type safety for migration definitions and support -for pagination over custom indexes with -[`customRange`](#migrating-a-subset-of-a-table-using-an-index). -As always, database operations in migrations will abide -by your schema definition at runtime. **Note**: if you use +The `schema` provides type safety for migration definitions. As always, database +operations in migrations will abide by your schema definition at runtime. +**Note**: if you use [custom functions](https://stack.convex.dev/custom-functions) to override `internalMutation`, see [below](#override-the-internalmutation-to-apply-custom-db-behavior). @@ -162,7 +160,9 @@ export const validateRequiredField = migrations.define({ }); ``` -Note: to use customRange, you must provide your schema to Migrations. +Note: if you have an in-progress `customRange` migration that was started on +versions 0.3.5-0.3.7 of this component, provide your schema to Migrations so it +can resume from its stored cursor. ## Running migrations one at a time diff --git a/example/convex/example.test.ts b/example/convex/example.test.ts index bf21306..164e007 100644 --- a/example/convex/example.test.ts +++ b/example/convex/example.test.ts @@ -3,7 +3,9 @@ import { initConvexTest } from "./setup.test"; import { components, internal } from "./_generated/api"; import { runToCompletion } from "@convex-dev/migrations"; import { createFunctionHandle, getFunctionName } from "convex/server"; +import { paginator } from "convex-helpers/server/pagination"; import { migrations } from "./example"; +import schema from "./schema"; describe("example", () => { beforeEach(async () => { @@ -22,7 +24,7 @@ describe("example", () => { expect(docs).toHaveLength(10); expect(docs.some((doc) => doc.optionalField === undefined)).toBe(true); }); - await t.run(async (ctx) => { + await t.action(async (ctx) => { await runToCompletion( ctx, components.migrations, @@ -41,7 +43,7 @@ describe("example", () => { const t = initConvexTest(); await t.mutation(internal.example.seed, { count: 10 }); await expect( - t.run(async (ctx) => { + t.action(async (ctx) => { await runToCompletion( ctx, components.migrations, @@ -91,7 +93,9 @@ describe("example", () => { internal.example.validateRequiredField, ]), ); - expect(second?.name).toBe(getFunctionName(internal.example.setDefaultValue)); + expect(second?.name).toBe( + getFunctionName(internal.example.setDefaultValue), + ); expect(second?.state).toBe("inProgress"); expect(second?.processed).toBe(first?.processed); }); @@ -138,7 +142,7 @@ describe("example", () => { expect(docs).toHaveLength(10); expect(docs.some((doc) => doc.optionalField === undefined)).toBe(true); }); - await t.run(async (ctx) => { + await t.action(async (ctx) => { const fnHandle = await createFunctionHandle( internal.example.setDefaultValue, ); @@ -153,4 +157,32 @@ describe("example", () => { expect(after.every((doc) => doc.optionalField !== undefined)).toBe(true); }); }); + + test("resumes from a legacy convex-helpers paginator cursor", async () => { + const t = initConvexTest(); + await t.mutation(internal.example.seed, { count: 10 }); + // Older versions of this component stored paginator cursors. + const { continueCursor } = await t.run((ctx) => + paginator(ctx.db, schema) + .query("myTable") + .paginate({ cursor: null, numItems: 4 }), + ); + expect(continueCursor.startsWith("[")).toBe(true); + const result = await t.mutation(internal.example.setDefaultValue, { + cursor: continueCursor, + batchSize: 100, + dryRun: false, + oneBatchOnly: true, + }); + expect(result).toMatchObject({ processed: 6, isDone: true }); + await t.run(async (ctx) => { + const docs = await ctx.db.query("myTable").collect(); + const migrated = docs.filter((doc) => doc.optionalField !== undefined); + expect(migrated.length).toBeGreaterThanOrEqual(6); + // The first 4 docs (before the cursor) were skipped. + for (const doc of docs.slice(4)) { + expect(doc.optionalField).toBeDefined(); + } + }); + }); }); diff --git a/example/convex/localComponent/example.ts b/example/convex/localComponent/example.ts index 8458c2b..23d3295 100644 --- a/example/convex/localComponent/example.ts +++ b/example/convex/localComponent/example.ts @@ -1,9 +1,8 @@ /** * Example: Migrations inside a Convex Component * - * When writing migrations for data inside a component, you MUST pass the schema - * to the Migrations constructor. This is because the built-in .paginate() method - * is not available in component contexts - we use the paginator from convex-helpers instead. + * Migrations work on data inside a component the same way as in an app. + * Passing the schema provides type safety for migration definitions. */ import { Migrations } from "@convex-dev/migrations"; import { components } from "./_generated/api.js"; diff --git a/package-lock.json b/package-lock.json index b25e4f8..640ff00 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,7 @@ "chokidar-cli": "3.0.0", "convex": "1.46.0", "convex-helpers": "0.1.128", - "convex-test": "0.0.62", + "convex-test": "https://pkg.pr.new/convex-test@163", "eslint": "9.39.4", "eslint-plugin-react": "7.37.5", "eslint-plugin-react-hooks": "7.1.1", @@ -2211,8 +2211,8 @@ }, "node_modules/convex-test": { "version": "0.0.62", - "resolved": "https://registry.npmjs.org/convex-test/-/convex-test-0.0.62.tgz", - "integrity": "sha512-gvFps/A1JhOlFToZKjStSX8lWQS8oPF4oKm6U5D/ZdmOtfC7bKqw5Ce5ccAf45jvxPKPcFiyWt8JrueE+byB2g==", + "resolved": "https://pkg.pr.new/convex-test@163", + "integrity": "sha512-hoq+HPdnwDs1BUwPe/VS+iwIzrKvtZJp6IvbhCaW7wDflviau0XVrY2A8Sd2yfxex7/GIy1XVsauYEECQcHM7Q==", "dev": true, "license": "Apache-2.0", "peerDependencies": { diff --git a/package.json b/package.json index 02eb772..d6267ef 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ "chokidar-cli": "3.0.0", "convex": "1.46.0", "convex-helpers": "0.1.128", - "convex-test": "0.0.62", + "convex-test": "https://pkg.pr.new/convex-test@163", "eslint": "9.39.4", "eslint-plugin-react": "7.37.5", "eslint-plugin-react-hooks": "7.1.1", diff --git a/src/client/index.test.ts b/src/client/index.test.ts index 2f47a57..a25a71e 100644 --- a/src/client/index.test.ts +++ b/src/client/index.test.ts @@ -1,5 +1,10 @@ import { describe, test, expect } from "vitest"; -import { Migrations, DEFAULT_BATCH_SIZE, isNewFormatCursor } from "./index.js"; +import { + Migrations, + DEFAULT_BATCH_SIZE, + isNewFormatCursor, + isPaginatorCursor, +} from "./index.js"; import type { ComponentApi } from "../component/_generated/component.js"; describe("Migrations class", () => { @@ -15,30 +20,55 @@ describe("DEFAULT_BATCH_SIZE", () => { }); }); -describe("isNewFormatCursor", () => { - test("null cursor returns true (starting fresh)", () => { - expect(isNewFormatCursor(null)).toBe(true); +describe("isPaginatorCursor", () => { + test("null or undefined cursor returns false (built-in pagination)", () => { + expect(isPaginatorCursor(null)).toBe(false); + expect(isPaginatorCursor(undefined)).toBe(false); }); - test("new format cursor (JSON array) returns true", () => { + test("legacy paginator cursor (JSON array) returns true", () => { // Example cursors from convex-helpers paginator - expect(isNewFormatCursor('["value", 1234567890, "documentId"]')).toBe(true); - expect(isNewFormatCursor("[]")).toBe(true); + expect(isPaginatorCursor('["value", 1234567890, "documentId"]')).toBe(true); + expect(isPaginatorCursor('[null, "undefined", 1234567890.5, "id"]')).toBe( + true, + ); expect( - isNewFormatCursor('[1719412234000,"k97d5ycbj3vgwcprmmxvmrm2dh7a5qhv"]'), + isPaginatorCursor('[1719412234000,"k97d5ycbj3vgwcprmmxvmrm2dh7a5qhv"]'), ).toBe(true); }); - test("old format cursor (encrypted string) returns false", () => { + test("built-in cursor (opaque string) returns false", () => { // Example cursors from built-in .paginate() - opaque encrypted strings - expect(isNewFormatCursor("u1oU8i23WMATVwA2CneZ")).toBe(false); - expect(isNewFormatCursor("encryptedOpaqueString123")).toBe(false); - expect(isNewFormatCursor("abc")).toBe(false); + expect(isPaginatorCursor("u1oU8i23WMATVwA2CneZ")).toBe(false); + expect(isPaginatorCursor("encryptedOpaqueString123")).toBe(false); + expect(isPaginatorCursor("abc")).toBe(false); + }); + + test("convex-test built-in cursor (JSON array of strings) returns false", () => { + expect( + isPaginatorCursor( + '["1792024584238.001","\\"0000000000000000000010001myTable\\""]', + ), + ).toBe(false); + expect(isPaginatorCursor("_end_cursor")).toBe(false); }); test("edge cases", () => { - expect(isNewFormatCursor("")).toBe(false); - expect(isNewFormatCursor("{}")).toBe(false); // object, not array - expect(isNewFormatCursor("null")).toBe(false); // string "null" + expect(isPaginatorCursor("")).toBe(false); + expect(isPaginatorCursor("{}")).toBe(false); // object, not array + expect(isPaginatorCursor("null")).toBe(false); // string "null" + expect(isPaginatorCursor("[]")).toBe(false); + expect(isPaginatorCursor("[not json")).toBe(false); + }); +}); + +describe("isNewFormatCursor (deprecated)", () => { + test("null or paginator cursor returns true", () => { + expect(isNewFormatCursor(null)).toBe(true); + expect(isNewFormatCursor("[]")).toBe(true); + }); + + test("built-in cursor returns false", () => { + expect(isNewFormatCursor("u1oU8i23WMATVwA2CneZ")).toBe(false); }); }); diff --git a/src/client/index.ts b/src/client/index.ts index ae35726..4c1b387 100644 --- a/src/client/index.ts +++ b/src/client/index.ts @@ -1,3 +1,5 @@ +// Only used to resume migrations with legacy paginator cursors. See +// isPaginatorCursor. import { paginator } from "convex-helpers/server/pagination"; import { createFunctionHandle, @@ -105,7 +107,11 @@ export class Migrations< */ migrationsLocationPrefix?: string; /** - * The schema for your database. Required when using customRange. + * The schema for your database. Provides type safety for migration + * definitions. At runtime it is only needed to resume an in-progress + * migration with a customRange that was started on versions + * 0.3.5-0.3.7 of this component (which stored convex-helpers + * paginator cursors). * ```ts * import schema from "./schema.js"; * @@ -424,36 +430,35 @@ export class Migrations< } } - // Determine which pagination method to use - const cursorIsOldFormat = - args.cursor !== null && - args.cursor !== undefined && - !isNewFormatCursor(args.cursor); - - // Use paginator when cursor is compatible. - // Falls back to built-in db.query for old-format cursors from - // in-progress migrations that started before the paginator was used. - const useNewPaginator = !cursorIsOldFormat; - if (useNewPaginator && customRange && !this.options?.schema) { - throw new Error( - `You must provide your schema to use a custom range.`, - ); + // Use built-in db.query + .paginate() by default. + // Legacy fallback: migrations started with @convex-dev/migrations + // 0.3.5-0.3.7 stored cursors from the convex-helpers paginator (JSON + // arrays starting with "["). Those can't be passed to the built-in + // .paginate(), so we keep using the paginator to resume them. + const usePaginator = isPaginatorCursor(args.cursor); + let q: QueryInitializer>; + if (usePaginator) { + if (customRange && !this.options?.schema) { + throw new Error( + "This migration has an in-progress cursor from the " + + "convex-helpers paginator (older versions of this component). " + + "To resume it with a custom range, provide your schema to " + + "Migrations, or restart the migration with the cursor set to null.", + ); + } + const paginatorSchema = (this.options?.schema ?? + defineSchema({ [table]: defineTable({}) })) as SchemaDefinition< + Record, + true + >; + q = paginator(ctx.db as any, paginatorSchema).query( + table, + ) as unknown as QueryInitializer< + NamedTableInfo + >; + } else { + q = ctx.db.query(table); } - const paginatorSchema = (this.options?.schema ?? - defineSchema({ [table]: defineTable({}) })) as SchemaDefinition< - Record, - true - >; - - // Build the query using either paginator or built-in db.query - // Both implement compatible QueryInitializer interfaces - const q = useNewPaginator - ? (paginator(ctx.db as any, paginatorSchema).query( - table, - ) as unknown as QueryInitializer< - NamedTableInfo - >) - : ctx.db.query(table); const range = customRange ? customRange(q) : q; let continueCursor: string; let page: DocumentByName[]; @@ -467,7 +472,7 @@ export class Migrations< console.error( "Error paginating. This can happen if the migration " + "was initially run on a different table, different custom range, " + - "or you upgraded convex-helpers with in-progress migrations. " + + "or you upgraded this component with in-progress migrations. " + "This creates an invalid pagination cursor. " + "Run all migrations to completion on the old cursor, or re-run " + "them explicitly with the cursor set to null. " + @@ -837,15 +842,37 @@ export async function runToCompletion( } /** - * Detects if a cursor is in the new paginator format (JSON array starting with "[") - * or old built-in format (encrypted opaque string). + * Detects if a cursor is from the convex-helpers paginator, which versions + * 0.3.5-0.3.7 of this component used. Those cursors are the JSON-serialized + * index key of the last document, ending in `[..., _creationTime, _id]`, e.g. + * '[1719412234000,"k97d5ycbj3vgwcprmmxvmrm2dh7a5qhv"]'. * - * New format cursors from convex-helpers paginator are JSON-serialized arrays like: - * '["value", 1234567890, "documentId"]' + * Cursors from the built-in .paginate() are opaque strings. (convex-test's + * built-in cursors are also JSON arrays, but every element is a string, so + * they don't have a numeric `_creationTime`.) * - * Old format cursors from built-in .paginate() are encrypted opaque strings. - * - * @returns true if this is a new-format cursor (or null for starting fresh) + * @returns true if this is a legacy paginator cursor. False for null. + */ +export function isPaginatorCursor( + cursor: string | null | undefined, +): cursor is string { + if (typeof cursor !== "string" || !cursor.startsWith("[")) return false; + try { + const key = JSON.parse(cursor); + return ( + Array.isArray(key) && + key.length >= 2 && + typeof key[key.length - 2] === "number" && + typeof key[key.length - 1] === "string" + ); + } catch { + return false; + } +} + +/** + * @deprecated Use {@link isPaginatorCursor}. Returns true for null or a + * cursor that starts with "[". */ export function isNewFormatCursor(cursor: string | null): boolean { if (cursor === null) return true;