Use built-in pagination instead of paginator

README.md

@@ -98,11 +98,9 @@ import schema from "./_generated/schema.js";
 export const migrations = new Migrations(components.migrations, { schema });
 ```
 
-The `schema` provides type safety for migration definitions and support
-for pagination over custom indexes with
-[`customRange`](#migrating-a-subset-of-a-table-using-an-index).
-As always, database operations in migrations will abide
-by your schema definition at runtime. **Note**: if you use
+The `schema` provides type safety for migration definitions. As always, database
+operations in migrations will abide by your schema definition at runtime.
+**Note**: if you use
 [custom functions](https://stack.convex.dev/custom-functions) to override
 `internalMutation`, see
 [below](#override-the-internalmutation-to-apply-custom-db-behavior).
@@ -162,7 +160,9 @@ export const validateRequiredField = migrations.define({
 });
 ```
 
-Note: to use customRange, you must provide your schema to Migrations.
+Note: if you have an in-progress `customRange` migration that was started on
+versions 0.3.5-0.3.7 of this component, provide your schema to Migrations so it
+can resume from its stored cursor.
 
 ## Running migrations one at a time
 

example/convex/example.test.ts

@@ -3,7 +3,9 @@ import { initConvexTest } from "./setup.test";
 import { components, internal } from "./_generated/api";
 import { runToCompletion } from "@convex-dev/migrations";
 import { createFunctionHandle, getFunctionName } from "convex/server";
+import { paginator } from "convex-helpers/server/pagination";
 import { migrations } from "./example";
+import schema from "./schema";
 
 describe("example", () => {
   beforeEach(async () => {
@@ -22,7 +24,7 @@ describe("example", () => {
       expect(docs).toHaveLength(10);
       expect(docs.some((doc) => doc.optionalField === undefined)).toBe(true);
     });
-    await t.run(async (ctx) => {
+    await t.action(async (ctx) => {
       await runToCompletion(
         ctx,
         components.migrations,
@@ -41,7 +43,7 @@ describe("example", () => {
     const t = initConvexTest();
     await t.mutation(internal.example.seed, { count: 10 });
     await expect(
-      t.run(async (ctx) => {
+      t.action(async (ctx) => {
         await runToCompletion(
           ctx,
           components.migrations,
@@ -91,7 +93,9 @@ describe("example", () => {
         internal.example.validateRequiredField,
       ]),
     );
-    expect(second?.name).toBe(getFunctionName(internal.example.setDefaultValue));
+    expect(second?.name).toBe(
+      getFunctionName(internal.example.setDefaultValue),
+    );
     expect(second?.state).toBe("inProgress");
     expect(second?.processed).toBe(first?.processed);
   });
@@ -138,7 +142,7 @@ describe("example", () => {
       expect(docs).toHaveLength(10);
       expect(docs.some((doc) => doc.optionalField === undefined)).toBe(true);
     });
-    await t.run(async (ctx) => {
+    await t.action(async (ctx) => {
       const fnHandle = await createFunctionHandle(
         internal.example.setDefaultValue,
       );
@@ -153,4 +157,32 @@ describe("example", () => {
       expect(after.every((doc) => doc.optionalField !== undefined)).toBe(true);
     });
   });
+
+  test("resumes from a legacy convex-helpers paginator cursor", async () => {
+    const t = initConvexTest();
+    await t.mutation(internal.example.seed, { count: 10 });
+    // Older versions of this component stored paginator cursors.
+    const { continueCursor } = await t.run((ctx) =>
+      paginator(ctx.db, schema)
+        .query("myTable")
+        .paginate({ cursor: null, numItems: 4 }),
+    );
+    expect(continueCursor.startsWith("[")).toBe(true);
+    const result = await t.mutation(internal.example.setDefaultValue, {
+      cursor: continueCursor,
+      batchSize: 100,
+      dryRun: false,
+      oneBatchOnly: true,
+    });
+    expect(result).toMatchObject({ processed: 6, isDone: true });
+    await t.run(async (ctx) => {
+      const docs = await ctx.db.query("myTable").collect();
+      const migrated = docs.filter((doc) => doc.optionalField !== undefined);
+      expect(migrated.length).toBeGreaterThanOrEqual(6);
+      // The first 4 docs (before the cursor) were skipped.
+      for (const doc of docs.slice(4)) {
+        expect(doc.optionalField).toBeDefined();
+      }
+    });
+  });
 });

example/convex/localComponent/example.ts

@@ -1,9 +1,8 @@
 /**
  * Example: Migrations inside a Convex Component
  *
- * When writing migrations for data inside a component, you MUST pass the schema
- * to the Migrations constructor. This is because the built-in .paginate() method
- * is not available in component contexts - we use the paginator from convex-helpers instead.
+ * Migrations work on data inside a component the same way as in an app.
+ * Passing the schema provides type safety for migration definitions.
  */
 import { Migrations } from "@convex-dev/migrations";
 import { components } from "./_generated/api.js";

package.json

@@ -72,7 +72,7 @@
     "chokidar-cli": "3.0.0",
     "convex": "1.46.0",
     "convex-helpers": "0.1.128",
-    "convex-test": "0.0.62",
+    "convex-test": "https://pkg.pr.new/convex-test@163",
     "eslint": "9.39.4",
     "eslint-plugin-react": "7.37.5",
     "eslint-plugin-react-hooks": "7.1.1",

src/client/index.test.ts

@@ -1,5 +1,10 @@
 import { describe, test, expect } from "vitest";
-import { Migrations, DEFAULT_BATCH_SIZE, isNewFormatCursor } from "./index.js";
+import {
+  Migrations,
+  DEFAULT_BATCH_SIZE,
+  isNewFormatCursor,
+  isPaginatorCursor,
+} from "./index.js";
 import type { ComponentApi } from "../component/_generated/component.js";
 
 describe("Migrations class", () => {
@@ -15,30 +20,55 @@ describe("DEFAULT_BATCH_SIZE", () => {
   });
 });
 
-describe("isNewFormatCursor", () => {
-  test("null cursor returns true (starting fresh)", () => {
-    expect(isNewFormatCursor(null)).toBe(true);
+describe("isPaginatorCursor", () => {
+  test("null or undefined cursor returns false (built-in pagination)", () => {
+    expect(isPaginatorCursor(null)).toBe(false);
+    expect(isPaginatorCursor(undefined)).toBe(false);
   });
 
-  test("new format cursor (JSON array) returns true", () => {
+  test("legacy paginator cursor (JSON array) returns true", () => {
     // Example cursors from convex-helpers paginator
-    expect(isNewFormatCursor('["value", 1234567890, "documentId"]')).toBe(true);
-    expect(isNewFormatCursor("[]")).toBe(true);
+    expect(isPaginatorCursor('["value", 1234567890, "documentId"]')).toBe(true);
+    expect(isPaginatorCursor('[null, "undefined", 1234567890.5, "id"]')).toBe(
+      true,
+    );
     expect(
-      isNewFormatCursor('[1719412234000,"k97d5ycbj3vgwcprmmxvmrm2dh7a5qhv"]'),
+      isPaginatorCursor('[1719412234000,"k97d5ycbj3vgwcprmmxvmrm2dh7a5qhv"]'),
     ).toBe(true);
   });
 
-  test("old format cursor (encrypted string) returns false", () => {
+  test("built-in cursor (opaque string) returns false", () => {
     // Example cursors from built-in .paginate() - opaque encrypted strings
-    expect(isNewFormatCursor("u1oU8i23WMATVwA2CneZ")).toBe(false);
-    expect(isNewFormatCursor("encryptedOpaqueString123")).toBe(false);
-    expect(isNewFormatCursor("abc")).toBe(false);
+    expect(isPaginatorCursor("u1oU8i23WMATVwA2CneZ")).toBe(false);
+    expect(isPaginatorCursor("encryptedOpaqueString123")).toBe(false);
+    expect(isPaginatorCursor("abc")).toBe(false);
+  });
+
+  test("convex-test built-in cursor (JSON array of strings) returns false", () => {
+    expect(
+      isPaginatorCursor(
+        '["1792024584238.001","\\"0000000000000000000010001myTable\\""]',
+      ),
+    ).toBe(false);
+    expect(isPaginatorCursor("_end_cursor")).toBe(false);
   });
 
   test("edge cases", () => {
-    expect(isNewFormatCursor("")).toBe(false);
-    expect(isNewFormatCursor("{}")).toBe(false); // object, not array
-    expect(isNewFormatCursor("null")).toBe(false); // string "null"
+    expect(isPaginatorCursor("")).toBe(false);
+    expect(isPaginatorCursor("{}")).toBe(false); // object, not array
+    expect(isPaginatorCursor("null")).toBe(false); // string "null"
+    expect(isPaginatorCursor("[]")).toBe(false);
+    expect(isPaginatorCursor("[not json")).toBe(false);
+  });
+});
+
+describe("isNewFormatCursor (deprecated)", () => {
+  test("null or paginator cursor returns true", () => {
+    expect(isNewFormatCursor(null)).toBe(true);
+    expect(isNewFormatCursor("[]")).toBe(true);
+  });
+
+  test("built-in cursor returns false", () => {
+    expect(isNewFormatCursor("u1oU8i23WMATVwA2CneZ")).toBe(false);
   });
 });

src/client/index.ts

@@ -1,3 +1,5 @@
+// Only used to resume migrations with legacy paginator cursors. See
+// isPaginatorCursor.
 import { paginator } from "convex-helpers/server/pagination";
 import {
   createFunctionHandle,
@@ -105,7 +107,11 @@ export class Migrations<
        */
       migrationsLocationPrefix?: string;
       /**
-       * The schema for your database. Required when using customRange.
+       * The schema for your database. Provides type safety for migration
+       * definitions. At runtime it is only needed to resume an in-progress
+       * migration with a customRange that was started on versions
+       * 0.3.5-0.3.7 of this component (which stored convex-helpers
+       * paginator cursors).
        * ```ts
        * import schema from "./schema.js";
        *
@@ -424,36 +430,35 @@ export class Migrations<
           }
         }
 
-        // Determine which pagination method to use
-        const cursorIsOldFormat =
-          args.cursor !== null &&
-          args.cursor !== undefined &&
-          !isNewFormatCursor(args.cursor);
-
-        // Use paginator when cursor is compatible.
-        // Falls back to built-in db.query for old-format cursors from
-        // in-progress migrations that started before the paginator was used.
-        const useNewPaginator = !cursorIsOldFormat;
-        if (useNewPaginator && customRange && !this.options?.schema) {
-          throw new Error(
-            `You must provide your schema to use a custom range.`,
-          );
+        // Use built-in db.query + .paginate() by default.
+        // Legacy fallback: migrations started with @convex-dev/migrations
+        // 0.3.5-0.3.7 stored cursors from the convex-helpers paginator (JSON
+        // arrays starting with "["). Those can't be passed to the built-in
+        // .paginate(), so we keep using the paginator to resume them.
+        const usePaginator = isPaginatorCursor(args.cursor);
+        let q: QueryInitializer<NamedTableInfo<DataModel, TableName>>;
+        if (usePaginator) {
+          if (customRange && !this.options?.schema) {
+            throw new Error(
+              "This migration has an in-progress cursor from the " +
+                "convex-helpers paginator (older versions of this component). " +
+                "To resume it with a custom range, provide your schema to " +
+                "Migrations, or restart the migration with the cursor set to null.",
+            );
+          }
+          const paginatorSchema = (this.options?.schema ??
+            defineSchema({ [table]: defineTable({}) })) as SchemaDefinition<
+            Record<TableName, TableDefinition>,
+            true
+          >;
+          q = paginator(ctx.db as any, paginatorSchema).query(
+            table,
+          ) as unknown as QueryInitializer<
+            NamedTableInfo<DataModel, TableName>
+          >;
+        } else {
+          q = ctx.db.query(table);
         }
-        const paginatorSchema = (this.options?.schema ??
-          defineSchema({ [table]: defineTable({}) })) as SchemaDefinition<
-          Record<TableName, TableDefinition>,
-          true
-        >;
-
-        // Build the query using either paginator or built-in db.query
-        // Both implement compatible QueryInitializer interfaces
-        const q = useNewPaginator
-          ? (paginator(ctx.db as any, paginatorSchema).query(
-              table,
-            ) as unknown as QueryInitializer<
-              NamedTableInfo<DataModel, TableName>
-            >)
-          : ctx.db.query(table);
         const range = customRange ? customRange(q) : q;
         let continueCursor: string;
         let page: DocumentByName<DataModel, TableName>[];
@@ -467,7 +472,7 @@ export class Migrations<
           console.error(
             "Error paginating. This can happen if the migration " +
               "was initially run on a different table, different custom range, " +
-              "or you upgraded convex-helpers with in-progress migrations. " +
+              "or you upgraded this component with in-progress migrations. " +
               "This creates an invalid pagination cursor. " +
               "Run all migrations to completion on the old cursor, or re-run " +
               "them explicitly with the cursor set to null. " +
@@ -837,15 +842,37 @@ export async function runToCompletion(
 }
 
 /**
- * Detects if a cursor is in the new paginator format (JSON array starting with "[")
- * or old built-in format (encrypted opaque string).
+ * Detects if a cursor is from the convex-helpers paginator, which versions
+ * 0.3.5-0.3.7 of this component used. Those cursors are the JSON-serialized
+ * index key of the last document, ending in `[..., _creationTime, _id]`, e.g.
+ * '[1719412234000,"k97d5ycbj3vgwcprmmxvmrm2dh7a5qhv"]'.
  *
- * New format cursors from convex-helpers paginator are JSON-serialized arrays like:
- * '["value", 1234567890, "documentId"]'
+ * Cursors from the built-in .paginate() are opaque strings. (convex-test's
+ * built-in cursors are also JSON arrays, but every element is a string, so
+ * they don't have a numeric `_creationTime`.)
  *
- * Old format cursors from built-in .paginate() are encrypted opaque strings.
- *
- * @returns true if this is a new-format cursor (or null for starting fresh)
+ * @returns true if this is a legacy paginator cursor. False for null.
+ */
+export function isPaginatorCursor(
+  cursor: string | null | undefined,
+): cursor is string {
+  if (typeof cursor !== "string" || !cursor.startsWith("[")) return false;
+  try {
+    const key = JSON.parse(cursor);
+    return (
+      Array.isArray(key) &&
+      key.length >= 2 &&
+      typeof key[key.length - 2] === "number" &&
+      typeof key[key.length - 1] === "string"
+    );
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * @deprecated Use {@link isPaginatorCursor}. Returns true for null or a
+ * cursor that starts with "[".
  */
 export function isNewFormatCursor(cursor: string | null): boolean {
   if (cursor === null) return true;