fix: batch and compress UID FETCH for large mailboxes

src/imap/fetcher.ts

@@ -7,6 +7,7 @@ import { selectBodies, structureToBodyParts, type EmailBodyPart } from "../mappi
 import { flagsToKeywords } from "../mapping/flags.js";
 import { encodeBlobId, encodeEmailId, encodeMailboxId } from "../mapping/ids.js";
 import { parseHeaderBlock, asMessageIds, computeThreadIdFromHeaders, type ParsedHeader } from "./headers.js";
+import { fetchByUid } from "./uids.js";
 import type { AccountRow, MailboxRow, Store, EmailCacheUpsert } from "../state/store.js";
 
 // The header fields threading and the default Email/get property set need:
@@ -313,10 +314,10 @@ async function fetchBodyValuesBatched(
     const byUid = new Map(g.members.map((m) => [m.uid, m]));
     const rawByUid = new Map<number, Map<string, Buffer>>();
     try {
-      for await (const msg of client.fetch(
+      for await (const msg of fetchByUid(
+        client,
         g.members.map((m) => m.uid),
         { uid: true, bodyParts: g.partIds },
-        { uid: true },
       )) {
         if (msg.uid != null && msg.bodyParts) rawByUid.set(msg.uid, msg.bodyParts);
       }
@@ -391,10 +392,10 @@ async function fetchPreviewsBatched(
   for (const [partId, group] of byPartId) {
     const taskByUid = new Map(group.map((t) => [t.uid, t]));
     try {
-      for await (const m of client.fetch(
+      for await (const m of fetchByUid(
+        client,
         group.map((t) => t.uid),
         { uid: true, bodyParts: [{ key: partId, start: 0, maxLength: PREVIEW_FETCH_BYTES }] },
-        { uid: true },
       )) {
         if (m.uid == null) continue;
         const buf = m.bodyParts?.get(partId);
@@ -464,7 +465,7 @@ export async function fetchEmailsBatch(
   }
 
   if (missing.length > 0) {
-    for await (const m of client.fetch(missing, metaQuery(needsFull), { uid: true })) {
+    for await (const m of fetchByUid(client, missing, metaQuery(needsFull))) {
       if (m.uid == null) continue;
       raws.set(m.uid, rawFromFetch(m, needsFull));
     }
@@ -473,7 +474,7 @@ export async function fetchEmailsBatch(
   // Flags for cache-served messages: one flags-only FETCH for the whole set.
   const flagUids = uids.filter((uid) => raws.get(uid)?.flags === null);
   if (flagUids.length > 0) {
-    for await (const m of client.fetch(flagUids, { uid: true, flags: true }, { uid: true })) {
+    for await (const m of fetchByUid(client, flagUids, { uid: true, flags: true })) {
       if (m.uid == null) continue;
       const r = raws.get(m.uid);
       if (r) r.flags = new Set(m.flags ?? []);

src/imap/uids.ts

@@ -0,0 +1,51 @@
+import type { FetchMessageObject, FetchQueryObject, ImapFlow } from "imapflow";
+
+// Some IMAP servers cap the command line (MDaemon and Kerio Connect reject or
+// drop the connection; RFC 7162 §4 asks clients to stay under 8192 octets), so
+// a UID list must never go out as one flat comma list. imapflow does not guard
+// against this: a number[] range is sent as `range.join(",")`, uncompressed.
+const MAX_SET_BYTES = 4000;
+
+/**
+ * Pack UIDs into IMAP sequence sets ("1:3,5,9:12"), split so no set exceeds
+ * `maxBytes`. Input may be unsorted or contain duplicates. Contiguous runs
+ * collapse into one range, so a dense folder of any size stays one command;
+ * only sparse sets (a search hitting every other message) need several.
+ */
+export function uidSets(uids: Iterable<number>, maxBytes = MAX_SET_BYTES): string[] {
+  const sorted = [...new Set(uids)]
+    .filter((u) => Number.isInteger(u) && u > 0)
+    .sort((a, b) => a - b);
+  const sets: string[] = [];
+  let cur = "";
+  let i = 0;
+  while (i < sorted.length) {
+    const start = sorted[i]!;
+    let end = start;
+    while (sorted[i + 1] === end + 1) end = sorted[++i]!;
+    i++;
+    const part = start === end ? `${start}` : `${start}:${end}`;
+    if (cur && cur.length + 1 + part.length > maxBytes) {
+      sets.push(cur);
+      cur = part;
+    } else {
+      cur = cur ? `${cur},${part}` : part;
+    }
+  }
+  if (cur) sets.push(cur);
+  return sets;
+}
+
+/**
+ * UID FETCH over an arbitrary UID list, one bounded command per sequence set.
+ * Drop-in for `client.fetch(uids, query, { uid: true })`.
+ */
+export async function* fetchByUid(
+  client: ImapFlow,
+  uids: Iterable<number>,
+  query: FetchQueryObject,
+): AsyncGenerator<FetchMessageObject> {
+  for (const set of uidSets(uids)) {
+    yield* client.fetch(set, query, { uid: true });
+  }
+}

src/jmap/methods/email.ts

@@ -5,6 +5,7 @@ import { decodeEmailId, decodeMailboxId, encodeBlobId, encodeEmailId, encodeMail
 import { encodeEmailState } from "../../state/states.js";
 import { fetchEmailsBatch, type JmapEmail } from "../../imap/fetcher.js";
 import { withMailbox } from "../../imap/client.js";
+import { fetchByUid } from "../../imap/uids.js";
 import { JmapError, accountNotFound, invalidArguments, notFound, unsupportedFilter, unsupportedSort } from "../errors.js";
 import { compileFilter, UnsupportedFilter, type Filter } from "../../imap/search.js";
 import { listMailboxes, refreshMailboxCounts } from "./mailbox.js";
@@ -348,7 +349,8 @@ function compareHits(a: SortableHit, b: SortableHit, sort: SortSpec[]): number {
 }
 
 // Enrich UIDs with whatever metadata the requested sort needs. Non-receivedAt
-// sorts FETCH envelope/size/flags from IMAP — one batched FETCH per mailbox.
+// sorts FETCH envelope/size/flags from IMAP — batched per mailbox, split into
+// bounded UID sets so huge folders don't overflow the server's line limit.
 async function fetchSortMetadata(
   client: ImapFlow,
   uids: number[],
@@ -359,9 +361,7 @@ async function fetchSortMetadata(
   const query = needsFetch
     ? { uid: true, internalDate: true, size: true, envelope: true, flags: true }
     : { uid: true, internalDate: true };
-  // Pass the array, not a joined string: imapflow compresses number[] into
-  // range syntax (1:5000), keeping the command line bounded on huge folders.
-  for await (const msg of client.fetch(uids, query, { uid: true })) {
+  for await (const msg of fetchByUid(client, uids, query)) {
     const uid = Number(msg.uid);
     out.set(uid, {
       receivedAt: msg.internalDate ? new Date(msg.internalDate).getTime() : 0,
@@ -440,7 +440,7 @@ async function postFilterByHasAttachment(
     try {
       await withMailbox(ctx.client, row.name, async () => {
         const upserts: EmailCacheUpsert[] = [];
-        for await (const msg of ctx.client.fetch(missingUids, { uid: true, bodyStructure: true }, { uid: true })) {
+        for await (const msg of fetchByUid(ctx.client, missingUids, { uid: true, bodyStructure: true })) {
           const uid = Number(msg.uid);
           const eid = uidByEid.get(uid);
           if (!eid) continue;

src/jmap/methods/threads.ts

@@ -15,6 +15,7 @@ import { decodeEmailId, decodeMailboxId, encodeEmailId } from "../../mapping/ids
 import { decodeEmailState, encodeEmailState } from "../../state/states.js";
 import { accountNotFound, cannotCalculateChanges } from "../errors.js";
 import { withMailbox } from "../../imap/client.js";
+import { fetchByUid } from "../../imap/uids.js";
 import { listMailboxes } from "./mailbox.js";
 import { parseHeaderBlock, computeThreadIdFromHeaders } from "../../imap/headers.js";
 import { THREAD_HEADER_FIELDS } from "../../imap/fetcher.js";
@@ -128,10 +129,10 @@ async function syncMailboxScan(
     const gaps = [...live].filter((uid) => !cachedSet.has(uid));
     if (gaps.length > 0) {
       const upserts: EmailCacheUpsert[] = [];
-      for await (const msg of client.fetch(
+      for await (const msg of fetchByUid(
+        client,
         gaps,
         { uid: true, envelope: true, internalDate: true, headers: THREAD_HEADER_FIELDS },
-        { uid: true },
       )) {
         if (msg.uid == null) continue;
         const hb =

test/unit/uids.spec.ts

@@ -0,0 +1,75 @@
+// Email/query on a folder with a few thousand messages used to send every UID
+// in one UID FETCH line; servers with a line-length cap rejected it and the
+// folder came back empty. These pin the packing and the per-command bound.
+
+import type { ImapFlow } from "imapflow";
+import { describe, expect, it } from "vitest";
+import { fetchByUid, uidSets } from "../../src/imap/uids.js";
+
+function expand(sets: string[]): number[] {
+  const out: number[] = [];
+  for (const set of sets) {
+    for (const part of set.split(",")) {
+      const [a, b] = part.split(":").map(Number);
+      if (b === undefined) out.push(a!);
+      else for (let u = a!; u <= b; u++) out.push(u);
+    }
+  }
+  return out;
+}
+
+describe("uidSets", () => {
+  it("returns nothing for an empty list", () => {
+    expect(uidSets([])).toEqual([]);
+  });
+
+  it("collapses contiguous runs into ranges", () => {
+    expect(uidSets([1, 2, 3, 5, 7, 8])).toEqual(["1:3,5,7:8"]);
+  });
+
+  it("sorts and de-duplicates its input", () => {
+    expect(uidSets([5, 3, 4, 4, 1])).toEqual(["1,3:5"]);
+  });
+
+  it("drops values that are not valid UIDs", () => {
+    expect(uidSets([0, -1, Number.NaN, 1.5, 2])).toEqual(["2"]);
+  });
+
+  it("keeps a dense folder of any size to a single command", () => {
+    const uids = Array.from({ length: 2500 }, (_, i) => 100000 + i);
+    expect(uidSets(uids)).toEqual(["100000:102499"]);
+  });
+
+  it("splits a sparse set so no command exceeds the byte budget", () => {
+    const uids = Array.from({ length: 20000 }, (_, i) => 100000 + i * 2);
+    const sets = uidSets(uids);
+    expect(sets.length).toBeGreaterThan(1);
+    for (const s of sets) expect(s.length).toBeLessThanOrEqual(4000);
+    expect(expand(sets)).toEqual(uids);
+  });
+
+  it("splits only between parts, never inside one", () => {
+    expect(uidSets([1, 3, 5, 7], 3)).toEqual(["1,3", "5,7"]);
+    expect(uidSets([123456], 3)).toEqual(["123456"]);
+  });
+});
+
+describe("fetchByUid", () => {
+  it("issues one UID FETCH per set and yields every message", async () => {
+    const calls: { range: string; options: unknown }[] = [];
+    const client = {
+      async *fetch(range: string, _query: unknown, options: unknown) {
+        calls.push({ range, options });
+        for (const uid of expand([range])) yield { uid, seq: uid };
+      },
+    } as unknown as ImapFlow;
+
+    const uids = Array.from({ length: 3000 }, (_, i) => 1 + i * 2);
+    const seen: number[] = [];
+    for await (const msg of fetchByUid(client, uids, { uid: true })) seen.push(msg.uid);
+
+    expect(calls.length).toBeGreaterThan(1);
+    for (const c of calls) expect(c.options).toEqual({ uid: true });
+    expect(seen).toEqual(uids);
+  });
+});