From 9b373f6e573dc4850211159b363b5b7b6453b808 Mon Sep 17 00:00:00 2001 From: Valery Lutoshkin Date: Sat, 4 Jul 2026 09:39:44 +0000 Subject: [PATCH 1/2] fix: batch and compress UID FETCH for large mailboxes fetchSortMetadata() and the hasAttachment post-filter each built a single UID FETCH command by joining every matching UID with commas (e.g. UID FETCH 1,2,3,...,2505 (...)). On mailboxes with a few thousand messages this produces a command line long enough that some IMAP servers (observed with MDaemon) reject it or drop the connection outright. The resulting IMAP error wasn't surfaced as a JMAP error, so JMAP clients saw an empty mailbox instead of a fetch failure. Fix: sort UIDs, chunk them into batches of 200, and compress each batch into IMAP sequence-set ranges (e.g. "100:105,110,204:210") instead of a flat comma list. Cuts both the number of round-trips and the command length for large, mostly-contiguous mailboxes. Applies to both fetchSortMetadata() (Email/query sort/fetch path) and the hasAttachment post-filter, which had the same pattern. --- src/jmap/methods/email.ts | 79 ++++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 21 deletions(-) diff --git a/src/jmap/methods/email.ts b/src/jmap/methods/email.ts index 46b8ee8..9debc9d 100644 --- a/src/jmap/methods/email.ts +++ b/src/jmap/methods/email.ts @@ -336,6 +336,35 @@ function compareHits(a: SortableHit, b: SortableHit, sort: SortSpec[]): number { // Enrich UIDs with whatever metadata the requested sort needs. Non-receivedAt // sorts FETCH envelope/size/flags from IMAP โ€” one batched FETCH per mailbox. +// Compress a sorted array of UIDs into IMAP sequence-set ranges, e.g. +// [1,2,3,5,6,9] -> "1:3,5:6,9". Cuts command length dramatically for large, +// mostly-contiguous mailboxes compared to a flat comma-joined list. +function compressUidsToRanges(sortedUids: number[]): string { + if (sortedUids.length === 0) return ""; + const parts: string[] = []; + let start = sortedUids[0]!; + let prev = sortedUids[0]!; + for (let i = 1; i <= sortedUids.length; i++) { + const cur = sortedUids[i]; + if (cur !== undefined && cur === prev + 1) { + prev = cur; + continue; + } + parts.push(start === prev ? `${start}` : `${start}:${prev}`); + if (cur === undefined) break; + start = cur; + prev = cur; + } + return parts.join(","); +} + +// Max UIDs per FETCH command. Kept conservative because some IMAP servers +// (observed with Kerio Connect and MDaemon) enforce stricter command-line +// length limits than Dovecot/Gmail; a single unbatched FETCH over a large +// mailbox (thousands of UIDs) can be rejected or drop the connection outright, +// which previously surfaced to JMAP clients as a silently empty result. +const FETCH_BATCH_SIZE = 200; + async function fetchSortMetadata( client: ImapFlow, uids: number[], @@ -343,23 +372,27 @@ async function fetchSortMetadata( ): Promise }>> { const out = new Map }>(); if (uids.length === 0) return out; - const range = uids.join(","); const query = needsFetch ? { uid: true, internalDate: true, size: true, envelope: true, flags: true } : { uid: true, internalDate: true }; - for await (const msg of client.fetch(range, query, { uid: true })) { - const uid = Number(msg.uid); - out.set(uid, { - receivedAt: msg.internalDate ? new Date(msg.internalDate).getTime() : 0, - size: typeof msg.size === "number" ? msg.size : undefined, - from: pickFirstAddr((msg.envelope as { from?: { address?: string }[] } | undefined)?.from), - to: pickFirstAddr((msg.envelope as { to?: { address?: string }[] } | undefined)?.to), - subject: (msg.envelope as { subject?: string } | undefined)?.subject, - sentAt: (msg.envelope as { date?: Date | string } | undefined)?.date - ? new Date((msg.envelope as { date: Date | string }).date).getTime() - : 0, - flags: msg.flags ? new Set(msg.flags) : undefined, - }); + const sorted = uids.slice().sort((a, b) => a - b); + for (let i = 0; i < sorted.length; i += FETCH_BATCH_SIZE) { + const batch = sorted.slice(i, i + FETCH_BATCH_SIZE); + const range = compressUidsToRanges(batch); + for await (const msg of client.fetch(range, query, { uid: true })) { + const uid = Number(msg.uid); + out.set(uid, { + receivedAt: msg.internalDate ? new Date(msg.internalDate).getTime() : 0, + size: typeof msg.size === "number" ? msg.size : undefined, + from: pickFirstAddr((msg.envelope as { from?: { address?: string }[] } | undefined)?.from), + to: pickFirstAddr((msg.envelope as { to?: { address?: string }[] } | undefined)?.to), + subject: (msg.envelope as { subject?: string } | undefined)?.subject, + sentAt: (msg.envelope as { date?: Date | string } | undefined)?.date + ? new Date((msg.envelope as { date: Date | string }).date).getTime() + : 0, + flags: msg.flags ? new Set(msg.flags) : undefined, + }); + } } return out; } @@ -404,13 +437,17 @@ async function postFilterByHasAttachment( if (uidByEid.size === 0) continue; try { await withMailbox(ctx.client, row.name, async () => { - const range = [...uidByEid.keys()].join(","); - for await (const msg of ctx.client.fetch(range, { uid: true, bodyStructure: true }, { uid: true })) { - const uid = Number(msg.uid); - const eid = uidByEid.get(uid); - if (!eid) continue; - const has = bodyStructureHasAttachment(msg.bodyStructure); - if (has === want) keep.add(eid); + const sortedUids = [...uidByEid.keys()].sort((a, b) => a - b); + for (let i = 0; i < sortedUids.length; i += FETCH_BATCH_SIZE) { + const batch = sortedUids.slice(i, i + FETCH_BATCH_SIZE); + const range = compressUidsToRanges(batch); + for await (const msg of ctx.client.fetch(range, { uid: true, bodyStructure: true }, { uid: true })) { + const uid = Number(msg.uid); + const eid = uidByEid.get(uid); + if (!eid) continue; + const has = bodyStructureHasAttachment(msg.bodyStructure); + if (has === want) keep.add(eid); + } } }); } catch { From 214a445df71dd7082833dc27fd6c565f9c12b569 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Mon, 14 Sep 2026 19:43:50 +0200 Subject: [PATCH 2/2] fix: split UID FETCH into bounded sequence sets for large mailboxes --- src/imap/fetcher.ts | 13 ++++--- src/imap/uids.ts | 51 +++++++++++++++++++++++++ src/jmap/methods/email.ts | 10 ++--- src/jmap/methods/threads.ts | 5 ++- test/unit/uids.spec.ts | 75 +++++++++++++++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 13 deletions(-) create mode 100644 src/imap/uids.ts create mode 100644 test/unit/uids.spec.ts diff --git a/src/imap/fetcher.ts b/src/imap/fetcher.ts index 9a6e7a1..6f83623 100644 --- a/src/imap/fetcher.ts +++ b/src/imap/fetcher.ts @@ -7,6 +7,7 @@ import { selectBodies, structureToBodyParts, type EmailBodyPart } from "../mappi import { flagsToKeywords } from "../mapping/flags.js"; import { encodeBlobId, encodeEmailId, encodeMailboxId } from "../mapping/ids.js"; import { parseHeaderBlock, asMessageIds, computeThreadIdFromHeaders, type ParsedHeader } from "./headers.js"; +import { fetchByUid } from "./uids.js"; import type { AccountRow, MailboxRow, Store, EmailCacheUpsert } from "../state/store.js"; // The header fields threading and the default Email/get property set need: @@ -313,10 +314,10 @@ async function fetchBodyValuesBatched( const byUid = new Map(g.members.map((m) => [m.uid, m])); const rawByUid = new Map>(); try { - for await (const msg of client.fetch( + for await (const msg of fetchByUid( + client, g.members.map((m) => m.uid), { uid: true, bodyParts: g.partIds }, - { uid: true }, )) { if (msg.uid != null && msg.bodyParts) rawByUid.set(msg.uid, msg.bodyParts); } @@ -391,10 +392,10 @@ async function fetchPreviewsBatched( for (const [partId, group] of byPartId) { const taskByUid = new Map(group.map((t) => [t.uid, t])); try { - for await (const m of client.fetch( + for await (const m of fetchByUid( + client, group.map((t) => t.uid), { uid: true, bodyParts: [{ key: partId, start: 0, maxLength: PREVIEW_FETCH_BYTES }] }, - { uid: true }, )) { if (m.uid == null) continue; const buf = m.bodyParts?.get(partId); @@ -464,7 +465,7 @@ export async function fetchEmailsBatch( } if (missing.length > 0) { - for await (const m of client.fetch(missing, metaQuery(needsFull), { uid: true })) { + for await (const m of fetchByUid(client, missing, metaQuery(needsFull))) { if (m.uid == null) continue; raws.set(m.uid, rawFromFetch(m, needsFull)); } @@ -473,7 +474,7 @@ export async function fetchEmailsBatch( // Flags for cache-served messages: one flags-only FETCH for the whole set. const flagUids = uids.filter((uid) => raws.get(uid)?.flags === null); if (flagUids.length > 0) { - for await (const m of client.fetch(flagUids, { uid: true, flags: true }, { uid: true })) { + for await (const m of fetchByUid(client, flagUids, { uid: true, flags: true })) { if (m.uid == null) continue; const r = raws.get(m.uid); if (r) r.flags = new Set(m.flags ?? []); diff --git a/src/imap/uids.ts b/src/imap/uids.ts new file mode 100644 index 0000000..37837d8 --- /dev/null +++ b/src/imap/uids.ts @@ -0,0 +1,51 @@ +import type { FetchMessageObject, FetchQueryObject, ImapFlow } from "imapflow"; + +// Some IMAP servers cap the command line (MDaemon and Kerio Connect reject or +// drop the connection; RFC 7162 ยง4 asks clients to stay under 8192 octets), so +// a UID list must never go out as one flat comma list. imapflow does not guard +// against this: a number[] range is sent as `range.join(",")`, uncompressed. +const MAX_SET_BYTES = 4000; + +/** + * Pack UIDs into IMAP sequence sets ("1:3,5,9:12"), split so no set exceeds + * `maxBytes`. Input may be unsorted or contain duplicates. Contiguous runs + * collapse into one range, so a dense folder of any size stays one command; + * only sparse sets (a search hitting every other message) need several. + */ +export function uidSets(uids: Iterable, maxBytes = MAX_SET_BYTES): string[] { + const sorted = [...new Set(uids)] + .filter((u) => Number.isInteger(u) && u > 0) + .sort((a, b) => a - b); + const sets: string[] = []; + let cur = ""; + let i = 0; + while (i < sorted.length) { + const start = sorted[i]!; + let end = start; + while (sorted[i + 1] === end + 1) end = sorted[++i]!; + i++; + const part = start === end ? `${start}` : `${start}:${end}`; + if (cur && cur.length + 1 + part.length > maxBytes) { + sets.push(cur); + cur = part; + } else { + cur = cur ? `${cur},${part}` : part; + } + } + if (cur) sets.push(cur); + return sets; +} + +/** + * UID FETCH over an arbitrary UID list, one bounded command per sequence set. + * Drop-in for `client.fetch(uids, query, { uid: true })`. + */ +export async function* fetchByUid( + client: ImapFlow, + uids: Iterable, + query: FetchQueryObject, +): AsyncGenerator { + for (const set of uidSets(uids)) { + yield* client.fetch(set, query, { uid: true }); + } +} diff --git a/src/jmap/methods/email.ts b/src/jmap/methods/email.ts index 9cc6375..7ffe3d5 100644 --- a/src/jmap/methods/email.ts +++ b/src/jmap/methods/email.ts @@ -5,6 +5,7 @@ import { decodeEmailId, decodeMailboxId, encodeBlobId, encodeEmailId, encodeMail import { encodeEmailState } from "../../state/states.js"; import { fetchEmailsBatch, type JmapEmail } from "../../imap/fetcher.js"; import { withMailbox } from "../../imap/client.js"; +import { fetchByUid } from "../../imap/uids.js"; import { JmapError, accountNotFound, invalidArguments, notFound, unsupportedFilter, unsupportedSort } from "../errors.js"; import { compileFilter, UnsupportedFilter, type Filter } from "../../imap/search.js"; import { listMailboxes, refreshMailboxCounts } from "./mailbox.js"; @@ -348,7 +349,8 @@ function compareHits(a: SortableHit, b: SortableHit, sort: SortSpec[]): number { } // Enrich UIDs with whatever metadata the requested sort needs. Non-receivedAt -// sorts FETCH envelope/size/flags from IMAP โ€” one batched FETCH per mailbox. +// sorts FETCH envelope/size/flags from IMAP โ€” batched per mailbox, split into +// bounded UID sets so huge folders don't overflow the server's line limit. async function fetchSortMetadata( client: ImapFlow, uids: number[], @@ -359,9 +361,7 @@ async function fetchSortMetadata( const query = needsFetch ? { uid: true, internalDate: true, size: true, envelope: true, flags: true } : { uid: true, internalDate: true }; - // Pass the array, not a joined string: imapflow compresses number[] into - // range syntax (1:5000), keeping the command line bounded on huge folders. - for await (const msg of client.fetch(uids, query, { uid: true })) { + for await (const msg of fetchByUid(client, uids, query)) { const uid = Number(msg.uid); out.set(uid, { receivedAt: msg.internalDate ? new Date(msg.internalDate).getTime() : 0, @@ -440,7 +440,7 @@ async function postFilterByHasAttachment( try { await withMailbox(ctx.client, row.name, async () => { const upserts: EmailCacheUpsert[] = []; - for await (const msg of ctx.client.fetch(missingUids, { uid: true, bodyStructure: true }, { uid: true })) { + for await (const msg of fetchByUid(ctx.client, missingUids, { uid: true, bodyStructure: true })) { const uid = Number(msg.uid); const eid = uidByEid.get(uid); if (!eid) continue; diff --git a/src/jmap/methods/threads.ts b/src/jmap/methods/threads.ts index 403b446..7c28cde 100644 --- a/src/jmap/methods/threads.ts +++ b/src/jmap/methods/threads.ts @@ -15,6 +15,7 @@ import { decodeEmailId, decodeMailboxId, encodeEmailId } from "../../mapping/ids import { decodeEmailState, encodeEmailState } from "../../state/states.js"; import { accountNotFound, cannotCalculateChanges } from "../errors.js"; import { withMailbox } from "../../imap/client.js"; +import { fetchByUid } from "../../imap/uids.js"; import { listMailboxes } from "./mailbox.js"; import { parseHeaderBlock, computeThreadIdFromHeaders } from "../../imap/headers.js"; import { THREAD_HEADER_FIELDS } from "../../imap/fetcher.js"; @@ -128,10 +129,10 @@ async function syncMailboxScan( const gaps = [...live].filter((uid) => !cachedSet.has(uid)); if (gaps.length > 0) { const upserts: EmailCacheUpsert[] = []; - for await (const msg of client.fetch( + for await (const msg of fetchByUid( + client, gaps, { uid: true, envelope: true, internalDate: true, headers: THREAD_HEADER_FIELDS }, - { uid: true }, )) { if (msg.uid == null) continue; const hb = diff --git a/test/unit/uids.spec.ts b/test/unit/uids.spec.ts new file mode 100644 index 0000000..581b8db --- /dev/null +++ b/test/unit/uids.spec.ts @@ -0,0 +1,75 @@ +// Email/query on a folder with a few thousand messages used to send every UID +// in one UID FETCH line; servers with a line-length cap rejected it and the +// folder came back empty. These pin the packing and the per-command bound. + +import type { ImapFlow } from "imapflow"; +import { describe, expect, it } from "vitest"; +import { fetchByUid, uidSets } from "../../src/imap/uids.js"; + +function expand(sets: string[]): number[] { + const out: number[] = []; + for (const set of sets) { + for (const part of set.split(",")) { + const [a, b] = part.split(":").map(Number); + if (b === undefined) out.push(a!); + else for (let u = a!; u <= b; u++) out.push(u); + } + } + return out; +} + +describe("uidSets", () => { + it("returns nothing for an empty list", () => { + expect(uidSets([])).toEqual([]); + }); + + it("collapses contiguous runs into ranges", () => { + expect(uidSets([1, 2, 3, 5, 7, 8])).toEqual(["1:3,5,7:8"]); + }); + + it("sorts and de-duplicates its input", () => { + expect(uidSets([5, 3, 4, 4, 1])).toEqual(["1,3:5"]); + }); + + it("drops values that are not valid UIDs", () => { + expect(uidSets([0, -1, Number.NaN, 1.5, 2])).toEqual(["2"]); + }); + + it("keeps a dense folder of any size to a single command", () => { + const uids = Array.from({ length: 2500 }, (_, i) => 100000 + i); + expect(uidSets(uids)).toEqual(["100000:102499"]); + }); + + it("splits a sparse set so no command exceeds the byte budget", () => { + const uids = Array.from({ length: 20000 }, (_, i) => 100000 + i * 2); + const sets = uidSets(uids); + expect(sets.length).toBeGreaterThan(1); + for (const s of sets) expect(s.length).toBeLessThanOrEqual(4000); + expect(expand(sets)).toEqual(uids); + }); + + it("splits only between parts, never inside one", () => { + expect(uidSets([1, 3, 5, 7], 3)).toEqual(["1,3", "5,7"]); + expect(uidSets([123456], 3)).toEqual(["123456"]); + }); +}); + +describe("fetchByUid", () => { + it("issues one UID FETCH per set and yields every message", async () => { + const calls: { range: string; options: unknown }[] = []; + const client = { + async *fetch(range: string, _query: unknown, options: unknown) { + calls.push({ range, options }); + for (const uid of expand([range])) yield { uid, seq: uid }; + }, + } as unknown as ImapFlow; + + const uids = Array.from({ length: 3000 }, (_, i) => 1 + i * 2); + const seen: number[] = []; + for await (const msg of fetchByUid(client, uids, { uid: true })) seen.push(msg.uid); + + expect(calls.length).toBeGreaterThan(1); + for (const c of calls) expect(c.options).toEqual({ uid: true }); + expect(seen).toEqual(uids); + }); +});