diff --git a/src/imap/fetcher.ts b/src/imap/fetcher.ts index 9a6e7a1..6f83623 100644 --- a/src/imap/fetcher.ts +++ b/src/imap/fetcher.ts @@ -7,6 +7,7 @@ import { selectBodies, structureToBodyParts, type EmailBodyPart } from "../mappi import { flagsToKeywords } from "../mapping/flags.js"; import { encodeBlobId, encodeEmailId, encodeMailboxId } from "../mapping/ids.js"; import { parseHeaderBlock, asMessageIds, computeThreadIdFromHeaders, type ParsedHeader } from "./headers.js"; +import { fetchByUid } from "./uids.js"; import type { AccountRow, MailboxRow, Store, EmailCacheUpsert } from "../state/store.js"; // The header fields threading and the default Email/get property set need: @@ -313,10 +314,10 @@ async function fetchBodyValuesBatched( const byUid = new Map(g.members.map((m) => [m.uid, m])); const rawByUid = new Map>(); try { - for await (const msg of client.fetch( + for await (const msg of fetchByUid( + client, g.members.map((m) => m.uid), { uid: true, bodyParts: g.partIds }, - { uid: true }, )) { if (msg.uid != null && msg.bodyParts) rawByUid.set(msg.uid, msg.bodyParts); } @@ -391,10 +392,10 @@ async function fetchPreviewsBatched( for (const [partId, group] of byPartId) { const taskByUid = new Map(group.map((t) => [t.uid, t])); try { - for await (const m of client.fetch( + for await (const m of fetchByUid( + client, group.map((t) => t.uid), { uid: true, bodyParts: [{ key: partId, start: 0, maxLength: PREVIEW_FETCH_BYTES }] }, - { uid: true }, )) { if (m.uid == null) continue; const buf = m.bodyParts?.get(partId); @@ -464,7 +465,7 @@ export async function fetchEmailsBatch( } if (missing.length > 0) { - for await (const m of client.fetch(missing, metaQuery(needsFull), { uid: true })) { + for await (const m of fetchByUid(client, missing, metaQuery(needsFull))) { if (m.uid == null) continue; raws.set(m.uid, rawFromFetch(m, needsFull)); } @@ -473,7 +474,7 @@ export async function fetchEmailsBatch( // Flags for cache-served messages: one flags-only FETCH for the whole set. const flagUids = uids.filter((uid) => raws.get(uid)?.flags === null); if (flagUids.length > 0) { - for await (const m of client.fetch(flagUids, { uid: true, flags: true }, { uid: true })) { + for await (const m of fetchByUid(client, flagUids, { uid: true, flags: true })) { if (m.uid == null) continue; const r = raws.get(m.uid); if (r) r.flags = new Set(m.flags ?? []); diff --git a/src/imap/uids.ts b/src/imap/uids.ts new file mode 100644 index 0000000..37837d8 --- /dev/null +++ b/src/imap/uids.ts @@ -0,0 +1,51 @@ +import type { FetchMessageObject, FetchQueryObject, ImapFlow } from "imapflow"; + +// Some IMAP servers cap the command line (MDaemon and Kerio Connect reject or +// drop the connection; RFC 7162 §4 asks clients to stay under 8192 octets), so +// a UID list must never go out as one flat comma list. imapflow does not guard +// against this: a number[] range is sent as `range.join(",")`, uncompressed. +const MAX_SET_BYTES = 4000; + +/** + * Pack UIDs into IMAP sequence sets ("1:3,5,9:12"), split so no set exceeds + * `maxBytes`. Input may be unsorted or contain duplicates. Contiguous runs + * collapse into one range, so a dense folder of any size stays one command; + * only sparse sets (a search hitting every other message) need several. + */ +export function uidSets(uids: Iterable, maxBytes = MAX_SET_BYTES): string[] { + const sorted = [...new Set(uids)] + .filter((u) => Number.isInteger(u) && u > 0) + .sort((a, b) => a - b); + const sets: string[] = []; + let cur = ""; + let i = 0; + while (i < sorted.length) { + const start = sorted[i]!; + let end = start; + while (sorted[i + 1] === end + 1) end = sorted[++i]!; + i++; + const part = start === end ? `${start}` : `${start}:${end}`; + if (cur && cur.length + 1 + part.length > maxBytes) { + sets.push(cur); + cur = part; + } else { + cur = cur ? `${cur},${part}` : part; + } + } + if (cur) sets.push(cur); + return sets; +} + +/** + * UID FETCH over an arbitrary UID list, one bounded command per sequence set. + * Drop-in for `client.fetch(uids, query, { uid: true })`. + */ +export async function* fetchByUid( + client: ImapFlow, + uids: Iterable, + query: FetchQueryObject, +): AsyncGenerator { + for (const set of uidSets(uids)) { + yield* client.fetch(set, query, { uid: true }); + } +} diff --git a/src/jmap/methods/email.ts b/src/jmap/methods/email.ts index 9cc6375..7ffe3d5 100644 --- a/src/jmap/methods/email.ts +++ b/src/jmap/methods/email.ts @@ -5,6 +5,7 @@ import { decodeEmailId, decodeMailboxId, encodeBlobId, encodeEmailId, encodeMail import { encodeEmailState } from "../../state/states.js"; import { fetchEmailsBatch, type JmapEmail } from "../../imap/fetcher.js"; import { withMailbox } from "../../imap/client.js"; +import { fetchByUid } from "../../imap/uids.js"; import { JmapError, accountNotFound, invalidArguments, notFound, unsupportedFilter, unsupportedSort } from "../errors.js"; import { compileFilter, UnsupportedFilter, type Filter } from "../../imap/search.js"; import { listMailboxes, refreshMailboxCounts } from "./mailbox.js"; @@ -348,7 +349,8 @@ function compareHits(a: SortableHit, b: SortableHit, sort: SortSpec[]): number { } // Enrich UIDs with whatever metadata the requested sort needs. Non-receivedAt -// sorts FETCH envelope/size/flags from IMAP — one batched FETCH per mailbox. +// sorts FETCH envelope/size/flags from IMAP — batched per mailbox, split into +// bounded UID sets so huge folders don't overflow the server's line limit. async function fetchSortMetadata( client: ImapFlow, uids: number[], @@ -359,9 +361,7 @@ async function fetchSortMetadata( const query = needsFetch ? { uid: true, internalDate: true, size: true, envelope: true, flags: true } : { uid: true, internalDate: true }; - // Pass the array, not a joined string: imapflow compresses number[] into - // range syntax (1:5000), keeping the command line bounded on huge folders. - for await (const msg of client.fetch(uids, query, { uid: true })) { + for await (const msg of fetchByUid(client, uids, query)) { const uid = Number(msg.uid); out.set(uid, { receivedAt: msg.internalDate ? new Date(msg.internalDate).getTime() : 0, @@ -440,7 +440,7 @@ async function postFilterByHasAttachment( try { await withMailbox(ctx.client, row.name, async () => { const upserts: EmailCacheUpsert[] = []; - for await (const msg of ctx.client.fetch(missingUids, { uid: true, bodyStructure: true }, { uid: true })) { + for await (const msg of fetchByUid(ctx.client, missingUids, { uid: true, bodyStructure: true })) { const uid = Number(msg.uid); const eid = uidByEid.get(uid); if (!eid) continue; diff --git a/src/jmap/methods/threads.ts b/src/jmap/methods/threads.ts index 403b446..7c28cde 100644 --- a/src/jmap/methods/threads.ts +++ b/src/jmap/methods/threads.ts @@ -15,6 +15,7 @@ import { decodeEmailId, decodeMailboxId, encodeEmailId } from "../../mapping/ids import { decodeEmailState, encodeEmailState } from "../../state/states.js"; import { accountNotFound, cannotCalculateChanges } from "../errors.js"; import { withMailbox } from "../../imap/client.js"; +import { fetchByUid } from "../../imap/uids.js"; import { listMailboxes } from "./mailbox.js"; import { parseHeaderBlock, computeThreadIdFromHeaders } from "../../imap/headers.js"; import { THREAD_HEADER_FIELDS } from "../../imap/fetcher.js"; @@ -128,10 +129,10 @@ async function syncMailboxScan( const gaps = [...live].filter((uid) => !cachedSet.has(uid)); if (gaps.length > 0) { const upserts: EmailCacheUpsert[] = []; - for await (const msg of client.fetch( + for await (const msg of fetchByUid( + client, gaps, { uid: true, envelope: true, internalDate: true, headers: THREAD_HEADER_FIELDS }, - { uid: true }, )) { if (msg.uid == null) continue; const hb = diff --git a/test/unit/uids.spec.ts b/test/unit/uids.spec.ts new file mode 100644 index 0000000..581b8db --- /dev/null +++ b/test/unit/uids.spec.ts @@ -0,0 +1,75 @@ +// Email/query on a folder with a few thousand messages used to send every UID +// in one UID FETCH line; servers with a line-length cap rejected it and the +// folder came back empty. These pin the packing and the per-command bound. + +import type { ImapFlow } from "imapflow"; +import { describe, expect, it } from "vitest"; +import { fetchByUid, uidSets } from "../../src/imap/uids.js"; + +function expand(sets: string[]): number[] { + const out: number[] = []; + for (const set of sets) { + for (const part of set.split(",")) { + const [a, b] = part.split(":").map(Number); + if (b === undefined) out.push(a!); + else for (let u = a!; u <= b; u++) out.push(u); + } + } + return out; +} + +describe("uidSets", () => { + it("returns nothing for an empty list", () => { + expect(uidSets([])).toEqual([]); + }); + + it("collapses contiguous runs into ranges", () => { + expect(uidSets([1, 2, 3, 5, 7, 8])).toEqual(["1:3,5,7:8"]); + }); + + it("sorts and de-duplicates its input", () => { + expect(uidSets([5, 3, 4, 4, 1])).toEqual(["1,3:5"]); + }); + + it("drops values that are not valid UIDs", () => { + expect(uidSets([0, -1, Number.NaN, 1.5, 2])).toEqual(["2"]); + }); + + it("keeps a dense folder of any size to a single command", () => { + const uids = Array.from({ length: 2500 }, (_, i) => 100000 + i); + expect(uidSets(uids)).toEqual(["100000:102499"]); + }); + + it("splits a sparse set so no command exceeds the byte budget", () => { + const uids = Array.from({ length: 20000 }, (_, i) => 100000 + i * 2); + const sets = uidSets(uids); + expect(sets.length).toBeGreaterThan(1); + for (const s of sets) expect(s.length).toBeLessThanOrEqual(4000); + expect(expand(sets)).toEqual(uids); + }); + + it("splits only between parts, never inside one", () => { + expect(uidSets([1, 3, 5, 7], 3)).toEqual(["1,3", "5,7"]); + expect(uidSets([123456], 3)).toEqual(["123456"]); + }); +}); + +describe("fetchByUid", () => { + it("issues one UID FETCH per set and yields every message", async () => { + const calls: { range: string; options: unknown }[] = []; + const client = { + async *fetch(range: string, _query: unknown, options: unknown) { + calls.push({ range, options }); + for (const uid of expand([range])) yield { uid, seq: uid }; + }, + } as unknown as ImapFlow; + + const uids = Array.from({ length: 3000 }, (_, i) => 1 + i * 2); + const seen: number[] = []; + for await (const msg of fetchByUid(client, uids, { uid: true })) seen.push(msg.uid); + + expect(calls.length).toBeGreaterThan(1); + for (const c of calls) expect(c.options).toEqual({ uid: true }); + expect(seen).toEqual(uids); + }); +});