From f87a925417745e425e91ab5744ac4c69fe86459e Mon Sep 17 00:00:00 2001 From: Divide-By-0 Date: Sat, 3 Oct 2026 17:13:49 -0700 Subject: [PATCH] fix(verify): decode quoted-printable bodies even when the encoding header is hidden Proofs that hide the body and reveal a few sentences also hide the MIME part's Content-Transfer-Encoding header, so the verify page and CLI showed raw QP ("samples=E2=80=94whether iden=" + soft line breaks). Detect QP from the revealed text too (soft breaks not after '=', =XX escapes that decode to valid UTF-8 between hidden bytes; never when another encoding is declared) and add a 'Show raw' toggle. Display only: the proof still covers the encoded bytes. Co-Authored-By: Claude Opus 5.5 --- .../redacted-email-proof/scripts/redacted.mjs | 43 +++++++++++++++-- src/lib.ts | 11 ++++- src/pages/VerifyPage.tsx | 17 +++++-- src/utils/qp.ts | 47 +++++++++++++++++-- tests/qp.test.ts | 37 ++++++++++++++- 5 files changed, 142 insertions(+), 13 deletions(-) diff --git a/skills/redacted-email-proof/scripts/redacted.mjs b/skills/redacted-email-proof/scripts/redacted.mjs index 4481106..4cebae8 100755 --- a/skills/redacted-email-proof/scripts/redacted.mjs +++ b/skills/redacted-email-proof/scripts/redacted.mjs @@ -763,9 +763,10 @@ export function outputsOf(publicInputs, prefix, maxHeaderLength) { const header = latin1(headerBytes.slice(0, headerLen)); const body = latin1(bodyBytes.slice(0, bodyLen)); const pretty = (s) => compact(Buffer.from(s, "latin1").toString("utf8").replace(/\u0000/g, BLOCK).replace(/\r\n/g, "\n")); - const qpBody = hasQuotedPrintablePart(Buffer.from(body, "latin1")); + const qpEvidence = quotedPrintableEvidence(Buffer.from(body, "latin1")); + const qpBody = qpEvidence !== null; const bodyText = qpBody ? pretty(displayDecodeMasked(Buffer.from(body, "latin1")).toString("latin1")) : pretty(body); - return { header, body, headerText: pretty(header), bodyText, bodyDecodedFromQp: qpBody }; + return { header, body, headerText: pretty(header), bodyText, bodyRawText: pretty(body), bodyDecodedFromQp: qpBody, qpEvidence }; } // REASON: noir-bignum >= v0.9 (used by the v2 circuits) defines the Barrett parameter as @@ -805,6 +806,37 @@ export function qpDecode(bytes) { return Buffer.from(out); } // Decode a masked raw body (0x00 = hidden) for display; an escape with a hidden byte -> one 0x00. +// Same as src/utils/qp.ts quotedPrintableEvidence: "header" | "content" | null. Decide display +// decoding from a revealed Content-Transfer-Encoding header, else from the text itself (soft +// breaks not after "=", uppercase =XX runs that are valid UTF-8). See the REASON note there. +export function quotedPrintableEvidence(maskedBody) { + const text = Buffer.from(maskedBody).toString("latin1"); + const ctes = [...text.matchAll(/content-transfer-encoding:\s*([a-z0-9-]+)/gi)].map((m) => m[1].toLowerCase()); + if (ctes.includes("quoted-printable")) return "header"; + if (ctes.length) return null; + let softBreaks = 0; + for (const m of text.matchAll(/=\r\n/g)) { + const prev = text[m.index - 1]; + if (prev !== "=" && prev !== "\0" && prev !== undefined) softBreaks++; + } + const escapes = (text.match(/=[0-9A-F]{2}/g) ?? []).length; + if (escapes) { + // validate the decoded text between hidden bytes (QP can split a UTF-8 char across "=\r\n") + const decoded = displayDecodeMasked(Buffer.from(maskedBody)); + let start = 0; + for (let i = 0; i <= decoded.length; i++) { + if (i < decoded.length && decoded[i] !== 0) continue; + try { + new TextDecoder("utf-8", { fatal: true }).decode(decoded.subarray(start, i)); + } catch { + return null; + } + start = i + 1; + } + } + return softBreaks >= 2 || (softBreaks >= 1 && escapes >= 1) || escapes >= 3 ? "content" : null; +} + export function displayDecodeMasked(bytes) { const out = []; for (let i = 0; i < bytes.length; i++) { @@ -1175,6 +1207,8 @@ async function cmdVerify(ref, opts) { fromAlignedWithDkimDomain: binding?.fromAligned ?? null, maskedHeader: outs.headerText, maskedBody: outs.bodyText, + maskedBodyRaw: outs.bodyRawText ?? outs.bodyText, + bodyDecodedFromQp: outs.bodyDecodedFromQp ?? false, }; if (opts.json) { console.log(JSON.stringify(result, null, 2)); @@ -1209,7 +1243,10 @@ async function cmdVerify(ref, opts) { ); } } - console.log("\n----- masked header -----\n" + outs.headerText + "\n----- masked body -----\n" + outs.bodyText); + const qpNote = outs.bodyDecodedFromQp + ? ` (decoded from quoted-printable${outs.qpEvidence === "content" ? ", detected from the text" : ""}; the proof covers the encoded bytes, see maskedBodyRaw in --json)` + : ""; + console.log("\n----- masked header -----\n" + outs.headerText + `\n----- masked body${qpNote} -----\n` + outs.bodyText); } if (!result.proofValid) process.exit(2); if (!result.keyMatches) process.exit(3); diff --git a/src/lib.ts b/src/lib.ts index a6445f7..f597a56 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -14,7 +14,7 @@ import type { DKIMResult } from "./utils/emlParser"; import circuitConfigs from "./circuit-configs.json"; import { canonicalPublicInputs, decodeMaskedBytes, V2_PREFIX } from "./utils/proofOutputs"; import { dkimFieldSequence } from "./utils/dkimFields"; -import { bodyViewFor, charMaskToByteMask, viewMaskToRawMask, displayDecodeMasked, looksQuotedPrintable } from "./utils/qp"; +import { bodyViewFor, charMaskToByteMask, viewMaskToRawMask, displayDecodeMasked, quotedPrintableEvidence } from "./utils/qp"; /** * Circuit versions @@ -385,6 +385,10 @@ export function extractMaskedDataFromProof(proof: ProofWithCircuit): { emailNullifier: Uint8Array; version: CircuitVersion; bodyDecodedFromQp: boolean; + /** why the body was decoded: a revealed Content-Transfer-Encoding header, or the text itself */ + qpEvidence: "header" | "content" | null; + /** the proven body bytes as text, undecoded (for "show raw") */ + maskedBodyRaw: string; } | null { try { // Decode exactly what handleVerifyProof verifies (throws on non-canonical inputs). @@ -415,11 +419,14 @@ export function extractMaskedDataFromProof(proof: ProofWithCircuit): { const decoder = new TextDecoder("utf-8", { fatal: false }); // Display-only decoding of the proven raw bytes (quoted-printable is a public function of them). - const bodyDecodedFromQp = looksQuotedPrintable(body); + const qpEvidence = quotedPrintableEvidence(body); + const bodyDecodedFromQp = qpEvidence !== null; return { maskedHeader: decoder.decode(header), maskedBody: decoder.decode(bodyDecodedFromQp ? displayDecodeMasked(body) : body), + maskedBodyRaw: decoder.decode(body), bodyDecodedFromQp, + qpEvidence, publicKeyHash, emailNullifier, version: layout.version, diff --git a/src/pages/VerifyPage.tsx b/src/pages/VerifyPage.tsx index 589028b..44468b4 100644 --- a/src/pages/VerifyPage.tsx +++ b/src/pages/VerifyPage.tsx @@ -48,6 +48,9 @@ export default function VerifyPage() { } | null>(null); const [maskedBody, setMaskedBody] = useState(''); const [bodyDecodedFromQp, setBodyDecodedFromQp] = useState(false); + const [qpEvidence, setQpEvidence] = useState<"header" | "content" | null>(null); + const [maskedBodyRaw, setMaskedBodyRaw] = useState(""); + const [showRawBody, setShowRawBody] = useState(false); const [showVerificationModal, setShowVerificationModal] = useState(false); useEffect(() => { @@ -183,6 +186,8 @@ export default function VerifyPage() { // Store the masked body setMaskedBody(maskedData.maskedBody); setBodyDecodedFromQp(maskedData.bodyDecodedFromQp); + setQpEvidence(maskedData.qpEvidence); + setMaskedBodyRaw(maskedData.maskedBodyRaw); } catch (err) { console.error("Error loading verification data:", err); @@ -379,13 +384,19 @@ export default function VerifyPage() {
{bodyDecodedFromQp && (

- Shown decoded from quoted-printable. The proof covers the encoded bytes; decoding - them is a fixed public step anyone can repeat. + {showRawBody + ? "Shown as the raw quoted-printable bytes the proof covers." + : qpEvidence === "content" + ? "Shown decoded from quoted-printable (detected from the text; its Content-Transfer-Encoding header is hidden). The proof covers the encoded bytes; decoding them is a fixed public step anyone can repeat." + : "Shown decoded from quoted-printable. The proof covers the encoded bytes; decoding them is a fixed public step anyone can repeat."}{" "} +

)}
diff --git a/src/utils/qp.ts b/src/utils/qp.ts index e732f71..0dd51cb 100644 --- a/src/utils/qp.ts +++ b/src/utils/qp.ts @@ -161,6 +161,47 @@ export function displayDecodeMasked(bytes: Uint8Array): Uint8Array { return Uint8Array.from(out); } -/** Should the verify page decode this (masked) body? Only if its revealed bytes say it's QP. */ -export const looksQuotedPrintable = (maskedBody: Uint8Array) => - /content-transfer-encoding:\s*quoted-printable/i.test(new TextDecoder("latin1").decode(maskedBody)); +/** + * Should the verify page decode this (masked) body, and why: "header" when a revealed + * Content-Transfer-Encoding header says quoted-printable, "content" when the revealed text itself + * is clearly QP, else null. + * REASON: provers often hide the whole body and reveal a few sentences, which hides the part's + * Content-Transfer-Encoding header. The page then showed raw "samples=E2=80=94whether iden=" text. + * Decoding is display-only (the proof covers the raw bytes, and "show raw" stays available), so a + * content heuristic is safe as long as it doesn't fire on ordinary text: + * - soft line breaks "=\r\n" not preceded by "=" (base64 padding "==" at a line end is not QP), and + * - uppercase "=XX" escapes, where every revealed stretch of the decoded text is valid UTF-8 + * (checked after decoding, because QP may split one UTF-8 character across a soft break: + * "=E2=80=\r\n=94" is an em dash), + * - and no revealed Content-Transfer-Encoding header naming another encoding. + */ +export function quotedPrintableEvidence(maskedBody: Uint8Array): "header" | "content" | null { + const text = new TextDecoder("latin1").decode(maskedBody); + const ctes = [...text.matchAll(/content-transfer-encoding:\s*([a-z0-9-]+)/gi)].map((m) => m[1].toLowerCase()); + if (ctes.includes("quoted-printable")) return "header"; + if (ctes.length) return null; + let softBreaks = 0; + for (const m of text.matchAll(/=\r\n/g)) { + const prev = text[m.index! - 1]; + if (prev !== "=" && prev !== "\0" && prev !== undefined) softBreaks++; + } + const escapes = (text.match(/=[0-9A-F]{2}/g) ?? []).length; + if (escapes) { + // Validate between hidden bytes (0x00): a hidden neighbour can cut a character in half. + const decoded = displayDecodeMasked(maskedBody); + let start = 0; + for (let i = 0; i <= decoded.length; i++) { + if (i < decoded.length && decoded[i] !== 0) continue; + try { + new TextDecoder("utf-8", { fatal: true }).decode(decoded.subarray(start, i)); + } catch { + return null; // escapes that aren't UTF-8 text: probably not QP, don't guess + } + start = i + 1; + } + } + return softBreaks >= 2 || (softBreaks >= 1 && escapes >= 1) || escapes >= 3 ? "content" : null; +} + +/** Should the verify page decode this (masked) body? See quotedPrintableEvidence. */ +export const looksQuotedPrintable = (maskedBody: Uint8Array) => quotedPrintableEvidence(maskedBody) !== null; diff --git a/tests/qp.test.ts b/tests/qp.test.ts index d0335a8..4525477 100644 --- a/tests/qp.test.ts +++ b/tests/qp.test.ts @@ -5,9 +5,9 @@ */ import { test } from "node:test"; import assert from "node:assert/strict"; -import { bodyViewFor, displayDecodeMasked, qpDecode, viewMaskToRawMask } from "../src/utils/qp"; +import { bodyViewFor, displayDecodeMasked, qpDecode, quotedPrintableEvidence, viewMaskToRawMask } from "../src/utils/qp"; // @ts-expect-error - plain JS module without types -import { qpDecode as cliQpDecode, displayDecodeMasked as cliDisplayDecodeMasked } from "../skills/redacted-email-proof/scripts/redacted.mjs"; +import { qpDecode as cliQpDecode, displayDecodeMasked as cliDisplayDecodeMasked, quotedPrintableEvidence as cliQpEvidence } from "../skills/redacted-email-proof/scripts/redacted.mjs"; const RAW = Buffer.from( "Content-Type: text/html; charset=utf-8\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n" + @@ -64,3 +64,36 @@ test("display decoding of masked output: partly hidden escapes show as one hidde assert.equal(Buffer.from(cliDisplayDecodeMasked(masked)).toString("latin1"), site); assert.equal(site, "xâ\u0082¬ y \0 z \0!"); }); + +// A proof that hides the part's Content-Transfer-Encoding header (e.g. --hide-body with a few +// sentences revealed) must still display decoded text, not "samples=E2=80=94whether iden=". +test("QP detected from the text when the Content-Transfer-Encoding header is hidden", () => { + const raw = Buffer.from( + "Content-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n" + + "we never sell data derived from our customers' samples=E2=80=94whether iden=\r\ntifiable or not=\r\n, ever=E2=80=\r\n=94really.\r\n", + "latin1" + ); + const masked = Buffer.from(raw); + masked.fill(0, 0, raw.indexOf("\r\n\r\n") + 4); // MIME part headers hidden + for (const [name, evidence] of [["site", quotedPrintableEvidence], ["cli", cliQpEvidence]] as const) { + assert.equal(evidence(raw), "header", name); + assert.equal(evidence(masked), "content", name); + } + const shownText = Buffer.from(displayDecodeMasked(masked)).toString("utf8"); + assert.ok(shownText.includes("samples—whether identifiable or not, ever—really."), shownText); +}); + +test("QP content detection does not fire on base64, plain text, or another declared encoding", () => { + const cases: [string, string][] = [ + ["base64 padding at line ends", "SGVsbG8gd29ybGQ=\r\nSGk==\r\nQUJD==\r\n"], + ["plain text with = signs", "x = 1\r\nif (a == b) total = 3D printer\r\n"], + ["one lone soft-break-like line", "price =\r\nsee below\r\n"], + ["escapes that aren't UTF-8", "bytes =FF=FE=FD here\r\n"], + ["declared base64 part", "Content-Transfer-Encoding: base64\r\n\r\nc2Ftc=\r\nGxl=E2=80=94=\r\n"], + ]; + for (const [label, text] of cases) { + const b = Buffer.from(text, "latin1"); + assert.equal(quotedPrintableEvidence(b), null, `site: ${label}`); + assert.equal(cliQpEvidence(b), null, `cli: ${label}`); + } +});