diff --git a/skills/redacted-email-proof/SKILL.md b/skills/redacted-email-proof/SKILL.md index 25ce412..15c96d1 100644 --- a/skills/redacted-email-proof/SKILL.md +++ b/skills/redacted-email-proof/SKILL.md @@ -97,6 +97,21 @@ $R prove email.eml --dry-run \ command **fails if a hide target isn't found** (use `--allow-missing` to override). Treat that failure as a signal: the text may be encoded differently, so read the `inspect` output. - Hides always win over reveals. +- **Reveal each passage once, in its most readable copy.** Most HTML email is + `multipart/alternative`: the same message appears as a `text/plain` part and again as a + `text/html` part. Revealing a passage in both publishes it twice, the second time wrapped in + tags and entities (`can't`, `
`), which is noisy and exposes the HTML layout around it. + - The CLI handles the common case: when a `--reveal` matches the plain-text part, its copies in + the HTML part stay hidden and a warning says so. Pass `--reveal-all-parts` only if the user + wants both. + - When choosing what to reveal, read `inspect`'s body and pick the plain-text wording. Prefer + `--reveal` phrases over broad `--reveal-regex` patterns that also catch the HTML copy. + - If two passages are near-duplicates (a quoted reply, a signature repeated at the bottom, a + plain and an HTML rendering), reveal only the one a person would read: plain text over HTML, + the original over the quoted copy. + - If the text exists *only* in the HTML part, reveal it there, and keep it to the text itself: + don't widen the reveal to the surrounding tags. + - Check the preview: the same sentence should not appear twice. - **Show the preview to the user and get their OK before proving.** Everything not shown as █ in the preview becomes public if the proof is published. diff --git a/skills/redacted-email-proof/scripts/redacted.mjs b/skills/redacted-email-proof/scripts/redacted.mjs index 4cebae8..6af3699 100755 --- a/skills/redacted-email-proof/scripts/redacted.mjs +++ b/skills/redacted-email-proof/scripts/redacted.mjs @@ -114,6 +114,8 @@ Mask options for prove (hidden bytes become 0x00 in the proof and show as ${BLOC --reveal With --hide-body: re-reveal occurrences of this text. Repeatable. Hides always win over reveals. --reveal-regex Regex form of --reveal. + --reveal-all-parts Also reveal matches inside the text/html part when the text is in the + plain-text part too (default: reveal only the plain-text copy). --reveal-dkim-hashes Keep DKIM-Signature bh= and b= visible (hidden by default because bh= lets anyone confirm a guess of a fully hidden body). --allow-missing Don't fail when a --hide target is not found. @@ -560,6 +562,37 @@ function headerLines(h) { return lines.filter((l) => l.name); } +// Content byte ranges ([start, end)) of MIME parts of one media type (e.g. "text/html"), found +// from their delimiter lines ("--"), so nested multiparts work too. Part headers are +// excluded: only the content after each part's blank line counts. +export function mimePartRanges(body, type) { + const delims = [...body.matchAll(/(?:^|\r\n)--[^\r\n\s]{1,200}(?=\r\n|$)/g)].map((m) => m.index + (m[0].startsWith("\r\n") ? 2 : 0)); + const re = new RegExp(`^content-type:\\s*${type.replace("/", "\\/")}`, "im"); + const out = []; + for (let i = 0; i < delims.length; i++) { + const partStart = body.indexOf("\r\n", delims[i]) + 2; + const headersEnd = body.indexOf("\r\n\r\n", partStart); + const partEnd = i + 1 < delims.length ? delims[i + 1] : body.length; + if (partStart < 2 || headersEnd < 0 || headersEnd > partEnd) continue; + if (re.test(body.slice(partStart, headersEnd))) out.push([headersEnd + 4, partEnd]); + } + return out; +} +export const htmlPartRanges = (body) => mimePartRanges(body, "text/html"); + +// Every match of `matcher` in the body (raw, quoted-printable and base64 views), as byte spans +// plus the matched (decoded) text, so reveals can be deduplicated per passage. +function matchSpans(s, matcher) { + const out = []; + for (const view of viewsOf(s)) { + for (const [a, b] of matcher(view.text)) { + const [lo, hi] = view.span ? view.span(a, b) : [view.map(a)[0], view.map(b - 1)[1]]; + out.push({ lo, hi, text: view.text.slice(a, b).replace(/\s+/g, " ").trim() }); + } + } + return out; +} + export function buildMasks(dkim, opts) { const h = lat1(dkim.headers); const b = lat1(dkim.body); @@ -600,11 +633,37 @@ export function buildMasks(dkim, opts) { } // reveals first, hides last: hides always win + // REASON: multipart/alternative mail carries the same message twice, as text/plain and as + // text/html. Revealing a passage in both publishes it twice, the second time wrapped in markup + // and entities ("can't"), which is noisy and reveals the HTML structure around it for nothing. + // Deduplicated PER MATCH: an HTML match is skipped only when the same matched text also occurs + // in real text/plain content (not MIME headers or attachments, which aren't a readable copy). + // HTML-only passages, including other matches of the same regex, are still revealed. + // --reveal-all-parts keeps every copy. + const html = htmlPartRanges(b); + const plain = mimePartRanges(b, "text/plain"); + const within = (ranges, lo, hi) => ranges.some(([s0, e0]) => lo >= s0 && hi <= e0); + const reveal = (label, matcher) => { + const spans = matchSpans(b, matcher); + const plainTexts = new Set(spans.filter((m) => within(plain, m.lo, m.hi)).map((m) => m.text)); + let revealed = 0; + let skipped = 0; + for (const m of spans) { + if (!opts["reveal-all-parts"] && within(html, m.lo, m.hi) && plainTexts.has(m.text)) { + skipped++; + continue; + } + for (let k = m.lo; k < m.hi; k++) bodyMask[k] = 1; + revealed++; + } + if (skipped) warnings.push(`${label}: ${skipped} copy(ies) in the HTML part left hidden; the same text is revealed in the plain-text part (--reveal-all-parts reveals both)`); + return revealed; + }; for (const t of [].concat(opts.reveal || [])) { - if (!applyMatches(bodyMask, b, literalMatcher(needleLatin1(t)), 1)) warnings.push(`--reveal "${t}" not found in body`); + if (!reveal(`--reveal "${t}"`, literalMatcher(needleLatin1(t)))) warnings.push(`--reveal "${t}" not found in body`); } for (const r of [].concat(opts["reveal-regex"] || [])) { - if (!applyMatches(bodyMask, b, regexMatcher(r), 1)) warnings.push(`--reveal-regex /${r}/ matched nothing in body`); + if (!reveal(`--reveal-regex /${r}/`, regexMatcher(r))) warnings.push(`--reveal-regex /${r}/ matched nothing in body`); } // header name bytes are structural; only hide inside values @@ -623,7 +682,17 @@ export function buildMasks(dkim, opts) { // Long hidden runs (usually whole hidden lines, newlines included) become one "█×N" marker so the // preview stays readable in a terminal or an agent's context window. -const compact = (s) => s.replace(new RegExp(`${BLOCK}{12,}`, "g"), (m) => `${BLOCK}×${m.length}`); +// Shorten redactions for the terminal: a long run of blocks becomes "█×N", and 3+ consecutive +// lines that are entirely redacted become one "█ ⋯ N redacted characters (L lines) ⋯" line +// (same idea as the verify page's collapsed redactions, src/utils/maskedRuns.ts). +export const compact = (s) => + s + .replace(new RegExp(`${BLOCK}{12,}`, "g"), (m) => `${BLOCK}×${m.length}`) + .replace(new RegExp(`(?:^[ \\t]*${BLOCK}+(?:×(\\d+))?[ \\t]*(?:\\n|$)){3,}`, "gm"), (m) => { + const lines = m.split("\n").filter((l) => l.trim()); + const n = lines.reduce((sum, l) => sum + (Number(/×(\d+)/.exec(l)?.[1]) || l.trim().length), 0); + return `${BLOCK} ⋯ ${n.toLocaleString("en-US")} redacted characters (${lines.length} lines) ⋯${m.endsWith("\n") ? "\n" : ""}`; + }); function renderDecoded(s, mask) { const bytes = Buffer.from(s, "latin1").map((v, i) => (mask[i] ? v : 0)); @@ -1271,6 +1340,7 @@ if (isMain) { reveal: { type: "string", multiple: true }, "reveal-regex": { type: "string", multiple: true }, "reveal-dkim-hashes": { type: "boolean" }, + "reveal-all-parts": { type: "boolean" }, "allow-missing": { type: "boolean" }, domain: { type: "string" }, "dry-run": { type: "boolean" }, diff --git a/src/components/MaskedText.tsx b/src/components/MaskedText.tsx index bf1ad9e..85d7d1f 100644 --- a/src/components/MaskedText.tsx +++ b/src/components/MaskedText.tsx @@ -1,67 +1,64 @@ +import { useState } from 'react'; +import { maskedSegments } from '../utils/maskedRuns'; + interface MaskedTextProps { text: string; className?: string; } -/** - * Component that displays text with null bytes rendered as black blocks - * - * Null bytes (0x00) in the text are displayed as black rectangles, - * representing masked/redacted content from the ZK proof. - */ -export default function MaskedText({ text, className = '' }: MaskedTextProps) { - // Split text into segments: regular text and null byte sequences - const segments: { type: 'text' | 'masked'; content: string }[] = []; - let currentSegment = ''; - let inMaskedSection = false; - - for (let i = 0; i < text.length; i++) { - const char = text[i]; - const isNull = char === '\x00'; - - if (isNull !== inMaskedSection) { - // Transition between masked and unmasked - if (currentSegment) { - segments.push({ - type: inMaskedSection ? 'masked' : 'text', - content: currentSegment, - }); - } - currentSegment = char; - inMaskedSection = isNull; - } else { - currentSegment += char; - } - } +const Blocks = ({ count }: { count: number }) => ( + + {'█'.repeat(count)} + +); - // Push final segment - if (currentSegment) { - segments.push({ - type: inMaskedSection ? 'masked' : 'text', - content: currentSegment, - }); +/** A long redacted stretch, collapsed to one row; click to show every block. */ +function CollapsedRedaction({ content, hidden, lines }: { content: string; hidden: number; lines: number }) { + const [open, setOpen] = useState(false); + if (open) { + return ( + <> + {content.split(/(\0+)/).map((part, i) => + part.startsWith('\0') ? : {part} + )} + + + ); } + return ( + + ); +} +/** + * Text with null bytes (0x00, masked content from the ZK proof) rendered as black blocks. + * Long redacted stretches are collapsed to one row (see utils/maskedRuns.ts). + */ +export default function MaskedText({ text, className = '' }: MaskedTextProps) { return ( - {segments.map((segment, idx) => { - if (segment.type === 'masked') { - // Render black blocks for masked content - // Each null byte becomes one block character - return ( - - {'\u2588'.repeat(segment.content.length)} - - ); - } + {maskedSegments(text).map((segment, idx) => { + if (segment.type === 'collapsed') return ; + if (segment.type === 'masked') return ; return {segment.content}; })} diff --git a/src/utils/maskedRuns.ts b/src/utils/maskedRuns.ts new file mode 100644 index 0000000..d86d3da --- /dev/null +++ b/src/utils/maskedRuns.ts @@ -0,0 +1,64 @@ +/** + * Split masked text (0x00 = hidden byte) into display segments, collapsing long redacted stretches. + * + * REASON: a proof that hides most of a body shows tens of thousands of █ blocks, which bury the + * few revealed sentences. A redacted stretch is either + * - one long run of 0x00 (the hidden newlines are hidden too, so its "lines" are only wrapping), or + * - many fully hidden lines whose line breaks were revealed: 0x00 runs separated by whitespace only. + * Both become one "collapsed" segment the page can show as "⋯ N redacted characters (~L lines) ⋯" + * with a toggle to expand. Display only: the proof and the decoded bytes are unchanged. + */ +export type MaskedSegment = + | { type: "text"; content: string } + | { type: "masked"; content: string } + /** a long redacted stretch: `content` is the original text of the stretch (hidden bytes and the whitespace between them) */ + | { type: "collapsed"; content: string; hidden: number; lines: number }; + +/** Collapse redacted stretches with at least this many hidden characters (~3 wrapped lines). */ +export const COLLAPSE_MIN_HIDDEN = 240; +/** Characters per line used to estimate the line count of a run whose newlines are hidden. */ +export const CHARS_PER_LINE = 80; + +export function maskedSegments(text: string, minHidden = COLLAPSE_MIN_HIDDEN): MaskedSegment[] { + // 1. plain runs: text vs 0x00 + const runs: { masked: boolean; content: string }[] = []; + for (const m of text.matchAll(/\0+|[^\0]+/g)) runs.push({ masked: m[0][0] === "\0", content: m[0] }); + + // 2. group masked runs that are separated only by whitespace (revealed line breaks between + // fully hidden lines) into one stretch + const out: MaskedSegment[] = []; + for (let i = 0; i < runs.length; ) { + if (!runs[i].masked) { + out.push({ type: "text", content: runs[i].content }); + i++; + continue; + } + let j = i; + let hidden = 0; + let content = ""; + let breaks = 0; + // extend while the next text run is whitespace only and is followed by another masked run + for (;;) { + hidden += runs[j].content.length; + content += runs[j].content; + const gap = runs[j + 1]; + if (gap && !gap.masked && /^\s+$/.test(gap.content) && runs[j + 2]?.masked) { + content += gap.content; + breaks += (gap.content.match(/\n/g) ?? []).length; + j += 2; + continue; + } + break; + } + if (hidden >= minHidden) { + // Revealed line breaks give exact lines; hidden newlines don't, so estimate from length. + const lines = Math.max(breaks + 1, Math.ceil(hidden / CHARS_PER_LINE)); + out.push({ type: "collapsed", content, hidden, lines }); + } else { + // short: keep the original runs as they were + for (let k = i; k <= j; k++) out.push({ type: runs[k].masked ? "masked" : "text", content: runs[k].content }); + } + i = j + 1; + } + return out; +} diff --git a/tests/masked-runs.test.ts b/tests/masked-runs.test.ts new file mode 100644 index 0000000..6e0fde0 --- /dev/null +++ b/tests/masked-runs.test.ts @@ -0,0 +1,86 @@ +/** Verify page: long redacted stretches collapse to one row (src/utils/maskedRuns.ts). */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { maskedSegments } from "../src/utils/maskedRuns"; + +const H = (n: number) => "\0".repeat(n); + +test("one long hidden run (hidden newlines) collapses with an estimated line count", () => { + const segs = maskedSegments(`Hi${H(27430)}omissions.${H(10)}`); + assert.deepEqual(segs.map((s) => s.type), ["text", "collapsed", "text", "masked"]); + const c = segs[1] as Extract<(typeof segs)[number], { type: "collapsed" }>; + assert.equal(c.hidden, 27430); + assert.equal(c.lines, Math.ceil(27430 / 80)); +}); + +test("many fully hidden lines with revealed line breaks merge into one stretch with the exact line count", () => { + const lines = Array.from({ length: 111 }, () => H(70)).join("\r\n"); + const segs = maskedSegments(`Dear X,\r\n${lines}\r\nThanks`); + assert.deepEqual(segs.map((s) => s.type), ["text", "collapsed", "text"]); + const c = segs[1] as Extract<(typeof segs)[number], { type: "collapsed" }>; + assert.equal(c.hidden, 111 * 70); + assert.equal(c.lines, 111); + // nothing is lost: the collapsed stretch keeps the original bytes for "show" + assert.equal(segs.map((s) => s.content).join(""), `Dear X,\r\n${lines}\r\nThanks`); +}); + +test("short redactions and stretches broken by revealed words stay inline", () => { + assert.deepEqual(maskedSegments(`To: ${H(30)}\r\n`).map((s) => s.type), ["text", "masked", "text"]); + const segs = maskedSegments(`${H(200)} word ${H(200)}`); + assert.deepEqual(segs.map((s) => s.type), ["masked", "text", "masked"]); +}); + +test("CLI compact: long runs become █×N and 3+ fully redacted lines merge into one", async () => { + // @ts-expect-error - plain JS module without types + const { compact } = await import("../skills/redacted-email-proof/scripts/redacted.mjs"); + const B = "█"; + const lines = [...Array.from({ length: 110 }, () => B.repeat(70)), B.repeat(5)].join("\n"); + assert.equal(compact(`Dear X,\n${lines}\nThanks ${B.repeat(5)}\n`), `Dear X,\n${B} ⋯ 7,705 redacted characters (111 lines) ⋯\nThanks ${B.repeat(5)}\n`); + assert.equal(compact(`a\n${B.repeat(20)}\n${B.repeat(20)}\nb`), `a\n${B}×20\n${B}×20\nb`, "two lines stay as they are"); +}); + +test("CLI --reveal: text in both the plain and HTML parts is revealed only in the plain-text part", async () => { + // @ts-expect-error - plain JS module without types + const { buildMasks, htmlPartRanges } = await import("../skills/redacted-email-proof/scripts/redacted.mjs"); + const body = + "--b1\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nWe can share the lab agreement.\r\n" + + "--b1\r\nContent-Type: text/html; charset=utf-8\r\n\r\n
We can share the lab agreement.

Only in html

\r\n--b1--\r\n"; + const dkim = { headers: Buffer.from("from:a@b.example\r\n"), body: Buffer.from(body) }; + const shown = (mask: number[]) => [...body].map((c, i) => (mask[i] ? c : "\0")).join("").replace(/\0+/g, "█"); + assert.equal(htmlPartRanges(body).length, 1); + + const m = buildMasks(dkim, { "hide-body": true, reveal: ["We can share the lab agreement.", "Only in html"] }); + assert.equal(shown(m.bodyMask).match(/We can share/g)?.length, 1, "revealed once"); + assert.ok(body.indexOf("We can share") === shown(m.bodyMask).indexOf("We can share") || shown(m.bodyMask).startsWith("█We can share"), "the plain-text copy"); + assert.match(shown(m.bodyMask), /Only in html/, "HTML-only text is still revealed"); + assert.ok(m.warnings.some((w: string) => /HTML part left hidden/.test(w))); + + const all = buildMasks(dkim, { "hide-body": true, reveal: ["We can share the lab agreement."], "reveal-all-parts": true }); + assert.equal(shown(all.bodyMask).match(/We can share/g)?.length, 2, "--reveal-all-parts keeps both"); +}); + +test("CLI --reveal dedupe is per match: a regex still reveals HTML-only passages", async () => { + // @ts-expect-error - plain JS module without types + const { buildMasks } = await import("../skills/redacted-email-proof/scripts/redacted.mjs"); + const body = + "--b1\r\nContent-Type: text/plain\r\n\r\nPublic agreement\r\n" + + "--b1\r\nContent-Type: text/html\r\n\r\n

Public agreement

Private agreement

\r\n--b1--\r\n"; + const dkim = { headers: Buffer.from("from:a@b.example\r\n"), body: Buffer.from(body) }; + const m = buildMasks(dkim, { "hide-body": true, "reveal-regex": ["(Public|Private) agreement"] }); + const shown = [...body].map((c, i) => (m.bodyMask[i] ? c : "\0")).join(""); + assert.equal(shown.match(/Public agreement/g)?.length, 1, "duplicated passage revealed once"); + assert.match(shown, /Private agreement/, "HTML-only match still revealed"); +}); + +test("CLI --reveal: MIME headers/attachments are not a 'plain-text copy' (HTML-only message keeps its reveal)", async () => { + // @ts-expect-error - plain JS module without types + const { buildMasks } = await import("../skills/redacted-email-proof/scripts/redacted.mjs"); + const body = + "--m\r\nContent-Type: text/html\r\n\r\n

See agreement.pdf attached

\r\n" + + '--m\r\nContent-Type: application/pdf; name="agreement.pdf"\r\nContent-Transfer-Encoding: base64\r\n\r\nJVBERi0=\r\n--m--\r\n'; + const dkim = { headers: Buffer.from("from:a@b.example\r\n"), body: Buffer.from(body) }; + const m = buildMasks(dkim, { "hide-body": true, reveal: ["agreement.pdf"] }); + const at = body.indexOf("See agreement.pdf") + 4; + assert.ok(m.bodyMask.slice(at, at + "agreement.pdf".length).every((x: number) => x === 1), "the readable HTML occurrence is revealed"); + assert.ok(!m.warnings.some((w: string) => /HTML part left hidden/.test(w))); +});