From 316769739fa46ad1743ebc6ab8fc3edd62c356eb Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Tue, 10 Nov 2026 12:39:00 -0500 Subject: [PATCH 01/14] feat(dag-viewer): drill into large memo groups by any candidate dimension (#1262) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A memo group with more than 12 candidates used to collapse to one count per origin and expand to every candidate, so a 280-candidate CmsWithHeap group (depth × width × heap_size) was one box or 280 boxes. It now splits by its first varying dimension (origin, sketch family, each θ field, window width), and clicking a count drills into it until at most 12 remain. The group label shows the path, clicking the group goes back a level, and the group's details offer the other varying dimensions to split by. The dimensions come from the candidate's existing `parameters` and `operator`; the document is unchanged. Co-Authored-By: Claude Opus 5.5 --- tools/dag-viewer/README.md | 9 +- tools/dag-viewer/memo.js | 148 ++++++++++++++++++++++---------- tools/dag-viewer/memo.test.mjs | 50 ++++++++--- tools/dag-viewer/test_viewer.py | 25 ++++-- 4 files changed, 165 insertions(+), 67 deletions(-) diff --git a/tools/dag-viewer/README.md b/tools/dag-viewer/README.md index d4a7f207b..8b9ed7c63 100644 --- a/tools/dag-viewer/README.md +++ b/tools/dag-viewer/README.md @@ -73,8 +73,13 @@ To run it, see [RUNNING.md](RUNNING.md). Pass 2 pruned) and the queries it is the root of; its candidates sit inside it, bordered by origin, marked `✚ new` when the step added them and `✗ pruned` (dashed) when Pass 2 pruned them. A group with more than - 12 candidates is drawn as one node per origin with its count; clicking a - count lists them. Each candidate box shows its parameters θ. On the last + 12 candidates is split into counts by its first dimension that varies: + origin, sketch family, each θ field (e.g. `k`, `depth`, `width`), then + tumbling-window width. Clicking a count drills into it, splitting again + until at most 12 remain, which are then drawn one by one; the group's + label shows the path (`origin=Pass1 › depth=5`), clicking the group goes + back one level, and the group's details offer the other varying + dimensions to split by instead. Each candidate box shows its parameters θ. On the last tab the selected plan's candidates are filled green with `✓ selected` (a group may have several, one per execution context). Clicking a group shows its operator, children, parents, queries, its candidates with θ, diff --git a/tools/dag-viewer/memo.js b/tools/dag-viewer/memo.js index 11d692856..b0a08523a 100644 --- a/tools/dag-viewer/memo.js +++ b/tools/dag-viewer/memo.js @@ -271,18 +271,61 @@ function memoGroupRanks(groups) { // Sizes of the drawing, in cytoscape pixels. const MEMO_ITEM = { width: 190, height: 56, gap: 10, columns: 3, pad: 12, label: 34, rowGap: 70, groupGap: 36, rowWidth: 2200 }; +// A candidate's dimensions as [name, value] pairs, values as text: its +// origin, its sketch family and each θ field, and its tumbling-window width. +// A candidate without a dimension has the value "none" for it. +function memoCandidateDims(c) { + const dims = [['origin', MEMO_ORIGINS[c.origin].name]]; + Object.entries(c.parameters || {}).forEach(([family, fields]) => { + dims.push(['sketch', family]); + if (memoIsObject(fields)) Object.entries(fields).forEach(([name, v]) => dims.push([name, compactWire(v)])); + }); + const op = memoIsObject(c.operator) ? Object.values(Object.values(c.operator)[0] || {})[0] : null; + if (memoIsObject(op) && op.tumbling_window_width_ms != null) dims.push(['window_ms', compactWire(op.tumbling_window_width_ms)]); + return dims; +} + +// How a group with `candidates` is drawn under `state` (`{path, next}`: the +// [dimension, value] pairs the user drilled into, and the dimension to +// split by next). `shown` are the candidates matching `path`; `choices` the +// dimensions that still vary among them. While more than `collapseAbove` +// remain and a dimension varies, they split by `dim` (`next` when it +// varies, else the first that does) into `buckets` ([value, candidates], +// numeric values ascending with "none" last); otherwise `dim` is null and +// `shown` are drawn one by one. +function memoGroupDrill(candidates, state, collapseAbove) { + const path = (state && state.path) || []; + const dimsOf = new Map(candidates.map((c) => [c.index, new Map(memoCandidateDims(c))])); + const valueOf = (c, dim) => dimsOf.get(c.index).get(dim) ?? 'none'; + const shown = candidates.filter((c) => path.every(([dim, value]) => valueOf(c, dim) === value)); + const choices = [...new Set(shown.flatMap((c) => [...dimsOf.get(c.index).keys()]))] + .filter((dim) => !path.some(([d]) => d === dim) && new Set(shown.map((c) => valueOf(c, dim))).size > 1); + if (shown.length <= collapseAbove || !choices.length) return { path, shown, choices, dim: null, buckets: [] }; + const dim = choices.includes(state && state.next) ? state.next : choices[0]; + const byValue = new Map(); + shown.forEach((c) => { + const value = valueOf(c, dim); + if (!byValue.has(value)) byValue.set(value, []); + byValue.get(value).push(c); + }); + const buckets = [...byValue]; + const rank = (v) => (v === 'none' ? Infinity : Number(v)); + if (buckets.every(([v]) => v === 'none' || (v !== '' && !Number.isNaN(Number(v))))) buckets.sort(([a], [b]) => rank(a) - rank(b)); + return { path, shown, choices, dim, buckets }; +} + // Step `s` of `memo` as cytoscape elements with preset positions. A group // is node `g` (a compound node holding its candidates, or a plain box -// when it has none, as at Stage 0); a candidate is `gc`; a -// collapsed group holds one node per origin, `go`. An edge goes -// from a candidate (or origin node, or a group without candidates) to each -// child group it reads. Options: `expanded` (ids of large groups drawn with -// every candidate), `collapseAbove` (default MEMO_COLLAPSE_ABOVE), -// `queryIds` (the workload's query ids). The selected plan's candidates -// are marked at the last step only; Pass 2's eliminated candidates are -// marked `pruned`. +// when it has none, as at Stage 0); a candidate is `gc`; a large +// group holds one node per value of the dimension it splits by +// (memoGroupDrill), `gb`. An edge goes from a candidate (or bucket +// node, or a group without candidates) to each child group it reads. +// Options: `drill` (per group id, its memoGroupDrill state), +// `collapseAbove` (default MEMO_COLLAPSE_ABOVE), `queryIds` (the workload's +// query ids). The selected plan's candidates are marked at the last step +// only; Pass 2's eliminated candidates are marked `pruned`. function memoStepElements(memo, s, options) { - const { expanded = [], collapseAbove = MEMO_COLLAPSE_ABOVE, queryIds = [] } = options || {}; + const { drill = {}, collapseAbove = MEMO_COLLAPSE_ABOVE, queryIds = [] } = options || {}; const step = memo.steps[s]; const last = s === memo.steps.length - 1; const selected = last && memo.selected_candidates ? memo.selected_candidates : {}; @@ -298,37 +341,31 @@ function memoStepElements(memo, s, options) { edges.push({ data: { id, source, target: `g${child}`, queries }, classes: chosen ? 'chosen' : '' }); }; - // Each group's items (candidates or origin counts), laid out below. + // Each group's items (candidates or bucket counts), laid out below. const boxes = step.groups.map((group) => { const chosenIndices = selected[String(group.id)] || []; - const collapsible = group.candidates.length > collapseAbove; - const collapsed = collapsible && !expanded.includes(group.id); + const split = memoGroupDrill(group.candidates, drill[group.id], collapseAbove); const queries = group.queries; let items = []; - if (collapsed) { - const byOrigin = new Map(); - group.candidates.forEach((c) => { - if (!byOrigin.has(c.origin)) byOrigin.set(c.origin, { added: 0, pruned: 0, children: new Set(), chosen: false, chosenChildren: [] }); - const entry = byOrigin.get(c.origin); - if (c.added) entry.added += 1; - if (c.pruned) entry.pruned += 1; - c.children.forEach((child) => entry.children.add(child)); - if (chosenIndices.includes(c.index)) { entry.chosen = true; entry.chosenChildren.push(...c.children); } - }); - items = Object.entries(group.candidate_count_by_origin).filter(([, count]) => count > 0).map(([origin, count]) => { - const entry = byOrigin.get(origin); - const lines = [`${count} ${MEMO_ORIGINS[origin].name}`]; - const notes = [entry.added ? `✚ ${entry.added} new` : '', entry.pruned ? `✗ ${entry.pruned} pruned` : '', entry.chosen ? '✓ selected' : ''].filter(Boolean); + if (split.dim) { + items = split.buckets.map(([value, cs], k) => { + const chosen = cs.filter((c) => chosenIndices.includes(c.index)); + const added = cs.filter((c) => c.added).length; + const pruned = cs.filter((c) => c.pruned).length; + const origins = new Set(cs.map((c) => c.origin)); + const lines = [split.dim === 'origin' ? `${cs.length} ${value}` : `${cs.length} · ${split.dim}=${value}`]; + const notes = [added ? `✚ ${added} new` : '', pruned ? `✗ ${pruned} pruned` : '', chosen.length ? '✓ selected' : ''].filter(Boolean); lines.push(notes.length ? notes.join(' · ') : 'click to expand'); return { - data: { id: `g${group.id}o${origin}`, kind: 'origin', groupId: group.id, origin, count, color: MEMO_ORIGINS[origin].color, + data: { id: `g${group.id}b${k}`, kind: 'bucket', groupId: group.id, dim: split.dim, value, count: cs.length, + color: origins.size === 1 ? MEMO_ORIGINS[cs[0].origin].color : '#848b98', label: lines.join('\n'), queries, w: MEMO_ITEM.width, h: MEMO_ITEM.height }, - classes: ['memoItem', entry.chosen ? 'chosen' : '', entry.added ? 'added' : ''].filter(Boolean).join(' '), - children: [...entry.children], chosenChildren: entry.chosenChildren, chosen: entry.chosen, + classes: ['memoItem', chosen.length ? 'chosen' : '', added ? 'added' : ''].filter(Boolean).join(' '), + children: [...new Set(cs.flatMap((c) => c.children))], chosenChildren: chosen.flatMap((c) => c.children), chosen: chosen.length > 0, }; }); } else { - items = group.candidates.map((c) => { + items = split.shown.map((c) => { const chosen = chosenIndices.includes(c.index); const notes = [MEMO_ORIGINS[c.origin].name, c.added ? '✚ new' : '', c.pruned ? '✗ pruned' : '', chosen ? '✓ selected' : ''].filter(Boolean); return { @@ -342,13 +379,14 @@ function memoStepElements(memo, s, options) { const pruned = group.candidates.filter((c) => c.pruned).length; const count = `${group.candidate_count} candidate${group.candidate_count === 1 ? '' : 's'}${pruned ? ` (${pruned} pruned by Pass 2)` : ''}`; const roots = group.roots_of.length ? ` · root of ${group.roots_of.map(queryName).join(', ')}` : ''; - const toggle = collapsible ? (collapsed ? ' · click a count to expand' : ' · click the group to collapse') : ''; - const label = `G${group.id} · ${memoClip(group.label, 40)}\n${step.step !== 'stage0' ? count : 'no candidates yet'}${roots}${toggle}`; + const where = split.path.length ? `\n${split.path.map(([dim, value]) => `${dim}=${value}`).join(' › ')} (${split.shown.length})` : ''; + const toggle = (split.dim ? ' · click a count to expand' : '') + (split.path.length ? ' · click the group to go back' : ''); + const label = `G${group.id} · ${memoClip(group.label, 40)}\n${step.step !== 'stage0' ? count : 'no candidates yet'}${roots}${toggle}${where}`; const columns = Math.min(Math.max(items.length, 1), MEMO_ITEM.columns); const rows = Math.max(1, Math.ceil(items.length / MEMO_ITEM.columns)); const width = items.length ? columns * MEMO_ITEM.width + (columns - 1) * MEMO_ITEM.gap + 2 * MEMO_ITEM.pad : 230; - const height = items.length ? rows * MEMO_ITEM.height + (rows - 1) * MEMO_ITEM.gap + 2 * MEMO_ITEM.pad + MEMO_ITEM.label : 56; - return { group, items, label, width, height, collapsible, collapsed, queries }; + const height = items.length ? rows * MEMO_ITEM.height + (rows - 1) * MEMO_ITEM.gap + 2 * MEMO_ITEM.pad + MEMO_ITEM.label + (where ? 14 : 0) : 56; + return { group, items, label, width, height, drilled: split.path.length > 0, queries }; }); // Rank rows, highest rank on top; a rank wider than the row width wraps. @@ -388,7 +426,7 @@ function memoStepElements(memo, s, options) { boxes.forEach((box) => { const { group, items } = box; const data = { id: `g${group.id}`, kind: 'group', groupId: group.id, label: box.label, queries: box.queries, - collapsible: box.collapsible, collapsed: box.collapsed, w: box.width, h: box.height }; + drilled: box.drilled, w: box.width, h: box.height }; const chosenGroup = Object.prototype.hasOwnProperty.call(selected, String(group.id)); const node = { data, classes: ['memoGroup', items.length ? 'compound' : '', chosenGroup ? 'chosenGroup' : ''].filter(Boolean).join(' ') }; if (!items.length) node.position = { x: box.x + box.width / 2, y: box.y + box.height / 2 }; @@ -404,7 +442,7 @@ function memoStepElements(memo, s, options) { }, }); // Only the selected candidate's own edges are marked, not those of - // the other candidates sharing its origin count. + // the other candidates sharing its bucket. item.children.forEach((child) => edge(item.data.id, child, item.chosenChildren.includes(child), box.queries)); }); if (!items.length) group.children.forEach((child) => edge(`g${group.id}`, child, false, box.queries)); @@ -441,8 +479,8 @@ function memoViewController() { let cy = null; let elements = []; let highlighted = ''; - // Per step, the large groups the user expanded. - let expanded = []; + // Per step, per group id, its memoGroupDrill state. + let drill = []; function style() { return [ @@ -466,7 +504,7 @@ function memoViewController() { function draw() { if (cy) { cy.destroy(); cy = null; } - elements = memoStepElements(memo, stepIndex, { expanded: expanded[stepIndex], queryIds }); + elements = memoStepElements(memo, stepIndex, { drill: drill[stepIndex], queryIds }); cy = cytoscape({ container: $('cyMemo'), elements, style: style(), layout: { name: 'preset', fit: true, padding: 16 }, wheelSensitivity: 0.25, minZoom: 0.05, maxZoom: 2.5, boxSelectionEnabled: false, @@ -484,7 +522,7 @@ function memoViewController() { $('memoTabs').querySelectorAll('button').forEach((b) => b.addEventListener('click', () => selectStep(Number(b.dataset.step)))); } - const HINT = '

Click a memo candidate to see its properties, its logical edges (Pass 2 tab) and its physical candidates; click a count to expand its group.

'; + const HINT = '

Click a memo candidate to see its properties, its logical edges (Pass 2 tab) and its physical candidates; click a count to split its group further, and the group to go back.

'; function dim() { if (!cy || !cy.batch) return; @@ -498,10 +536,19 @@ function memoViewController() { const groupList = (ids) => (ids.length ? ids.map((id) => `G${id}`).join(', ') : 'none'); const queryList = (ids) => (ids.length ? ids.map((q) => queryIds[q] || `query #${q + 1}`).join(', ') : 'none'); + const drillOf = (id) => drill[stepIndex][id] || { path: [], next: null }; + // Sets group `id`'s drill state, redraws and shows the group. + function setDrill(id, state) { + drill[stepIndex] = { ...drill[stepIndex], [id]: state }; + draw(); + showGroup(groupOf(id)); + } + function tap(d) { const group = groupOf(d.groupId); - if (d.kind === 'origin') { expanded[stepIndex] = [...expanded[stepIndex], d.groupId]; draw(); showGroup(group); return; } - if (d.kind === 'group' && d.collapsible && !d.collapsed) { expanded[stepIndex] = expanded[stepIndex].filter((id) => id !== d.groupId); draw(); showGroup(group); return; } + const state = drillOf(d.groupId); + if (d.kind === 'bucket') { setDrill(d.groupId, { path: [...state.path, [d.dim, d.value]], next: null }); return; } + if (d.kind === 'group' && d.drilled) { setDrill(d.groupId, { path: state.path.slice(0, -1), next: null }); return; } if (d.kind === 'candidate') showCandidate(group, group.candidates.find((c) => c.index === d.index)); else showGroup(group); } @@ -528,10 +575,20 @@ function memoViewController() { const section = (title, items) => (items.length ? `
${esc(title)}
${list(items)}` : ''); + // While the group is split into counts, a button per dimension that still + // varies, to split by it instead. + function splitButtons(group) { + const state = drillOf(group.id); + const split = memoGroupDrill(group.candidates, state, MEMO_COLLAPSE_ABOVE); + if (!split.dim) return ''; + const buttons = split.choices.map((dim) => ``).join(' '); + return `
split by
${buttons}
`; + } + function showGroup(group) { const origins = Object.entries(group.candidate_count_by_origin).map(([o, n]) => `${n} ${MEMO_ORIGINS[o].name}`).join(', '); const pruned = memoPrunedLines(doc, group.id); - $('details').innerHTML = `

Memo group G${esc(group.id)} ${esc(memo.steps[stepIndex].title)}

${kv([ + $('details').innerHTML = `

Memo group G${esc(group.id)} ${esc(memo.steps[stepIndex].title)}

${splitButtons(group)}${kv([ ['label', group.label], ['operator', stageNodeLines(group.operator).join(' · ')], ['reads', groupList(group.children)], ['read by', groupList(group.parents)], ['queries reaching it', queryList(group.queries)], ['root of', queryList(group.roots_of)], @@ -543,6 +600,9 @@ function memoViewController() { ])}${list(group.candidates.map(candidateLine))} ${section('pruned by Pass 2', pruned.pass2)}${section('pruned by Stage 2', pruned.stage2)}${section('pruned by Stage 3', pruned.stage3)}
Group operator (JSON)
${esc(JSON.stringify(group.operator, null, 2))}
`; + $('details').querySelectorAll('.memo-split button').forEach((b) => b.addEventListener('click', () => { + setDrill(group.id, { ...drillOf(group.id), next: b.dataset.dim }); + })); } // At the Pass 2 step (the memo Stage 2 expands): why Pass 2 pruned the @@ -585,7 +645,7 @@ function memoViewController() { highlighted = ''; $('memoView').hidden = !memo; if (!memo) { $('memoTabs').innerHTML = ''; $('memoView-meta').textContent = ''; return; } - expanded = memo.steps.map(() => []); + drill = memo.steps.map(() => ({})); // The last step first: it is the memo the selected plan was chosen from. selectStep(memo.steps.length - 1); } diff --git a/tools/dag-viewer/memo.test.mjs b/tools/dag-viewer/memo.test.mjs index 102baa17e..e97028ec1 100644 --- a/tools/dag-viewer/memo.test.mjs +++ b/tools/dag-viewer/memo.test.mjs @@ -118,18 +118,42 @@ test("Pass 2's pruned candidates are marked and counted on their group", () => { assert.ok(!call('memoStepElements', fixture(), 1, {}).some((e) => /pruned/.test(e.classes || ''))); }); -test('a large group collapses to one count per origin and expands on request', () => { +test('a candidate\'s dimensions are its origin, sketch family, θ fields and window width', () => { + const step = fixture().steps[PASS2]; + assert.deepEqual(call('memoCandidateDims', step.groups[3].candidates[5]), [['origin', 'Pass1'], ['sketch', 'Kll'], ['k', '1404']]); + assert.deepEqual(call('memoCandidateDims', step.groups[3].candidates[7]), [['origin', 'TumblingWindows']]); + assert.deepEqual(call('memoCandidateDims', step.groups[5].candidates[0]).at(-1), ['window_ms', '60000']); +}); + +test('a large group splits by its first varying dimension; a count drills into it', () => { const memo = fixture(); const inG3 = (elements) => nodesOf(elements).filter((e) => e.data.parent === 'g3'); - const collapsed = call('memoStepElements', memo, PASS2, { collapseAbove: 4 }); - assert.deepEqual(inG3(collapsed).map((e) => e.data.label), ['7 Pass1\n✗ 4 pruned · ✓ selected', '1 TumblingWindows\n✚ 1 new']); - assert.match(byId(collapsed, 'g3').data.label, /click a count to expand/); - assert.match(byId(collapsed, 'g3opass1').classes, /chosen/); - assert.ok(byId(collapsed, 'g3otumbling_windows>g6')); - const open = call('memoStepElements', memo, PASS2, { collapseAbove: 4, expanded: [3] }); - assert.equal(inG3(open).length, 8); - assert.match(byId(open, 'g3').data.label, /click the group to collapse/); - assert.equal(cytoscape(open).getElementById('g3').children().length, 8); + const split = call('memoStepElements', memo, PASS2, { collapseAbove: 4 }); + assert.deepEqual(inG3(split).map((e) => e.data.label), ['7 Pass1\n✗ 4 pruned · ✓ selected', '1 TumblingWindows\n✚ 1 new']); + assert.match(byId(split, 'g3').data.label, /click a count to expand/); + assert.match(byId(split, 'g3b0').classes, /chosen/); + assert.ok(byId(split, 'g3b1>g6')); + // Pass1's 7 candidates still exceed 4, so they split by the next varying dimension, k. + const pass1 = [['origin', 'Pass1']]; + const deeper = call('memoStepElements', memo, PASS2, { collapseAbove: 4, drill: { 3: { path: pass1 } } }); + assert.deepEqual(inG3(deeper).map((e) => e.data.label.split('\n')[0]), + ['1 · k=26', '1 · k=52', '1 · k=132', '1 · k=269', '1 · k=547', '1 · k=1404', '1 · k=2863']); + assert.match(byId(deeper, 'g3').data.label, /click the group to go back\norigin=Pass1 \(7\)/); + // With room for all 7, the drilled group shows Pass1's candidates themselves. + const open = call('memoStepElements', memo, PASS2, { collapseAbove: 7, drill: { 3: { path: pass1 } } }); + assert.deepEqual(inG3(open).map((e) => e.data.id), ['g3c0', 'g3c1', 'g3c2', 'g3c3', 'g3c4', 'g3c5', 'g3c6']); + assert.equal(cytoscape(open).getElementById('g3').children().length, 7); +}); + +test('a group splits by the dimension chosen next; a candidate without it counts as "none", last', () => { + const g3 = fixture().steps[PASS2].groups[3]; + const split = call('memoGroupDrill', g3.candidates, { path: [], next: 'k' }, 4); + assert.equal(split.dim, 'k'); + assert.deepEqual(split.choices, ['origin', 'sketch', 'k']); + assert.deepEqual(split.buckets.map(([value, cs]) => [value, cs.length]), + [['26', 1], ['52', 1], ['132', 1], ['269', 1], ['547', 1], ['1404', 1], ['2863', 1], ['none', 1]]); + // A `next` that no longer varies falls back to the first that does. + assert.equal(call('memoGroupDrill', g3.candidates, { path: [['origin', 'Pass1']], next: 'origin' }, 4).dim, 'k'); }); test('the selected candidates are highlighted on the last step only', () => { @@ -143,13 +167,13 @@ test('the selected candidates are highlighted on the last step only', () => { assert.ok(!call('memoStepElements', memo, PASS2, {}).some((e) => /chosen/.test(e.classes || ''))); }); -test('a collapsed count marks only the selected candidate\'s edges', () => { +test('a count marks only the selected candidate\'s edges', () => { const memo = fixture(); // Another Pass1 candidate of G3 reads G6 instead of G1. memo.steps[PASS2].groups[3].candidates[6].children = [6]; const elements = call('memoStepElements', memo, PASS2, { collapseAbove: 4 }); - assert.match(byId(elements, 'g3opass1>g1').classes, /chosen/); - assert.equal(byId(elements, 'g3opass1>g6').classes, ''); + assert.match(byId(elements, 'g3b0>g1').classes, /chosen/); + assert.equal(byId(elements, 'g3b0>g6').classes, ''); }); test('highlighting a query dims the groups it does not reach', () => { diff --git a/tools/dag-viewer/test_viewer.py b/tools/dag-viewer/test_viewer.py index 34116bc47..ae3ae01d2 100644 --- a/tools/dag-viewer/test_viewer.py +++ b/tools/dag-viewer/test_viewer.py @@ -889,19 +889,28 @@ def test_the_memo_view_has_a_tab_per_step_and_opens_on_the_last(self): self.assertIn("3 groups · 0 candidates", self.text(js, "memoView-meta")) self.assertIn("shown on the last tab", self.text(js, "memoView-meta")) - def test_tapping_a_count_expands_its_group_and_the_group_collapses_it(self): - """Tapping an origin count expands its group; tapping the group collapses it.""" + def test_tapping_counts_drills_down_to_candidates_and_the_group_goes_back(self): + """Tapping a count drills into it until the group shows candidates + (at most 12); the details offer the dimensions to split by; tapping + the group goes back one level at a time.""" doc = example1() js = self.memo_page(doc) pass2 = doc["memo"]["steps"][-1] big = next(g for g in pass2["groups"] if g["candidate_count"] > 12) - counts = self.memo_ids(js, f"g{big['id']}") - self.assertEqual(len(counts), len([n for n in big["candidate_count_by_origin"].values() if n])) - self.tap_memo(js, counts[0]) - self.assertEqual(len(self.memo_ids(js, f"g{big['id']}")), big["candidate_count"]) + group = f"g{big['id']}" + levels = [self.memo_ids(js, group)] + self.assertTrue(all("b" in i for i in levels[0])) + self.tap_memo(js, group) + self.assertIn('data-dim="', js.eval("__els.details.innerHTML")) + while "b" in levels[-1][0]: + self.tap_memo(js, levels[-1][0]) + levels.append(self.memo_ids(js, group)) + self.assertTrue(all("c" in i for i in levels[-1])) + self.assertLessEqual(len(levels[-1]), 12) self.assertIn(f"Memo group G{big['id']}", self.text(js, "details")) - self.tap_memo(js, f"g{big['id']}") - self.assertEqual(self.memo_ids(js, f"g{big['id']}"), counts) + for level in reversed(levels[:-1]): + self.tap_memo(js, group) + self.assertEqual(self.memo_ids(js, group), level) def test_tapping_a_candidate_lists_its_logical_edges_and_physical_candidates(self): """Tapping a memo candidate shows its θ and required accuracy, its From 1afaeb37336e73ca0a064e1318cce3b49b3a7e84 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Tue, 10 Nov 2026 12:41:58 -0500 Subject: [PATCH 02/14] fix(dag-viewer): drilled-in candidates show the dimensions that tell them apart (#1262) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drilled to depth=5 › heap_size=10, a CmsWithHeap group's 7 candidates all read "θ CmsWithHeap(depth=5, heap_s…": the clipped θ hid width, the one dimension that differs. Drilled in, a candidate now shows the dimensions still varying among the shown ones ("θ width=28"). The split-by buttons use the page's .btn style, so the dimension in use shows as pressed. Co-Authored-By: Claude Opus 5.5 --- tools/dag-viewer/memo.js | 8 ++++++-- tools/dag-viewer/memo.test.mjs | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/dag-viewer/memo.js b/tools/dag-viewer/memo.js index b0a08523a..810b4da45 100644 --- a/tools/dag-viewer/memo.js +++ b/tools/dag-viewer/memo.js @@ -368,9 +368,13 @@ function memoStepElements(memo, s, options) { items = split.shown.map((c) => { const chosen = chosenIndices.includes(c.index); const notes = [MEMO_ORIGINS[c.origin].name, c.added ? '✚ new' : '', c.pruned ? '✗ pruned' : '', chosen ? '✓ selected' : ''].filter(Boolean); + // Drilled in, the full θ would clip; show what tells the shown candidates apart. + const dims = new Map(memoCandidateDims(c)); + const theta = split.path.length && split.choices.length + ? `θ ${split.choices.map((dim) => `${dim}=${dims.get(dim) ?? 'none'}`).join(', ')}` : memoParametersText(c.parameters); return { data: { id: `g${group.id}c${c.index}`, kind: 'candidate', groupId: group.id, index: c.index, origin: c.origin, color: MEMO_ORIGINS[c.origin].color, - label: `${memoClip(`#${c.index} ${c.label}`, 30)}\n${memoClip(memoParametersText(c.parameters) || '—', 30)}\n${memoClip(notes.join(' · '), 30)}`, queries, w: MEMO_ITEM.width, h: MEMO_ITEM.height }, + label: `${memoClip(`#${c.index} ${c.label}`, 30)}\n${memoClip(theta || '—', 30)}\n${memoClip(notes.join(' · '), 30)}`, queries, w: MEMO_ITEM.width, h: MEMO_ITEM.height }, classes: ['memoItem', chosen ? 'chosen' : '', c.added ? 'added' : '', c.pruned ? 'pruned' : ''].filter(Boolean).join(' '), children: c.children, chosenChildren: chosen ? c.children : [], chosen, }; @@ -581,7 +585,7 @@ function memoViewController() { const state = drillOf(group.id); const split = memoGroupDrill(group.candidates, state, MEMO_COLLAPSE_ABOVE); if (!split.dim) return ''; - const buttons = split.choices.map((dim) => ``).join(' '); + const buttons = split.choices.map((dim) => ``).join(' '); return `
split by
${buttons}
`; } diff --git a/tools/dag-viewer/memo.test.mjs b/tools/dag-viewer/memo.test.mjs index e97028ec1..5df2159be 100644 --- a/tools/dag-viewer/memo.test.mjs +++ b/tools/dag-viewer/memo.test.mjs @@ -143,6 +143,8 @@ test('a large group splits by its first varying dimension; a count drills into i const open = call('memoStepElements', memo, PASS2, { collapseAbove: 7, drill: { 3: { path: pass1 } } }); assert.deepEqual(inG3(open).map((e) => e.data.id), ['g3c0', 'g3c1', 'g3c2', 'g3c3', 'g3c4', 'g3c5', 'g3c6']); assert.equal(cytoscape(open).getElementById('g3').children().length, 7); + // Drilled in, a candidate shows the dimensions that still tell the shown ones apart. + assert.equal(byId(open, 'g3c5').data.label.split('\n')[1], 'θ k=1404'); }); test('a group splits by the dimension chosen next; a candidate without it counts as "none", last', () => { From a956791af3392d9133f6489b92bdec143acd77e4 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Tue, 10 Nov 2026 15:40:21 -0500 Subject: [PATCH 03/14] fix(dag-viewer): deployment line names each summary's summary_type, not undefined (#1262) Co-Authored-By: Claude Opus 5.5 --- tools/dag-viewer/stages.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dag-viewer/stages.js b/tools/dag-viewer/stages.js index 0e506c6b3..585b0a837 100644 --- a/tools/dag-viewer/stages.js +++ b/tools/dag-viewer/stages.js @@ -749,7 +749,7 @@ function usageText(usage) { function deploymentText(capabilities) { if (!capabilities || typeof capabilities !== 'object') return ''; const summaries = Array.isArray(capabilities.supported_summaries) - ? `summaries: ${capabilities.supported_summaries.map((s) => `${s.family} (${(s.readouts || []).join(', ')})`).join(', ')}` + ? `summaries: ${capabilities.supported_summaries.map((s) => `${s.summary_type} (${(s.readouts || []).join(', ')})`).join(', ')}` : 'every summary the planner builds'; return [capabilities.raw_data_retained === false ? 'raw data not retained' : 'raw data retained', capabilities.query_time_results_retained ? 'query-time results retained' : 'query-time results not retained', summaries].join(' · '); From 9c1d70514c6e8d6646b999a99b7bec8f7c440fda Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Tue, 10 Nov 2026 15:40:22 -0500 Subject: [PATCH 04/14] feat(dag-viewer): one memo edge per group when its candidates share inputs, a how-to-read line, light theme by default with a toggle (#1262) Co-Authored-By: Claude Opus 5.5 --- tools/dag-viewer/README.md | 6 +++++- tools/dag-viewer/app.js | 8 ++++++++ tools/dag-viewer/index.html | 4 +++- tools/dag-viewer/memo.js | 11 ++++++++--- tools/dag-viewer/memo.test.mjs | 2 ++ 5 files changed, 26 insertions(+), 5 deletions(-) diff --git a/tools/dag-viewer/README.md b/tools/dag-viewer/README.md index 8b9ed7c63..47d24d05f 100644 --- a/tools/dag-viewer/README.md +++ b/tools/dag-viewer/README.md @@ -79,7 +79,9 @@ To run it, see [RUNNING.md](RUNNING.md). until at most 12 remain, which are then drawn one by one; the group's label shows the path (`origin=Pass1 › depth=5`), clicking the group goes back one level, and the group's details offer the other varying - dimensions to split by instead. Each candidate box shows its parameters θ. On the last + dimensions to split by instead. An arrow goes from a candidate (or count) to + each group it reads, or once from the group when all its candidates read the + same groups; a line above the drawing says how to read it. Each candidate box shows its parameters θ. On the last tab the selected plan's candidates are filled green with `✓ selected` (a group may have several, one per execution context). Clicking a group shows its operator, children, parents, queries, its candidates with θ, @@ -94,6 +96,8 @@ To run it, see [RUNNING.md](RUNNING.md). physical edges per input (`✓ C`, or `✗ C: α^phys = 0: …`; `selected` when the plan uses it), and the physical candidates of it Stage 2 and Stage 3 pruned. +- **Theme**: light by default; **Dark theme** switches it and is remembered + in the browser. - **Details**: click a node for its operator, the queries reaching it, timing, lifetime, output schema, guarantee and Stage 3 cost; click an edge for its schema and data state. diff --git a/tools/dag-viewer/app.js b/tools/dag-viewer/app.js index d7024702b..e366ef858 100644 --- a/tools/dag-viewer/app.js +++ b/tools/dag-viewer/app.js @@ -343,6 +343,14 @@ $('queryPick').addEventListener('change', (e) => highlightQuery(e.target.value)); window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', restyle); new MutationObserver(restyle).observe(document.documentElement, { attributes: true, attributeFilter: ['data-theme'] }); + // Light unless the viewer picked dark (remembered per browser). + const setTheme = (dark) => { + document.documentElement.dataset.theme = dark ? 'dark' : 'light'; + $('themeToggle').setAttribute('aria-pressed', String(dark)); + try { localStorage.setItem('dagViewerTheme', dark ? 'dark' : 'light'); } catch (e) { /* storage blocked */ } + }; + try { if (localStorage.getItem('dagViewerTheme') === 'dark') setTheme(true); } catch (e) { /* storage blocked */ } + $('themeToggle').addEventListener('click', () => setTheme(document.documentElement.dataset.theme !== 'dark')); $('details').innerHTML = HINT; window.StageViewer = { planWorkload }; diff --git a/tools/dag-viewer/index.html b/tools/dag-viewer/index.html index 54d02d8f0..97a8b0837 100644 --- a/tools/dag-viewer/index.html +++ b/tools/dag-viewer/index.html @@ -1,5 +1,5 @@ - + @@ -138,6 +138,7 @@

ASAPPlanner Stage Viewer

+ diff --git a/tools/dag-viewer/memo.js b/tools/dag-viewer/memo.js index 810b4da45..7f685d444 100644 --- a/tools/dag-viewer/memo.js +++ b/tools/dag-viewer/memo.js @@ -319,7 +319,8 @@ function memoGroupDrill(candidates, state, collapseAbove) { // when it has none, as at Stage 0); a candidate is `gc`; a large // group holds one node per value of the dimension it splits by // (memoGroupDrill), `gb`. An edge goes from a candidate (or bucket -// node, or a group without candidates) to each child group it reads. +// node, or a group without candidates) to each child group it reads; when +// all of a group's items read the same groups, from the group instead. // Options: `drill` (per group id, its memoGroupDrill state), // `collapseAbove` (default MEMO_COLLAPSE_ABOVE), `queryIds` (the workload's // query ids). The selected plan's candidates are marked at the last step @@ -429,6 +430,8 @@ function memoStepElements(memo, s, options) { boxes.forEach((box) => { const { group, items } = box; + const key = (item) => [...new Set(item.children)].sort().join(','); + const shared = items.every((item) => key(item) === key(items[0])); const data = { id: `g${group.id}`, kind: 'group', groupId: group.id, label: box.label, queries: box.queries, drilled: box.drilled, w: box.width, h: box.height }; const chosenGroup = Object.prototype.hasOwnProperty.call(selected, String(group.id)); @@ -447,9 +450,11 @@ function memoStepElements(memo, s, options) { }); // Only the selected candidate's own edges are marked, not those of // the other candidates sharing its bucket. - item.children.forEach((child) => edge(item.data.id, child, item.chosenChildren.includes(child), box.queries)); + if (!shared) item.children.forEach((child) => edge(item.data.id, child, item.chosenChildren.includes(child), box.queries)); }); - if (!items.length) group.children.forEach((child) => edge(`g${group.id}`, child, false, box.queries)); + // When every item reads the same groups, one edge per child from the group says it all. + if (shared) (items.length ? items[0].children : group.children) + .forEach((child) => edge(`g${group.id}`, child, items.some((i) => i.chosenChildren.includes(child)), box.queries)); }); return nodes.concat(edges); } diff --git a/tools/dag-viewer/memo.test.mjs b/tools/dag-viewer/memo.test.mjs index 5df2159be..2c5231336 100644 --- a/tools/dag-viewer/memo.test.mjs +++ b/tools/dag-viewer/memo.test.mjs @@ -104,6 +104,8 @@ test('each candidate sits inside its group, colored by origin, with θ, and an e assert.equal(call('memoClip', 'abcdef', 4), 'abc…'); assert.ok(byId(elements, 'g3c5>g1')); assert.ok(byId(elements, 'g3c7>g6')); + // G5's three candidates all read G0: one edge, from the group. + assert.deepEqual(edgesOf(elements).filter((e) => e.data.source.startsWith('g5')).map((e) => e.data.id), ['g5>g0']); const cy = cytoscape(elements); assert.equal(cy.getElementById('g3').children().length, 8); assert.equal(cy.edges().length, edgesOf(elements).length); From 0e8f82a99d1fd578a84c3e539846fa204cf3f4c6 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Tue, 10 Nov 2026 16:18:28 -0500 Subject: [PATCH 05/14] feat(dag-viewer): hide Pass 2's pruned memo candidates until asked; theme toggle top right (#1262) Co-Authored-By: Claude Opus 5.5 --- tools/dag-viewer/index.html | 8 +++++--- tools/dag-viewer/memo.js | 17 ++++++++++++----- tools/dag-viewer/memo.test.mjs | 9 +++++++++ tools/dag-viewer/test_viewer.py | 17 ++++++++++++++--- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/tools/dag-viewer/index.html b/tools/dag-viewer/index.html index 97a8b0837..81879cd0a 100644 --- a/tools/dag-viewer/index.html +++ b/tools/dag-viewer/index.html @@ -110,6 +110,7 @@ footer { color: var(--muted); font-size: 12px; max-width: 100ch; } @media (max-width: 820px) { .main { grid-template-columns: minmax(0, 1fr); } .rank-panel { position: static; } .rank { max-height: 340px; } .cy { height: 460px; } } @media (prefers-reduced-motion: reduce) { * { transition: none !important; } } +.title-row { display: flex; justify-content: space-between; align-items: center; gap: 12px; flex-wrap: wrap; } .toolbar { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; } .btn { font: inherit; font-size: 13px; padding: 5px 11px; border-radius: 6px; border: 1px solid var(--line); background: var(--panel); color: var(--fg); cursor: pointer; } .btn:hover { background: var(--panel-2); } @@ -131,14 +132,14 @@
-

ASAPPlanner Stage Viewer

+

ASAPPlanner Stage Viewer

+

The planner's stages for one workload: the query-language frontends lower the queries into one logical DAG (Stage 0), Pass 1 and Pass 2 propose logical ASAP candidates in a memo and Pass 2 prunes those no consumer can use (Stage 1), each remaining candidate becomes physical candidates (Stage 2), and Stage 3 selects the cheapest plan from them.

-
How to read it: a big box is a group, one sub-result (an e-class); the small boxes in it are its candidates, interchangeable ways to compute that result (e-nodes), bordered by origin; an arrow points to the group a candidate reads as input, drawn once from the group when all its candidates read the same groups.
diff --git a/tools/dag-viewer/memo.js b/tools/dag-viewer/memo.js index 7f685d444..e472c48c0 100644 --- a/tools/dag-viewer/memo.js +++ b/tools/dag-viewer/memo.js @@ -326,7 +326,7 @@ function memoGroupDrill(candidates, state, collapseAbove) { // query ids). The selected plan's candidates are marked at the last step // only; Pass 2's eliminated candidates are marked `pruned`. function memoStepElements(memo, s, options) { - const { drill = {}, collapseAbove = MEMO_COLLAPSE_ABOVE, queryIds = [] } = options || {}; + const { drill = {}, collapseAbove = MEMO_COLLAPSE_ABOVE, queryIds = [], hidePruned = false } = options || {}; const step = memo.steps[s]; const last = s === memo.steps.length - 1; const selected = last && memo.selected_candidates ? memo.selected_candidates : {}; @@ -335,7 +335,12 @@ function memoStepElements(memo, s, options) { const nodes = []; const edges = []; const edgeIds = new Set(); + // With `hidePruned`, Pass 2's pruned candidates are left out, and so is a + // group whose candidates were all pruned, with the edges into it. + const shownOf = (group) => (hidePruned ? group.candidates.filter((c) => !c.pruned) : group.candidates); + const hidden = new Set(step.groups.filter((g) => g.candidates.length && !shownOf(g).length).map((g) => g.id)); const edge = (source, child, chosen, queries) => { + if (hidden.has(child)) return; const id = `${source}>g${child}`; if (edgeIds.has(id)) return; edgeIds.add(id); @@ -343,9 +348,9 @@ function memoStepElements(memo, s, options) { }; // Each group's items (candidates or bucket counts), laid out below. - const boxes = step.groups.map((group) => { + const boxes = step.groups.filter((group) => !hidden.has(group.id)).map((group) => { const chosenIndices = selected[String(group.id)] || []; - const split = memoGroupDrill(group.candidates, drill[group.id], collapseAbove); + const split = memoGroupDrill(shownOf(group), drill[group.id], collapseAbove); const queries = group.queries; let items = []; if (split.dim) { @@ -382,7 +387,7 @@ function memoStepElements(memo, s, options) { }); } const pruned = group.candidates.filter((c) => c.pruned).length; - const count = `${group.candidate_count} candidate${group.candidate_count === 1 ? '' : 's'}${pruned ? ` (${pruned} pruned by Pass 2)` : ''}`; + const count = `${group.candidate_count} candidate${group.candidate_count === 1 ? '' : 's'}${pruned ? ` (${pruned} pruned by Pass 2${hidePruned ? ', hidden' : ''})` : ''}`; const roots = group.roots_of.length ? ` · root of ${group.roots_of.map(queryName).join(', ')}` : ''; const where = split.path.length ? `\n${split.path.map(([dim, value]) => `${dim}=${value}`).join(' › ')} (${split.shown.length})` : ''; const toggle = (split.dim ? ' · click a count to expand' : '') + (split.path.length ? ' · click the group to go back' : ''); @@ -513,7 +518,7 @@ function memoViewController() { function draw() { if (cy) { cy.destroy(); cy = null; } - elements = memoStepElements(memo, stepIndex, { drill: drill[stepIndex], queryIds }); + elements = memoStepElements(memo, stepIndex, { drill: drill[stepIndex], queryIds, hidePruned: !($('memoShowPruned') || {}).checked }); cy = cytoscape({ container: $('cyMemo'), elements, style: style(), layout: { name: 'preset', fit: true, padding: 16 }, wheelSensitivity: 0.25, minZoom: 0.05, maxZoom: 2.5, boxSelectionEnabled: false, @@ -641,6 +646,8 @@ function memoViewController() {
Candidate operator (JSON)
${esc(JSON.stringify(candidate.operator, null, 2))}
`; } + if (typeof document !== 'undefined' && $('memoShowPruned')) $('memoShowPruned').addEventListener('change', () => { if (memo) draw(); }); + function selectStep(i) { stepIndex = i; draw(); diff --git a/tools/dag-viewer/memo.test.mjs b/tools/dag-viewer/memo.test.mjs index 2c5231336..02f18efe6 100644 --- a/tools/dag-viewer/memo.test.mjs +++ b/tools/dag-viewer/memo.test.mjs @@ -120,6 +120,15 @@ test("Pass 2's pruned candidates are marked and counted on their group", () => { assert.ok(!call('memoStepElements', fixture(), 1, {}).some((e) => /pruned/.test(e.classes || ''))); }); +test('hiding pruned candidates leaves out them, groups with none left, and the edges into those groups', () => { + const memo = fixture(); + memo.steps[PASS2].groups[4].candidates[0].pruned = true; + const elements = call('memoStepElements', memo, PASS2, { hidePruned: true }); + assert.deepEqual(nodesOf(elements).filter((e) => e.data.parent === 'g3').map((e) => e.data.id), ['g3c4', 'g3c5', 'g3c6', 'g3c7']); + assert.match(byId(elements, 'g3').data.label, /\(4 pruned by Pass 2, hidden\)/); + assert.ok(!byId(elements, 'g4') && !elements.some((e) => e.data.target === 'g4')); +}); + test('a candidate\'s dimensions are its origin, sketch family, θ fields and window width', () => { const step = fixture().steps[PASS2]; assert.deepEqual(call('memoCandidateDims', step.groups[3].candidates[5]), [['origin', 'Pass1'], ['sketch', 'Kll'], ['k', '1404']]); diff --git a/tools/dag-viewer/test_viewer.py b/tools/dag-viewer/test_viewer.py index ae3ae01d2..cbd3a67d9 100644 --- a/tools/dag-viewer/test_viewer.py +++ b/tools/dag-viewer/test_viewer.py @@ -854,9 +854,20 @@ def test_an_invalid_document_shows_an_error(self): js = self.page("?doc=bad.json", {"bad.json": json.dumps({"format": "nope"})}) self.assertIn("Could not load", js.eval("__els.queries.innerHTML")) - def memo_page(self, doc=None): - """The p90 fixture document (or `doc`) on the page.""" - return self.page("?doc=m.json", {"m.json": json.dumps(doc or p90_kll())}) + def memo_page(self, doc=None, show_pruned=True): + """The p90 fixture document (or `doc`) on the page, with Pass 2's + pruned candidates shown unless `show_pruned` is false.""" + js = self.page("?doc=m.json", {"m.json": json.dumps(doc or p90_kll())}) + if show_pruned: + js.eval("__els.memoShowPruned.checked = true; __els.memoShowPruned.__on.change()") + return js + + def test_the_memo_view_hides_pruned_candidates_until_asked(self): + """Pass 2's pruned candidates are hidden until the box is ticked.""" + js = self.memo_page(show_pruned=False) + self.assertEqual(self.memo_ids(js, "g3"), ["g3c4", "g3c5", "g3c6", "g3c7"]) + js.eval("__els.memoShowPruned.checked = true; __els.memoShowPruned.__on.change()") + self.assertEqual(len(self.memo_ids(js, "g3")), 8) def tap_memo(self, js, element_id): js.eval(f"__taps['cyMemo node']({{ target: {{ data() {{ return __rendered.cyMemo.find((e) => e.data.id === {json.dumps(element_id)}).data; }} }} }})") From a7bd22af188b2b77e44cc775c16619d090e0b0d1 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Tue, 10 Nov 2026 16:18:34 -0500 Subject: [PATCH 06/14] docs(dag-viewer): README says pruned candidates are hidden until asked (#1262) Co-Authored-By: Claude Opus 5.5 --- tools/dag-viewer/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/dag-viewer/README.md b/tools/dag-viewer/README.md index 47d24d05f..4c75ffb4b 100644 --- a/tools/dag-viewer/README.md +++ b/tools/dag-viewer/README.md @@ -72,7 +72,8 @@ To run it, see [RUNNING.md](RUNNING.md). group is a box `G ·