From 50c03f3f124c5bd0f66d11f4602538e8cde9c409 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 09:16:36 -0500 Subject: [PATCH 1/7] perf(web): Go to file, the language bar, compare and the cold home without serial reads QW-028: Go to file answers from the history index at once (then the tree walk, now 16 trees at a time), ranks fuzzily as GitHub's finder does, takes the arrow keys, Enter and the t hotkey, and is on every directory. QW-027: chunk reads asked for in one turn share a query; a history walk reads the next pack blocks ahead; the merge-base search asks for its frontier's parents ahead; counting runs 24 files at a time and counts changes too large to show. dash v22.0.0...v23.0.0 counts in ~14 s and 42 requests (was 176 s, 1,287 and partial). QW-059: the rename read budget (400 blobs, 16 at a time) reaches git's pairs on that compare: 1,530 files, +112310 -69084. QW-087: a cold read of a long ref timeline goes out as 8 key ranges at once, not six pages in a row. The page-budget spec holds these to request and time budgets on the dash mirror. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/components/repo/diff-view.tsx | 14 +- forge-web/components/repo/go-to-file.test.tsx | 105 ++++++++++ forge-web/components/repo/go-to-file.tsx | 184 ++++++++++++++++++ .../components/repo/repo-home-content.tsx | 85 +------- forge-web/components/repo/tree-content.tsx | 12 +- forge-web/e2e/page-budget.spec.ts | 128 ++++++++++++ forge-web/lib/browse/read-ahead.test.ts | 17 ++ forge-web/lib/browse/reader.ts | 29 ++- forge-web/lib/repo/chrome.test.ts | 47 ++++- forge-web/lib/repo/chrome.ts | 11 +- forge-web/lib/repo/refs.ts | 24 +++ forge-web/lib/view/blame.test.ts | 4 +- forge-web/lib/view/browse-fallback.test.ts | 9 +- forge-web/lib/view/browse-source.ts | 53 ++++- forge-web/lib/view/chunk-gather.test.ts | 45 +++++ forge-web/lib/view/commit-log.ts | 2 +- forge-web/lib/view/file-diff.test.ts | 20 +- forge-web/lib/view/file-diff.ts | 5 +- forge-web/lib/view/fuzzy.test.ts | 59 ++++++ forge-web/lib/view/fuzzy.ts | 136 +++++++++++++ forge-web/lib/view/pull-diff.ts | 18 ++ forge-web/lib/view/renames.ts | 4 +- forge-web/lib/view/repo-facts.ts | 26 +++ forge-web/lib/view/text-diff.ts | 30 +++ forge-web/lib/view/zip.ts | 52 ++++- 25 files changed, 997 insertions(+), 122 deletions(-) create mode 100644 forge-web/components/repo/go-to-file.test.tsx create mode 100644 forge-web/components/repo/go-to-file.tsx create mode 100644 forge-web/lib/view/chunk-gather.test.ts create mode 100644 forge-web/lib/view/fuzzy.test.ts create mode 100644 forge-web/lib/view/fuzzy.ts diff --git a/forge-web/components/repo/diff-view.tsx b/forge-web/components/repo/diff-view.tsx index 7866e0b48..0a2ab0c14 100644 --- a/forge-web/components/repo/diff-view.tsx +++ b/forge-web/components/repo/diff-view.tsx @@ -67,11 +67,17 @@ export const InlineCommentsContext = createContext(null) /** Files whose patches load per page. */ const FILE_PAGE = 25 /** A change set of at most this many files has its lines counted as it opens; a bigger one on request. */ -const AUTO_COUNT = 100 +const AUTO_COUNT = 300 /** Rows a file renders before a "show more lines" button. */ const LINE_PAGE = 400 -/** Patch reads in flight at once. */ -const PATCH_CONCURRENCY = 4 +/** + * Patch reads in flight at once: those of files on screen, and of a whole change set being counted + * (QW-027). Blob reads that run side by side share their chunk queries (`queueChunkSeqs`), so a + * wider pool costs fewer round trips, not more requests: dashpay/dash v22.0.0...v23.0.0's 1,530 + * files were 1,287 requests over 176 s four at a time, and are ~80 over ~20 s. + */ +const PATCH_CONCURRENCY = 12 +const COUNT_CONCURRENCY = 24 /** Finished patches are committed to state in batches at most this often (ms). */ const PATCH_FLUSH_MS = 100 @@ -163,7 +169,7 @@ export function DiffView({ const key = (path: string): string => `${ignoreWhitespace ? 'w' : 'x'}:${path}` const todo = (countAll ? changes : changes.slice(0, shown)).filter((c) => !requested.current.has(key(c.path))) for (const c of todo) requested.current.add(key(c.path)) - void mapPooled(todo, PATCH_CONCURRENCY, async (change) => { + void mapPooled(todo, countAll ? COUNT_CONCURRENCY : PATCH_CONCURRENCY, async (change) => { // loadFilePatch turns read failures into placeholders; anything else must still settle // the file, or it would sit on "Reading file…" with its key already requested. const patch = await loadFilePatch(sides, change, { ignoreWhitespace }).catch( diff --git a/forge-web/components/repo/go-to-file.test.tsx b/forge-web/components/repo/go-to-file.test.tsx new file mode 100644 index 000000000..9f4cf1151 --- /dev/null +++ b/forge-web/components/repo/go-to-file.test.tsx @@ -0,0 +1,105 @@ +// @vitest-environment jsdom +/** + * Go to file (QW-028): results from the history index at once, then the walk's exact list; fuzzy + * ranking; the arrow keys and Enter open a result; `t` focuses the box. + */ + +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const push = vi.fn() +vi.mock('next/link', () => ({ default: ({ href, children, ...rest }: { href: string; children: React.ReactNode }) => {children} })) +vi.mock('next/navigation', () => ({ useRouter: () => ({ push, replace: () => undefined }) })) + +/** The walk resolves when the test says; the index answers at once. */ +let finishWalk: (files: string[]) => void = () => undefined +vi.mock('@/lib/view/repo-facts', () => ({ + indexedFilePaths: async () => ['src/validation.cpp', 'src/net_processing.cpp', 'src/deleted_since.cpp'], + repoFilesWalk: () => + new Promise((resolve) => { + finishWalk = (files) => resolve({ files: files.map((path) => ({ path, oid: '', mode: 0o100644, size: 0 })), truncated: false }) + }), +})) + +import { GoToFile } from './go-to-file' + +let root: Root +let el: HTMLDivElement +beforeEach(() => { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + push.mockClear() + // jsdom has no layout, so no scrollIntoView. + Element.prototype.scrollIntoView = () => undefined + el = document.createElement('div') + document.body.append(el) + root = createRoot(el) +}) +afterEach(() => { + act(() => root.unmount()) + el.remove() +}) + +const input = (): HTMLInputElement => el.querySelector('[data-testid="go-to-file"]') as HTMLInputElement +const results = (): string[] => [...el.querySelectorAll('[data-testid="go-to-file-result"]')].map((a) => a.textContent ?? '') + +async function type(text: string): Promise { + await act(async () => { + const box = input() + box.focus() + // React listens for the native setter's input event. + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call(box, text) + box.dispatchEvent(new Event('input', { bubbles: true })) + }) +} + +async function key(k: string, target: EventTarget = input()): Promise { + await act(async () => { + target.dispatchEvent(new KeyboardEvent('keydown', { key: k, bubbles: true })) + }) +} + +async function render(): Promise { + await act(async () => { + root.render( + 'tree'} addr={{ owner: 'o', name: 'n' }} refParam="" />, + ) + }) +} + +describe('GoToFile', () => { + it('answers from the history index before the walk, then from the walk', async () => { + await render() + await type('deleted') + await vi.waitFor(() => expect(results()).toEqual(['src/deleted_since.cpp'])) + // The walk's list is exact: a file the index still named is gone. + await act(async () => finishWalk(['src/validation.cpp', 'src/net_processing.cpp'])) + await vi.waitFor(() => expect(el.textContent).toContain('No matching file.')) + }) + + it('ranks fuzzily, moves with the arrow keys and opens the choice with Enter', async () => { + await render() + await type('netproc') + await vi.waitFor(() => expect(results()).toEqual(['src/net_processing.cpp'])) + await type('src') + await vi.waitFor(() => expect(results()).toHaveLength(3)) + const first = results()[0] + await key('ArrowDown') + expect(el.querySelector('[role=option][aria-selected=true]')?.textContent).toBe(results()[1]) + await key('ArrowUp') + await key('Enter') + expect(push).toHaveBeenCalledWith(expect.stringContaining(`path=${encodeURIComponent(first as string)}`)) + }) + + it('focuses on t, and Escape clears it', async () => { + await render() + // jsdom lays nothing out: give the box a client rect so it counts as visible. + input().getClientRects = () => [{}] as unknown as DOMRectList + await key('t', document.body) + expect(document.activeElement).toBe(input()) + await type('val') + await key('Escape') + expect(input().value).toBe('') + expect(document.activeElement).not.toBe(input()) + }) +}) diff --git a/forge-web/components/repo/go-to-file.tsx b/forge-web/components/repo/go-to-file.tsx new file mode 100644 index 000000000..3b833d946 --- /dev/null +++ b/forge-web/components/repo/go-to-file.tsx @@ -0,0 +1,184 @@ +'use client' + +/** + * Go to file (QW-028): GitHub's file finder on the repo home and every directory. Fuzzy matching + * ({@link fuzzyRank}: `netproc` finds `src/net_processing.cpp`), the arrow keys and Enter to open + * a result, Escape to leave, and `t` anywhere on the page to start typing. + * + * The list comes from the history index first: the index the file list's column already loaded + * names every path at the tip, so the first results need no read at all. The tree walk the + * language bar shares ({@link repoFilesWalk}, its reads in parallel) starts on first focus and, + * once done, is the list: it is exact where an index a delta extends can still name a deleted file. + */ + +import { useEffect, useId, useMemo, useRef, useState, type KeyboardEvent as ReactKeyboardEvent } from 'react' +import Link from 'next/link' +import { useRouter } from 'next/navigation' +import { Search } from 'lucide-react' +import type { ObjectReader } from '@/lib/view/tree-nav' +import { indexedFilePaths, repoFilesWalk } from '@/lib/view/repo-facts' +import { fuzzyRank, type FuzzyHit } from '@/lib/view/fuzzy' +import { plural } from '@/lib/view/format' +import { isPageShortcut } from '@/lib/focus' +import { useAsync } from '@/hooks/use-async' +import { Input } from '@/components/ui/input' +import { repoHref, type RepoAddress } from '@/hooks/use-query-param' +import { cn } from '@/lib/utils' + +/** Results shown at once (the list scrolls). */ +const RESULTS = 20 + +/** A path with its matched characters marked. */ +function Marked({ hit }: { hit: FuzzyHit }): JSX.Element { + const at = new Set(hit.positions) + const parts: JSX.Element[] = [] + let run = '' + let marked = false + const flush = (key: number): void => { + if (run === '') return + parts.push(marked ? {run} : {run}) + run = '' + } + for (let i = 0; i < hit.path.length; i++) { + if (at.has(i) !== marked) { + flush(i) + marked = at.has(i) + } + run += hit.path[i] + } + flush(hit.path.length) + return <>{parts} +} + +export function GoToFile({ + reader, + repoKey: key, + tipOid, + rootTree, + addr, + refParam, + className, +}: { + reader: ObjectReader & { readonly memoScope?: object } + repoKey: string + tipOid: string + /** The tip's root tree, read when the walk starts. */ + rootTree: () => Promise + addr: RepoAddress + refParam: string + className?: string +}): JSX.Element { + const [started, setStarted] = useState(false) + const [query, setQuery] = useState('') + const [active, setActive] = useState(0) + const input = useRef(null) + const listId = useId() + const router = useRouter() + // The index's list (no read when the column already loaded it), then the walk's exact one. + const indexed = useAsync(() => indexedFilePaths(reader, tipOid), [key, tipOid], { enabled: started }) + const walk = useAsync(async () => repoFilesWalk(key, tipOid, reader, await rootTree()), [key, tipOid], { enabled: started }) + const paths = useMemo(() => walk.data?.files.map((f) => f.path) ?? indexed.data ?? null, [walk.data, indexed.data]) + const q = query.trim() + const hits = useMemo(() => (q === '' || paths === null ? [] : fuzzyRank(q, paths, RESULTS)), [q, paths]) + const hrefOf = (path: string): string => repoHref('/repo/blob', addr, { path, ...(refParam ? { ref: refParam } : {}) }) + const optionId = (i: number): string => `${listId}-${i}` + + // `t` starts typing a file name, as on GitHub. + useEffect(() => { + const onKey = (e: KeyboardEvent): void => { + if (!isPageShortcut(e, 't')) return + const box = input.current + if (box === null || box.getClientRects().length === 0) return + e.preventDefault() + box.focus() + box.select() + } + document.addEventListener('keydown', onKey) + return () => document.removeEventListener('keydown', onKey) + }, []) + + // A new query starts from its best match. + useEffect(() => setActive(0), [q]) + + const onKeyDown = (e: ReactKeyboardEvent): void => { + if (e.key === 'ArrowDown' || e.key === 'ArrowUp') { + if (hits.length === 0) return + e.preventDefault() + const next = (active + (e.key === 'ArrowDown' ? 1 : -1) + hits.length) % hits.length + setActive(next) + document.getElementById(optionId(next))?.scrollIntoView({ block: 'nearest' }) + } else if (e.key === 'Enter') { + const hit = hits[active] + if (hit === undefined) return + e.preventDefault() + router.push(hrefOf(hit.path)) + } else if (e.key === 'Escape') { + e.preventDefault() + setQuery('') + input.current?.blur() + } + } + + const searching = paths === null && (walk.loading || indexed.loading || !started) + const open = q !== '' + return ( +
+ + + setStarted(true)} + onChange={(e) => setQuery(e.target.value)} + onKeyDown={onKeyDown} + placeholder="Go to file" + className="h-8 py-1 pl-8 pr-7" + role="combobox" + aria-expanded={open} + aria-controls={listId} + aria-autocomplete="list" + aria-activedescendant={open && hits[active] !== undefined ? optionId(active) : undefined} + autoComplete="off" + spellCheck={false} + data-testid="go-to-file" + /> + + t + + {open ? ( +
    + {searching ?
  • Listing files…
  • : null} + {walk.error && paths === null ?
  • {walk.error}
  • : null} + {!searching && paths !== null && hits.length === 0 ?
  • No matching file.
  • : null} + {hits.map((hit, i) => ( +
  • setActive(i)}> + + + +
  • + ))} + {walk.data?.truncated ? ( +
  • Searched the first {plural(walk.data.files.length, 'file')}.
  • + ) : null} +
+ ) : null} +
+ ) +} diff --git a/forge-web/components/repo/repo-home-content.tsx b/forge-web/components/repo/repo-home-content.tsx index ea7861041..d1ce66eb2 100644 --- a/forge-web/components/repo/repo-home-content.tsx +++ b/forge-web/components/repo/repo-home-content.tsx @@ -4,23 +4,22 @@ * RepoHomeContent — the Code tab landing (`ux-dx-spec.md` §5.3): the ref bar (branch switcher, * `n commits`, Go to file), the root file list with a lazily loaded commit column, and the * README. Reads are size-independent (locator ranged object reads); `flatIndex` is never - * loaded here. A repo with no refs shows the empty state (§5.5); a private repo the viewer + * loaded here ({@link GoToFile} lists the files from the history index, then a walk). A repo with no refs shows the empty state (§5.5); a private repo the viewer * cannot decrypt never reaches here (the scaffold shows `PrivateRepoState`); unreadable storage degrades via * {@link BrowseBoundary}. */ -import { useEffect, useMemo, useState } from 'react' +import { useCallback, useEffect, useMemo } from 'react' import Link from 'next/link' -import { AlertTriangle, FileText, GitCommit, Rocket, Search } from 'lucide-react' +import { AlertTriangle, FileText, GitCommit, Rocket } from 'lucide-react' import { CopyRow } from '@/components/ui/copy-row' import { ACTIVE_NETWORK } from '@/lib/constants' import { dashRange, PUSH_COST_DASH } from '@/lib/sdk/cost' import { repoCommands, shellWord } from '@/lib/view/repo-commands' import type { BrowseReader } from '@/lib/browse' -import { loadRepoFacts, repoFilesWalk } from '@/lib/view/repo-facts' +import { loadRepoFacts } from '@/lib/view/repo-facts' import { repoKey } from '@/lib/repo' import type { RepoHome, SelectedRef } from '@/lib/view' -import { plural } from '@/lib/view/format' import { commitRootTree, decodeTextBlob, @@ -49,7 +48,7 @@ import { FileList } from '@/components/repo/file-list' import { RefDeletedState, RefNotFoundState, RefSwitcher } from '@/components/repo/ref-switcher' import { MarkdownView, type MarkdownRepoContext } from '@/components/markdown-view' import { ErrorState, LoadingBlock } from '@/components/ui/states' -import { Input } from '@/components/ui/input' +import { GoToFile } from '@/components/repo/go-to-file' import { Oid } from '@/components/ui/oid' import { repoHref, type RepoAddress } from '@/hooks/use-query-param' import { pinnedHref, usePermalinkKey } from '@/components/repo/permalink' @@ -150,6 +149,7 @@ function RootBody({ [tipOid], { enabled: data !== null }, ) + const rootTreeOf = useCallback(() => Promise.resolve(data?.tree ?? ''), [data?.tree]) const readmeRepo = useMemo(() => ({ addr, refParam, dir: '', reader, tipOid }), [addr, refParam, reader, tipOid]) // The About card's LICENSE and language bar (F-5): worked out only once everything the page // shows has settled (list, README, commit count and column), so they never delay it. @@ -187,7 +187,7 @@ function RootBody({ {commitCountLabel(commits.data)} - + repoFilesWalk(key, tipOid, reader, rootTree), [key, tipOid], { enabled: started }) - const q = query.trim().toLowerCase() - const hits = q === '' ? [] : (walk.data?.files ?? []).map((f) => f.path).filter((p) => p.toLowerCase().includes(q)).slice(0, 12) - return ( -
- - - setStarted(true)} - onChange={(e) => setQuery(e.target.value)} - placeholder="Go to file" - className="h-8 py-1 pl-8" - role="combobox" - aria-expanded={q !== ''} - aria-controls="go-to-file-results" - autoComplete="off" - /> - {q !== '' ? ( -
    - {walk.loading ?
  • Listing files…
  • : null} - {walk.error ?
  • {walk.error}
  • : null} - {!walk.loading && hits.length === 0 ?
  • No matching file.
  • : null} - {hits.map((p) => ( -
  • - - {p} - -
  • - ))} - {walk.data?.truncated ? ( -
  • Searched the first {plural(walk.data.files.length, 'file')}.
  • - ) : null} -
- ) : null} -
- ) -} - /** * The empty-repository state (`ux-dx-spec.md` §5.5): the commands that push an existing * repository, or start from scratch, and where the bytes will go. diff --git a/forge-web/components/repo/tree-content.tsx b/forge-web/components/repo/tree-content.tsx index a3c1f261a..40b7e2773 100644 --- a/forge-web/components/repo/tree-content.tsx +++ b/forge-web/components/repo/tree-content.tsx @@ -2,19 +2,20 @@ /** * TreeContent — a directory listing at an arbitrary path (browse plane, ranged reads), with the - * last-commit column the repo home has (from the history index, else a walk). + * last-commit column the repo home has (from the history index, else a walk), and Go to file. */ -import { useMemo } from 'react' +import { useCallback, useMemo } from 'react' import type { BrowseReader } from '@/lib/browse' import type { RepoHome } from '@/lib/view' import { readTree, selectedTip, selectRef, treeAtPath, type TreeEntry } from '@/lib/view' import { rootTreeOf, type PeeledTip } from '@/lib/view/tip' -import type { RepoRef } from '@/lib/repo' +import { repoKey, type RepoRef } from '@/lib/repo' import { useAsync } from '@/hooks/use-async' import { BrowseBoundary } from '@/components/repo/browse-boundary' import { ReadErrorState, ResolvedTip } from '@/components/repo/resolved-tip' import { FileList } from '@/components/repo/file-list' +import { GoToFile } from '@/components/repo/go-to-file' import { CommitCell, SearchOlderHistory, useLastCommits } from '@/components/repo/commit-column' import { PathBreadcrumb } from '@/components/repo/path-breadcrumb' import { PathActions } from '@/components/repo/path-actions' @@ -93,6 +94,7 @@ function DirBody({ // The commit column needs a commit's history: a tag of a tree has none, so it has no column. const names = useMemo(() => (data === null || tip.type !== 'commit' ? null : data.map((e) => e.name)), [data, tip.type]) const lastCommits = useLastCommits(reader, tip.oid, path, names) + const rootTree = useCallback(() => rootTreeOf(reader, tip), [reader, tip]) if (loading) return // A missing path is deterministic (common right after a ref switch) — no point retrying. if (error?.includes('path not found')) { @@ -102,6 +104,10 @@ function DirBody({ if (!data) return return (
+ {/* GitHub offers Go to file on every directory, not only the root (QW-028). */} +
+ +
{ }) }) +/** + * Code browsing on a large repo (QW-027, QW-028, QW-087), on the dash mirror: request and time + * budgets for what used to take tens of seconds. Measured on bonsia (2026-09-30), and the budgets + * are about twice that, so a slow devnet node passes and a return of the old serial reads fails: + * + * - the cold home's ref timeline (~700 updates): 8 key ranges in one round trip, where six + * `$createdAt` pages went one after the other (1.0–1.4 s before anything else could start); + * - Go to file: first results from the history index with no read (measured 8 ms after typing, + * was 14.5 s), the exact walk after it in ~1.9 s and 17 requests (was 100–120, one at a time); + * - the language bar: ~2.4 s and 19 requests once the About card is in view (was 12.8 s); + * - compare v22.0.0...v23.0.0: listed in ~16 s and 74 requests (was 19–21 s and 257), its 1,530 + * files (git's own count, renames paired) counted in ~14 s and 42 requests, complete (was 176 s, + * 1,287 requests and still partial). + * + * Run where the dash mirror is: `E2E_DEVNET=moutai`, or elsewhere with its owner in + * `E2E_SHOWCASE_DASHPAY` (on bonsia: 7A1MEuLjzcHZq8bLBzGYSUkpb2VM9dv7gtNuNYrPxKt3). + */ +/** `CHROME_KEYSET_SPLITS` (lib/repo/refs.ts): the key ranges the chrome reads a long ref timeline as. */ +const DASH_HOME_REF_READS_MAX = 8 + DAPI_RESEND_SLACK +/** The ranges go out together: from the first to the last, well under one round trip's worth. */ +const DASH_HOME_REF_SPREAD_MS = 600 +const GOTO_FIRST_RESULTS_MS = 1_000 +const GOTO_WALK_MS = 10_000 +const GOTO_WALK_REQUESTS = 40 +const LANGUAGES_MS = 10_000 +const LANGUAGES_REQUESTS = 40 +const COMPARE_LISTED_MS = 40_000 +const COMPARE_LISTED_REQUESTS = 150 +const COMPARE_COUNT_MS = 40_000 +const COMPARE_COUNT_REQUESTS = 90 + +test.describe('code browsing budgets on the dash mirror (QW-027, QW-028, QW-087)', () => { + test.skip(E2E_DEVNET !== 'moutai' && !process.env['E2E_SHOWCASE_DASHPAY'], 'the dash mirror is imported on moutai (elsewhere, set E2E_SHOWCASE_DASHPAY)') + test.describe.configure({ timeout: 240_000 }) + + test('cb-1. the cold home reads its ref timeline in one round trip; Go to file and the language bar answer in seconds', async ({ browser }) => { + const dash = await showcaseRepo('DASHPAY', 'dash') + const context = await browser.newContext() + const page = await context.newPage() + const refReads: number[] = [] + page.on('request', (r) => { + if (DAPI_METHOD.exec(r.url())?.[1] !== 'getDocuments') return + if (decodeDocumentsRequest(r.postDataBuffer())?.documentType === 'refUpdate') refReads.push(Date.now()) + }) + const dapi = recordDapi(page) + await page.goto(repoUrl('', '', dash), { waitUntil: 'domcontentloaded' }) + await waitForRepoResolved(page) + await expect(fileRows(page).first()).toBeVisible({ timeout: 60_000 }) + const spread = refReads.length === 0 ? 0 : Math.max(...refReads) - Math.min(...refReads) + test.info().annotations.push({ type: 'dapi', description: `dash cold home: ${refReads.length} refUpdate reads over ${spread} ms` }) + expect(refReads.length).toBeLessThanOrEqual(DASH_HOME_REF_READS_MAX) + expect(spread, 'the ref ranges go out side by side').toBeLessThanOrEqual(DASH_HOME_REF_SPREAD_MS) + await expect(page.getByTestId('commit-count')).toContainText(/\d/, { timeout: 60_000 }) + await settle(page) + + // Go to file: `t` focuses it; results from the history index at once, fuzzy and ranked. + await page.locator('main').click({ position: { x: 5, y: 5 } }) + await page.keyboard.press('t') + const box = page.getByTestId('go-to-file') + await expect(box).toBeFocused() + const beforeGoto = dapi.all().length + const typed = Date.now() + await box.fill('validation.cpp') + await expect(page.getByTestId('go-to-file-result').first()).toHaveText('src/validation.cpp', { timeout: GOTO_FIRST_RESULTS_MS }) + await box.fill('netproc') + await expect(page.getByTestId('go-to-file-result').first()).toContainText('net_processing') + await expect(page.getByText('Listing files')).toHaveCount(0, { timeout: GOTO_WALK_MS }) + await settle(page) + const walk = dapi.all().length - beforeGoto + test.info().annotations.push({ type: 'dapi', description: `Go to file: walk done in ${Date.now() - typed} ms (settled), ${walk} requests` }) + expect(walk).toBeLessThanOrEqual(GOTO_WALK_REQUESTS) + await page.keyboard.press('ArrowDown') + await page.keyboard.press('Enter') + await expect(page).toHaveURL(/\/repo\/blob\/.*net_processing/) + await shot(page, 'cb-01-goto-file-enter') + await context.close() + + // The language bar, in a fresh context (its walk not shared with Go to file's). + const fresh = await browser.newContext() + const lang = await fresh.newPage() + const langDapi = recordDapi(lang) + await lang.goto(repoUrl('', '', dash), { waitUntil: 'domcontentloaded' }) + await waitForRepoResolved(lang) + await expect(fileRows(lang).first()).toBeVisible({ timeout: 60_000 }) + await expect(lang.getByTestId('commit-count')).toContainText(/\d/, { timeout: 60_000 }) + await settle(lang) + const beforeLang = langDapi.all().length + const scrolled = Date.now() + const bar = lang.getByTestId('language-bar') + for (let y = 0; y < 4_000 && !(await bar.isVisible()); y += 400) { + await lang.evaluate((top) => window.scrollTo(0, top), y) + await lang.waitForTimeout(150) + } + await expect(bar.getByRole('img')).toBeVisible({ timeout: LANGUAGES_MS }) + const langMs = Date.now() - scrolled + await settle(lang) + const langReads = langDapi.all().length - beforeLang + test.info().annotations.push({ type: 'dapi', description: `language bar: ${langMs} ms, ${langReads} requests` }) + expect(langReads).toBeLessThanOrEqual(LANGUAGES_REQUESTS) + await shot(lang, 'cb-02-language-bar') + await fresh.close() + }) + + test('cb-2. compare v22.0.0...v23.0.0: listed, then every line counted, within budget', async ({ browser }) => { + const dash = await showcaseRepo('DASHPAY', 'dash') + const context = await browser.newContext() + const page = await context.newPage() + const dapi = recordDapi(page) + const opened = Date.now() + await page.goto(repoUrl('compare', '&base=v22.0.0&head=v23.0.0', dash), { waitUntil: 'domcontentloaded' }) + await expect(page.getByTestId('compare-summary')).toContainText('1,530 files changed', { timeout: COMPARE_LISTED_MS }) + const listed = dapi.all().length + test.info().annotations.push({ type: 'dapi', description: `compare listed: ${Date.now() - opened} ms, ${listed} requests` }) + expect(listed).toBeLessThanOrEqual(COMPARE_LISTED_REQUESTS) + const counting = Date.now() + await page.getByTestId('count-lines').click() + await expect(page.getByTestId('diff-totals')).toBeVisible({ timeout: COMPARE_COUNT_MS }) + const counted = dapi.all().length - listed + test.info().annotations.push({ type: 'dapi', description: `compare counted: ${Date.now() - counting} ms, ${counted} requests` }) + // git diff -M --shortstat v22.0.0...v23.0.0, and nothing left out. + await expect(page.getByTestId('diff-totals')).toContainText('+112310 −69084') + await expect(page.getByTestId('diff-totals-partial')).toHaveCount(0) + expect(counted).toBeLessThanOrEqual(COMPARE_COUNT_REQUESTS) + await shot(page, 'cb-03-compare-counted') + await context.close() + }) +}) + /** * Whether a request is one read-ahead block of a history walk: a `chunk` read of 17-19 seqs * (256 KiB / 14,700 B per chunk). The locator is read 100 seqs at a time, an object alone 1-2. diff --git a/forge-web/lib/browse/read-ahead.test.ts b/forge-web/lib/browse/read-ahead.test.ts index c2f46442e..f008b0d88 100644 --- a/forge-web/lib/browse/read-ahead.test.ts +++ b/forge-web/lib/browse/read-ahead.test.ts @@ -83,4 +83,21 @@ describe('readAheadSource', () => { await ra.fetchRange(0, 65, 66) // block 1 fetched again expect(inner.calls.map(([s]) => s)).toEqual([0, 64, 128, 64]) }) + + it('reads the next blocks ahead once reads run through the pack in order (QW-027), and only then', async () => { + const inner = source(1000) + const ra = readAheadSource(inner, 64, 64, 2) + await ra.fetchRange(0, 0, 1) // block 0: nothing ahead yet + await ra.fetchRange(0, 200, 201) // block 3: a jump, not a run + expect(inner.calls.map(([s]) => s)).toEqual([0, 192]) + await ra.fetchRange(0, 256, 257) // block 4 after block 3: blocks 5 and 6 are asked for too + expect(inner.calls.map(([s]) => s)).toEqual([0, 192, 320, 384, 256]) + // The walk reaches them from memory, and keeps two blocks ahead of itself. + await ra.fetchRange(0, 330, 331) // block 5 + expect(inner.calls.map(([s]) => s)).toEqual([0, 192, 320, 384, 256, 448]) + // Never past the end of the pack (blocks 0..15). + await ra.fetchRange(0, 900, 901) + await ra.fetchRange(0, 960, 961) + expect(Math.max(...inner.calls.map(([s]) => s))).toBe(960) + }) }) diff --git a/forge-web/lib/browse/reader.ts b/forge-web/lib/browse/reader.ts index e29d09aa8..ec8ed2fcf 100644 --- a/forge-web/lib/browse/reader.ts +++ b/forge-web/lib/browse/reader.ts @@ -57,6 +57,13 @@ export interface PackSource { export const READ_AHEAD_BLOCK = 256 * 1024 /** Blocks a {@link readAheadSource} keeps (so at most 16 MiB). */ const READ_AHEAD_BLOCKS = 64 +/** + * Blocks a history walk's source fetches ahead once its reads run through a pack in order + * (QW-027): a merge-base search over dashpay/dash v22.0.0...v23.0.0 reads 18,000 commits one block + * after the next, ~60 round trips in a row. With the next blocks asked for as the walk enters + * one, they arrive side by side (and in one chunk query, gathered with the demanded block). + */ +export const READ_AHEAD_SEQUENTIAL = 3 /** * A {@link PackSource} that fetches aligned {@link READ_AHEAD_BLOCK}-byte blocks and serves @@ -69,13 +76,27 @@ const READ_AHEAD_BLOCKS = 64 * fail to load go straight to `inner`, so nothing it could read before becomes unreadable. * Every object read through it is still hash-checked by the reader. */ -export function readAheadSource(inner: PackSource, block = READ_AHEAD_BLOCK, maxBlocks = READ_AHEAD_BLOCKS): PackSource { +export function readAheadSource(inner: PackSource, block = READ_AHEAD_BLOCK, maxBlocks = READ_AHEAD_BLOCKS, ahead = 0): PackSource { const blocks = new Map>() - const blockOf = (packRef: number, index: number, size: number, copy: number | undefined): Promise => { + /** Per pack copy, the block the last read asked for (a read of the next one is a sequential run). */ + const lastBlock = new Map() + const blockOf = (packRef: number, index: number, size: number, copy: number | undefined, demand = true): Promise => { // A single-copy pack reads the same bytes whether or not a copy is named: one key, so the // reader pinning copy 0 after its first verified object does not refetch the block. const single = (inner.copyCount?.(packRef) ?? 1) === 1 - const key = `${packRef}:${single ? 0 : copy ?? ''}:${index}` + const copyKey = `${packRef}:${single ? 0 : copy ?? ''}` + const key = `${copyKey}:${index}` + if (demand && ahead > 0) { + // Reads running through the pack in order: ask for the next blocks now, side by side. + const last = lastBlock.get(copyKey) + lastBlock.set(copyKey, index) + if (last === index - 1) { + const blocksInPack = Math.ceil(size / block) + for (let next = index + 1; next <= index + ahead && next < blocksInPack; next++) { + if (!blocks.has(`${copyKey}:${next}`)) blockOf(packRef, next, size, copy, false).catch(() => undefined) + } + } + } const hit = blocks.get(key) if (hit !== undefined) { blocks.delete(key) @@ -394,7 +415,7 @@ export class BrowseReader { const verdicts = this.opts.onObject ? new BatchedVerdicts(this.opts.onObject) : null const walker = new BrowseReader( this.locator, - readAheadSource(this.packs), + readAheadSource(this.packs, READ_AHEAD_BLOCK, READ_AHEAD_BLOCKS, READ_AHEAD_SEQUENTIAL), { ...this.opts, ...(verdicts ? { onObject: (v: ObjectVerdict) => verdicts.note(v) } : {}) }, this.view, // The same object memos: what the walk verified, the page does not read again. diff --git a/forge-web/lib/repo/chrome.test.ts b/forge-web/lib/repo/chrome.test.ts index a1e705ea4..eb7c7f2a5 100644 --- a/forge-web/lib/repo/chrome.test.ts +++ b/forge-web/lib/repo/chrome.test.ts @@ -16,7 +16,8 @@ vi.mock('../constants', async (importOriginal) => { }) import type { ForgeIds } from '../deployments' -import { bytesToBase64, hexToBase64, setPlatformVersion, type DocumentQuery } from '../sdk' +import { base64ToHex, bytesToBase64, hexToBase64, setPlatformVersion, type DocumentQuery } from '../sdk' +import { CHROME_KEYSET_SPLITS } from './refs' import { loadRepoHome } from '../view/repo-view' import { cachedDpnsName, clearDpnsCache } from '../view/dpns' import { chromeFallbacks, readRepoChrome, repoTimelines, resetRepoTimelines, staleRepoTimelines } from './chrome' @@ -78,19 +79,30 @@ function fixture(tags: number): Store { const field = (d: Doc, f: string): unknown => f.split('.').reduce((o, k) => (o as Doc | undefined)?.[k], d) +/** A byteArray field compares by its bytes (hex), as Drive's index does, not by its base64 text. */ +const keyOf = (f: string, x: unknown): unknown => (f === 'refNameHash' && typeof x === 'string' ? base64ToHex(x) : x) + function filter(rows: Doc[], where: readonly (readonly unknown[])[] = []): Doc[] { return rows.filter((d) => where.every(([f, op, v]) => { - const x = field(d, f as string) - if (op === '==') return x === v + const x = keyOf(f as string, field(d, f as string)) as number + const y = keyOf(f as string, v) as number + if (op === '==') return x === y if (op === 'in') return (v as unknown[]).includes(x) - if (op === '>=') return (x as number) >= (v as number) - if (op === '>') return (x as number) > (v as number) + if (op === '>=') return x >= y + if (op === '>') return x > y throw new Error(`fake: ${String(op)}`) }), ) } +/** The `refState` index's order: `refNameHash` bytes, then `$createdAt`. */ +const byHash = (a: Doc, b: Doc): number => { + const x = base64ToHex(a['refNameHash'] as string) + const y = base64ToHex(b['refNameHash'] as string) + return x < y ? -1 : x > y ? 1 : byTime(a, b) +} + const byTime = (a: Doc, b: Doc): number => (a['$createdAt'] as number) - (b['$createdAt'] as number) || (String(a['$id']) < String(b['$id']) ? -1 : 1) interface Fake { @@ -101,12 +113,14 @@ interface Fake { } /** Plain queries and composites over `store`; `calls` records one entry per request. */ -function fakeSdk(store: Store): Fake { +function fakeSdk(store: Store, { keyset = true }: { readonly keyset?: boolean } = {}): Fake { const calls: string[] = [] const fake: { hold: Promise | null } = { hold: null } const rowsOf = (contract: string, type: string): Doc[] => [...(store[contract]?.[type] ?? [])] const plain = (q: DocumentQuery): Doc[] => { - let rows = filter(rowsOf(q.dataContractId, q.documentTypeName), q.where).sort(byTime) + // `keyset: false`: a node that ignores a `refNameHash` order (answers in `$createdAt` order). + const order = keyset && q.orderBy?.[0]?.[0] === 'refNameHash' ? byHash : byTime + let rows = filter(rowsOf(q.dataContractId, q.documentTypeName), q.where).sort(order) if (q.startAfter !== undefined) rows = rows.slice(rows.findIndex((d) => d['$id'] === q.startAfter) + 1) return rows.slice(0, Math.min(q.limit ?? 100, 100)) } @@ -187,13 +201,26 @@ describe('repo chrome: one composite for the whole chrome', () => { expect(await readRepoChrome(sdk, FORGE, OWNER, 'nope', 'devnet')).toBeNull() }) - it('a timeline past one page is read on to the end with plain pages', async () => { + it('a ref timeline past one page is read on as key ranges side by side (QW-087)', async () => { const { sdk, calls } = fakeSdk(fixture(250)) const t = await (await readRepoChrome(sdk, FORGE, OWNER, 'demo', 'devnet'))!.timelines! expect(t.refUpdate).toHaveLength(251) expect(new Set(t.refUpdate.map((d) => d['$id'])).size).toBe(251) - // The composite's first page, then ⌈251/100⌉ − 1 = 2 continuation pages. - expect(calls).toEqual(['composite:repo', 'query:refUpdate', 'query:refUpdate']) + // Oldest first, as the `$createdAt` pages were. + const times = t.refUpdate.map((d) => d['$createdAt'] as number) + expect(times).toEqual([...times].sort((a, b) => a - b)) + // The composite's first page, then one page per key range, all sent at once: one round trip + // where ⌈251/100⌉ − 1 = 2 continuation pages went one after the other (six on dashpay/dash). + expect(calls).toEqual(['composite:repo', ...Array(CHROME_KEYSET_SPLITS).fill('query:refUpdate')]) + }) + + it('a node that does not honor the key ranges: read on to the end with plain pages', async () => { + const { sdk, calls } = fakeSdk(fixture(250), { keyset: false }) + const t = await (await readRepoChrome(sdk, FORGE, OWNER, 'demo', 'devnet'))!.timelines! + expect(t.refUpdate).toHaveLength(251) + expect(new Set(t.refUpdate.map((d) => d['$id'])).size).toBe(251) + // The ranges were refused, then ⌈251/100⌉ − 1 = 2 continuation pages. + expect(calls.slice(-2)).toEqual(['query:refUpdate', 'query:refUpdate']) }) it('a later read asks only for what is new, and keeps what it held', async () => { diff --git a/forge-web/lib/repo/chrome.ts b/forge-web/lib/repo/chrome.ts index e35c21727..3e7a26f31 100644 --- a/forge-web/lib/repo/chrome.ts +++ b/forge-web/lib/repo/chrome.ts @@ -36,10 +36,14 @@ import { cursorPadded, queryAllDocuments, shareInFlight, type PlainDocument, typ import { DOC, type RepoRef } from './contract' import { membersGeneration, membershipsFromDocs, seedMemberships } from './members' import { noteForkOf } from './fork-parent' +import { readRefRowsInRanges } from './refs' import { repoRefOf, toRepoDoc, type RepoDoc } from './resolveRepo' import { seedTargetCounts } from './social' import { repoSource } from './source' +/** The timelines keyed by `refNameHash` too, so a long one can be read as key ranges in parallel. */ +const REF_TYPES: ReadonlySet = new Set([DOC.refUpdate, DOC.protectedRefUpdate]) + /** The append-only timelines the store keeps. */ export const TIMELINE_TYPES = [DOC.config, DOC.refUpdate, DOC.protectedRefUpdate, DOC.packManifest] as const export type TimelineType = (typeof TIMELINE_TYPES)[number] @@ -176,8 +180,13 @@ async function completeTimelines( const read = await Promise.all( TIMELINE_TYPES.map(async (type, i) => { const page = pages[i] ?? [] - const rest = (): Promise => + const serial = (): Promise => queryAllDocuments(sdk, repoSource(repo).repoQuery(type, { ...sinceWhere(known, type), orderBy: [['$createdAt', 'asc']] }), { firstPage: page }) + // A first read of a repo's ref updates past a page (dashpay/dash: ~700) reads the rest as + // key ranges side by side, not page after page (QW-087); a node that does not honor them, + // and every delta read, continues the `$createdAt` page instead. + const rest = async (): Promise => + (known === undefined && REF_TYPES.has(type) ? await readRefRowsInRanges(sdk, repo, type) : null) ?? serial() const rows = page.length < PAGE ? page : await rest() return [type, mergeRows(known?.[type] ?? [], rows)] as const }), diff --git a/forge-web/lib/repo/refs.ts b/forge-web/lib/repo/refs.ts index 0b66115ae..7dd0184f5 100644 --- a/forge-web/lib/repo/refs.ts +++ b/forge-web/lib/repo/refs.ts @@ -273,6 +273,30 @@ export async function keysetScan( ) } +/** + * Every row of one ref-update type, read as `splits` keyset ranges side by side from the start + * (QW-087): for a reader that already knows the type has more than a page (the repo chrome's + * `$createdAt` page came back full) and so need not read a first page to learn it. dashpay/dash's + * ~700 updates were six serial continuation pages (a 1.0–1.4 s waterfall on every cold page); as + * ranges they are one round trip, each range read on (by key) only past its own page. Null when a + * node did not honor a range (see {@link keysetScan}): the caller then reads the plain way. + */ +export async function readRefRowsInRanges( + sdk: EvoSDK, + repo: RepoRef, + documentTypeName: string, + splits = CHROME_KEYSET_SPLITS, +): Promise { + const ceilings = splitHashRange('0'.repeat(64), splits) + const starts: (HashKey | null)[] = [null, ...ceilings.map(hashKey)] + const parts = await Promise.all(starts.map((start, i) => keysetScan(sdk, repo, documentTypeName, start, ceilings[i] ?? null))) + if (parts.some((p) => p === null)) return null + return dedupeById(parts.flatMap((p) => p as PlainDocument[])) +} + +/** Ranges {@link readRefRowsInRanges} reads by default: a page each covers ~800 updates in one round trip. */ +export const CHROME_KEYSET_SPLITS = 8 + /** * Whether some update's non-null `prevOid` is no update's `newOid` in the same ref — a parent * that should have been read and was not. Parity: forge-core `refs::has_missing_parent`. diff --git a/forge-web/lib/view/blame.test.ts b/forge-web/lib/view/blame.test.ts index d735145cc..1edc09f40 100644 --- a/forge-web/lib/view/blame.test.ts +++ b/forge-web/lib/view/blame.test.ts @@ -213,10 +213,10 @@ describe('blame bounds', () => { }) it('a change too large to check for a rename leaves the lines it added unresolved, not on it', async () => { - // 70 files deleted with the path's addition: scoring them all is over the rename read budget. + // 450 files deleted with the path's addition: scoring them all is over the rename read budget. const s = new Store() const gone: Record = {} - for (let i = 0; i < 70; i++) gone[`d/f${i}`] = `file ${i}\n${'filler\n'.repeat(i % 5)}` + for (let i = 0; i < 450; i++) gone[`d/f${i}`] = `file ${i}\n${'filler\n'.repeat(i % 5)}` const c1 = s.commit(s.files(gone), [], 'one') const c2 = s.commit(s.files({ 'new.txt': 'a\nb\n' }), [c1], 'replace everything') const c3 = s.commit(s.files({ 'new.txt': 'a\nb\nc\n' }), [c2], 'three') diff --git a/forge-web/lib/view/browse-fallback.test.ts b/forge-web/lib/view/browse-fallback.test.ts index 61e9082e3..1b5d587e6 100644 --- a/forge-web/lib/view/browse-fallback.test.ts +++ b/forge-web/lib/view/browse-fallback.test.ts @@ -673,7 +673,14 @@ describe('startFallback with external-storage packs', () => { const external = manifestFor(ext.pack, 1, { storage: 1, uris: ['https://flaky.example/p'], createdAt: 1, documentId: 'b' }) const repo = testRepo('fallback-expiry') vi.stubGlobal('fetch', () => Promise.resolve(new Response('nope', { status: 503 }))) - await startFallback(mockSdk(new Map([[platform.packHash, plat.pack]])), repo, [platform, external]) + // Platform chunk reads are gathered for one timer turn (`queueChunkSeqs`): let the fake clock + // run that turn, a millisecond at a time, until the clone is built. + let built = false + const cloning = startFallback(mockSdk(new Map([[platform.packHash, plat.pack]])), repo, [platform, external]).finally(() => { + built = true + }) + while (!built) await vi.advanceTimersByTimeAsync(1) + await cloning expect(cachedFallback(repo.repoId, [platform, external])).not.toBeNull() await vi.advanceTimersByTimeAsync(61_000) expect(cachedFallback(repo.repoId, [platform, external])).toBeNull() diff --git a/forge-web/lib/view/browse-source.ts b/forge-web/lib/view/browse-source.ts index 24b5ca273..a6c5f609e 100644 --- a/forge-web/lib/view/browse-source.ts +++ b/forge-web/lib/view/browse-source.ts @@ -140,6 +140,7 @@ function evictChunks(): void { export function clearChunkCache(): void { chunkCache.clear() chunkCacheBytes = 0 + chunkQueriesSent = 0 } /** Query one batch of chunk docs (uncached) and return payloads keyed by seq. */ @@ -189,6 +190,56 @@ async function queryChunkBatch( return bySeq } +/** + * Chunk seqs asked for in the same turn of the event loop, per copy of an artifact, gathered into + * one query (QW-027/QW-028): reads that run side by side (a pool of blob reads counting a diff's + * lines, a tree walk's parallel reads, Blame's read-ahead) each need a chunk or two, and each used to + * cost a query of its own. Gathered, they share queries of up to {@link CHUNK_QUERY_MAX} seqs + * ({@link queryChunkBatch} splits and pools past that). A copy's seqs are unique per `packHash`, + * so a gathered answer serves each asker exactly its own seqs. + */ +interface PendingChunkQuery { + readonly seqs: Set + readonly result: Promise> +} +const pendingChunkQueries = new Map() + +let chunkQueriesSent = 0 + +/** Test hook: how many chunk queries {@link queueChunkSeqs} has sent since {@link clearChunkCache}. */ +export function chunkQueryCount(): number { + return chunkQueriesSent +} + +/** + * `seqs` of the copy keyed `key`, fetched with the other seqs asked for this turn (one macrotask: + * the continuations of reads that just landed together get to ask first), in one query while they + * fit one ({@link CHUNK_QUERY_MAX}); past that, the next asker starts a new query rather than make + * everyone wait for a bigger one. `query` fetches a gathered set; the first asker's is used. + */ +export function queueChunkSeqs( + key: string, + seqs: readonly number[], + query: (seqs: readonly number[]) => Promise>, +): Promise> { + let pending = pendingChunkQueries.get(key) + if (pending !== undefined && pending.seqs.size + seqs.filter((q) => !pending?.seqs.has(q)).length > CHUNK_QUERY_MAX) pending = undefined + if (pending === undefined) { + const gathered = new Set() + const result = new Promise>((resolve, reject) => { + setTimeout(() => { + if (pendingChunkQueries.get(key)?.seqs === gathered) pendingChunkQueries.delete(key) + chunkQueriesSent += Math.ceil(gathered.size / CHUNK_QUERY_MAX) + query([...gathered].sort((a, b) => a - b)).then(resolve, reject) + }, 0) + }) + pending = { seqs: gathered, result } + pendingChunkQueries.set(key, pending) + } + for (const seq of seqs) pending.seqs.add(seq) + return pending.result +} + /** * Fetch a contiguous `[start, end)` range of a platform-stored artifact by `packHash` (hex). * @@ -231,7 +282,7 @@ async function fetchPlatformRange( else missing.push(seq) } if (missing.length > 0) { - const batch = queryChunkBatch(sdk, repo, manifest, missing) + const batch = queueChunkSeqs(cachePrefix, missing, (seqs) => queryChunkBatch(sdk, repo, manifest, seqs)) for (const seq of missing) { const promise = batch.then((bySeq) => { const payload = bySeq.get(seq) diff --git a/forge-web/lib/view/chunk-gather.test.ts b/forge-web/lib/view/chunk-gather.test.ts new file mode 100644 index 000000000..a2d6ba6a9 --- /dev/null +++ b/forge-web/lib/view/chunk-gather.test.ts @@ -0,0 +1,45 @@ +/** + * Platform chunk reads asked for in the same turn share one query (QW-027, QW-028): a pool of + * blob reads counting a diff's lines, or a tree walk's parallel reads, each need a chunk or two. + */ + +import { describe, expect, it } from 'vitest' + +import { chunkQueryCount, clearChunkCache, queueChunkSeqs } from './browse-source' + +const answer = (seqs: readonly number[]): Promise> => Promise.resolve(new Map(seqs.map((s) => [s, Uint8Array.of(s)]))) + +describe('queueChunkSeqs', () => { + it('gathers one turn\'s asks of a copy into one query, each asker served its own seqs', async () => { + clearChunkCache() + const sent: number[][] = [] + const query = (seqs: readonly number[]) => (sent.push([...seqs]), answer(seqs)) + const [a, b, c] = await Promise.all([queueChunkSeqs('copy', [5], query), queueChunkSeqs('copy', [2, 3], query), queueChunkSeqs('copy', [3], query)]) + expect(sent).toEqual([[2, 3, 5]]) + for (const got of [a, b, c]) expect([...got.keys()]).toEqual([2, 3, 5]) + expect(chunkQueryCount()).toBe(1) + }) + + it('keeps copies apart, and starts a new query rather than grow one past a page', async () => { + clearChunkCache() + const sent: number[][] = [] + const query = (seqs: readonly number[]) => (sent.push([...seqs]), answer(seqs)) + const many = Array.from({ length: 90 }, (_, i) => i) + await Promise.all([ + queueChunkSeqs('one', many, query), + queueChunkSeqs('two', [1], query), + // 90 + 20 is over a page (100): its own query. + queueChunkSeqs('one', Array.from({ length: 20 }, (_, i) => 200 + i), query), + ]) + expect(sent.map((s) => s.length).sort((x, y) => x - y)).toEqual([1, 20, 90]) + }) + + it('a failed query fails its askers, and the next turn asks again', async () => { + clearChunkCache() + let fail = true + const query = (seqs: readonly number[]) => (fail ? Promise.reject(new Error('node down')) : answer(seqs)) + await expect(queueChunkSeqs('copy', [1], query)).rejects.toThrow('node down') + fail = false + await expect(queueChunkSeqs('copy', [1], query)).resolves.toEqual(new Map([[1, Uint8Array.of(1)]])) + }) +}) diff --git a/forge-web/lib/view/commit-log.ts b/forge-web/lib/view/commit-log.ts index b5002ab07..769eef197 100644 --- a/forge-web/lib/view/commit-log.ts +++ b/forge-web/lib/view/commit-log.ts @@ -75,7 +75,7 @@ export async function diffTreesWithRenames(sides: DiffSides, baseTreeOid: string const DIFF_NODE_CAP = 2000 /** Tree reads in flight at once while walking changed subtrees. */ -const TREE_READ_POOL = 6 +const TREE_READ_POOL = 16 async function readTreeMap(reader: ObjectReader, treeOid: string | null): Promise> { // A tree that cannot be read is an error, not an empty directory: treating it as empty diff --git a/forge-web/lib/view/file-diff.test.ts b/forge-web/lib/view/file-diff.test.ts index 9c89060ec..c39f13665 100644 --- a/forge-web/lib/view/file-diff.test.ts +++ b/forge-web/lib/view/file-diff.test.ts @@ -193,13 +193,25 @@ describe('loadFilePatch over a pack (the live read path)', () => { if (patch.kind === 'placeholder') expect(patch.note).toMatch(/stored size as 20 MB.*has not been checked/) }) - it('calls an over-256 KiB file with too many changes too complex, not too large', async () => { + it('calls an over-256 KiB file with too many changes too complex, not too large, and still counts it (QW-027)', async () => { const s = new Store() const r = s.reader() const patch = await loadFilePatch({ base: r, head: r }, change({ baseOid: s.blob(lines(12_000, 'p')), headOid: s.blob(lines(12_000, 'q')) })) - expect(patch).toMatchObject({ kind: 'placeholder', reason: 'too-complex' }) - if (patch.kind === 'placeholder') expect(patch.note).toMatch(/^Large file \(\d+ KB\) not shown.*lines are not counted/) - expect(uncountedReasons([patch.change], () => patch)).toBe('1 too many changes to diff') + expect(patch).toMatchObject({ kind: 'placeholder', reason: 'too-complex', added: 12_000, deleted: 12_000 }) + if (patch.kind === 'placeholder') expect(patch.note).toMatch(/^Large file \(\d+ KB\) not shown.*its lines are counted/) + // The change set's totals stay whole: nothing is left uncounted. + expect(diffTotals([patch.change], () => patch)).toMatchObject({ added: 12_000, deleted: 12_000, uncounted: 0 }) + expect(uncountedReasons([patch.change], () => patch)).toBe('') + }) + + it('counts a change past the shown diff\'s edit bound (QW-027: dash v22…v23 left 16 files out)', async () => { + const s = new Store() + const r = s.reader() + // 3,000 changed lines in a small file: over the 2,000 a diff shows, well within what is counted. + const before = Array.from({ length: 3_000 }, (_, i) => `old ${i}\n`).join('') + const after = Array.from({ length: 3_000 }, (_, i) => (i % 2 === 0 ? `old ${i}\n` : `new ${i}\n`)).join('') + const patch = await loadFilePatch({ base: r, head: r }, change({ baseOid: s.blob(before), headOid: s.blob(after) })) + expect(patch).toMatchObject({ kind: 'placeholder', reason: 'too-complex', added: 1_500, deleted: 1_500 }) }) it('calls a file binary when one side is, whichever side fails first', async () => { diff --git a/forge-web/lib/view/file-diff.ts b/forge-web/lib/view/file-diff.ts index cf10c0c3d..065ec7d72 100644 --- a/forge-web/lib/view/file-diff.ts +++ b/forge-web/lib/view/file-diff.ts @@ -10,7 +10,7 @@ import { decodeTextBlob } from './git-objects' import { isGitlink, type DiffSides, type FileChange } from './commit-log' import { ObjectTooLargeError } from '../browse' import { knownMinSize, type ObjectReader } from './tree-nav' -import { diffStat, diffTextLines, type TextDiffLine } from './text-diff' +import { countTextChanges, diffStat, diffTextLines, type TextDiffLine } from './text-diff' import { formatBytes } from './format' /** Blobs above this are not diffed inline. */ @@ -177,6 +177,9 @@ export async function loadFilePatch( const lines = diffTextLines(before.text, after.text, undefined, { ignoreWhitespace: options.ignoreWhitespace === true }) if (lines === null) { const large = size > INLINE_BLOB_MAX_BYTES ? `${largeNote(size)} ` : '' + // Too large to show, but counted (QW-027): the change set's totals stay whole. + const counted = countTextChanges(before.text, after.text, { ignoreWhitespace: options.ignoreWhitespace === true }) + if (counted !== null) return { ...placeholder(change, 'too-complex', `${large}This change is too large to show in the browser; its lines are counted.`), ...counted } return placeholder(change, 'too-complex', `${large}This change is too large to diff in the browser, so its lines are not counted.`) } if (size <= INLINE_BLOB_MAX_BYTES) return textPatch(change, lines) diff --git a/forge-web/lib/view/fuzzy.test.ts b/forge-web/lib/view/fuzzy.test.ts new file mode 100644 index 000000000..cc4bb0650 --- /dev/null +++ b/forge-web/lib/view/fuzzy.test.ts @@ -0,0 +1,59 @@ +/** Go to file's fuzzy ranking (QW-028): GitHub's file finder, which dash's QA compared against. */ + +import { describe, expect, it } from 'vitest' + +import { fuzzyMatch, fuzzyRank } from './fuzzy' + +const DASH = [ + 'src/test/evo_trivialvalidation.cpp', + 'src/test/util/validation.cpp', + 'src/validation.cpp', + 'src/validation.h', + 'src/net_processing.cpp', + 'src/net_processing.h', + 'src/net.cpp', + 'src/qt/locale/dash_de.ts', + 'doc/release-notes.md', +] + +describe('fuzzyRank', () => { + it('puts the file named exactly first, before deeper and longer names', () => { + expect(fuzzyRank('validation.cpp', DASH, 3).map((h) => h.path)).toEqual([ + 'src/validation.cpp', + 'src/test/util/validation.cpp', + 'src/test/evo_trivialvalidation.cpp', + ]) + }) + + it('matches characters in order across word breaks, as GitHub does', () => { + expect(fuzzyRank('netproc', DASH, 5).map((h) => h.path).sort()).toEqual(['src/net_processing.cpp', 'src/net_processing.h']) + expect(fuzzyRank('NETPROC.h', DASH, 5)[0]?.path).toBe('src/net_processing.h') + // Spaces are ignored. + expect(fuzzyRank('net proc.cpp', DASH, 1)[0]?.path).toBe('src/net_processing.cpp') + }) + + it('finds nothing for an empty query or characters out of order', () => { + expect(fuzzyRank(' ', DASH, 5)).toEqual([]) + expect(fuzzyRank('ppcten', DASH, 5)).toEqual([]) + }) + + it('bounds what it returns', () => { + expect(fuzzyRank('s', DASH, 2)).toHaveLength(2) + }) +}) + +describe('fuzzyMatch', () => { + it('reports the matched positions, preferring a run at the start of a name', () => { + const m = fuzzyMatch('val', 'src/validation.cpp') + expect(m?.positions).toEqual([4, 5, 6]) + expect(fuzzyMatch('np', 'src/net_processing.cpp')?.positions).toEqual([4, 8]) + }) + + it('is null when a character is missing', () => { + expect(fuzzyMatch('xyz', 'src/validation.cpp')).toBeNull() + }) + + it('scores a whole-path match highest', () => { + expect(fuzzyMatch('src/net.cpp', 'src/net.cpp')?.score).toBe(Infinity) + }) +}) diff --git a/forge-web/lib/view/fuzzy.ts b/forge-web/lib/view/fuzzy.ts new file mode 100644 index 000000000..c032eda41 --- /dev/null +++ b/forge-web/lib/view/fuzzy.ts @@ -0,0 +1,136 @@ +/** + * Fuzzy file-name matching for Go to file (QW-028), GitHub's file finder: the query's characters + * in order anywhere in the path (`netproc` finds `src/net_processing.cpp`), case-insensitive, + * ranked so a match at the start of a name, a run of adjacent characters and a shorter path come + * first. The scoring is fzy's (github.com/jhawthorn/fzy, MIT), which GitHub-style finders use: + * a dynamic programme over (query, path) that rewards a match after `/`, `_`, `-`, `.` or a + * lower-to-upper case change and a consecutive match, and charges for every skipped character. + */ + +const SCORE_GAP_LEADING = -0.005 +const SCORE_GAP_TRAILING = -0.005 +const SCORE_GAP_INNER = -0.01 +const SCORE_MATCH_CONSECUTIVE = 1.0 +const SCORE_MATCH_SLASH = 0.9 +const SCORE_MATCH_WORD = 0.8 +const SCORE_MATCH_CAPITAL = 0.7 +const SCORE_MATCH_DOT = 0.6 +/** Paths longer than this are matched but not scored (fzy's own bound). */ +const MATCH_MAX_LEN = 1024 + +/** A match: its score (higher is better) and the matched positions in the path. */ +export interface FuzzyMatch { + readonly score: number + readonly positions: readonly number[] +} + +/** Whether every character of `needle` (lowercase) appears in `haystack` (lowercase), in order. */ +function hasMatch(needle: string, haystack: string): boolean { + let at = 0 + for (const ch of needle) { + at = haystack.indexOf(ch, at) + if (at === -1) return false + at += 1 + } + return true +} + +/** The bonus for a match at `i`, by the character before it. */ +function bonusAt(path: string, i: number): number { + if (i === 0) return SCORE_MATCH_SLASH + const prev = path[i - 1] as string + const ch = path[i] as string + if (prev === '/') return SCORE_MATCH_SLASH + if (prev === '-' || prev === '_' || prev === ' ') return SCORE_MATCH_WORD + if (prev === '.') return SCORE_MATCH_DOT + if (prev >= 'a' && prev <= 'z' && ch >= 'A' && ch <= 'Z') return SCORE_MATCH_CAPITAL + return 0 +} + +/** + * The best way `query` matches `path`, or null when it does not (spaces in the query are ignored, + * as GitHub's finder ignores them). An empty query matches nothing. + */ +export function fuzzyMatch(query: string, path: string): FuzzyMatch | null { + const needle = query.replace(/\s+/g, '').toLowerCase() + if (needle === '') return null + const lower = path.toLowerCase() + if (!hasMatch(needle, lower)) return null + const n = needle.length + const m = path.length + if (n === m) return { score: Infinity, positions: [...Array(m).keys()] } + if (m > MATCH_MAX_LEN) return { score: -Infinity, positions: [] } + + // D[i][j]: the best score of needle[0..i] ending with a match at j; M[i][j]: the best score of + // needle[0..i] within path[0..j]. + const D: Float64Array[] = [] + const M: Float64Array[] = [] + const bonus = new Float64Array(m) + for (let j = 0; j < m; j++) bonus[j] = bonusAt(path, j) + for (let i = 0; i < n; i++) { + const d = new Float64Array(m) + const mm = new Float64Array(m) + const gap = i === n - 1 ? SCORE_GAP_TRAILING : SCORE_GAP_INNER + let prev = -Infinity + for (let j = 0; j < m; j++) { + if (needle[i] === lower[j]) { + let score = -Infinity + if (i === 0) score = j * SCORE_GAP_LEADING + (bonus[j] as number) + else if (j > 0) { + const pm = (M[i - 1] as Float64Array)[j - 1] as number + const pd = (D[i - 1] as Float64Array)[j - 1] as number + score = Math.max(pm + (bonus[j] as number), pd + SCORE_MATCH_CONSECUTIVE) + } + d[j] = score + prev = Math.max(score, prev + gap) + } else { + d[j] = -Infinity + prev += gap + } + mm[j] = prev + } + D.push(d) + M.push(mm) + } + + // Walk back for the positions: the latest match on the best path, preferring a consecutive run. + const positions = new Array(n) + let matchRequired = false + for (let i = n - 1, j = m - 1; i >= 0; i--) { + for (; j >= 0; j--) { + const d = (D[i] as Float64Array)[j] as number + const mm = (M[i] as Float64Array)[j] as number + if (d !== -Infinity && (matchRequired || d === mm)) { + // Consecutive with the next match: this one must be taken too. + matchRequired = i > 0 && j > 0 && mm === ((D[i - 1] as Float64Array)[j - 1] as number) + SCORE_MATCH_CONSECUTIVE + positions[i] = j + j -= 1 + break + } + } + } + return { score: (M[n - 1] as Float64Array)[m - 1] as number, positions } +} + +/** One ranked hit. */ +export interface FuzzyHit extends FuzzyMatch { + readonly path: string +} + +/** + * The `limit` best matches of `query` among `paths`, best first; ties go to the shorter path, then + * path order. + */ +export function fuzzyRank(query: string, paths: readonly string[], limit: number): FuzzyHit[] { + const needle = query.replace(/\s+/g, '').toLowerCase() + if (needle === '') return [] + const hits: FuzzyHit[] = [] + for (const path of paths) { + // The cheap in-order test first: most paths fail it, and scoring is the costly part. + if (!hasMatch(needle, path.toLowerCase())) continue + const match = fuzzyMatch(needle, path) + if (match !== null) hits.push({ path, ...match }) + } + hits.sort((a, b) => b.score - a.score || a.path.length - b.path.length || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) + return hits.slice(0, limit) +} diff --git a/forge-web/lib/view/pull-diff.ts b/forge-web/lib/view/pull-diff.ts index e6feb6464..1b66b7b4c 100644 --- a/forge-web/lib/view/pull-diff.ts +++ b/forge-web/lib/view/pull-diff.ts @@ -314,6 +314,23 @@ export async function findMergeBases( queue.push({ oid, when, seq: seq++ }) } + // The walk takes one commit at a time, so each read of a commit in a pack block not yet fetched + // was a round trip of its own: ~70 in a row on dashpay/dash v22.0.0...v23.0.0 (QW-027). The + // parents of everything waiting in the queue are what it reads next (the loop walks every + // queued commit until all are stale), so they are asked for ahead, side by side, through the + // reader's memo. They count toward the cap only once walked; what is read ahead in all is + // bounded by the cap too. + const prefetched = new Set() + const prefetchFrontier = (): void => { + for (const q of queue) { + for (const p of commits.get(q.oid)?.parents ?? []) { + if (commits.has(p) || prefetched.has(p) || prefetched.size >= cap) continue + prefetched.add(p) + readCommit(reader, p).catch(() => undefined) + } + } + } + const candidates: string[] = [] try { await Promise.all([push(baseOid, BASE), push(headOid, HEAD)]) @@ -333,6 +350,7 @@ export async function findMergeBases( flags.set(oid, flagsOf(oid) | STALE) } const parents = (commits.get(oid) as { parents: readonly string[] }).parents + prefetchFrontier() // A commit's parents are independent reads — fetch them together. await Promise.all(parents.filter((p) => (flagsOf(p) & flag) !== flag).map((p) => push(p, flag))) } diff --git a/forge-web/lib/view/renames.ts b/forge-web/lib/view/renames.ts index 328819974..3c9ccbb0c 100644 --- a/forge-web/lib/view/renames.ts +++ b/forge-web/lib/view/renames.ts @@ -41,12 +41,12 @@ const FIRST_FEW_BYTES = 8000 const HASHBASE = 107927 /** Blobs the inexact phases may read (each a ranged read over the network). */ -const RENAME_READ_BUDGET = 64 +const RENAME_READ_BUDGET = 400 /** A blob larger than this is not scored for similarity. */ const RENAME_MAX_BLOB_BYTES = 1024 * 1024 const NOT_LOOKED = 'Renames with edits were not all looked for:' /** Blob reads in flight at once. */ -const READ_POOL = 6 +const READ_POOL = 16 const S_IFMT = 0o170000 const S_IFREG = 0o100000 diff --git a/forge-web/lib/view/repo-facts.ts b/forge-web/lib/view/repo-facts.ts index ebbb6ec22..6bd682c2f 100644 --- a/forge-web/lib/view/repo-facts.ts +++ b/forge-web/lib/view/repo-facts.ts @@ -25,6 +25,7 @@ import { onRepoContentWritten } from '../repo/push' import { readReleaseCount, readReleases, releaseCountOf } from '../repo/releases' import { invalidateSessionCache, sessionCached } from './session-cache' import { historyWalker } from './commit-log' +import { historyOf } from './history-source' import { readBlob, type ObjectReader } from './tree-nav' import { decodeTextBlob, type TreeEntry } from './git-objects' import { formatBytes } from './format' @@ -114,6 +115,31 @@ export function repoFilesWalk( return walk } +/** Each history index's file paths, worked out once. */ +const indexedFiles = new WeakMap() + +/** + * The files at `tipOid` as the history index lists them (QW-028), with no tree read: the index the + * file list's column already loaded names every path of the tip (directories too, which are left + * out here). Null when no index covers the tip. A delta over a full index can still name a file + * deleted since that full index (the reader drops such paths when a tree lists names), so a caller + * takes these as a first answer until the tree walk ({@link repoFilesWalk}) has the exact one. + */ +export async function indexedFilePaths(reader: { readonly memoScope?: object }, tipOid: string): Promise { + const history = historyOf(reader) + if (history === null || !history.covers(tipOid)) return null + const index = await history.load(tipOid) + const hit = indexedFiles.get(index) + if (hit !== undefined) return hit + const dirs = new Set() + for (const path of index.paths.keys()) { + for (let at = path.indexOf('/'); at !== -1; at = path.indexOf('/', at + 1)) dirs.add(path.slice(0, at)) + } + const files = [...index.paths.keys()].filter((p) => !dirs.has(p)).sort() + indexedFiles.set(index, files) + return files +} + /** * Work out and publish the facts for this tip: the license from the root's license files (a few * KiB each, through the page's reader), then the language bar from the shared file walk. Facts diff --git a/forge-web/lib/view/text-diff.ts b/forge-web/lib/view/text-diff.ts index 077c22f1d..89a861276 100644 --- a/forge-web/lib/view/text-diff.ts +++ b/forge-web/lib/view/text-diff.ts @@ -101,6 +101,36 @@ function gitOps(a: readonly string[], b: readonly string[], limits: DiffLimits): return edits > limits.maxEdits ? null : ops } +/** + * How far {@link countTextChanges} searches: a change too large to show is still counted (QW-027), + * so its search may run further than a shown diff's before it gives up. + */ +export const COUNT_DIFF_LIMITS: DiffLimits = { maxEdits: Infinity, maxWork: 100_000_000 } + +/** + * The lines a change adds and deletes, for one too large to show ({@link diffTextLines} said null): + * the same diff, with no bound on its edits, so a change set's totals stay whole. Null only when + * the search itself runs past {@link COUNT_DIFF_LIMITS}. + */ +export function countTextChanges( + before: string, + after: string, + options: { readonly ignoreWhitespace?: boolean } = {}, +): { readonly added: number; readonly deleted: number } | null { + const key = options.ignoreWhitespace ? (l: string): string => l.replace(/\s+/g, '') : null + const oldLines = splitLines(before) + const newLines = splitLines(after) + const ops = gitOps(key ? oldLines.map(key) : oldLines, key ? newLines.map(key) : newLines, COUNT_DIFF_LIMITS) + if (ops === null) return null + let added = 0 + let deleted = 0 + for (const op of ops) { + if (op === 1) deleted += 1 + else if (op === 2) added += 1 + } + return { added, deleted } +} + /** * Compute a line-level diff. `null` means the change is too large for an in-browser review; * callers should surface that honestly rather than freezing the page. diff --git a/forge-web/lib/view/zip.ts b/forge-web/lib/view/zip.ts index 1f963a90a..4b3e62019 100644 --- a/forge-web/lib/view/zip.ts +++ b/forge-web/lib/view/zip.ts @@ -9,7 +9,7 @@ import { MODE_GITLINK, MODE_TREE } from '../browse' import type { ObjectReader } from './tree-nav' import { ObjectTypeError, peel, readBlob, readTree } from './tree-nav' -import { decodeTextBlob } from './git-objects' +import { decodeTextBlob, type TreeEntry } from './git-objects' import { historyWalker } from './commit-log' import { autoAbbrevLength, @@ -74,24 +74,31 @@ export function isSafeName(name: string): boolean { } /** - * Every blob (and symlink) under a tree, gitlinks and unsafe names skipped, breadth first, sorted - * by path, each with its stored size from the locator (tree reads only). Stops after `maxTrees` - * trees or `maxFiles` files (`truncated`, also when one tree alone holds more than the bound), so - * Go to file and the language bar can list a large repo without walking all of it. + * Tree reads a walk keeps in flight (QW-028): a level of dashpay/dash's tree is dozens of + * directories, and one read at a time made Go to file and the language bar wait ~14 s for ~100 + * round trips. Reads of the same pack's neighbouring trees share their queries (the read-ahead + * walker's blocks, the chunk queries gathered per turn). + */ +export const WALK_POOL = 16 + +/** + * Every blob (and symlink) under a tree, gitlinks and unsafe names skipped, breadth first with up + * to `pool` trees read at once, sorted by path, each with its stored size from the locator (tree + * reads only). Stops after `maxTrees` trees or `maxFiles` files (`truncated`, also when one tree + * alone holds more than the bound), so Go to file and the language bar can list a large repo + * without walking all of it. */ export async function walkFiles( reader: ObjectReader, treeOid: string, - { maxTrees = Infinity, maxFiles = Infinity }: { readonly maxTrees?: number; readonly maxFiles?: number } = {}, + { maxTrees = Infinity, maxFiles = Infinity, pool = WALK_POOL }: { readonly maxTrees?: number; readonly maxFiles?: number; readonly pool?: number } = {}, ): Promise { const files: ZipFile[] = [] const queue: [string, string][] = [[treeOid, '']] let trees = 0 let dropped = false - while (queue.length > 0 && trees < maxTrees && files.length < maxFiles) { - const [oid, prefix] = queue.shift() as [string, string] - trees += 1 - for (const e of await readTree(reader, oid)) { + const take = (entries: readonly TreeEntry[], prefix: string): void => { + for (const e of entries) { // A tree is hash-checked, not sane: a hostile pusher can name an entry `..` (zip-slip). if (!isSafeName(e.name)) continue const path = prefix ? `${prefix}/${e.name}` : e.name @@ -101,6 +108,31 @@ export async function walkFiles( else dropped = true } } + await new Promise((resolve, reject) => { + let active = 0 + let failed = false + const pump = (): void => { + while (!failed && active < pool && queue.length > 0 && trees < maxTrees && files.length < maxFiles) { + const [oid, prefix] = queue.shift() as [string, string] + trees += 1 + active += 1 + readTree(reader, oid).then( + (entries) => { + active -= 1 + if (failed) return + take(entries, prefix) + pump() + }, + (e: unknown) => { + failed = true + reject(e) + }, + ) + } + if (active === 0 && !failed) resolve() + } + pump() + }) files.sort((a, b) => (a.path < b.path ? -1 : 1)) return { files, truncated: dropped || queue.length > 0 } } From 1ae346cda777600e51b93b065e2426547d7b1c3c Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 09:43:06 -0500 Subject: [PATCH 2/7] fix(web): count a too-large change from its one diff, and harden the parallel walk Review: a change with more edits than a diff shows is now counted from the same xdiff pass (no second, larger search on the main thread); an exception inside the walk's tree-read callback rejects the walk instead of leaving it pending; clearing the chunk cache drops gathered queries too; Go to file keeps matched positions on the path when lowercasing would change its length. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/lib/view/browse-source.ts | 1 + forge-web/lib/view/file-diff.ts | 11 +++--- forge-web/lib/view/fuzzy.test.ts | 5 +++ forge-web/lib/view/fuzzy.ts | 23 ++++++++++-- forge-web/lib/view/text-diff.ts | 57 ++++++++++++----------------- forge-web/lib/view/zip.ts | 16 ++++---- 6 files changed, 61 insertions(+), 52 deletions(-) diff --git a/forge-web/lib/view/browse-source.ts b/forge-web/lib/view/browse-source.ts index a6c5f609e..0553ad7d1 100644 --- a/forge-web/lib/view/browse-source.ts +++ b/forge-web/lib/view/browse-source.ts @@ -141,6 +141,7 @@ export function clearChunkCache(): void { chunkCache.clear() chunkCacheBytes = 0 chunkQueriesSent = 0 + pendingChunkQueries.clear() } /** Query one batch of chunk docs (uncached) and return payloads keyed by seq. */ diff --git a/forge-web/lib/view/file-diff.ts b/forge-web/lib/view/file-diff.ts index 065ec7d72..f60f8069d 100644 --- a/forge-web/lib/view/file-diff.ts +++ b/forge-web/lib/view/file-diff.ts @@ -10,7 +10,7 @@ import { decodeTextBlob } from './git-objects' import { isGitlink, type DiffSides, type FileChange } from './commit-log' import { ObjectTooLargeError } from '../browse' import { knownMinSize, type ObjectReader } from './tree-nav' -import { countTextChanges, diffStat, diffTextLines, type TextDiffLine } from './text-diff' +import { diffStat, diffTextLinesOrCount, type TextDiffLine } from './text-diff' import { formatBytes } from './format' /** Blobs above this are not diffed inline. */ @@ -174,12 +174,11 @@ export async function loadFilePatch( try { const [before, after] = await readSides(sides, change, options) const size = Math.max(before.size, after.size) - const lines = diffTextLines(before.text, after.text, undefined, { ignoreWhitespace: options.ignoreWhitespace === true }) - if (lines === null) { + const lines = diffTextLinesOrCount(before.text, after.text, undefined, { ignoreWhitespace: options.ignoreWhitespace === true }) + if (!Array.isArray(lines)) { const large = size > INLINE_BLOB_MAX_BYTES ? `${largeNote(size)} ` : '' - // Too large to show, but counted (QW-027): the change set's totals stay whole. - const counted = countTextChanges(before.text, after.text, { ignoreWhitespace: options.ignoreWhitespace === true }) - if (counted !== null) return { ...placeholder(change, 'too-complex', `${large}This change is too large to show in the browser; its lines are counted.`), ...counted } + // Too many changes to show, but counted from the same diff (QW-027): the totals stay whole. + if (lines !== null) return { ...placeholder(change, 'too-complex', `${large}This change is too large to show in the browser; its lines are counted.`), added: lines.added, deleted: lines.deleted } return placeholder(change, 'too-complex', `${large}This change is too large to diff in the browser, so its lines are not counted.`) } if (size <= INLINE_BLOB_MAX_BYTES) return textPatch(change, lines) diff --git a/forge-web/lib/view/fuzzy.test.ts b/forge-web/lib/view/fuzzy.test.ts index cc4bb0650..c44380e62 100644 --- a/forge-web/lib/view/fuzzy.test.ts +++ b/forge-web/lib/view/fuzzy.test.ts @@ -49,6 +49,11 @@ describe('fuzzyMatch', () => { expect(fuzzyMatch('np', 'src/net_processing.cpp')?.positions).toEqual([4, 8]) }) + it('keeps positions on the path when lowercasing would change its length', () => { + const m = fuzzyMatch('md', 'İstanbul/README.md') + expect(m?.positions.map((p) => 'İstanbul/README.md'[p])).toEqual(['m', 'd']) + }) + it('is null when a character is missing', () => { expect(fuzzyMatch('xyz', 'src/validation.cpp')).toBeNull() }) diff --git a/forge-web/lib/view/fuzzy.ts b/forge-web/lib/view/fuzzy.ts index c032eda41..db19f1d5b 100644 --- a/forge-web/lib/view/fuzzy.ts +++ b/forge-web/lib/view/fuzzy.ts @@ -24,6 +24,21 @@ export interface FuzzyMatch { readonly positions: readonly number[] } +/** + * `text` lowercased one UTF-16 unit at a time, so each position stays where it was: a character + * whose lowercase form is longer (`İ`) is kept as it is, and matched positions stay the path's. + */ +function foldCase(text: string): string { + const lower = text.toLowerCase() + if (lower.length === text.length) return lower + let out = '' + for (const ch of text) { + const l = ch.toLowerCase() + out += l.length === ch.length ? l : ch + } + return out +} + /** Whether every character of `needle` (lowercase) appears in `haystack` (lowercase), in order. */ function hasMatch(needle: string, haystack: string): boolean { let at = 0 @@ -52,9 +67,9 @@ function bonusAt(path: string, i: number): number { * as GitHub's finder ignores them). An empty query matches nothing. */ export function fuzzyMatch(query: string, path: string): FuzzyMatch | null { - const needle = query.replace(/\s+/g, '').toLowerCase() + const needle = foldCase(query.replace(/\s+/g, '')) if (needle === '') return null - const lower = path.toLowerCase() + const lower = foldCase(path) if (!hasMatch(needle, lower)) return null const n = needle.length const m = path.length @@ -122,12 +137,12 @@ export interface FuzzyHit extends FuzzyMatch { * path order. */ export function fuzzyRank(query: string, paths: readonly string[], limit: number): FuzzyHit[] { - const needle = query.replace(/\s+/g, '').toLowerCase() + const needle = foldCase(query.replace(/\s+/g, '')) if (needle === '') return [] const hits: FuzzyHit[] = [] for (const path of paths) { // The cheap in-order test first: most paths fail it, and scoring is the costly part. - if (!hasMatch(needle, path.toLowerCase())) continue + if (!hasMatch(needle, foldCase(path))) continue const match = fuzzyMatch(needle, path) if (match !== null) hits.push({ path, ...match }) } diff --git a/forge-web/lib/view/text-diff.ts b/forge-web/lib/view/text-diff.ts index 89a861276..2206f20f2 100644 --- a/forge-web/lib/view/text-diff.ts +++ b/forge-web/lib/view/text-diff.ts @@ -66,9 +66,11 @@ type Op = 0 | 1 | 2 // equal | delete | insert /** * git's edit script for `a` → `b` ({@link xdiffChanges}, then {@link compactChanges}): each hunk's - * deletions before its additions, as git prints them. Null past `limits`. + * deletions before its additions, as git prints them. Null when the search runs past + * `limits.maxWork`; `{ added, deleted }` alone when the script has more than `limits.maxEdits` + * edits (too many to show, still counted: QW-027). */ -function gitOps(a: readonly string[], b: readonly string[], limits: DiffLimits): Op[] | null { +function gitOps(a: readonly string[], b: readonly string[], limits: DiffLimits): Op[] | { readonly added: number; readonly deleted: number } | null { if (a.length === 0 || b.length === 0) return new Array(a.length + b.length).fill(a.length === 0 ? 2 : 1) const marks = xdiffChanges(a, b, limits) if (marks === null) return null @@ -79,68 +81,55 @@ function gitOps(a: readonly string[], b: readonly string[], limits: DiffLimits): // Compaction asserts what xdiff asserts; the uncompacted alignment is still a valid diff. c = marks } + let deleted = 0 + let added = 0 + for (let k = 0; k < a.length; k++) deleted += c.oldChanged[k] as number + for (let k = 0; k < b.length; k++) added += c.newChanged[k] as number + if (added + deleted > limits.maxEdits) return { added, deleted } const ops: Op[] = [] - let edits = 0 let i = 0 let j = 0 while (i < a.length || j < b.length) { if (i < a.length && c.oldChanged[i] === 1) { ops.push(1) i++ - edits++ } else if (j < b.length && c.newChanged[j] === 1) { ops.push(2) j++ - edits++ } else { ops.push(0) i++ j++ } } - return edits > limits.maxEdits ? null : ops + return ops } /** - * How far {@link countTextChanges} searches: a change too large to show is still counted (QW-027), - * so its search may run further than a shown diff's before it gives up. - */ -export const COUNT_DIFF_LIMITS: DiffLimits = { maxEdits: Infinity, maxWork: 100_000_000 } - -/** - * The lines a change adds and deletes, for one too large to show ({@link diffTextLines} said null): - * the same diff, with no bound on its edits, so a change set's totals stay whole. Null only when - * the search itself runs past {@link COUNT_DIFF_LIMITS}. + * Compute a line-level diff. `null` means the change is too large for an in-browser review; + * callers should surface that honestly rather than freezing the page. */ -export function countTextChanges( +export function diffTextLines( before: string, after: string, + limits: DiffLimits = DEFAULT_DIFF_LIMITS, options: { readonly ignoreWhitespace?: boolean } = {}, -): { readonly added: number; readonly deleted: number } | null { - const key = options.ignoreWhitespace ? (l: string): string => l.replace(/\s+/g, '') : null - const oldLines = splitLines(before) - const newLines = splitLines(after) - const ops = gitOps(key ? oldLines.map(key) : oldLines, key ? newLines.map(key) : newLines, COUNT_DIFF_LIMITS) - if (ops === null) return null - let added = 0 - let deleted = 0 - for (const op of ops) { - if (op === 1) deleted += 1 - else if (op === 2) added += 1 - } - return { added, deleted } +): TextDiffLine[] | null { + const got = diffTextLinesOrCount(before, after, limits, options) + return Array.isArray(got) ? got : null } /** - * Compute a line-level diff. `null` means the change is too large for an in-browser review; - * callers should surface that honestly rather than freezing the page. + * {@link diffTextLines}, or for a change with more edits than `limits.maxEdits` (too many to + * show) the lines it adds and deletes, from the same diff (QW-027: a change set's totals stay + * whole). Null only when the search runs past `limits.maxWork`. */ -export function diffTextLines( +export function diffTextLinesOrCount( before: string, after: string, limits: DiffLimits = DEFAULT_DIFF_LIMITS, options: { readonly ignoreWhitespace?: boolean } = {}, -): TextDiffLine[] | null { +): TextDiffLine[] | { readonly added: number; readonly deleted: number } | null { const oldLines = splitLines(before) const newLines = splitLines(after) // Ignoring whitespace (`git diff -w`) compares lines with all whitespace removed; the lines @@ -152,7 +141,7 @@ export function diffTextLines( // xdiff sees the whole files: it trims their common ends itself, after counting every record // (which lines its cleanup drops depends on how often they occur in the whole file). const ops = gitOps(oldKeys, newKeys, limits) - if (ops === null) return null + if (ops === null || !Array.isArray(ops)) return ops const lines: TextDiffLine[] = [] let o = 0 diff --git a/forge-web/lib/view/zip.ts b/forge-web/lib/view/zip.ts index 4b3e62019..ef4c86f9e 100644 --- a/forge-web/lib/view/zip.ts +++ b/forge-web/lib/view/zip.ts @@ -116,18 +116,18 @@ export async function walkFiles( const [oid, prefix] = queue.shift() as [string, string] trees += 1 active += 1 - readTree(reader, oid).then( - (entries) => { + const fail = (e: unknown): void => { + failed = true + reject(e) + } + readTree(reader, oid) + .then((entries) => { active -= 1 if (failed) return take(entries, prefix) pump() - }, - (e: unknown) => { - failed = true - reject(e) - }, - ) + }) + .catch(fail) } if (active === 0 && !failed) resolve() } From e56eab40fc2e9700f2826d678b4616f76da7c401 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 11:26:45 -0500 Subject: [PATCH 3/7] fix(web): check key-range ref reads for lost rows before keeping them The repo chrome read a long ref timeline as keyset ranges but only fell back to the $createdAt read when a range was refused, so rows a node silently left out were kept for the whole session (later reads fetch only newer rows). Group both ref-update types per ref, run the prevOid completeness check readAllRefUpdates runs, and re-read the ranged types serially when it fails. A range read that throws now falls back too instead of failing the timeline. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/lib/repo/chrome.test.ts | 80 +++++++++++++++++++++++++++++-- forge-web/lib/repo/chrome.ts | 53 +++++++++++++++----- forge-web/lib/repo/refs.ts | 14 +++++- 3 files changed, 130 insertions(+), 17 deletions(-) diff --git a/forge-web/lib/repo/chrome.test.ts b/forge-web/lib/repo/chrome.test.ts index eb7c7f2a5..6126cdd00 100644 --- a/forge-web/lib/repo/chrome.test.ts +++ b/forge-web/lib/repo/chrome.test.ts @@ -113,16 +113,28 @@ interface Fake { } /** Plain queries and composites over `store`; `calls` records one entry per request. */ -function fakeSdk(store: Store, { keyset = true }: { readonly keyset?: boolean } = {}): Fake { +function fakeSdk( + store: Store, + { + keyset = true, + rangeAnswer, + }: { + readonly keyset?: boolean + /** Rewrites (or throws on) the answer to a key-ordered (`refNameHash`) read. */ + readonly rangeAnswer?: (q: DocumentQuery, rows: Doc[]) => Doc[] + } = {}, +): Fake { const calls: string[] = [] const fake: { hold: Promise | null } = { hold: null } const rowsOf = (contract: string, type: string): Doc[] => [...(store[contract]?.[type] ?? [])] const plain = (q: DocumentQuery): Doc[] => { + const keyed = q.orderBy?.[0]?.[0] === 'refNameHash' // `keyset: false`: a node that ignores a `refNameHash` order (answers in `$createdAt` order). - const order = keyset && q.orderBy?.[0]?.[0] === 'refNameHash' ? byHash : byTime + const order = keyset && keyed ? byHash : byTime let rows = filter(rowsOf(q.dataContractId, q.documentTypeName), q.where).sort(order) if (q.startAfter !== undefined) rows = rows.slice(rows.findIndex((d) => d['$id'] === q.startAfter) + 1) - return rows.slice(0, Math.min(q.limit ?? 100, 100)) + rows = rows.slice(0, Math.min(q.limit ?? 100, 100)) + return keyed && rangeAnswer !== undefined ? rangeAnswer(q, rows) : rows } const wrap = (rows: Doc[]) => new Map(rows.map((d) => [String(d['$id']), d])) const sdk = { @@ -223,6 +235,68 @@ describe('repo chrome: one composite for the whole chrome', () => { expect(calls.slice(-2)).toEqual(['query:refUpdate', 'query:refUpdate']) }) + /** + * `tags` tags each created then moved (the move's `prevOid` is the creation's `newOid`), so a + * lost creation is a missing parent; plus `main` (one plain update, {@link TIP}). `protectedMoves`: + * each move is a `protectedRefUpdate` whose parent is the plain creation. + */ + function movedTags(tags: number, protectedMoves = false): { store: Store; creations: Doc[] } { + const store = fixture(0) + const creations: Doc[] = [] + for (let i = 0; i < tags; i++) { + const refName = `refs/tags/m${i}` + const refNameHash = bytesToBase64(sha256(new TextEncoder().encode(refName))) + const first = hexToBase64(i.toString(16).padStart(40, '0')) + const created = doc({ $ownerId: OWNER, repoId: REPO, refName, refNameHash, newOid: first }) + creations.push(created) + store.CORE!.refUpdate!.push(created) + const move = doc({ $ownerId: OWNER, repoId: REPO, refName, refNameHash, prevOid: first, newOid: hexToBase64('ee'.repeat(20)), force: true }) + store.CORE![protectedMoves ? 'protectedRefUpdate' : 'refUpdate']!.push(move) + } + return { store, creations } + } + + it('key ranges that lost a row (in order, so not refused) fail the parent check: read the plain way', async () => { + const { store, creations } = movedTags(120) + // A creation the composite's first page does not hold, so only the ranges could have read it. + const lost = creations[100]!['$id'] + const { sdk, calls } = fakeSdk(store, { rangeAnswer: (_q, rows) => rows.filter((d) => d['$id'] !== lost) }) + const t = await (await readRepoChrome(sdk, FORGE, OWNER, 'demo', 'devnet'))!.timelines! + expect(t.refUpdate).toHaveLength(241) + expect(t.refUpdate.map((d) => d['$id'])).toContain(lost) + // The ranges, then the `$createdAt` continuation of the composite's page: ⌈241/100⌉ − 1 = 2. + expect(calls).toEqual(['composite:repo', ...Array(CHROME_KEYSET_SPLITS + 2).fill('query:refUpdate')]) + }) + + it('a key range that fails outright: read the plain way, not fail the timeline', async () => { + let first = true + const { sdk, calls } = fakeSdk(fixture(250), { + rangeAnswer: (_q, rows) => { + if (first) { + first = false + throw new Error('node down') + } + return rows + }, + }) + const t = await (await readRepoChrome(sdk, FORGE, OWNER, 'demo', 'devnet'))!.timelines! + expect(t.refUpdate).toHaveLength(251) + expect(new Set(t.refUpdate.map((d) => d['$id'])).size).toBe(251) + expect(calls).toEqual(['composite:repo', ...Array(CHROME_KEYSET_SPLITS + 2).fill('query:refUpdate')]) + }) + + it('protected updates whose parents are plain updates are whole: the check groups both types, so no re-read', async () => { + // Both types past a page, both read as ranges; checked per type, every move would lack its parent. + const { store } = movedTags(120, true) + const { sdk, calls } = fakeSdk(store) + const t = await (await readRepoChrome(sdk, FORGE, OWNER, 'demo', 'devnet'))!.timelines! + expect(t.refUpdate).toHaveLength(121) + expect(t.protectedRefUpdate).toHaveLength(120) + expect([...calls].sort()).toEqual( + ['composite:repo', ...Array(CHROME_KEYSET_SPLITS).fill('query:protectedRefUpdate'), ...Array(CHROME_KEYSET_SPLITS).fill('query:refUpdate')].sort(), + ) + }) + it('a later read asks only for what is new, and keeps what it held', async () => { const store = fixture(250) const { sdk, calls } = fakeSdk(store) diff --git a/forge-web/lib/repo/chrome.ts b/forge-web/lib/repo/chrome.ts index 3e7a26f31..8df36f50f 100644 --- a/forge-web/lib/repo/chrome.ts +++ b/forge-web/lib/repo/chrome.ts @@ -36,7 +36,7 @@ import { cursorPadded, queryAllDocuments, shareInFlight, type PlainDocument, typ import { DOC, type RepoRef } from './contract' import { membersGeneration, membershipsFromDocs, seedMemberships } from './members' import { noteForkOf } from './fork-parent' -import { readRefRowsInRanges } from './refs' +import { readRefRowsInRanges, refRowsMissParent } from './refs' import { repoRefOf, toRepoDoc, type RepoDoc } from './resolveRepo' import { seedTargetCounts } from './social' import { repoSource } from './source' @@ -177,21 +177,48 @@ async function completeTimelines( known: RepoTimelines | undefined, pages: readonly (readonly PlainDocument[])[], ): Promise { + const pageOf = (type: TimelineType): readonly PlainDocument[] => pages[TIMELINE_TYPES.indexOf(type)] ?? [] + // A short page is the whole delta; a full one is continued `$createdAt` page after page. + const serial = async (type: TimelineType): Promise => { + const page = pageOf(type) + if (page.length < PAGE) return [...page] + return queryAllDocuments(sdk, repoSource(repo).repoQuery(type, { ...sinceWhere(known, type), orderBy: [['$createdAt', 'asc']] }), { firstPage: page }) + } + // A first read of a repo's ref updates past a page (dashpay/dash: ~700) reads the rest as key + // ranges side by side, not page after page (QW-087). Null: read serially instead — every delta + // read, and a node that does not honor the ranges or fails one outright. + const ranged = async (type: TimelineType): Promise => { + if (known !== undefined || !REF_TYPES.has(type) || pageOf(type).length < PAGE) return null + const rows = await readRefRowsInRanges(sdk, repo, type).catch(() => null) + return rows === null ? null : [...pageOf(type), ...rows] + } const read = await Promise.all( - TIMELINE_TYPES.map(async (type, i) => { - const page = pages[i] ?? [] - const serial = (): Promise => - queryAllDocuments(sdk, repoSource(repo).repoQuery(type, { ...sinceWhere(known, type), orderBy: [['$createdAt', 'asc']] }), { firstPage: page }) - // A first read of a repo's ref updates past a page (dashpay/dash: ~700) reads the rest as - // key ranges side by side, not page after page (QW-087); a node that does not honor them, - // and every delta read, continues the `$createdAt` page instead. - const rest = async (): Promise => - (known === undefined && REF_TYPES.has(type) ? await readRefRowsInRanges(sdk, repo, type) : null) ?? serial() - const rows = page.length < PAGE ? page : await rest() - return [type, mergeRows(known?.[type] ?? [], rows)] as const + TIMELINE_TYPES.map(async (type) => { + const fromRanges = await ranged(type) + return { type, fromRanges: fromRanges !== null, rows: fromRanges ?? (await serial(type)) } }), ) - return Object.fromEntries(read) as unknown as RepoTimelines + const rows = Object.fromEntries(read.map((r) => [r.type, r.rows])) as Record + // Ranges that answered in order can still have lost rows, and the loss would stick: later reads + // ask only for newer rows. So the ref rows get the `prevOid` check the full ref read runs (across + // both types), and a failure re-reads the ranged types the `$createdAt` way (the rest already were). + if (read.some((r) => r.fromRanges) && lostParent(repo, rows)) { + await Promise.all( + read.filter((r) => r.fromRanges).map(async (r) => { + rows[r.type] = await serial(r.type) + }), + ) + } + return Object.fromEntries(TIMELINE_TYPES.map((type) => [type, mergeRows(known?.[type] ?? [], rows[type])])) as unknown as RepoTimelines +} + +/** {@link refRowsMissParent} over the ref timelines; a row it cannot attribute counts as a loss. */ +function lostParent(repo: RepoRef, rows: Readonly>): boolean { + try { + return refRowsMissParent(repo, rows) + } catch { + return true + } } /** diff --git a/forge-web/lib/repo/refs.ts b/forge-web/lib/repo/refs.ts index 7dd0184f5..023aa66d1 100644 --- a/forge-web/lib/repo/refs.ts +++ b/forge-web/lib/repo/refs.ts @@ -309,7 +309,7 @@ export function hasMissingParent(updates: readonly RefUpdate[]): boolean { interface TypeScan { readonly type: string readonly isProtected: boolean - readonly rows: PlainDocument[] + readonly rows: readonly PlainDocument[] } /** Group rows per ref: plain before protected, each in read order (the fold re-sorts). */ @@ -330,6 +330,18 @@ function groupByRef(repo: RepoRef, scans: readonly TypeScan[]): Map>): boolean { + const scans = REF_UPDATE_TYPES.map(([type, isProtected]): TypeScan => ({ type, isProtected, rows: rowsByType[type] ?? [] })) + return [...groupByRef(repo, scans).values()].some(hasMissingParent) +} + /** `sha256(refName)` hex: the key a private ref is grouped under once its name is decrypted. */ export function publicRefKey(refName: string): string { return bytesToHex(sha256(new TextEncoder().encode(refName))) From c2a9a185b026a26ea8f86ea3e8c77cc43619aba5 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 11:26:58 -0500 Subject: [PATCH 4/7] fix(web): gather chunk reads off the timer queue and cap chunk queries in flight queueChunkSeqs waited on setTimeout(0), which background tabs clamp to ~1 s, stalling every serial chunk round; it now yields one macrotask through a MessageChannel post (setTimeout only where there is none). Chunk query sends also share one session-wide limit of 16 in flight, so a diff counting many files over several packs plus rename detection cannot burst ~50 proof-verified queries at once. The per-origin slot helper now hands a released slot straight to the next waiter. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/lib/view/browse-source.test.ts | 26 ++++++++++++ forge-web/lib/view/browse-source.ts | 54 +++++++++++++++++++----- forge-web/lib/view/chunk-gather.test.ts | 17 +++++++- 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/forge-web/lib/view/browse-source.test.ts b/forge-web/lib/view/browse-source.test.ts index ef1c03cd8..73dcc615f 100644 --- a/forge-web/lib/view/browse-source.test.ts +++ b/forge-web/lib/view/browse-source.test.ts @@ -22,6 +22,7 @@ import { serializeLocator, type IndexedObject } from '../browse/indexer' import { artifactRangeFetch, buildPackSource, + CHUNK_QUERIES_IN_FLIGHT, clearChunkCache, loadArtifactBytesProgress, loadBrowseContext, @@ -188,6 +189,31 @@ describe('chunk LRU', () => { fail = false expect(Array.from(await fetchRange(0, 10))).toEqual(Array.from(full.subarray(0, 10))) }) + + it('keeps at most CHUNK_QUERIES_IN_FLIGHT chunk queries out across every pack read at once', async () => { + const inner = mockSdk(() => full) as unknown as { documents: { query: (q: unknown) => Promise> } } + let inFlight = 0 + let peak = 0 + let sent = 0 + const sdk = { + documents: { + query: async (q: unknown): Promise> => { + sent++ + inFlight++ + peak = Math.max(peak, inFlight) + await new Promise((r) => setTimeout(r, 2)) + inFlight-- + return inner.documents.query(q) + }, + }, + } as unknown as EvoSDK + // 48 packs (a diff's count pool over several packs), each read at once: 48 separate queries. + const packs = Array.from({ length: 48 }, (_, i) => gitPack((0x40 + i).toString(16), 0, `p${i}`)) + const got = await Promise.all(packs.map((p) => artifactRangeFetch(sdk, REPO, p)(0, 10))) + for (const bytes of got) expect(Array.from(bytes)).toEqual(Array.from(full.subarray(0, 10))) + expect(sent).toBe(48) + expect(peak).toBe(CHUNK_QUERIES_IN_FLIGHT) + }) }) // L-15: a 10 MB locator is ~7 windows of 100 chunks; read one after another they were ~2 s of diff --git a/forge-web/lib/view/browse-source.ts b/forge-web/lib/view/browse-source.ts index 0553ad7d1..95cd192cd 100644 --- a/forge-web/lib/view/browse-source.ts +++ b/forge-web/lib/view/browse-source.ts @@ -65,6 +65,33 @@ import { onPrivateSessionEnded } from '../repo/private-session' /** Chunk queries in flight at once when one range spans more than a single query. */ const CHUNK_QUERY_POOL = 6 +/** + * Chunk queries in flight at once across every read of the session. Each range pools its own + * batches ({@link CHUNK_QUERY_POOL}), but a diff counting two dozen files over several packs, or + * rename detection reading candidates, runs many ranges side by side; without one global bound a + * turn could send ~50 proof-verified queries at once. + */ +export const CHUNK_QUERIES_IN_FLIGHT = 16 +const chunkQuerySlots = { active: 0, waiting: [] as (() => void)[] } + +/** + * Run `run` on the next macrotask. A `MessageChannel` post, not `setTimeout(0)`: browsers clamp + * timers in background tabs to ~1 s, which would stall every serial round of chunk reads while + * the tab is hidden. `setTimeout` only where there is no `MessageChannel`. + */ +function nextMacrotask(run: () => void): void { + if (typeof MessageChannel === 'undefined') { + setTimeout(run, 0) + return + } + const channel = new MessageChannel() + channel.port1.onmessage = () => { + channel.port1.close() + run() + } + channel.port2.postMessage(null) +} + /** Concatenate the `d0..d2` byteArray fields (base64) of one chunk row, in order. */ function chunkPayload(doc: Record): Uint8Array { const parts: Uint8Array[] = [] @@ -175,9 +202,8 @@ async function queryChunkBatch( const i = next++ const batch = batches[i] if (batch === undefined) return - const { documents } = await queryDocumentsWithProof( - sdk, - source.chunkQuery(manifest.packHash, manifest.uploader, batch), + const { documents } = await withSlotOf(chunkQuerySlots, CHUNK_QUERIES_IN_FLIGHT, () => + queryDocumentsWithProof(sdk, source.chunkQuery(manifest.packHash, manifest.uploader, batch)), ) for (const doc of documents) { const raw = doc['seq'] @@ -228,11 +254,11 @@ export function queueChunkSeqs( if (pending === undefined) { const gathered = new Set() const result = new Promise>((resolve, reject) => { - setTimeout(() => { + nextMacrotask(() => { if (pendingChunkQueries.get(key)?.seqs === gathered) pendingChunkQueries.delete(key) chunkQueriesSent += Math.ceil(gathered.size / CHUNK_QUERY_MAX) query([...gathered].sort((a, b) => a - b)).then(resolve, reject) - }, 0) + }) }) pending = { seqs: gathered, result } pendingChunkQueries.set(key, pending) @@ -425,14 +451,22 @@ async function withSlot(key: string, run: () => Promise): Promise { slot = { active: 0, waiting: [] } originSlots.set(key, slot) } - const s = slot - if (s.active >= PER_ORIGIN_CONCURRENCY) await new Promise((resolve) => s.waiting.push(resolve)) - s.active += 1 + return withSlotOf(slot, PER_ORIGIN_CONCURRENCY, run) +} + +/** + * At most `limit` `run`s of `slots` in flight. A finishing run hands its slot straight to the + * next waiter, so a newcomer arriving in between cannot take it too and overshoot the limit. + */ +async function withSlotOf(slots: { active: number; readonly waiting: (() => void)[] }, limit: number, run: () => Promise): Promise { + if (slots.active >= limit) await new Promise((resolve) => slots.waiting.push(resolve)) + else slots.active += 1 try { return await run() } finally { - s.active -= 1 - s.waiting.shift()?.() + const next = slots.waiting.shift() + if (next !== undefined) next() + else slots.active -= 1 } } diff --git a/forge-web/lib/view/chunk-gather.test.ts b/forge-web/lib/view/chunk-gather.test.ts index a2d6ba6a9..8a8685cb8 100644 --- a/forge-web/lib/view/chunk-gather.test.ts +++ b/forge-web/lib/view/chunk-gather.test.ts @@ -3,7 +3,7 @@ * blob reads counting a diff's lines, or a tree walk's parallel reads, each need a chunk or two. */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { chunkQueryCount, clearChunkCache, queueChunkSeqs } from './browse-source' @@ -34,6 +34,21 @@ describe('queueChunkSeqs', () => { expect(sent.map((s) => s.length).sort((x, y) => x - y)).toEqual([1, 20, 90]) }) + it('gathers on a macrotask that is not a timer (background tabs clamp timers to ~1 s)', async () => { + clearChunkCache() + // Every timer frozen: a gather waiting on `setTimeout` would never send. + vi.useFakeTimers({ toFake: ['setTimeout', 'setInterval', 'setImmediate'] }) + try { + const sent: number[][] = [] + const query = (seqs: readonly number[]) => (sent.push([...seqs]), answer(seqs)) + const got = await Promise.all([queueChunkSeqs('copy', [1], query), queueChunkSeqs('copy', [4], query)]) + expect(sent).toEqual([[1, 4]]) + for (const g of got) expect([...g.keys()]).toEqual([1, 4]) + } finally { + vi.useRealTimers() + } + }) + it('a failed query fails its askers, and the next turn asks again', async () => { clearChunkCache() let fail = true From 6bfa946043e9b3e17d2150ad244d6691ce5cb94e Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 11:26:58 -0500 Subject: [PATCH 5/7] fix(web): keep a complete file index over a truncated walk in Go to file On a repo past the walk's 5000-file cap the partial walk replaced the complete index and results shrank. The walk is now the list only when it finished or there is no index, the 'Searched the first N files' note shows only when the truncated walk is in use, and the highlighted row resets whenever the results change so Enter always opens one. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/components/repo/go-to-file.test.tsx | 43 +++++++++++++++++-- forge-web/components/repo/go-to-file.tsx | 13 ++++-- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/forge-web/components/repo/go-to-file.test.tsx b/forge-web/components/repo/go-to-file.test.tsx index 9f4cf1151..3b5e9a9f4 100644 --- a/forge-web/components/repo/go-to-file.test.tsx +++ b/forge-web/components/repo/go-to-file.test.tsx @@ -12,13 +12,15 @@ const push = vi.fn() vi.mock('next/link', () => ({ default: ({ href, children, ...rest }: { href: string; children: React.ReactNode }) => {children} })) vi.mock('next/navigation', () => ({ useRouter: () => ({ push, replace: () => undefined }) })) -/** The walk resolves when the test says; the index answers at once. */ -let finishWalk: (files: string[]) => void = () => undefined +/** The walk resolves when the test says; the index answers at once (null: the repo has none). */ +const INDEX = ['src/validation.cpp', 'src/net_processing.cpp', 'src/deleted_since.cpp'] +let index: string[] | null = INDEX +let finishWalk: (files: string[], truncated?: boolean) => void = () => undefined vi.mock('@/lib/view/repo-facts', () => ({ - indexedFilePaths: async () => ['src/validation.cpp', 'src/net_processing.cpp', 'src/deleted_since.cpp'], + indexedFilePaths: async () => index, repoFilesWalk: () => new Promise((resolve) => { - finishWalk = (files) => resolve({ files: files.map((path) => ({ path, oid: '', mode: 0o100644, size: 0 })), truncated: false }) + finishWalk = (files, truncated = false) => resolve({ files: files.map((path) => ({ path, oid: '', mode: 0o100644, size: 0 })), truncated }) }), })) @@ -29,6 +31,7 @@ let el: HTMLDivElement beforeEach(() => { ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true push.mockClear() + index = INDEX // jsdom has no layout, so no scrollIntoView. Element.prototype.scrollIntoView = () => undefined el = document.createElement('div') @@ -77,6 +80,38 @@ describe('GoToFile', () => { await vi.waitFor(() => expect(el.textContent).toContain('No matching file.')) }) + it('a walk cut off at its file cap does not replace a complete index', async () => { + await render() + await type('src') + await vi.waitFor(() => expect(results()).toHaveLength(3)) + // Past the walk's cap: the index names more of the tip than the partial walk. + await act(async () => finishWalk(['src/validation.cpp'], true)) + expect(results()).toHaveLength(3) + expect(el.textContent).not.toContain('Searched the first') + }) + + it('with no index, a cut-off walk is the list, and says so', async () => { + index = null + await render() + await type('src') + await act(async () => finishWalk(['src/validation.cpp', 'src/init.cpp'], true)) + await vi.waitFor(() => expect(results()).toHaveLength(2)) + expect(el.textContent).toContain('Searched the first 2 files') + }) + + it('a new list of results resets the highlight, so Enter still opens one', async () => { + await render() + await type('src') + await vi.waitFor(() => expect(results()).toHaveLength(3)) + await key('ArrowDown') + await key('ArrowDown') + // The walk's exact list replaces the index's under the same query: one result now. + await act(async () => finishWalk(['src/validation.cpp'])) + await vi.waitFor(() => expect(results()).toEqual(['src/validation.cpp'])) + await key('Enter') + expect(push).toHaveBeenCalledWith(expect.stringContaining(`path=${encodeURIComponent('src/validation.cpp')}`)) + }) + it('ranks fuzzily, moves with the arrow keys and opens the choice with Enter', async () => { await render() await type('netproc') diff --git a/forge-web/components/repo/go-to-file.tsx b/forge-web/components/repo/go-to-file.tsx index 3b833d946..9ccbde777 100644 --- a/forge-web/components/repo/go-to-file.tsx +++ b/forge-web/components/repo/go-to-file.tsx @@ -77,7 +77,11 @@ export function GoToFile({ // The index's list (no read when the column already loaded it), then the walk's exact one. const indexed = useAsync(() => indexedFilePaths(reader, tipOid), [key, tipOid], { enabled: started }) const walk = useAsync(async () => repoFilesWalk(key, tipOid, reader, await rootTree()), [key, tipOid], { enabled: started }) - const paths = useMemo(() => walk.data?.files.map((f) => f.path) ?? indexed.data ?? null, [walk.data, indexed.data]) + // A walk that stopped at its file cap names fewer paths than a complete index: the index then + // stays the list, and the walk's partial one answers only when there is no index. + const walkPaths = useMemo(() => walk.data?.files.map((f) => f.path) ?? null, [walk.data]) + const walkInUse = walkPaths !== null && (!walk.data?.truncated || indexed.data == null) + const paths = walkInUse ? walkPaths : (indexed.data ?? walkPaths) const q = query.trim() const hits = useMemo(() => (q === '' || paths === null ? [] : fuzzyRank(q, paths, RESULTS)), [q, paths]) const hrefOf = (path: string): string => repoHref('/repo/blob', addr, { path, ...(refParam ? { ref: refParam } : {}) }) @@ -97,8 +101,9 @@ export function GoToFile({ return () => document.removeEventListener('keydown', onKey) }, []) - // A new query starts from its best match. - useEffect(() => setActive(0), [q]) + // A new list of results (a new query, or the walk's list replacing the index's) starts from its + // best match, so the highlighted row always exists and Enter always opens something. + useEffect(() => setActive(0), [hits]) const onKeyDown = (e: ReactKeyboardEvent): void => { if (e.key === 'ArrowDown' || e.key === 'ArrowUp') { @@ -174,7 +179,7 @@ export function GoToFile({ ))} - {walk.data?.truncated ? ( + {walkInUse && walk.data?.truncated ? (
  • Searched the first {plural(walk.data.files.length, 'file')}.
  • ) : null} From a73e5dbcac13e6cb75fdfe2cce04f792fc95c354 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 11:26:58 -0500 Subject: [PATCH 6/7] fix(web): take a parallel tree walk's reads in the order they started walkFiles queued child trees and counted files in read-completion order, so a truncated walk (the language bar, Go to file) kept a timing-dependent set. Reads still run up to 16 at once, but their entries are taken from the head of the started list, as a serial walk would, so the set is the same on every visit. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/lib/view/zip.test.ts | 33 ++++++++++++++++++++++++++++++++- forge-web/lib/view/zip.ts | 21 +++++++++++++++------ 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/forge-web/lib/view/zip.test.ts b/forge-web/lib/view/zip.test.ts index d3311d732..f8e2fa646 100644 --- a/forge-web/lib/view/zip.test.ts +++ b/forge-web/lib/view/zip.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from 'vitest' import { MODE_GITLINK } from '../browse' import { Store } from './diff-fixtures' -import { isSafeName, listFiles, readZipFiles, ZIP_MAX_BYTES, zipFileName, ZipTooLargeError } from './zip' +import { isSafeName, listFiles, readZipFiles, walkFiles, ZIP_MAX_BYTES, zipFileName, ZipTooLargeError } from './zip' describe('zip of a ref', () => { it('lists every file under the commit and skips submodules', async () => { @@ -41,6 +41,37 @@ describe('zip of a ref', () => { expect(new Set(Object.values(entries).map((b) => b.buffer)).size).toBe(2) }) + it('a truncated parallel walk keeps the same files whatever order its reads land in', async () => { + const s = new Store() + // 12 top-level directories, each with 3 files and a subdirectory of 2: 60 files in 25 trees. + const layout: Record = {} + for (let d = 0; d < 12; d++) { + for (let f = 0; f < 3; f++) layout[`d${d}/f${f}.txt`] = `${d}.${f}` + for (let f = 0; f < 2; f++) layout[`d${d}/sub/g${f}.txt`] = `${d}.sub.${f}` + } + const root = s.files(layout) + const inner = s.reader() + /** A reader whose reads land after `delay(n)` ms, `n` counting reads as they are asked. */ + const timed = (delay: (n: number) => number) => { + let n = 0 + return { + readObject: async (oid: string) => { + const wait = delay(n++) + await new Promise((r) => setTimeout(r, wait)) + return inner.readObject(oid) + }, + } + } + const walk = async (delay: (n: number) => number, pool?: number) => + (await walkFiles(timed(delay), root, { maxFiles: 25, ...(pool === undefined ? {} : { pool }) })).files.map((f) => f.path) + const serial = await walk(() => 0, 1) + expect(serial).toHaveLength(25) + // Later-asked reads landing first, first-asked landing first, and a scramble: one set. + expect(await walk((n) => 30 - n)).toEqual(serial) + expect(await walk((n) => n)).toEqual(serial) + expect(await walk((n) => (n * 7) % 11)).toEqual(serial) + }) + it('skips tree entries that would escape the zip root', () => { for (const bad of ['..', '.', '', 'a/b', 'a\\b', 'a\0b']) expect(isSafeName(bad)).toBe(false) expect(isSafeName('..a')).toBe(true) diff --git a/forge-web/lib/view/zip.ts b/forge-web/lib/view/zip.ts index ef4c86f9e..9fe21a23d 100644 --- a/forge-web/lib/view/zip.ts +++ b/forge-web/lib/view/zip.ts @@ -108,6 +108,10 @@ export async function walkFiles( else dropped = true } } + // Reads land in any order, but are taken in the order they were started: children are queued, + // and files counted against `maxFiles`, as a one-at-a-time walk would, so a truncated walk keeps + // the same set on every visit (the language bar must not vary with network timing). + const started: { readonly prefix: string; entries?: readonly TreeEntry[] }[] = [] await new Promise((resolve, reject) => { let active = 0 let failed = false @@ -116,18 +120,23 @@ export async function walkFiles( const [oid, prefix] = queue.shift() as [string, string] trees += 1 active += 1 - const fail = (e: unknown): void => { - failed = true - reject(e) - } + const read: (typeof started)[number] = { prefix } + started.push(read) readTree(reader, oid) .then((entries) => { active -= 1 if (failed) return - take(entries, prefix) + read.entries = entries + for (let head = started[0]; head?.entries !== undefined; head = started[0]) { + started.shift() + take(head.entries, head.prefix) + } pump() }) - .catch(fail) + .catch((e: unknown) => { + failed = true + reject(e) + }) } if (active === 0 && !failed) resolve() } From 6926b3916f9765367d506901e0646ae25143d7fd Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 11:26:58 -0500 Subject: [PATCH 7/7] fix(web): give rename detection a 64 MiB byte budget With the blob budget at 400 and 1 MiB per blob, rename detection could download ~400 MiB. Bytes are now counted as reads land and no further candidate is read once 64 MiB is spent: the basename phase keeps the pairs it read (the two sides are read interleaved, pair by pair) and skips the matrix; a matrix whose reads ran out is skipped rather than pair with a worse source than git. Typical source trees stay far below the cap. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/lib/view/renames.test.ts | 39 ++++++++++++++++++++++++++ forge-web/lib/view/renames.ts | 44 +++++++++++++++++++++++++----- 2 files changed, 76 insertions(+), 7 deletions(-) diff --git a/forge-web/lib/view/renames.test.ts b/forge-web/lib/view/renames.test.ts index e3cf49fa3..0d0441fde 100644 --- a/forge-web/lib/view/renames.test.ts +++ b/forge-web/lib/view/renames.test.ts @@ -222,6 +222,45 @@ describe('detectRenames', () => { expect(ours(got.changes).filter((l) => l.startsWith('R'))).toEqual(['R075 a/0.txt b/0.txt', 'R079 a/1.txt b/1.txt', 'R100 same.txt dir/same.txt']) }) + describe('within its byte budget (counted as reads land)', () => { + const files = (prefix: string, n: number, edit = ''): Record => + Object.fromEntries(Array.from({ length: n }, (_, i) => [`${prefix}${i}.txt`, `file ${i}\n${'body\n'.repeat(i + 3)}${edit}`])) + + it('same-named pairs: stops reading once spent, still pairs what it read, and skips the matrix', async () => { + const s = new Store() + // 20 same-named pairs (40 blobs); the pool starts 16 reads before any lands. + const a = s.files({ ...files('a/', 20), 'x/old.txt': 'moved\n'.repeat(20) }) + const b = s.files({ ...files('b/', 20, 'edited\n'), 'y/new.txt': `${'moved\n'.repeat(20)}more\n` }) + const r = s.reader() + const { changes } = await diffTrees({ base: r, head: r }, a, b) + const before = s.reads.length + const got = await detectRenames({ base: r, head: r }, changes, { readBytes: 1 }) + expect(s.reads.length - before).toBe(16) + expect(got.limited).toMatch(/kept their names would download more than/) + // The 16 reads were the first 8 pairs, pair by pair: each scored, each a pair git makes too. + // x/old.txt → y/new.txt needs the matrix, which is skipped. + const renamed = got.changes.filter((c) => c.status === 'renamed').map((c) => `${c.oldPath} ${c.path}`) + expect(renamed).toHaveLength(8) + expect(renamed).not.toContain('x/old.txt y/new.txt') + for (const pair of renamed) expect(pair).toMatch(/^a\/(\d+)\.txt b\/\1\.txt$/) + }) + + it('the matrix: a part of its blobs is not enough, so it is skipped rather than pair worse', async () => { + const s = new Store() + const a = s.files(files('a/', 10)) + const b = s.files(files('b/n', 10, 'edited\n')) + const r = s.reader() + const { changes } = await diffTrees({ base: r, head: r }, a, b) + const got = await detectRenames({ base: r, head: r }, changes, { readBytes: 1 }) + expect(got.changes.some((c) => c.status === 'renamed')).toBe(false) + expect(got.limited).toMatch(/comparing 10 deleted files with 10 added files would download more than/) + // With the default budget the same change pairs. + const full = await detectRenames({ base: r, head: r }, changes) + expect(full.limited).toBeNull() + expect(full.changes.filter((c) => c.status === 'renamed')).toHaveLength(10) + }) + }) + it('a blob that cannot be read is not scored, and the result says renames may be missing', async () => { const s = new Store() const text = 'shared\n'.repeat(40) diff --git a/forge-web/lib/view/renames.ts b/forge-web/lib/view/renames.ts index 3c9ccbb0c..064b29762 100644 --- a/forge-web/lib/view/renames.ts +++ b/forge-web/lib/view/renames.ts @@ -15,8 +15,9 @@ * * Two limits are this browser's, not git's, and the result says when either applied * ({@link RenameResult.limited}): reading blobs costs network requests here, so the inexact - * phases read at most {@link RENAME_READ_BUDGET} blobs (a phase that needs more is skipped), and a - * blob over {@link RENAME_MAX_BLOB_BYTES} is not scored. `.gitattributes` `diff` drivers are not + * phases read at most {@link RENAME_READ_BUDGET} blobs (a phase that needs more is skipped) and + * {@link RENAME_READ_BYTES} bytes of them (reading stops once spent), and a blob over + * {@link RENAME_MAX_BLOB_BYTES} is not scored. `.gitattributes` `diff` drivers are not * read: binary means a NUL in the first 8000 bytes, git's own default test. */ @@ -42,6 +43,12 @@ const HASHBASE = 107927 /** Blobs the inexact phases may read (each a ranged read over the network). */ const RENAME_READ_BUDGET = 400 +/** + * Blob bytes the inexact phases may download in all. Blob sizes are not known before a read, so + * this is counted as reads land: once it is spent no further candidate is read. 400 typical source + * files (tens of KiB each) stay far below it; 400 blobs near {@link RENAME_MAX_BLOB_BYTES} do not. + */ +const RENAME_READ_BYTES = 64 * 1024 * 1024 /** A blob larger than this is not scored for similarity. */ const RENAME_MAX_BLOB_BYTES = 1024 * 1024 const NOT_LOOKED = 'Renames with edits were not all looked for:' @@ -55,6 +62,8 @@ const isRegular = (mode: number): boolean => (mode & S_IFMT) === S_IFREG export interface RenameOptions { /** Blobs the inexact phases may read (default {@link RENAME_READ_BUDGET}). */ readonly readBudget?: number + /** Blob bytes the inexact phases may download (default {@link RENAME_READ_BYTES}). */ + readonly readBytes?: number } export interface RenameResult { @@ -181,7 +190,7 @@ function scoreCompare(a: Candidate | null, b: Candidate | null): number { * blob that cannot be read is not scored, and the result says renames may be missing. */ export async function detectRenames(sides: DiffSides, changes: readonly FileChange[], options: RenameOptions = {}): Promise { - const { readBudget = RENAME_READ_BUDGET } = options + const { readBudget = RENAME_READ_BUDGET, readBytes = RENAME_READ_BYTES } = options const sorted = [...changes].sort(byPath) const srcs: Spec[] = [] const dsts: Spec[] = [] @@ -236,12 +245,27 @@ export async function detectRenames(sides: DiffSides, changes: readonly FileChan const seen = new Set() return specs.filter((s) => isRegular(s.mode) && !blobs.has(s.oid) && !seen.has(s.oid) && (seen.add(s.oid), true)) } - const readAll = async (base: readonly Spec[], head: readonly Spec[]): Promise => { - const todo = [...unread(base).map((spec) => ({ spec, reader: sides.base })), ...unread(head).map((spec) => ({ spec, reader: sides.head }))] + let bytesRead = 0 + /** + * Read the not yet read blobs of `base` and `head`, the two sides interleaved (so the basename + * phase's pairs, passed side by side, are read pair by pair and a spent budget still leaves whole + * pairs). False when the byte budget ran out first: the blobs left unread are not scored (and + * not in `blobs`). Which blobs made it then depends on when reads landed; the result says so. + */ + const readAll = async (base: readonly Spec[], head: readonly Spec[]): Promise => { + const b = unread(base).map((spec) => ({ spec, reader: sides.base })) + const h = unread(head).map((spec) => ({ spec, reader: sides.head })) + const todo = Array.from({ length: Math.max(b.length, h.length) }, (_, i) => [b[i], h[i]]).flat().filter((t) => t !== undefined) + let complete = true await mapPooled(todo, READ_POOL, async ({ spec, reader }) => { if (blobs.has(spec.oid)) return + if (bytesRead >= readBytes) { + complete = false + return + } try { const o = await reader.readObject(spec.oid, { maxBytes: RENAME_MAX_BLOB_BYTES }) + bytesRead += o.bytes.length blobs.set(spec.oid, o.type === 'blob' ? { size: o.bytes.length, spans: spanHash(o.bytes) } : null) } catch (e) { if (e instanceof ObjectTooLargeError) tooLarge = true @@ -249,7 +273,9 @@ export async function detectRenames(sides: DiffSides, changes: readonly FileChan blobs.set(spec.oid, null) } }) + return complete } + const tooManyBytes = (what: string): string => `${NOT_LOOKED} ${what} would download more than ${formatBytes(readBytes)} in the browser.` const score = (s: Spec, d: Spec, minScore: number): number => { if (!isRegular(s.mode) || !isRegular(d.mode)) return 0 const a = blobs.get(s.oid) @@ -287,10 +313,12 @@ export async function detectRenames(sides: DiffSides, changes: readonly FileChan skipped = `${NOT_LOOKED} comparing the ${plural(byName.length, 'file')} that kept their names would read too many files in the browser, so ${n === 0 ? 'none' : `only ${n}`} of them ${n === 1 ? 'was' : 'were'} compared.` } if (reachable.length > 0) { - await readAll( + // Out of bytes part way: the pairs whose blobs were read still stand alone; the matrix is skipped. + const complete = await readAll( reachable.map(([x]) => x), reachable.map(([, y]) => y), ) + if (!complete) skipped ??= tooManyBytes(`comparing the ${plural(byName.length, 'file')} that kept their names`) for (const [src, dst] of reachable) { if (dst.used) continue // git's "already used in a rename"; cannot happen with unique names. const sc = score(src, dst, MIN_BASENAME_SCORE) @@ -307,8 +335,10 @@ export async function detectRenames(sides: DiffSides, changes: readonly FileChan skipped = `${NOT_LOOKED} ${left.length} deleted and ${rest.length} added files are over git's rename limit.` } else if (blobs.size + unread([...left, ...rest]).length > readBudget) { skipped = `${NOT_LOOKED} comparing ${plural(left.length, 'deleted file')} with ${plural(rest.length, 'added file')} would read too many files in the browser.` + } else if (!(await readAll(left, rest))) { + // A matrix over a part of the blobs could pair a file with a worse source than git does. + skipped = tooManyBytes(`comparing ${plural(left.length, 'deleted file')} with ${plural(rest.length, 'added file')}`) } else { - await readAll(left, rest) const matrix: (Candidate | null)[] = [] for (const d of rest) { const m = new Array(CANDIDATES).fill(null)