diff --git a/forge-web/components/repo/diff-view.tsx b/forge-web/components/repo/diff-view.tsx index 7866e0b48..0a2ab0c14 100644 --- a/forge-web/components/repo/diff-view.tsx +++ b/forge-web/components/repo/diff-view.tsx @@ -67,11 +67,17 @@ export const InlineCommentsContext = createContext(null) /** Files whose patches load per page. */ const FILE_PAGE = 25 /** A change set of at most this many files has its lines counted as it opens; a bigger one on request. */ -const AUTO_COUNT = 100 +const AUTO_COUNT = 300 /** Rows a file renders before a "show more lines" button. */ const LINE_PAGE = 400 -/** Patch reads in flight at once. */ -const PATCH_CONCURRENCY = 4 +/** + * Patch reads in flight at once: those of files on screen, and of a whole change set being counted + * (QW-027). Blob reads that run side by side share their chunk queries (`queueChunkSeqs`), so a + * wider pool costs fewer round trips, not more requests: dashpay/dash v22.0.0...v23.0.0's 1,530 + * files were 1,287 requests over 176 s four at a time, and are ~80 over ~20 s. + */ +const PATCH_CONCURRENCY = 12 +const COUNT_CONCURRENCY = 24 /** Finished patches are committed to state in batches at most this often (ms). */ const PATCH_FLUSH_MS = 100 @@ -163,7 +169,7 @@ export function DiffView({ const key = (path: string): string => `${ignoreWhitespace ? 'w' : 'x'}:${path}` const todo = (countAll ? changes : changes.slice(0, shown)).filter((c) => !requested.current.has(key(c.path))) for (const c of todo) requested.current.add(key(c.path)) - void mapPooled(todo, PATCH_CONCURRENCY, async (change) => { + void mapPooled(todo, countAll ? COUNT_CONCURRENCY : PATCH_CONCURRENCY, async (change) => { // loadFilePatch turns read failures into placeholders; anything else must still settle // the file, or it would sit on "Reading file…" with its key already requested. const patch = await loadFilePatch(sides, change, { ignoreWhitespace }).catch( diff --git a/forge-web/components/repo/go-to-file.test.tsx b/forge-web/components/repo/go-to-file.test.tsx new file mode 100644 index 000000000..3b5e9a9f4 --- /dev/null +++ b/forge-web/components/repo/go-to-file.test.tsx @@ -0,0 +1,140 @@ +// @vitest-environment jsdom +/** + * Go to file (QW-028): results from the history index at once, then the walk's exact list; fuzzy + * ranking; the arrow keys and Enter open a result; `t` focuses the box. + */ + +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const push = vi.fn() +vi.mock('next/link', () => ({ default: ({ href, children, ...rest }: { href: string; children: React.ReactNode }) => {children} })) +vi.mock('next/navigation', () => ({ useRouter: () => ({ push, replace: () => undefined }) })) + +/** The walk resolves when the test says; the index answers at once (null: the repo has none). */ +const INDEX = ['src/validation.cpp', 'src/net_processing.cpp', 'src/deleted_since.cpp'] +let index: string[] | null = INDEX +let finishWalk: (files: string[], truncated?: boolean) => void = () => undefined +vi.mock('@/lib/view/repo-facts', () => ({ + indexedFilePaths: async () => index, + repoFilesWalk: () => + new Promise((resolve) => { + finishWalk = (files, truncated = false) => resolve({ files: files.map((path) => ({ path, oid: '', mode: 0o100644, size: 0 })), truncated }) + }), +})) + +import { GoToFile } from './go-to-file' + +let root: Root +let el: HTMLDivElement +beforeEach(() => { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + push.mockClear() + index = INDEX + // jsdom has no layout, so no scrollIntoView. + Element.prototype.scrollIntoView = () => undefined + el = document.createElement('div') + document.body.append(el) + root = createRoot(el) +}) +afterEach(() => { + act(() => root.unmount()) + el.remove() +}) + +const input = (): HTMLInputElement => el.querySelector('[data-testid="go-to-file"]') as HTMLInputElement +const results = (): string[] => [...el.querySelectorAll('[data-testid="go-to-file-result"]')].map((a) => a.textContent ?? '') + +async function type(text: string): Promise { + await act(async () => { + const box = input() + box.focus() + // React listens for the native setter's input event. + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call(box, text) + box.dispatchEvent(new Event('input', { bubbles: true })) + }) +} + +async function key(k: string, target: EventTarget = input()): Promise { + await act(async () => { + target.dispatchEvent(new KeyboardEvent('keydown', { key: k, bubbles: true })) + }) +} + +async function render(): Promise { + await act(async () => { + root.render( + 'tree'} addr={{ owner: 'o', name: 'n' }} refParam="" />, + ) + }) +} + +describe('GoToFile', () => { + it('answers from the history index before the walk, then from the walk', async () => { + await render() + await type('deleted') + await vi.waitFor(() => expect(results()).toEqual(['src/deleted_since.cpp'])) + // The walk's list is exact: a file the index still named is gone. + await act(async () => finishWalk(['src/validation.cpp', 'src/net_processing.cpp'])) + await vi.waitFor(() => expect(el.textContent).toContain('No matching file.')) + }) + + it('a walk cut off at its file cap does not replace a complete index', async () => { + await render() + await type('src') + await vi.waitFor(() => expect(results()).toHaveLength(3)) + // Past the walk's cap: the index names more of the tip than the partial walk. + await act(async () => finishWalk(['src/validation.cpp'], true)) + expect(results()).toHaveLength(3) + expect(el.textContent).not.toContain('Searched the first') + }) + + it('with no index, a cut-off walk is the list, and says so', async () => { + index = null + await render() + await type('src') + await act(async () => finishWalk(['src/validation.cpp', 'src/init.cpp'], true)) + await vi.waitFor(() => expect(results()).toHaveLength(2)) + expect(el.textContent).toContain('Searched the first 2 files') + }) + + it('a new list of results resets the highlight, so Enter still opens one', async () => { + await render() + await type('src') + await vi.waitFor(() => expect(results()).toHaveLength(3)) + await key('ArrowDown') + await key('ArrowDown') + // The walk's exact list replaces the index's under the same query: one result now. + await act(async () => finishWalk(['src/validation.cpp'])) + await vi.waitFor(() => expect(results()).toEqual(['src/validation.cpp'])) + await key('Enter') + expect(push).toHaveBeenCalledWith(expect.stringContaining(`path=${encodeURIComponent('src/validation.cpp')}`)) + }) + + it('ranks fuzzily, moves with the arrow keys and opens the choice with Enter', async () => { + await render() + await type('netproc') + await vi.waitFor(() => expect(results()).toEqual(['src/net_processing.cpp'])) + await type('src') + await vi.waitFor(() => expect(results()).toHaveLength(3)) + const first = results()[0] + await key('ArrowDown') + expect(el.querySelector('[role=option][aria-selected=true]')?.textContent).toBe(results()[1]) + await key('ArrowUp') + await key('Enter') + expect(push).toHaveBeenCalledWith(expect.stringContaining(`path=${encodeURIComponent(first as string)}`)) + }) + + it('focuses on t, and Escape clears it', async () => { + await render() + // jsdom lays nothing out: give the box a client rect so it counts as visible. + input().getClientRects = () => [{}] as unknown as DOMRectList + await key('t', document.body) + expect(document.activeElement).toBe(input()) + await type('val') + await key('Escape') + expect(input().value).toBe('') + expect(document.activeElement).not.toBe(input()) + }) +}) diff --git a/forge-web/components/repo/go-to-file.tsx b/forge-web/components/repo/go-to-file.tsx new file mode 100644 index 000000000..9ccbde777 --- /dev/null +++ b/forge-web/components/repo/go-to-file.tsx @@ -0,0 +1,189 @@ +'use client' + +/** + * Go to file (QW-028): GitHub's file finder on the repo home and every directory. Fuzzy matching + * ({@link fuzzyRank}: `netproc` finds `src/net_processing.cpp`), the arrow keys and Enter to open + * a result, Escape to leave, and `t` anywhere on the page to start typing. + * + * The list comes from the history index first: the index the file list's column already loaded + * names every path at the tip, so the first results need no read at all. The tree walk the + * language bar shares ({@link repoFilesWalk}, its reads in parallel) starts on first focus and, + * once done, is the list: it is exact where an index a delta extends can still name a deleted file. + */ + +import { useEffect, useId, useMemo, useRef, useState, type KeyboardEvent as ReactKeyboardEvent } from 'react' +import Link from 'next/link' +import { useRouter } from 'next/navigation' +import { Search } from 'lucide-react' +import type { ObjectReader } from '@/lib/view/tree-nav' +import { indexedFilePaths, repoFilesWalk } from '@/lib/view/repo-facts' +import { fuzzyRank, type FuzzyHit } from '@/lib/view/fuzzy' +import { plural } from '@/lib/view/format' +import { isPageShortcut } from '@/lib/focus' +import { useAsync } from '@/hooks/use-async' +import { Input } from '@/components/ui/input' +import { repoHref, type RepoAddress } from '@/hooks/use-query-param' +import { cn } from '@/lib/utils' + +/** Results shown at once (the list scrolls). */ +const RESULTS = 20 + +/** A path with its matched characters marked. */ +function Marked({ hit }: { hit: FuzzyHit }): JSX.Element { + const at = new Set(hit.positions) + const parts: JSX.Element[] = [] + let run = '' + let marked = false + const flush = (key: number): void => { + if (run === '') return + parts.push(marked ? {run} : {run}) + run = '' + } + for (let i = 0; i < hit.path.length; i++) { + if (at.has(i) !== marked) { + flush(i) + marked = at.has(i) + } + run += hit.path[i] + } + flush(hit.path.length) + return <>{parts} +} + +export function GoToFile({ + reader, + repoKey: key, + tipOid, + rootTree, + addr, + refParam, + className, +}: { + reader: ObjectReader & { readonly memoScope?: object } + repoKey: string + tipOid: string + /** The tip's root tree, read when the walk starts. */ + rootTree: () => Promise + addr: RepoAddress + refParam: string + className?: string +}): JSX.Element { + const [started, setStarted] = useState(false) + const [query, setQuery] = useState('') + const [active, setActive] = useState(0) + const input = useRef(null) + const listId = useId() + const router = useRouter() + // The index's list (no read when the column already loaded it), then the walk's exact one. + const indexed = useAsync(() => indexedFilePaths(reader, tipOid), [key, tipOid], { enabled: started }) + const walk = useAsync(async () => repoFilesWalk(key, tipOid, reader, await rootTree()), [key, tipOid], { enabled: started }) + // A walk that stopped at its file cap names fewer paths than a complete index: the index then + // stays the list, and the walk's partial one answers only when there is no index. + const walkPaths = useMemo(() => walk.data?.files.map((f) => f.path) ?? null, [walk.data]) + const walkInUse = walkPaths !== null && (!walk.data?.truncated || indexed.data == null) + const paths = walkInUse ? walkPaths : (indexed.data ?? walkPaths) + const q = query.trim() + const hits = useMemo(() => (q === '' || paths === null ? [] : fuzzyRank(q, paths, RESULTS)), [q, paths]) + const hrefOf = (path: string): string => repoHref('/repo/blob', addr, { path, ...(refParam ? { ref: refParam } : {}) }) + const optionId = (i: number): string => `${listId}-${i}` + + // `t` starts typing a file name, as on GitHub. + useEffect(() => { + const onKey = (e: KeyboardEvent): void => { + if (!isPageShortcut(e, 't')) return + const box = input.current + if (box === null || box.getClientRects().length === 0) return + e.preventDefault() + box.focus() + box.select() + } + document.addEventListener('keydown', onKey) + return () => document.removeEventListener('keydown', onKey) + }, []) + + // A new list of results (a new query, or the walk's list replacing the index's) starts from its + // best match, so the highlighted row always exists and Enter always opens something. + useEffect(() => setActive(0), [hits]) + + const onKeyDown = (e: ReactKeyboardEvent): void => { + if (e.key === 'ArrowDown' || e.key === 'ArrowUp') { + if (hits.length === 0) return + e.preventDefault() + const next = (active + (e.key === 'ArrowDown' ? 1 : -1) + hits.length) % hits.length + setActive(next) + document.getElementById(optionId(next))?.scrollIntoView({ block: 'nearest' }) + } else if (e.key === 'Enter') { + const hit = hits[active] + if (hit === undefined) return + e.preventDefault() + router.push(hrefOf(hit.path)) + } else if (e.key === 'Escape') { + e.preventDefault() + setQuery('') + input.current?.blur() + } + } + + const searching = paths === null && (walk.loading || indexed.loading || !started) + const open = q !== '' + return ( +
+ + + setStarted(true)} + onChange={(e) => setQuery(e.target.value)} + onKeyDown={onKeyDown} + placeholder="Go to file" + className="h-8 py-1 pl-8 pr-7" + role="combobox" + aria-expanded={open} + aria-controls={listId} + aria-autocomplete="list" + aria-activedescendant={open && hits[active] !== undefined ? optionId(active) : undefined} + autoComplete="off" + spellCheck={false} + data-testid="go-to-file" + /> + + t + + {open ? ( +
    + {searching ?
  • Listing files…
  • : null} + {walk.error && paths === null ?
  • {walk.error}
  • : null} + {!searching && paths !== null && hits.length === 0 ?
  • No matching file.
  • : null} + {hits.map((hit, i) => ( +
  • setActive(i)}> + + + +
  • + ))} + {walkInUse && walk.data?.truncated ? ( +
  • Searched the first {plural(walk.data.files.length, 'file')}.
  • + ) : null} +
+ ) : null} +
+ ) +} diff --git a/forge-web/components/repo/repo-home-content.tsx b/forge-web/components/repo/repo-home-content.tsx index ea7861041..d1ce66eb2 100644 --- a/forge-web/components/repo/repo-home-content.tsx +++ b/forge-web/components/repo/repo-home-content.tsx @@ -4,23 +4,22 @@ * RepoHomeContent — the Code tab landing (`ux-dx-spec.md` §5.3): the ref bar (branch switcher, * `n commits`, Go to file), the root file list with a lazily loaded commit column, and the * README. Reads are size-independent (locator ranged object reads); `flatIndex` is never - * loaded here. A repo with no refs shows the empty state (§5.5); a private repo the viewer + * loaded here ({@link GoToFile} lists the files from the history index, then a walk). A repo with no refs shows the empty state (§5.5); a private repo the viewer * cannot decrypt never reaches here (the scaffold shows `PrivateRepoState`); unreadable storage degrades via * {@link BrowseBoundary}. */ -import { useEffect, useMemo, useState } from 'react' +import { useCallback, useEffect, useMemo } from 'react' import Link from 'next/link' -import { AlertTriangle, FileText, GitCommit, Rocket, Search } from 'lucide-react' +import { AlertTriangle, FileText, GitCommit, Rocket } from 'lucide-react' import { CopyRow } from '@/components/ui/copy-row' import { ACTIVE_NETWORK } from '@/lib/constants' import { dashRange, PUSH_COST_DASH } from '@/lib/sdk/cost' import { repoCommands, shellWord } from '@/lib/view/repo-commands' import type { BrowseReader } from '@/lib/browse' -import { loadRepoFacts, repoFilesWalk } from '@/lib/view/repo-facts' +import { loadRepoFacts } from '@/lib/view/repo-facts' import { repoKey } from '@/lib/repo' import type { RepoHome, SelectedRef } from '@/lib/view' -import { plural } from '@/lib/view/format' import { commitRootTree, decodeTextBlob, @@ -49,7 +48,7 @@ import { FileList } from '@/components/repo/file-list' import { RefDeletedState, RefNotFoundState, RefSwitcher } from '@/components/repo/ref-switcher' import { MarkdownView, type MarkdownRepoContext } from '@/components/markdown-view' import { ErrorState, LoadingBlock } from '@/components/ui/states' -import { Input } from '@/components/ui/input' +import { GoToFile } from '@/components/repo/go-to-file' import { Oid } from '@/components/ui/oid' import { repoHref, type RepoAddress } from '@/hooks/use-query-param' import { pinnedHref, usePermalinkKey } from '@/components/repo/permalink' @@ -150,6 +149,7 @@ function RootBody({ [tipOid], { enabled: data !== null }, ) + const rootTreeOf = useCallback(() => Promise.resolve(data?.tree ?? ''), [data?.tree]) const readmeRepo = useMemo(() => ({ addr, refParam, dir: '', reader, tipOid }), [addr, refParam, reader, tipOid]) // The About card's LICENSE and language bar (F-5): worked out only once everything the page // shows has settled (list, README, commit count and column), so they never delay it. @@ -187,7 +187,7 @@ function RootBody({ {commitCountLabel(commits.data)} - + repoFilesWalk(key, tipOid, reader, rootTree), [key, tipOid], { enabled: started }) - const q = query.trim().toLowerCase() - const hits = q === '' ? [] : (walk.data?.files ?? []).map((f) => f.path).filter((p) => p.toLowerCase().includes(q)).slice(0, 12) - return ( -
- - - setStarted(true)} - onChange={(e) => setQuery(e.target.value)} - placeholder="Go to file" - className="h-8 py-1 pl-8" - role="combobox" - aria-expanded={q !== ''} - aria-controls="go-to-file-results" - autoComplete="off" - /> - {q !== '' ? ( -
    - {walk.loading ?
  • Listing files…
  • : null} - {walk.error ?
  • {walk.error}
  • : null} - {!walk.loading && hits.length === 0 ?
  • No matching file.
  • : null} - {hits.map((p) => ( -
  • - - {p} - -
  • - ))} - {walk.data?.truncated ? ( -
  • Searched the first {plural(walk.data.files.length, 'file')}.
  • - ) : null} -
- ) : null} -
- ) -} - /** * The empty-repository state (`ux-dx-spec.md` §5.5): the commands that push an existing * repository, or start from scratch, and where the bytes will go. diff --git a/forge-web/components/repo/tree-content.tsx b/forge-web/components/repo/tree-content.tsx index a3c1f261a..40b7e2773 100644 --- a/forge-web/components/repo/tree-content.tsx +++ b/forge-web/components/repo/tree-content.tsx @@ -2,19 +2,20 @@ /** * TreeContent — a directory listing at an arbitrary path (browse plane, ranged reads), with the - * last-commit column the repo home has (from the history index, else a walk). + * last-commit column the repo home has (from the history index, else a walk), and Go to file. */ -import { useMemo } from 'react' +import { useCallback, useMemo } from 'react' import type { BrowseReader } from '@/lib/browse' import type { RepoHome } from '@/lib/view' import { readTree, selectedTip, selectRef, treeAtPath, type TreeEntry } from '@/lib/view' import { rootTreeOf, type PeeledTip } from '@/lib/view/tip' -import type { RepoRef } from '@/lib/repo' +import { repoKey, type RepoRef } from '@/lib/repo' import { useAsync } from '@/hooks/use-async' import { BrowseBoundary } from '@/components/repo/browse-boundary' import { ReadErrorState, ResolvedTip } from '@/components/repo/resolved-tip' import { FileList } from '@/components/repo/file-list' +import { GoToFile } from '@/components/repo/go-to-file' import { CommitCell, SearchOlderHistory, useLastCommits } from '@/components/repo/commit-column' import { PathBreadcrumb } from '@/components/repo/path-breadcrumb' import { PathActions } from '@/components/repo/path-actions' @@ -93,6 +94,7 @@ function DirBody({ // The commit column needs a commit's history: a tag of a tree has none, so it has no column. const names = useMemo(() => (data === null || tip.type !== 'commit' ? null : data.map((e) => e.name)), [data, tip.type]) const lastCommits = useLastCommits(reader, tip.oid, path, names) + const rootTree = useCallback(() => rootTreeOf(reader, tip), [reader, tip]) if (loading) return // A missing path is deterministic (common right after a ref switch) — no point retrying. if (error?.includes('path not found')) { @@ -102,6 +104,10 @@ function DirBody({ if (!data) return return (
+ {/* GitHub offers Go to file on every directory, not only the root (QW-028). */} +
+ +
{ }) }) +/** + * Code browsing on a large repo (QW-027, QW-028, QW-087), on the dash mirror: request and time + * budgets for what used to take tens of seconds. Measured on bonsia (2026-09-30), and the budgets + * are about twice that, so a slow devnet node passes and a return of the old serial reads fails: + * + * - the cold home's ref timeline (~700 updates): 8 key ranges in one round trip, where six + * `$createdAt` pages went one after the other (1.0–1.4 s before anything else could start); + * - Go to file: first results from the history index with no read (measured 8 ms after typing, + * was 14.5 s), the exact walk after it in ~1.9 s and 17 requests (was 100–120, one at a time); + * - the language bar: ~2.4 s and 19 requests once the About card is in view (was 12.8 s); + * - compare v22.0.0...v23.0.0: listed in ~16 s and 74 requests (was 19–21 s and 257), its 1,530 + * files (git's own count, renames paired) counted in ~14 s and 42 requests, complete (was 176 s, + * 1,287 requests and still partial). + * + * Run where the dash mirror is: `E2E_DEVNET=moutai`, or elsewhere with its owner in + * `E2E_SHOWCASE_DASHPAY` (on bonsia: 7A1MEuLjzcHZq8bLBzGYSUkpb2VM9dv7gtNuNYrPxKt3). + */ +/** `CHROME_KEYSET_SPLITS` (lib/repo/refs.ts): the key ranges the chrome reads a long ref timeline as. */ +const DASH_HOME_REF_READS_MAX = 8 + DAPI_RESEND_SLACK +/** The ranges go out together: from the first to the last, well under one round trip's worth. */ +const DASH_HOME_REF_SPREAD_MS = 600 +const GOTO_FIRST_RESULTS_MS = 1_000 +const GOTO_WALK_MS = 10_000 +const GOTO_WALK_REQUESTS = 40 +const LANGUAGES_MS = 10_000 +const LANGUAGES_REQUESTS = 40 +const COMPARE_LISTED_MS = 40_000 +const COMPARE_LISTED_REQUESTS = 150 +const COMPARE_COUNT_MS = 40_000 +const COMPARE_COUNT_REQUESTS = 90 + +test.describe('code browsing budgets on the dash mirror (QW-027, QW-028, QW-087)', () => { + test.skip(E2E_DEVNET !== 'moutai' && !process.env['E2E_SHOWCASE_DASHPAY'], 'the dash mirror is imported on moutai (elsewhere, set E2E_SHOWCASE_DASHPAY)') + test.describe.configure({ timeout: 240_000 }) + + test('cb-1. the cold home reads its ref timeline in one round trip; Go to file and the language bar answer in seconds', async ({ browser }) => { + const dash = await showcaseRepo('DASHPAY', 'dash') + const context = await browser.newContext() + const page = await context.newPage() + const refReads: number[] = [] + page.on('request', (r) => { + if (DAPI_METHOD.exec(r.url())?.[1] !== 'getDocuments') return + if (decodeDocumentsRequest(r.postDataBuffer())?.documentType === 'refUpdate') refReads.push(Date.now()) + }) + const dapi = recordDapi(page) + await page.goto(repoUrl('', '', dash), { waitUntil: 'domcontentloaded' }) + await waitForRepoResolved(page) + await expect(fileRows(page).first()).toBeVisible({ timeout: 60_000 }) + const spread = refReads.length === 0 ? 0 : Math.max(...refReads) - Math.min(...refReads) + test.info().annotations.push({ type: 'dapi', description: `dash cold home: ${refReads.length} refUpdate reads over ${spread} ms` }) + expect(refReads.length).toBeLessThanOrEqual(DASH_HOME_REF_READS_MAX) + expect(spread, 'the ref ranges go out side by side').toBeLessThanOrEqual(DASH_HOME_REF_SPREAD_MS) + await expect(page.getByTestId('commit-count')).toContainText(/\d/, { timeout: 60_000 }) + await settle(page) + + // Go to file: `t` focuses it; results from the history index at once, fuzzy and ranked. + await page.locator('main').click({ position: { x: 5, y: 5 } }) + await page.keyboard.press('t') + const box = page.getByTestId('go-to-file') + await expect(box).toBeFocused() + const beforeGoto = dapi.all().length + const typed = Date.now() + await box.fill('validation.cpp') + await expect(page.getByTestId('go-to-file-result').first()).toHaveText('src/validation.cpp', { timeout: GOTO_FIRST_RESULTS_MS }) + await box.fill('netproc') + await expect(page.getByTestId('go-to-file-result').first()).toContainText('net_processing') + await expect(page.getByText('Listing files')).toHaveCount(0, { timeout: GOTO_WALK_MS }) + await settle(page) + const walk = dapi.all().length - beforeGoto + test.info().annotations.push({ type: 'dapi', description: `Go to file: walk done in ${Date.now() - typed} ms (settled), ${walk} requests` }) + expect(walk).toBeLessThanOrEqual(GOTO_WALK_REQUESTS) + await page.keyboard.press('ArrowDown') + await page.keyboard.press('Enter') + await expect(page).toHaveURL(/\/repo\/blob\/.*net_processing/) + await shot(page, 'cb-01-goto-file-enter') + await context.close() + + // The language bar, in a fresh context (its walk not shared with Go to file's). + const fresh = await browser.newContext() + const lang = await fresh.newPage() + const langDapi = recordDapi(lang) + await lang.goto(repoUrl('', '', dash), { waitUntil: 'domcontentloaded' }) + await waitForRepoResolved(lang) + await expect(fileRows(lang).first()).toBeVisible({ timeout: 60_000 }) + await expect(lang.getByTestId('commit-count')).toContainText(/\d/, { timeout: 60_000 }) + await settle(lang) + const beforeLang = langDapi.all().length + const scrolled = Date.now() + const bar = lang.getByTestId('language-bar') + for (let y = 0; y < 4_000 && !(await bar.isVisible()); y += 400) { + await lang.evaluate((top) => window.scrollTo(0, top), y) + await lang.waitForTimeout(150) + } + await expect(bar.getByRole('img')).toBeVisible({ timeout: LANGUAGES_MS }) + const langMs = Date.now() - scrolled + await settle(lang) + const langReads = langDapi.all().length - beforeLang + test.info().annotations.push({ type: 'dapi', description: `language bar: ${langMs} ms, ${langReads} requests` }) + expect(langReads).toBeLessThanOrEqual(LANGUAGES_REQUESTS) + await shot(lang, 'cb-02-language-bar') + await fresh.close() + }) + + test('cb-2. compare v22.0.0...v23.0.0: listed, then every line counted, within budget', async ({ browser }) => { + const dash = await showcaseRepo('DASHPAY', 'dash') + const context = await browser.newContext() + const page = await context.newPage() + const dapi = recordDapi(page) + const opened = Date.now() + await page.goto(repoUrl('compare', '&base=v22.0.0&head=v23.0.0', dash), { waitUntil: 'domcontentloaded' }) + await expect(page.getByTestId('compare-summary')).toContainText('1,530 files changed', { timeout: COMPARE_LISTED_MS }) + const listed = dapi.all().length + test.info().annotations.push({ type: 'dapi', description: `compare listed: ${Date.now() - opened} ms, ${listed} requests` }) + expect(listed).toBeLessThanOrEqual(COMPARE_LISTED_REQUESTS) + const counting = Date.now() + await page.getByTestId('count-lines').click() + await expect(page.getByTestId('diff-totals')).toBeVisible({ timeout: COMPARE_COUNT_MS }) + const counted = dapi.all().length - listed + test.info().annotations.push({ type: 'dapi', description: `compare counted: ${Date.now() - counting} ms, ${counted} requests` }) + // git diff -M --shortstat v22.0.0...v23.0.0, and nothing left out. + await expect(page.getByTestId('diff-totals')).toContainText('+112310 −69084') + await expect(page.getByTestId('diff-totals-partial')).toHaveCount(0) + expect(counted).toBeLessThanOrEqual(COMPARE_COUNT_REQUESTS) + await shot(page, 'cb-03-compare-counted') + await context.close() + }) +}) + /** * Whether a request is one read-ahead block of a history walk: a `chunk` read of 17-19 seqs * (256 KiB / 14,700 B per chunk). The locator is read 100 seqs at a time, an object alone 1-2. diff --git a/forge-web/lib/browse/read-ahead.test.ts b/forge-web/lib/browse/read-ahead.test.ts index c2f46442e..f008b0d88 100644 --- a/forge-web/lib/browse/read-ahead.test.ts +++ b/forge-web/lib/browse/read-ahead.test.ts @@ -83,4 +83,21 @@ describe('readAheadSource', () => { await ra.fetchRange(0, 65, 66) // block 1 fetched again expect(inner.calls.map(([s]) => s)).toEqual([0, 64, 128, 64]) }) + + it('reads the next blocks ahead once reads run through the pack in order (QW-027), and only then', async () => { + const inner = source(1000) + const ra = readAheadSource(inner, 64, 64, 2) + await ra.fetchRange(0, 0, 1) // block 0: nothing ahead yet + await ra.fetchRange(0, 200, 201) // block 3: a jump, not a run + expect(inner.calls.map(([s]) => s)).toEqual([0, 192]) + await ra.fetchRange(0, 256, 257) // block 4 after block 3: blocks 5 and 6 are asked for too + expect(inner.calls.map(([s]) => s)).toEqual([0, 192, 320, 384, 256]) + // The walk reaches them from memory, and keeps two blocks ahead of itself. + await ra.fetchRange(0, 330, 331) // block 5 + expect(inner.calls.map(([s]) => s)).toEqual([0, 192, 320, 384, 256, 448]) + // Never past the end of the pack (blocks 0..15). + await ra.fetchRange(0, 900, 901) + await ra.fetchRange(0, 960, 961) + expect(Math.max(...inner.calls.map(([s]) => s))).toBe(960) + }) }) diff --git a/forge-web/lib/browse/reader.ts b/forge-web/lib/browse/reader.ts index e29d09aa8..ec8ed2fcf 100644 --- a/forge-web/lib/browse/reader.ts +++ b/forge-web/lib/browse/reader.ts @@ -57,6 +57,13 @@ export interface PackSource { export const READ_AHEAD_BLOCK = 256 * 1024 /** Blocks a {@link readAheadSource} keeps (so at most 16 MiB). */ const READ_AHEAD_BLOCKS = 64 +/** + * Blocks a history walk's source fetches ahead once its reads run through a pack in order + * (QW-027): a merge-base search over dashpay/dash v22.0.0...v23.0.0 reads 18,000 commits one block + * after the next, ~60 round trips in a row. With the next blocks asked for as the walk enters + * one, they arrive side by side (and in one chunk query, gathered with the demanded block). + */ +export const READ_AHEAD_SEQUENTIAL = 3 /** * A {@link PackSource} that fetches aligned {@link READ_AHEAD_BLOCK}-byte blocks and serves @@ -69,13 +76,27 @@ const READ_AHEAD_BLOCKS = 64 * fail to load go straight to `inner`, so nothing it could read before becomes unreadable. * Every object read through it is still hash-checked by the reader. */ -export function readAheadSource(inner: PackSource, block = READ_AHEAD_BLOCK, maxBlocks = READ_AHEAD_BLOCKS): PackSource { +export function readAheadSource(inner: PackSource, block = READ_AHEAD_BLOCK, maxBlocks = READ_AHEAD_BLOCKS, ahead = 0): PackSource { const blocks = new Map>() - const blockOf = (packRef: number, index: number, size: number, copy: number | undefined): Promise => { + /** Per pack copy, the block the last read asked for (a read of the next one is a sequential run). */ + const lastBlock = new Map() + const blockOf = (packRef: number, index: number, size: number, copy: number | undefined, demand = true): Promise => { // A single-copy pack reads the same bytes whether or not a copy is named: one key, so the // reader pinning copy 0 after its first verified object does not refetch the block. const single = (inner.copyCount?.(packRef) ?? 1) === 1 - const key = `${packRef}:${single ? 0 : copy ?? ''}:${index}` + const copyKey = `${packRef}:${single ? 0 : copy ?? ''}` + const key = `${copyKey}:${index}` + if (demand && ahead > 0) { + // Reads running through the pack in order: ask for the next blocks now, side by side. + const last = lastBlock.get(copyKey) + lastBlock.set(copyKey, index) + if (last === index - 1) { + const blocksInPack = Math.ceil(size / block) + for (let next = index + 1; next <= index + ahead && next < blocksInPack; next++) { + if (!blocks.has(`${copyKey}:${next}`)) blockOf(packRef, next, size, copy, false).catch(() => undefined) + } + } + } const hit = blocks.get(key) if (hit !== undefined) { blocks.delete(key) @@ -394,7 +415,7 @@ export class BrowseReader { const verdicts = this.opts.onObject ? new BatchedVerdicts(this.opts.onObject) : null const walker = new BrowseReader( this.locator, - readAheadSource(this.packs), + readAheadSource(this.packs, READ_AHEAD_BLOCK, READ_AHEAD_BLOCKS, READ_AHEAD_SEQUENTIAL), { ...this.opts, ...(verdicts ? { onObject: (v: ObjectVerdict) => verdicts.note(v) } : {}) }, this.view, // The same object memos: what the walk verified, the page does not read again. diff --git a/forge-web/lib/repo/chrome.test.ts b/forge-web/lib/repo/chrome.test.ts index a1e705ea4..6126cdd00 100644 --- a/forge-web/lib/repo/chrome.test.ts +++ b/forge-web/lib/repo/chrome.test.ts @@ -16,7 +16,8 @@ vi.mock('../constants', async (importOriginal) => { }) import type { ForgeIds } from '../deployments' -import { bytesToBase64, hexToBase64, setPlatformVersion, type DocumentQuery } from '../sdk' +import { base64ToHex, bytesToBase64, hexToBase64, setPlatformVersion, type DocumentQuery } from '../sdk' +import { CHROME_KEYSET_SPLITS } from './refs' import { loadRepoHome } from '../view/repo-view' import { cachedDpnsName, clearDpnsCache } from '../view/dpns' import { chromeFallbacks, readRepoChrome, repoTimelines, resetRepoTimelines, staleRepoTimelines } from './chrome' @@ -78,19 +79,30 @@ function fixture(tags: number): Store { const field = (d: Doc, f: string): unknown => f.split('.').reduce((o, k) => (o as Doc | undefined)?.[k], d) +/** A byteArray field compares by its bytes (hex), as Drive's index does, not by its base64 text. */ +const keyOf = (f: string, x: unknown): unknown => (f === 'refNameHash' && typeof x === 'string' ? base64ToHex(x) : x) + function filter(rows: Doc[], where: readonly (readonly unknown[])[] = []): Doc[] { return rows.filter((d) => where.every(([f, op, v]) => { - const x = field(d, f as string) - if (op === '==') return x === v + const x = keyOf(f as string, field(d, f as string)) as number + const y = keyOf(f as string, v) as number + if (op === '==') return x === y if (op === 'in') return (v as unknown[]).includes(x) - if (op === '>=') return (x as number) >= (v as number) - if (op === '>') return (x as number) > (v as number) + if (op === '>=') return x >= y + if (op === '>') return x > y throw new Error(`fake: ${String(op)}`) }), ) } +/** The `refState` index's order: `refNameHash` bytes, then `$createdAt`. */ +const byHash = (a: Doc, b: Doc): number => { + const x = base64ToHex(a['refNameHash'] as string) + const y = base64ToHex(b['refNameHash'] as string) + return x < y ? -1 : x > y ? 1 : byTime(a, b) +} + const byTime = (a: Doc, b: Doc): number => (a['$createdAt'] as number) - (b['$createdAt'] as number) || (String(a['$id']) < String(b['$id']) ? -1 : 1) interface Fake { @@ -101,14 +113,28 @@ interface Fake { } /** Plain queries and composites over `store`; `calls` records one entry per request. */ -function fakeSdk(store: Store): Fake { +function fakeSdk( + store: Store, + { + keyset = true, + rangeAnswer, + }: { + readonly keyset?: boolean + /** Rewrites (or throws on) the answer to a key-ordered (`refNameHash`) read. */ + readonly rangeAnswer?: (q: DocumentQuery, rows: Doc[]) => Doc[] + } = {}, +): Fake { const calls: string[] = [] const fake: { hold: Promise | null } = { hold: null } const rowsOf = (contract: string, type: string): Doc[] => [...(store[contract]?.[type] ?? [])] const plain = (q: DocumentQuery): Doc[] => { - let rows = filter(rowsOf(q.dataContractId, q.documentTypeName), q.where).sort(byTime) + const keyed = q.orderBy?.[0]?.[0] === 'refNameHash' + // `keyset: false`: a node that ignores a `refNameHash` order (answers in `$createdAt` order). + const order = keyset && keyed ? byHash : byTime + let rows = filter(rowsOf(q.dataContractId, q.documentTypeName), q.where).sort(order) if (q.startAfter !== undefined) rows = rows.slice(rows.findIndex((d) => d['$id'] === q.startAfter) + 1) - return rows.slice(0, Math.min(q.limit ?? 100, 100)) + rows = rows.slice(0, Math.min(q.limit ?? 100, 100)) + return keyed && rangeAnswer !== undefined ? rangeAnswer(q, rows) : rows } const wrap = (rows: Doc[]) => new Map(rows.map((d) => [String(d['$id']), d])) const sdk = { @@ -187,13 +213,88 @@ describe('repo chrome: one composite for the whole chrome', () => { expect(await readRepoChrome(sdk, FORGE, OWNER, 'nope', 'devnet')).toBeNull() }) - it('a timeline past one page is read on to the end with plain pages', async () => { + it('a ref timeline past one page is read on as key ranges side by side (QW-087)', async () => { const { sdk, calls } = fakeSdk(fixture(250)) const t = await (await readRepoChrome(sdk, FORGE, OWNER, 'demo', 'devnet'))!.timelines! expect(t.refUpdate).toHaveLength(251) expect(new Set(t.refUpdate.map((d) => d['$id'])).size).toBe(251) - // The composite's first page, then ⌈251/100⌉ − 1 = 2 continuation pages. - expect(calls).toEqual(['composite:repo', 'query:refUpdate', 'query:refUpdate']) + // Oldest first, as the `$createdAt` pages were. + const times = t.refUpdate.map((d) => d['$createdAt'] as number) + expect(times).toEqual([...times].sort((a, b) => a - b)) + // The composite's first page, then one page per key range, all sent at once: one round trip + // where ⌈251/100⌉ − 1 = 2 continuation pages went one after the other (six on dashpay/dash). + expect(calls).toEqual(['composite:repo', ...Array(CHROME_KEYSET_SPLITS).fill('query:refUpdate')]) + }) + + it('a node that does not honor the key ranges: read on to the end with plain pages', async () => { + const { sdk, calls } = fakeSdk(fixture(250), { keyset: false }) + const t = await (await readRepoChrome(sdk, FORGE, OWNER, 'demo', 'devnet'))!.timelines! + expect(t.refUpdate).toHaveLength(251) + expect(new Set(t.refUpdate.map((d) => d['$id'])).size).toBe(251) + // The ranges were refused, then ⌈251/100⌉ − 1 = 2 continuation pages. + expect(calls.slice(-2)).toEqual(['query:refUpdate', 'query:refUpdate']) + }) + + /** + * `tags` tags each created then moved (the move's `prevOid` is the creation's `newOid`), so a + * lost creation is a missing parent; plus `main` (one plain update, {@link TIP}). `protectedMoves`: + * each move is a `protectedRefUpdate` whose parent is the plain creation. + */ + function movedTags(tags: number, protectedMoves = false): { store: Store; creations: Doc[] } { + const store = fixture(0) + const creations: Doc[] = [] + for (let i = 0; i < tags; i++) { + const refName = `refs/tags/m${i}` + const refNameHash = bytesToBase64(sha256(new TextEncoder().encode(refName))) + const first = hexToBase64(i.toString(16).padStart(40, '0')) + const created = doc({ $ownerId: OWNER, repoId: REPO, refName, refNameHash, newOid: first }) + creations.push(created) + store.CORE!.refUpdate!.push(created) + const move = doc({ $ownerId: OWNER, repoId: REPO, refName, refNameHash, prevOid: first, newOid: hexToBase64('ee'.repeat(20)), force: true }) + store.CORE![protectedMoves ? 'protectedRefUpdate' : 'refUpdate']!.push(move) + } + return { store, creations } + } + + it('key ranges that lost a row (in order, so not refused) fail the parent check: read the plain way', async () => { + const { store, creations } = movedTags(120) + // A creation the composite's first page does not hold, so only the ranges could have read it. + const lost = creations[100]!['$id'] + const { sdk, calls } = fakeSdk(store, { rangeAnswer: (_q, rows) => rows.filter((d) => d['$id'] !== lost) }) + const t = await (await readRepoChrome(sdk, FORGE, OWNER, 'demo', 'devnet'))!.timelines! + expect(t.refUpdate).toHaveLength(241) + expect(t.refUpdate.map((d) => d['$id'])).toContain(lost) + // The ranges, then the `$createdAt` continuation of the composite's page: ⌈241/100⌉ − 1 = 2. + expect(calls).toEqual(['composite:repo', ...Array(CHROME_KEYSET_SPLITS + 2).fill('query:refUpdate')]) + }) + + it('a key range that fails outright: read the plain way, not fail the timeline', async () => { + let first = true + const { sdk, calls } = fakeSdk(fixture(250), { + rangeAnswer: (_q, rows) => { + if (first) { + first = false + throw new Error('node down') + } + return rows + }, + }) + const t = await (await readRepoChrome(sdk, FORGE, OWNER, 'demo', 'devnet'))!.timelines! + expect(t.refUpdate).toHaveLength(251) + expect(new Set(t.refUpdate.map((d) => d['$id'])).size).toBe(251) + expect(calls).toEqual(['composite:repo', ...Array(CHROME_KEYSET_SPLITS + 2).fill('query:refUpdate')]) + }) + + it('protected updates whose parents are plain updates are whole: the check groups both types, so no re-read', async () => { + // Both types past a page, both read as ranges; checked per type, every move would lack its parent. + const { store } = movedTags(120, true) + const { sdk, calls } = fakeSdk(store) + const t = await (await readRepoChrome(sdk, FORGE, OWNER, 'demo', 'devnet'))!.timelines! + expect(t.refUpdate).toHaveLength(121) + expect(t.protectedRefUpdate).toHaveLength(120) + expect([...calls].sort()).toEqual( + ['composite:repo', ...Array(CHROME_KEYSET_SPLITS).fill('query:protectedRefUpdate'), ...Array(CHROME_KEYSET_SPLITS).fill('query:refUpdate')].sort(), + ) }) it('a later read asks only for what is new, and keeps what it held', async () => { diff --git a/forge-web/lib/repo/chrome.ts b/forge-web/lib/repo/chrome.ts index e35c21727..8df36f50f 100644 --- a/forge-web/lib/repo/chrome.ts +++ b/forge-web/lib/repo/chrome.ts @@ -36,10 +36,14 @@ import { cursorPadded, queryAllDocuments, shareInFlight, type PlainDocument, typ import { DOC, type RepoRef } from './contract' import { membersGeneration, membershipsFromDocs, seedMemberships } from './members' import { noteForkOf } from './fork-parent' +import { readRefRowsInRanges, refRowsMissParent } from './refs' import { repoRefOf, toRepoDoc, type RepoDoc } from './resolveRepo' import { seedTargetCounts } from './social' import { repoSource } from './source' +/** The timelines keyed by `refNameHash` too, so a long one can be read as key ranges in parallel. */ +const REF_TYPES: ReadonlySet = new Set([DOC.refUpdate, DOC.protectedRefUpdate]) + /** The append-only timelines the store keeps. */ export const TIMELINE_TYPES = [DOC.config, DOC.refUpdate, DOC.protectedRefUpdate, DOC.packManifest] as const export type TimelineType = (typeof TIMELINE_TYPES)[number] @@ -173,16 +177,48 @@ async function completeTimelines( known: RepoTimelines | undefined, pages: readonly (readonly PlainDocument[])[], ): Promise { + const pageOf = (type: TimelineType): readonly PlainDocument[] => pages[TIMELINE_TYPES.indexOf(type)] ?? [] + // A short page is the whole delta; a full one is continued `$createdAt` page after page. + const serial = async (type: TimelineType): Promise => { + const page = pageOf(type) + if (page.length < PAGE) return [...page] + return queryAllDocuments(sdk, repoSource(repo).repoQuery(type, { ...sinceWhere(known, type), orderBy: [['$createdAt', 'asc']] }), { firstPage: page }) + } + // A first read of a repo's ref updates past a page (dashpay/dash: ~700) reads the rest as key + // ranges side by side, not page after page (QW-087). Null: read serially instead — every delta + // read, and a node that does not honor the ranges or fails one outright. + const ranged = async (type: TimelineType): Promise => { + if (known !== undefined || !REF_TYPES.has(type) || pageOf(type).length < PAGE) return null + const rows = await readRefRowsInRanges(sdk, repo, type).catch(() => null) + return rows === null ? null : [...pageOf(type), ...rows] + } const read = await Promise.all( - TIMELINE_TYPES.map(async (type, i) => { - const page = pages[i] ?? [] - const rest = (): Promise => - queryAllDocuments(sdk, repoSource(repo).repoQuery(type, { ...sinceWhere(known, type), orderBy: [['$createdAt', 'asc']] }), { firstPage: page }) - const rows = page.length < PAGE ? page : await rest() - return [type, mergeRows(known?.[type] ?? [], rows)] as const + TIMELINE_TYPES.map(async (type) => { + const fromRanges = await ranged(type) + return { type, fromRanges: fromRanges !== null, rows: fromRanges ?? (await serial(type)) } }), ) - return Object.fromEntries(read) as unknown as RepoTimelines + const rows = Object.fromEntries(read.map((r) => [r.type, r.rows])) as Record + // Ranges that answered in order can still have lost rows, and the loss would stick: later reads + // ask only for newer rows. So the ref rows get the `prevOid` check the full ref read runs (across + // both types), and a failure re-reads the ranged types the `$createdAt` way (the rest already were). + if (read.some((r) => r.fromRanges) && lostParent(repo, rows)) { + await Promise.all( + read.filter((r) => r.fromRanges).map(async (r) => { + rows[r.type] = await serial(r.type) + }), + ) + } + return Object.fromEntries(TIMELINE_TYPES.map((type) => [type, mergeRows(known?.[type] ?? [], rows[type])])) as unknown as RepoTimelines +} + +/** {@link refRowsMissParent} over the ref timelines; a row it cannot attribute counts as a loss. */ +function lostParent(repo: RepoRef, rows: Readonly>): boolean { + try { + return refRowsMissParent(repo, rows) + } catch { + return true + } } /** diff --git a/forge-web/lib/repo/refs.ts b/forge-web/lib/repo/refs.ts index 0b66115ae..023aa66d1 100644 --- a/forge-web/lib/repo/refs.ts +++ b/forge-web/lib/repo/refs.ts @@ -273,6 +273,30 @@ export async function keysetScan( ) } +/** + * Every row of one ref-update type, read as `splits` keyset ranges side by side from the start + * (QW-087): for a reader that already knows the type has more than a page (the repo chrome's + * `$createdAt` page came back full) and so need not read a first page to learn it. dashpay/dash's + * ~700 updates were six serial continuation pages (a 1.0–1.4 s waterfall on every cold page); as + * ranges they are one round trip, each range read on (by key) only past its own page. Null when a + * node did not honor a range (see {@link keysetScan}): the caller then reads the plain way. + */ +export async function readRefRowsInRanges( + sdk: EvoSDK, + repo: RepoRef, + documentTypeName: string, + splits = CHROME_KEYSET_SPLITS, +): Promise { + const ceilings = splitHashRange('0'.repeat(64), splits) + const starts: (HashKey | null)[] = [null, ...ceilings.map(hashKey)] + const parts = await Promise.all(starts.map((start, i) => keysetScan(sdk, repo, documentTypeName, start, ceilings[i] ?? null))) + if (parts.some((p) => p === null)) return null + return dedupeById(parts.flatMap((p) => p as PlainDocument[])) +} + +/** Ranges {@link readRefRowsInRanges} reads by default: a page each covers ~800 updates in one round trip. */ +export const CHROME_KEYSET_SPLITS = 8 + /** * Whether some update's non-null `prevOid` is no update's `newOid` in the same ref — a parent * that should have been read and was not. Parity: forge-core `refs::has_missing_parent`. @@ -285,7 +309,7 @@ export function hasMissingParent(updates: readonly RefUpdate[]): boolean { interface TypeScan { readonly type: string readonly isProtected: boolean - readonly rows: PlainDocument[] + readonly rows: readonly PlainDocument[] } /** Group rows per ref: plain before protected, each in read order (the fold re-sorts). */ @@ -306,6 +330,18 @@ function groupByRef(repo: RepoRef, scans: readonly TypeScan[]): Map>): boolean { + const scans = REF_UPDATE_TYPES.map(([type, isProtected]): TypeScan => ({ type, isProtected, rows: rowsByType[type] ?? [] })) + return [...groupByRef(repo, scans).values()].some(hasMissingParent) +} + /** `sha256(refName)` hex: the key a private ref is grouped under once its name is decrypted. */ export function publicRefKey(refName: string): string { return bytesToHex(sha256(new TextEncoder().encode(refName))) diff --git a/forge-web/lib/view/blame.test.ts b/forge-web/lib/view/blame.test.ts index d735145cc..1edc09f40 100644 --- a/forge-web/lib/view/blame.test.ts +++ b/forge-web/lib/view/blame.test.ts @@ -213,10 +213,10 @@ describe('blame bounds', () => { }) it('a change too large to check for a rename leaves the lines it added unresolved, not on it', async () => { - // 70 files deleted with the path's addition: scoring them all is over the rename read budget. + // 450 files deleted with the path's addition: scoring them all is over the rename read budget. const s = new Store() const gone: Record = {} - for (let i = 0; i < 70; i++) gone[`d/f${i}`] = `file ${i}\n${'filler\n'.repeat(i % 5)}` + for (let i = 0; i < 450; i++) gone[`d/f${i}`] = `file ${i}\n${'filler\n'.repeat(i % 5)}` const c1 = s.commit(s.files(gone), [], 'one') const c2 = s.commit(s.files({ 'new.txt': 'a\nb\n' }), [c1], 'replace everything') const c3 = s.commit(s.files({ 'new.txt': 'a\nb\nc\n' }), [c2], 'three') diff --git a/forge-web/lib/view/browse-fallback.test.ts b/forge-web/lib/view/browse-fallback.test.ts index 61e9082e3..1b5d587e6 100644 --- a/forge-web/lib/view/browse-fallback.test.ts +++ b/forge-web/lib/view/browse-fallback.test.ts @@ -673,7 +673,14 @@ describe('startFallback with external-storage packs', () => { const external = manifestFor(ext.pack, 1, { storage: 1, uris: ['https://flaky.example/p'], createdAt: 1, documentId: 'b' }) const repo = testRepo('fallback-expiry') vi.stubGlobal('fetch', () => Promise.resolve(new Response('nope', { status: 503 }))) - await startFallback(mockSdk(new Map([[platform.packHash, plat.pack]])), repo, [platform, external]) + // Platform chunk reads are gathered for one timer turn (`queueChunkSeqs`): let the fake clock + // run that turn, a millisecond at a time, until the clone is built. + let built = false + const cloning = startFallback(mockSdk(new Map([[platform.packHash, plat.pack]])), repo, [platform, external]).finally(() => { + built = true + }) + while (!built) await vi.advanceTimersByTimeAsync(1) + await cloning expect(cachedFallback(repo.repoId, [platform, external])).not.toBeNull() await vi.advanceTimersByTimeAsync(61_000) expect(cachedFallback(repo.repoId, [platform, external])).toBeNull() diff --git a/forge-web/lib/view/browse-source.test.ts b/forge-web/lib/view/browse-source.test.ts index ef1c03cd8..73dcc615f 100644 --- a/forge-web/lib/view/browse-source.test.ts +++ b/forge-web/lib/view/browse-source.test.ts @@ -22,6 +22,7 @@ import { serializeLocator, type IndexedObject } from '../browse/indexer' import { artifactRangeFetch, buildPackSource, + CHUNK_QUERIES_IN_FLIGHT, clearChunkCache, loadArtifactBytesProgress, loadBrowseContext, @@ -188,6 +189,31 @@ describe('chunk LRU', () => { fail = false expect(Array.from(await fetchRange(0, 10))).toEqual(Array.from(full.subarray(0, 10))) }) + + it('keeps at most CHUNK_QUERIES_IN_FLIGHT chunk queries out across every pack read at once', async () => { + const inner = mockSdk(() => full) as unknown as { documents: { query: (q: unknown) => Promise> } } + let inFlight = 0 + let peak = 0 + let sent = 0 + const sdk = { + documents: { + query: async (q: unknown): Promise> => { + sent++ + inFlight++ + peak = Math.max(peak, inFlight) + await new Promise((r) => setTimeout(r, 2)) + inFlight-- + return inner.documents.query(q) + }, + }, + } as unknown as EvoSDK + // 48 packs (a diff's count pool over several packs), each read at once: 48 separate queries. + const packs = Array.from({ length: 48 }, (_, i) => gitPack((0x40 + i).toString(16), 0, `p${i}`)) + const got = await Promise.all(packs.map((p) => artifactRangeFetch(sdk, REPO, p)(0, 10))) + for (const bytes of got) expect(Array.from(bytes)).toEqual(Array.from(full.subarray(0, 10))) + expect(sent).toBe(48) + expect(peak).toBe(CHUNK_QUERIES_IN_FLIGHT) + }) }) // L-15: a 10 MB locator is ~7 windows of 100 chunks; read one after another they were ~2 s of diff --git a/forge-web/lib/view/browse-source.ts b/forge-web/lib/view/browse-source.ts index 24b5ca273..95cd192cd 100644 --- a/forge-web/lib/view/browse-source.ts +++ b/forge-web/lib/view/browse-source.ts @@ -65,6 +65,33 @@ import { onPrivateSessionEnded } from '../repo/private-session' /** Chunk queries in flight at once when one range spans more than a single query. */ const CHUNK_QUERY_POOL = 6 +/** + * Chunk queries in flight at once across every read of the session. Each range pools its own + * batches ({@link CHUNK_QUERY_POOL}), but a diff counting two dozen files over several packs, or + * rename detection reading candidates, runs many ranges side by side; without one global bound a + * turn could send ~50 proof-verified queries at once. + */ +export const CHUNK_QUERIES_IN_FLIGHT = 16 +const chunkQuerySlots = { active: 0, waiting: [] as (() => void)[] } + +/** + * Run `run` on the next macrotask. A `MessageChannel` post, not `setTimeout(0)`: browsers clamp + * timers in background tabs to ~1 s, which would stall every serial round of chunk reads while + * the tab is hidden. `setTimeout` only where there is no `MessageChannel`. + */ +function nextMacrotask(run: () => void): void { + if (typeof MessageChannel === 'undefined') { + setTimeout(run, 0) + return + } + const channel = new MessageChannel() + channel.port1.onmessage = () => { + channel.port1.close() + run() + } + channel.port2.postMessage(null) +} + /** Concatenate the `d0..d2` byteArray fields (base64) of one chunk row, in order. */ function chunkPayload(doc: Record): Uint8Array { const parts: Uint8Array[] = [] @@ -140,6 +167,8 @@ function evictChunks(): void { export function clearChunkCache(): void { chunkCache.clear() chunkCacheBytes = 0 + chunkQueriesSent = 0 + pendingChunkQueries.clear() } /** Query one batch of chunk docs (uncached) and return payloads keyed by seq. */ @@ -173,9 +202,8 @@ async function queryChunkBatch( const i = next++ const batch = batches[i] if (batch === undefined) return - const { documents } = await queryDocumentsWithProof( - sdk, - source.chunkQuery(manifest.packHash, manifest.uploader, batch), + const { documents } = await withSlotOf(chunkQuerySlots, CHUNK_QUERIES_IN_FLIGHT, () => + queryDocumentsWithProof(sdk, source.chunkQuery(manifest.packHash, manifest.uploader, batch)), ) for (const doc of documents) { const raw = doc['seq'] @@ -189,6 +217,56 @@ async function queryChunkBatch( return bySeq } +/** + * Chunk seqs asked for in the same turn of the event loop, per copy of an artifact, gathered into + * one query (QW-027/QW-028): reads that run side by side (a pool of blob reads counting a diff's + * lines, a tree walk's parallel reads, Blame's read-ahead) each need a chunk or two, and each used to + * cost a query of its own. Gathered, they share queries of up to {@link CHUNK_QUERY_MAX} seqs + * ({@link queryChunkBatch} splits and pools past that). A copy's seqs are unique per `packHash`, + * so a gathered answer serves each asker exactly its own seqs. + */ +interface PendingChunkQuery { + readonly seqs: Set + readonly result: Promise> +} +const pendingChunkQueries = new Map() + +let chunkQueriesSent = 0 + +/** Test hook: how many chunk queries {@link queueChunkSeqs} has sent since {@link clearChunkCache}. */ +export function chunkQueryCount(): number { + return chunkQueriesSent +} + +/** + * `seqs` of the copy keyed `key`, fetched with the other seqs asked for this turn (one macrotask: + * the continuations of reads that just landed together get to ask first), in one query while they + * fit one ({@link CHUNK_QUERY_MAX}); past that, the next asker starts a new query rather than make + * everyone wait for a bigger one. `query` fetches a gathered set; the first asker's is used. + */ +export function queueChunkSeqs( + key: string, + seqs: readonly number[], + query: (seqs: readonly number[]) => Promise>, +): Promise> { + let pending = pendingChunkQueries.get(key) + if (pending !== undefined && pending.seqs.size + seqs.filter((q) => !pending?.seqs.has(q)).length > CHUNK_QUERY_MAX) pending = undefined + if (pending === undefined) { + const gathered = new Set() + const result = new Promise>((resolve, reject) => { + nextMacrotask(() => { + if (pendingChunkQueries.get(key)?.seqs === gathered) pendingChunkQueries.delete(key) + chunkQueriesSent += Math.ceil(gathered.size / CHUNK_QUERY_MAX) + query([...gathered].sort((a, b) => a - b)).then(resolve, reject) + }) + }) + pending = { seqs: gathered, result } + pendingChunkQueries.set(key, pending) + } + for (const seq of seqs) pending.seqs.add(seq) + return pending.result +} + /** * Fetch a contiguous `[start, end)` range of a platform-stored artifact by `packHash` (hex). * @@ -231,7 +309,7 @@ async function fetchPlatformRange( else missing.push(seq) } if (missing.length > 0) { - const batch = queryChunkBatch(sdk, repo, manifest, missing) + const batch = queueChunkSeqs(cachePrefix, missing, (seqs) => queryChunkBatch(sdk, repo, manifest, seqs)) for (const seq of missing) { const promise = batch.then((bySeq) => { const payload = bySeq.get(seq) @@ -373,14 +451,22 @@ async function withSlot(key: string, run: () => Promise): Promise { slot = { active: 0, waiting: [] } originSlots.set(key, slot) } - const s = slot - if (s.active >= PER_ORIGIN_CONCURRENCY) await new Promise((resolve) => s.waiting.push(resolve)) - s.active += 1 + return withSlotOf(slot, PER_ORIGIN_CONCURRENCY, run) +} + +/** + * At most `limit` `run`s of `slots` in flight. A finishing run hands its slot straight to the + * next waiter, so a newcomer arriving in between cannot take it too and overshoot the limit. + */ +async function withSlotOf(slots: { active: number; readonly waiting: (() => void)[] }, limit: number, run: () => Promise): Promise { + if (slots.active >= limit) await new Promise((resolve) => slots.waiting.push(resolve)) + else slots.active += 1 try { return await run() } finally { - s.active -= 1 - s.waiting.shift()?.() + const next = slots.waiting.shift() + if (next !== undefined) next() + else slots.active -= 1 } } diff --git a/forge-web/lib/view/chunk-gather.test.ts b/forge-web/lib/view/chunk-gather.test.ts new file mode 100644 index 000000000..8a8685cb8 --- /dev/null +++ b/forge-web/lib/view/chunk-gather.test.ts @@ -0,0 +1,60 @@ +/** + * Platform chunk reads asked for in the same turn share one query (QW-027, QW-028): a pool of + * blob reads counting a diff's lines, or a tree walk's parallel reads, each need a chunk or two. + */ + +import { describe, expect, it, vi } from 'vitest' + +import { chunkQueryCount, clearChunkCache, queueChunkSeqs } from './browse-source' + +const answer = (seqs: readonly number[]): Promise> => Promise.resolve(new Map(seqs.map((s) => [s, Uint8Array.of(s)]))) + +describe('queueChunkSeqs', () => { + it('gathers one turn\'s asks of a copy into one query, each asker served its own seqs', async () => { + clearChunkCache() + const sent: number[][] = [] + const query = (seqs: readonly number[]) => (sent.push([...seqs]), answer(seqs)) + const [a, b, c] = await Promise.all([queueChunkSeqs('copy', [5], query), queueChunkSeqs('copy', [2, 3], query), queueChunkSeqs('copy', [3], query)]) + expect(sent).toEqual([[2, 3, 5]]) + for (const got of [a, b, c]) expect([...got.keys()]).toEqual([2, 3, 5]) + expect(chunkQueryCount()).toBe(1) + }) + + it('keeps copies apart, and starts a new query rather than grow one past a page', async () => { + clearChunkCache() + const sent: number[][] = [] + const query = (seqs: readonly number[]) => (sent.push([...seqs]), answer(seqs)) + const many = Array.from({ length: 90 }, (_, i) => i) + await Promise.all([ + queueChunkSeqs('one', many, query), + queueChunkSeqs('two', [1], query), + // 90 + 20 is over a page (100): its own query. + queueChunkSeqs('one', Array.from({ length: 20 }, (_, i) => 200 + i), query), + ]) + expect(sent.map((s) => s.length).sort((x, y) => x - y)).toEqual([1, 20, 90]) + }) + + it('gathers on a macrotask that is not a timer (background tabs clamp timers to ~1 s)', async () => { + clearChunkCache() + // Every timer frozen: a gather waiting on `setTimeout` would never send. + vi.useFakeTimers({ toFake: ['setTimeout', 'setInterval', 'setImmediate'] }) + try { + const sent: number[][] = [] + const query = (seqs: readonly number[]) => (sent.push([...seqs]), answer(seqs)) + const got = await Promise.all([queueChunkSeqs('copy', [1], query), queueChunkSeqs('copy', [4], query)]) + expect(sent).toEqual([[1, 4]]) + for (const g of got) expect([...g.keys()]).toEqual([1, 4]) + } finally { + vi.useRealTimers() + } + }) + + it('a failed query fails its askers, and the next turn asks again', async () => { + clearChunkCache() + let fail = true + const query = (seqs: readonly number[]) => (fail ? Promise.reject(new Error('node down')) : answer(seqs)) + await expect(queueChunkSeqs('copy', [1], query)).rejects.toThrow('node down') + fail = false + await expect(queueChunkSeqs('copy', [1], query)).resolves.toEqual(new Map([[1, Uint8Array.of(1)]])) + }) +}) diff --git a/forge-web/lib/view/commit-log.ts b/forge-web/lib/view/commit-log.ts index b5002ab07..769eef197 100644 --- a/forge-web/lib/view/commit-log.ts +++ b/forge-web/lib/view/commit-log.ts @@ -75,7 +75,7 @@ export async function diffTreesWithRenames(sides: DiffSides, baseTreeOid: string const DIFF_NODE_CAP = 2000 /** Tree reads in flight at once while walking changed subtrees. */ -const TREE_READ_POOL = 6 +const TREE_READ_POOL = 16 async function readTreeMap(reader: ObjectReader, treeOid: string | null): Promise> { // A tree that cannot be read is an error, not an empty directory: treating it as empty diff --git a/forge-web/lib/view/file-diff.test.ts b/forge-web/lib/view/file-diff.test.ts index 9c89060ec..c39f13665 100644 --- a/forge-web/lib/view/file-diff.test.ts +++ b/forge-web/lib/view/file-diff.test.ts @@ -193,13 +193,25 @@ describe('loadFilePatch over a pack (the live read path)', () => { if (patch.kind === 'placeholder') expect(patch.note).toMatch(/stored size as 20 MB.*has not been checked/) }) - it('calls an over-256 KiB file with too many changes too complex, not too large', async () => { + it('calls an over-256 KiB file with too many changes too complex, not too large, and still counts it (QW-027)', async () => { const s = new Store() const r = s.reader() const patch = await loadFilePatch({ base: r, head: r }, change({ baseOid: s.blob(lines(12_000, 'p')), headOid: s.blob(lines(12_000, 'q')) })) - expect(patch).toMatchObject({ kind: 'placeholder', reason: 'too-complex' }) - if (patch.kind === 'placeholder') expect(patch.note).toMatch(/^Large file \(\d+ KB\) not shown.*lines are not counted/) - expect(uncountedReasons([patch.change], () => patch)).toBe('1 too many changes to diff') + expect(patch).toMatchObject({ kind: 'placeholder', reason: 'too-complex', added: 12_000, deleted: 12_000 }) + if (patch.kind === 'placeholder') expect(patch.note).toMatch(/^Large file \(\d+ KB\) not shown.*its lines are counted/) + // The change set's totals stay whole: nothing is left uncounted. + expect(diffTotals([patch.change], () => patch)).toMatchObject({ added: 12_000, deleted: 12_000, uncounted: 0 }) + expect(uncountedReasons([patch.change], () => patch)).toBe('') + }) + + it('counts a change past the shown diff\'s edit bound (QW-027: dash v22…v23 left 16 files out)', async () => { + const s = new Store() + const r = s.reader() + // 3,000 changed lines in a small file: over the 2,000 a diff shows, well within what is counted. + const before = Array.from({ length: 3_000 }, (_, i) => `old ${i}\n`).join('') + const after = Array.from({ length: 3_000 }, (_, i) => (i % 2 === 0 ? `old ${i}\n` : `new ${i}\n`)).join('') + const patch = await loadFilePatch({ base: r, head: r }, change({ baseOid: s.blob(before), headOid: s.blob(after) })) + expect(patch).toMatchObject({ kind: 'placeholder', reason: 'too-complex', added: 1_500, deleted: 1_500 }) }) it('calls a file binary when one side is, whichever side fails first', async () => { diff --git a/forge-web/lib/view/file-diff.ts b/forge-web/lib/view/file-diff.ts index cf10c0c3d..f60f8069d 100644 --- a/forge-web/lib/view/file-diff.ts +++ b/forge-web/lib/view/file-diff.ts @@ -10,7 +10,7 @@ import { decodeTextBlob } from './git-objects' import { isGitlink, type DiffSides, type FileChange } from './commit-log' import { ObjectTooLargeError } from '../browse' import { knownMinSize, type ObjectReader } from './tree-nav' -import { diffStat, diffTextLines, type TextDiffLine } from './text-diff' +import { diffStat, diffTextLinesOrCount, type TextDiffLine } from './text-diff' import { formatBytes } from './format' /** Blobs above this are not diffed inline. */ @@ -174,9 +174,11 @@ export async function loadFilePatch( try { const [before, after] = await readSides(sides, change, options) const size = Math.max(before.size, after.size) - const lines = diffTextLines(before.text, after.text, undefined, { ignoreWhitespace: options.ignoreWhitespace === true }) - if (lines === null) { + const lines = diffTextLinesOrCount(before.text, after.text, undefined, { ignoreWhitespace: options.ignoreWhitespace === true }) + if (!Array.isArray(lines)) { const large = size > INLINE_BLOB_MAX_BYTES ? `${largeNote(size)} ` : '' + // Too many changes to show, but counted from the same diff (QW-027): the totals stay whole. + if (lines !== null) return { ...placeholder(change, 'too-complex', `${large}This change is too large to show in the browser; its lines are counted.`), added: lines.added, deleted: lines.deleted } return placeholder(change, 'too-complex', `${large}This change is too large to diff in the browser, so its lines are not counted.`) } if (size <= INLINE_BLOB_MAX_BYTES) return textPatch(change, lines) diff --git a/forge-web/lib/view/fuzzy.test.ts b/forge-web/lib/view/fuzzy.test.ts new file mode 100644 index 000000000..c44380e62 --- /dev/null +++ b/forge-web/lib/view/fuzzy.test.ts @@ -0,0 +1,64 @@ +/** Go to file's fuzzy ranking (QW-028): GitHub's file finder, which dash's QA compared against. */ + +import { describe, expect, it } from 'vitest' + +import { fuzzyMatch, fuzzyRank } from './fuzzy' + +const DASH = [ + 'src/test/evo_trivialvalidation.cpp', + 'src/test/util/validation.cpp', + 'src/validation.cpp', + 'src/validation.h', + 'src/net_processing.cpp', + 'src/net_processing.h', + 'src/net.cpp', + 'src/qt/locale/dash_de.ts', + 'doc/release-notes.md', +] + +describe('fuzzyRank', () => { + it('puts the file named exactly first, before deeper and longer names', () => { + expect(fuzzyRank('validation.cpp', DASH, 3).map((h) => h.path)).toEqual([ + 'src/validation.cpp', + 'src/test/util/validation.cpp', + 'src/test/evo_trivialvalidation.cpp', + ]) + }) + + it('matches characters in order across word breaks, as GitHub does', () => { + expect(fuzzyRank('netproc', DASH, 5).map((h) => h.path).sort()).toEqual(['src/net_processing.cpp', 'src/net_processing.h']) + expect(fuzzyRank('NETPROC.h', DASH, 5)[0]?.path).toBe('src/net_processing.h') + // Spaces are ignored. + expect(fuzzyRank('net proc.cpp', DASH, 1)[0]?.path).toBe('src/net_processing.cpp') + }) + + it('finds nothing for an empty query or characters out of order', () => { + expect(fuzzyRank(' ', DASH, 5)).toEqual([]) + expect(fuzzyRank('ppcten', DASH, 5)).toEqual([]) + }) + + it('bounds what it returns', () => { + expect(fuzzyRank('s', DASH, 2)).toHaveLength(2) + }) +}) + +describe('fuzzyMatch', () => { + it('reports the matched positions, preferring a run at the start of a name', () => { + const m = fuzzyMatch('val', 'src/validation.cpp') + expect(m?.positions).toEqual([4, 5, 6]) + expect(fuzzyMatch('np', 'src/net_processing.cpp')?.positions).toEqual([4, 8]) + }) + + it('keeps positions on the path when lowercasing would change its length', () => { + const m = fuzzyMatch('md', 'İstanbul/README.md') + expect(m?.positions.map((p) => 'İstanbul/README.md'[p])).toEqual(['m', 'd']) + }) + + it('is null when a character is missing', () => { + expect(fuzzyMatch('xyz', 'src/validation.cpp')).toBeNull() + }) + + it('scores a whole-path match highest', () => { + expect(fuzzyMatch('src/net.cpp', 'src/net.cpp')?.score).toBe(Infinity) + }) +}) diff --git a/forge-web/lib/view/fuzzy.ts b/forge-web/lib/view/fuzzy.ts new file mode 100644 index 000000000..db19f1d5b --- /dev/null +++ b/forge-web/lib/view/fuzzy.ts @@ -0,0 +1,151 @@ +/** + * Fuzzy file-name matching for Go to file (QW-028), GitHub's file finder: the query's characters + * in order anywhere in the path (`netproc` finds `src/net_processing.cpp`), case-insensitive, + * ranked so a match at the start of a name, a run of adjacent characters and a shorter path come + * first. The scoring is fzy's (github.com/jhawthorn/fzy, MIT), which GitHub-style finders use: + * a dynamic programme over (query, path) that rewards a match after `/`, `_`, `-`, `.` or a + * lower-to-upper case change and a consecutive match, and charges for every skipped character. + */ + +const SCORE_GAP_LEADING = -0.005 +const SCORE_GAP_TRAILING = -0.005 +const SCORE_GAP_INNER = -0.01 +const SCORE_MATCH_CONSECUTIVE = 1.0 +const SCORE_MATCH_SLASH = 0.9 +const SCORE_MATCH_WORD = 0.8 +const SCORE_MATCH_CAPITAL = 0.7 +const SCORE_MATCH_DOT = 0.6 +/** Paths longer than this are matched but not scored (fzy's own bound). */ +const MATCH_MAX_LEN = 1024 + +/** A match: its score (higher is better) and the matched positions in the path. */ +export interface FuzzyMatch { + readonly score: number + readonly positions: readonly number[] +} + +/** + * `text` lowercased one UTF-16 unit at a time, so each position stays where it was: a character + * whose lowercase form is longer (`İ`) is kept as it is, and matched positions stay the path's. + */ +function foldCase(text: string): string { + const lower = text.toLowerCase() + if (lower.length === text.length) return lower + let out = '' + for (const ch of text) { + const l = ch.toLowerCase() + out += l.length === ch.length ? l : ch + } + return out +} + +/** Whether every character of `needle` (lowercase) appears in `haystack` (lowercase), in order. */ +function hasMatch(needle: string, haystack: string): boolean { + let at = 0 + for (const ch of needle) { + at = haystack.indexOf(ch, at) + if (at === -1) return false + at += 1 + } + return true +} + +/** The bonus for a match at `i`, by the character before it. */ +function bonusAt(path: string, i: number): number { + if (i === 0) return SCORE_MATCH_SLASH + const prev = path[i - 1] as string + const ch = path[i] as string + if (prev === '/') return SCORE_MATCH_SLASH + if (prev === '-' || prev === '_' || prev === ' ') return SCORE_MATCH_WORD + if (prev === '.') return SCORE_MATCH_DOT + if (prev >= 'a' && prev <= 'z' && ch >= 'A' && ch <= 'Z') return SCORE_MATCH_CAPITAL + return 0 +} + +/** + * The best way `query` matches `path`, or null when it does not (spaces in the query are ignored, + * as GitHub's finder ignores them). An empty query matches nothing. + */ +export function fuzzyMatch(query: string, path: string): FuzzyMatch | null { + const needle = foldCase(query.replace(/\s+/g, '')) + if (needle === '') return null + const lower = foldCase(path) + if (!hasMatch(needle, lower)) return null + const n = needle.length + const m = path.length + if (n === m) return { score: Infinity, positions: [...Array(m).keys()] } + if (m > MATCH_MAX_LEN) return { score: -Infinity, positions: [] } + + // D[i][j]: the best score of needle[0..i] ending with a match at j; M[i][j]: the best score of + // needle[0..i] within path[0..j]. + const D: Float64Array[] = [] + const M: Float64Array[] = [] + const bonus = new Float64Array(m) + for (let j = 0; j < m; j++) bonus[j] = bonusAt(path, j) + for (let i = 0; i < n; i++) { + const d = new Float64Array(m) + const mm = new Float64Array(m) + const gap = i === n - 1 ? SCORE_GAP_TRAILING : SCORE_GAP_INNER + let prev = -Infinity + for (let j = 0; j < m; j++) { + if (needle[i] === lower[j]) { + let score = -Infinity + if (i === 0) score = j * SCORE_GAP_LEADING + (bonus[j] as number) + else if (j > 0) { + const pm = (M[i - 1] as Float64Array)[j - 1] as number + const pd = (D[i - 1] as Float64Array)[j - 1] as number + score = Math.max(pm + (bonus[j] as number), pd + SCORE_MATCH_CONSECUTIVE) + } + d[j] = score + prev = Math.max(score, prev + gap) + } else { + d[j] = -Infinity + prev += gap + } + mm[j] = prev + } + D.push(d) + M.push(mm) + } + + // Walk back for the positions: the latest match on the best path, preferring a consecutive run. + const positions = new Array(n) + let matchRequired = false + for (let i = n - 1, j = m - 1; i >= 0; i--) { + for (; j >= 0; j--) { + const d = (D[i] as Float64Array)[j] as number + const mm = (M[i] as Float64Array)[j] as number + if (d !== -Infinity && (matchRequired || d === mm)) { + // Consecutive with the next match: this one must be taken too. + matchRequired = i > 0 && j > 0 && mm === ((D[i - 1] as Float64Array)[j - 1] as number) + SCORE_MATCH_CONSECUTIVE + positions[i] = j + j -= 1 + break + } + } + } + return { score: (M[n - 1] as Float64Array)[m - 1] as number, positions } +} + +/** One ranked hit. */ +export interface FuzzyHit extends FuzzyMatch { + readonly path: string +} + +/** + * The `limit` best matches of `query` among `paths`, best first; ties go to the shorter path, then + * path order. + */ +export function fuzzyRank(query: string, paths: readonly string[], limit: number): FuzzyHit[] { + const needle = foldCase(query.replace(/\s+/g, '')) + if (needle === '') return [] + const hits: FuzzyHit[] = [] + for (const path of paths) { + // The cheap in-order test first: most paths fail it, and scoring is the costly part. + if (!hasMatch(needle, foldCase(path))) continue + const match = fuzzyMatch(needle, path) + if (match !== null) hits.push({ path, ...match }) + } + hits.sort((a, b) => b.score - a.score || a.path.length - b.path.length || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) + return hits.slice(0, limit) +} diff --git a/forge-web/lib/view/pull-diff.ts b/forge-web/lib/view/pull-diff.ts index e6feb6464..1b66b7b4c 100644 --- a/forge-web/lib/view/pull-diff.ts +++ b/forge-web/lib/view/pull-diff.ts @@ -314,6 +314,23 @@ export async function findMergeBases( queue.push({ oid, when, seq: seq++ }) } + // The walk takes one commit at a time, so each read of a commit in a pack block not yet fetched + // was a round trip of its own: ~70 in a row on dashpay/dash v22.0.0...v23.0.0 (QW-027). The + // parents of everything waiting in the queue are what it reads next (the loop walks every + // queued commit until all are stale), so they are asked for ahead, side by side, through the + // reader's memo. They count toward the cap only once walked; what is read ahead in all is + // bounded by the cap too. + const prefetched = new Set() + const prefetchFrontier = (): void => { + for (const q of queue) { + for (const p of commits.get(q.oid)?.parents ?? []) { + if (commits.has(p) || prefetched.has(p) || prefetched.size >= cap) continue + prefetched.add(p) + readCommit(reader, p).catch(() => undefined) + } + } + } + const candidates: string[] = [] try { await Promise.all([push(baseOid, BASE), push(headOid, HEAD)]) @@ -333,6 +350,7 @@ export async function findMergeBases( flags.set(oid, flagsOf(oid) | STALE) } const parents = (commits.get(oid) as { parents: readonly string[] }).parents + prefetchFrontier() // A commit's parents are independent reads — fetch them together. await Promise.all(parents.filter((p) => (flagsOf(p) & flag) !== flag).map((p) => push(p, flag))) } diff --git a/forge-web/lib/view/renames.test.ts b/forge-web/lib/view/renames.test.ts index e3cf49fa3..0d0441fde 100644 --- a/forge-web/lib/view/renames.test.ts +++ b/forge-web/lib/view/renames.test.ts @@ -222,6 +222,45 @@ describe('detectRenames', () => { expect(ours(got.changes).filter((l) => l.startsWith('R'))).toEqual(['R075 a/0.txt b/0.txt', 'R079 a/1.txt b/1.txt', 'R100 same.txt dir/same.txt']) }) + describe('within its byte budget (counted as reads land)', () => { + const files = (prefix: string, n: number, edit = ''): Record => + Object.fromEntries(Array.from({ length: n }, (_, i) => [`${prefix}${i}.txt`, `file ${i}\n${'body\n'.repeat(i + 3)}${edit}`])) + + it('same-named pairs: stops reading once spent, still pairs what it read, and skips the matrix', async () => { + const s = new Store() + // 20 same-named pairs (40 blobs); the pool starts 16 reads before any lands. + const a = s.files({ ...files('a/', 20), 'x/old.txt': 'moved\n'.repeat(20) }) + const b = s.files({ ...files('b/', 20, 'edited\n'), 'y/new.txt': `${'moved\n'.repeat(20)}more\n` }) + const r = s.reader() + const { changes } = await diffTrees({ base: r, head: r }, a, b) + const before = s.reads.length + const got = await detectRenames({ base: r, head: r }, changes, { readBytes: 1 }) + expect(s.reads.length - before).toBe(16) + expect(got.limited).toMatch(/kept their names would download more than/) + // The 16 reads were the first 8 pairs, pair by pair: each scored, each a pair git makes too. + // x/old.txt → y/new.txt needs the matrix, which is skipped. + const renamed = got.changes.filter((c) => c.status === 'renamed').map((c) => `${c.oldPath} ${c.path}`) + expect(renamed).toHaveLength(8) + expect(renamed).not.toContain('x/old.txt y/new.txt') + for (const pair of renamed) expect(pair).toMatch(/^a\/(\d+)\.txt b\/\1\.txt$/) + }) + + it('the matrix: a part of its blobs is not enough, so it is skipped rather than pair worse', async () => { + const s = new Store() + const a = s.files(files('a/', 10)) + const b = s.files(files('b/n', 10, 'edited\n')) + const r = s.reader() + const { changes } = await diffTrees({ base: r, head: r }, a, b) + const got = await detectRenames({ base: r, head: r }, changes, { readBytes: 1 }) + expect(got.changes.some((c) => c.status === 'renamed')).toBe(false) + expect(got.limited).toMatch(/comparing 10 deleted files with 10 added files would download more than/) + // With the default budget the same change pairs. + const full = await detectRenames({ base: r, head: r }, changes) + expect(full.limited).toBeNull() + expect(full.changes.filter((c) => c.status === 'renamed')).toHaveLength(10) + }) + }) + it('a blob that cannot be read is not scored, and the result says renames may be missing', async () => { const s = new Store() const text = 'shared\n'.repeat(40) diff --git a/forge-web/lib/view/renames.ts b/forge-web/lib/view/renames.ts index 328819974..064b29762 100644 --- a/forge-web/lib/view/renames.ts +++ b/forge-web/lib/view/renames.ts @@ -15,8 +15,9 @@ * * Two limits are this browser's, not git's, and the result says when either applied * ({@link RenameResult.limited}): reading blobs costs network requests here, so the inexact - * phases read at most {@link RENAME_READ_BUDGET} blobs (a phase that needs more is skipped), and a - * blob over {@link RENAME_MAX_BLOB_BYTES} is not scored. `.gitattributes` `diff` drivers are not + * phases read at most {@link RENAME_READ_BUDGET} blobs (a phase that needs more is skipped) and + * {@link RENAME_READ_BYTES} bytes of them (reading stops once spent), and a blob over + * {@link RENAME_MAX_BLOB_BYTES} is not scored. `.gitattributes` `diff` drivers are not * read: binary means a NUL in the first 8000 bytes, git's own default test. */ @@ -41,12 +42,18 @@ const FIRST_FEW_BYTES = 8000 const HASHBASE = 107927 /** Blobs the inexact phases may read (each a ranged read over the network). */ -const RENAME_READ_BUDGET = 64 +const RENAME_READ_BUDGET = 400 +/** + * Blob bytes the inexact phases may download in all. Blob sizes are not known before a read, so + * this is counted as reads land: once it is spent no further candidate is read. 400 typical source + * files (tens of KiB each) stay far below it; 400 blobs near {@link RENAME_MAX_BLOB_BYTES} do not. + */ +const RENAME_READ_BYTES = 64 * 1024 * 1024 /** A blob larger than this is not scored for similarity. */ const RENAME_MAX_BLOB_BYTES = 1024 * 1024 const NOT_LOOKED = 'Renames with edits were not all looked for:' /** Blob reads in flight at once. */ -const READ_POOL = 6 +const READ_POOL = 16 const S_IFMT = 0o170000 const S_IFREG = 0o100000 @@ -55,6 +62,8 @@ const isRegular = (mode: number): boolean => (mode & S_IFMT) === S_IFREG export interface RenameOptions { /** Blobs the inexact phases may read (default {@link RENAME_READ_BUDGET}). */ readonly readBudget?: number + /** Blob bytes the inexact phases may download (default {@link RENAME_READ_BYTES}). */ + readonly readBytes?: number } export interface RenameResult { @@ -181,7 +190,7 @@ function scoreCompare(a: Candidate | null, b: Candidate | null): number { * blob that cannot be read is not scored, and the result says renames may be missing. */ export async function detectRenames(sides: DiffSides, changes: readonly FileChange[], options: RenameOptions = {}): Promise { - const { readBudget = RENAME_READ_BUDGET } = options + const { readBudget = RENAME_READ_BUDGET, readBytes = RENAME_READ_BYTES } = options const sorted = [...changes].sort(byPath) const srcs: Spec[] = [] const dsts: Spec[] = [] @@ -236,12 +245,27 @@ export async function detectRenames(sides: DiffSides, changes: readonly FileChan const seen = new Set() return specs.filter((s) => isRegular(s.mode) && !blobs.has(s.oid) && !seen.has(s.oid) && (seen.add(s.oid), true)) } - const readAll = async (base: readonly Spec[], head: readonly Spec[]): Promise => { - const todo = [...unread(base).map((spec) => ({ spec, reader: sides.base })), ...unread(head).map((spec) => ({ spec, reader: sides.head }))] + let bytesRead = 0 + /** + * Read the not yet read blobs of `base` and `head`, the two sides interleaved (so the basename + * phase's pairs, passed side by side, are read pair by pair and a spent budget still leaves whole + * pairs). False when the byte budget ran out first: the blobs left unread are not scored (and + * not in `blobs`). Which blobs made it then depends on when reads landed; the result says so. + */ + const readAll = async (base: readonly Spec[], head: readonly Spec[]): Promise => { + const b = unread(base).map((spec) => ({ spec, reader: sides.base })) + const h = unread(head).map((spec) => ({ spec, reader: sides.head })) + const todo = Array.from({ length: Math.max(b.length, h.length) }, (_, i) => [b[i], h[i]]).flat().filter((t) => t !== undefined) + let complete = true await mapPooled(todo, READ_POOL, async ({ spec, reader }) => { if (blobs.has(spec.oid)) return + if (bytesRead >= readBytes) { + complete = false + return + } try { const o = await reader.readObject(spec.oid, { maxBytes: RENAME_MAX_BLOB_BYTES }) + bytesRead += o.bytes.length blobs.set(spec.oid, o.type === 'blob' ? { size: o.bytes.length, spans: spanHash(o.bytes) } : null) } catch (e) { if (e instanceof ObjectTooLargeError) tooLarge = true @@ -249,7 +273,9 @@ export async function detectRenames(sides: DiffSides, changes: readonly FileChan blobs.set(spec.oid, null) } }) + return complete } + const tooManyBytes = (what: string): string => `${NOT_LOOKED} ${what} would download more than ${formatBytes(readBytes)} in the browser.` const score = (s: Spec, d: Spec, minScore: number): number => { if (!isRegular(s.mode) || !isRegular(d.mode)) return 0 const a = blobs.get(s.oid) @@ -287,10 +313,12 @@ export async function detectRenames(sides: DiffSides, changes: readonly FileChan skipped = `${NOT_LOOKED} comparing the ${plural(byName.length, 'file')} that kept their names would read too many files in the browser, so ${n === 0 ? 'none' : `only ${n}`} of them ${n === 1 ? 'was' : 'were'} compared.` } if (reachable.length > 0) { - await readAll( + // Out of bytes part way: the pairs whose blobs were read still stand alone; the matrix is skipped. + const complete = await readAll( reachable.map(([x]) => x), reachable.map(([, y]) => y), ) + if (!complete) skipped ??= tooManyBytes(`comparing the ${plural(byName.length, 'file')} that kept their names`) for (const [src, dst] of reachable) { if (dst.used) continue // git's "already used in a rename"; cannot happen with unique names. const sc = score(src, dst, MIN_BASENAME_SCORE) @@ -307,8 +335,10 @@ export async function detectRenames(sides: DiffSides, changes: readonly FileChan skipped = `${NOT_LOOKED} ${left.length} deleted and ${rest.length} added files are over git's rename limit.` } else if (blobs.size + unread([...left, ...rest]).length > readBudget) { skipped = `${NOT_LOOKED} comparing ${plural(left.length, 'deleted file')} with ${plural(rest.length, 'added file')} would read too many files in the browser.` + } else if (!(await readAll(left, rest))) { + // A matrix over a part of the blobs could pair a file with a worse source than git does. + skipped = tooManyBytes(`comparing ${plural(left.length, 'deleted file')} with ${plural(rest.length, 'added file')}`) } else { - await readAll(left, rest) const matrix: (Candidate | null)[] = [] for (const d of rest) { const m = new Array(CANDIDATES).fill(null) diff --git a/forge-web/lib/view/repo-facts.ts b/forge-web/lib/view/repo-facts.ts index ebbb6ec22..6bd682c2f 100644 --- a/forge-web/lib/view/repo-facts.ts +++ b/forge-web/lib/view/repo-facts.ts @@ -25,6 +25,7 @@ import { onRepoContentWritten } from '../repo/push' import { readReleaseCount, readReleases, releaseCountOf } from '../repo/releases' import { invalidateSessionCache, sessionCached } from './session-cache' import { historyWalker } from './commit-log' +import { historyOf } from './history-source' import { readBlob, type ObjectReader } from './tree-nav' import { decodeTextBlob, type TreeEntry } from './git-objects' import { formatBytes } from './format' @@ -114,6 +115,31 @@ export function repoFilesWalk( return walk } +/** Each history index's file paths, worked out once. */ +const indexedFiles = new WeakMap() + +/** + * The files at `tipOid` as the history index lists them (QW-028), with no tree read: the index the + * file list's column already loaded names every path of the tip (directories too, which are left + * out here). Null when no index covers the tip. A delta over a full index can still name a file + * deleted since that full index (the reader drops such paths when a tree lists names), so a caller + * takes these as a first answer until the tree walk ({@link repoFilesWalk}) has the exact one. + */ +export async function indexedFilePaths(reader: { readonly memoScope?: object }, tipOid: string): Promise { + const history = historyOf(reader) + if (history === null || !history.covers(tipOid)) return null + const index = await history.load(tipOid) + const hit = indexedFiles.get(index) + if (hit !== undefined) return hit + const dirs = new Set() + for (const path of index.paths.keys()) { + for (let at = path.indexOf('/'); at !== -1; at = path.indexOf('/', at + 1)) dirs.add(path.slice(0, at)) + } + const files = [...index.paths.keys()].filter((p) => !dirs.has(p)).sort() + indexedFiles.set(index, files) + return files +} + /** * Work out and publish the facts for this tip: the license from the root's license files (a few * KiB each, through the page's reader), then the language bar from the shared file walk. Facts diff --git a/forge-web/lib/view/text-diff.ts b/forge-web/lib/view/text-diff.ts index 077c22f1d..2206f20f2 100644 --- a/forge-web/lib/view/text-diff.ts +++ b/forge-web/lib/view/text-diff.ts @@ -66,9 +66,11 @@ type Op = 0 | 1 | 2 // equal | delete | insert /** * git's edit script for `a` → `b` ({@link xdiffChanges}, then {@link compactChanges}): each hunk's - * deletions before its additions, as git prints them. Null past `limits`. + * deletions before its additions, as git prints them. Null when the search runs past + * `limits.maxWork`; `{ added, deleted }` alone when the script has more than `limits.maxEdits` + * edits (too many to show, still counted: QW-027). */ -function gitOps(a: readonly string[], b: readonly string[], limits: DiffLimits): Op[] | null { +function gitOps(a: readonly string[], b: readonly string[], limits: DiffLimits): Op[] | { readonly added: number; readonly deleted: number } | null { if (a.length === 0 || b.length === 0) return new Array(a.length + b.length).fill(a.length === 0 ? 2 : 1) const marks = xdiffChanges(a, b, limits) if (marks === null) return null @@ -79,26 +81,28 @@ function gitOps(a: readonly string[], b: readonly string[], limits: DiffLimits): // Compaction asserts what xdiff asserts; the uncompacted alignment is still a valid diff. c = marks } + let deleted = 0 + let added = 0 + for (let k = 0; k < a.length; k++) deleted += c.oldChanged[k] as number + for (let k = 0; k < b.length; k++) added += c.newChanged[k] as number + if (added + deleted > limits.maxEdits) return { added, deleted } const ops: Op[] = [] - let edits = 0 let i = 0 let j = 0 while (i < a.length || j < b.length) { if (i < a.length && c.oldChanged[i] === 1) { ops.push(1) i++ - edits++ } else if (j < b.length && c.newChanged[j] === 1) { ops.push(2) j++ - edits++ } else { ops.push(0) i++ j++ } } - return edits > limits.maxEdits ? null : ops + return ops } /** @@ -111,6 +115,21 @@ export function diffTextLines( limits: DiffLimits = DEFAULT_DIFF_LIMITS, options: { readonly ignoreWhitespace?: boolean } = {}, ): TextDiffLine[] | null { + const got = diffTextLinesOrCount(before, after, limits, options) + return Array.isArray(got) ? got : null +} + +/** + * {@link diffTextLines}, or for a change with more edits than `limits.maxEdits` (too many to + * show) the lines it adds and deletes, from the same diff (QW-027: a change set's totals stay + * whole). Null only when the search runs past `limits.maxWork`. + */ +export function diffTextLinesOrCount( + before: string, + after: string, + limits: DiffLimits = DEFAULT_DIFF_LIMITS, + options: { readonly ignoreWhitespace?: boolean } = {}, +): TextDiffLine[] | { readonly added: number; readonly deleted: number } | null { const oldLines = splitLines(before) const newLines = splitLines(after) // Ignoring whitespace (`git diff -w`) compares lines with all whitespace removed; the lines @@ -122,7 +141,7 @@ export function diffTextLines( // xdiff sees the whole files: it trims their common ends itself, after counting every record // (which lines its cleanup drops depends on how often they occur in the whole file). const ops = gitOps(oldKeys, newKeys, limits) - if (ops === null) return null + if (ops === null || !Array.isArray(ops)) return ops const lines: TextDiffLine[] = [] let o = 0 diff --git a/forge-web/lib/view/zip.test.ts b/forge-web/lib/view/zip.test.ts index d3311d732..f8e2fa646 100644 --- a/forge-web/lib/view/zip.test.ts +++ b/forge-web/lib/view/zip.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from 'vitest' import { MODE_GITLINK } from '../browse' import { Store } from './diff-fixtures' -import { isSafeName, listFiles, readZipFiles, ZIP_MAX_BYTES, zipFileName, ZipTooLargeError } from './zip' +import { isSafeName, listFiles, readZipFiles, walkFiles, ZIP_MAX_BYTES, zipFileName, ZipTooLargeError } from './zip' describe('zip of a ref', () => { it('lists every file under the commit and skips submodules', async () => { @@ -41,6 +41,37 @@ describe('zip of a ref', () => { expect(new Set(Object.values(entries).map((b) => b.buffer)).size).toBe(2) }) + it('a truncated parallel walk keeps the same files whatever order its reads land in', async () => { + const s = new Store() + // 12 top-level directories, each with 3 files and a subdirectory of 2: 60 files in 25 trees. + const layout: Record = {} + for (let d = 0; d < 12; d++) { + for (let f = 0; f < 3; f++) layout[`d${d}/f${f}.txt`] = `${d}.${f}` + for (let f = 0; f < 2; f++) layout[`d${d}/sub/g${f}.txt`] = `${d}.sub.${f}` + } + const root = s.files(layout) + const inner = s.reader() + /** A reader whose reads land after `delay(n)` ms, `n` counting reads as they are asked. */ + const timed = (delay: (n: number) => number) => { + let n = 0 + return { + readObject: async (oid: string) => { + const wait = delay(n++) + await new Promise((r) => setTimeout(r, wait)) + return inner.readObject(oid) + }, + } + } + const walk = async (delay: (n: number) => number, pool?: number) => + (await walkFiles(timed(delay), root, { maxFiles: 25, ...(pool === undefined ? {} : { pool }) })).files.map((f) => f.path) + const serial = await walk(() => 0, 1) + expect(serial).toHaveLength(25) + // Later-asked reads landing first, first-asked landing first, and a scramble: one set. + expect(await walk((n) => 30 - n)).toEqual(serial) + expect(await walk((n) => n)).toEqual(serial) + expect(await walk((n) => (n * 7) % 11)).toEqual(serial) + }) + it('skips tree entries that would escape the zip root', () => { for (const bad of ['..', '.', '', 'a/b', 'a\\b', 'a\0b']) expect(isSafeName(bad)).toBe(false) expect(isSafeName('..a')).toBe(true) diff --git a/forge-web/lib/view/zip.ts b/forge-web/lib/view/zip.ts index 1f963a90a..9fe21a23d 100644 --- a/forge-web/lib/view/zip.ts +++ b/forge-web/lib/view/zip.ts @@ -9,7 +9,7 @@ import { MODE_GITLINK, MODE_TREE } from '../browse' import type { ObjectReader } from './tree-nav' import { ObjectTypeError, peel, readBlob, readTree } from './tree-nav' -import { decodeTextBlob } from './git-objects' +import { decodeTextBlob, type TreeEntry } from './git-objects' import { historyWalker } from './commit-log' import { autoAbbrevLength, @@ -74,24 +74,31 @@ export function isSafeName(name: string): boolean { } /** - * Every blob (and symlink) under a tree, gitlinks and unsafe names skipped, breadth first, sorted - * by path, each with its stored size from the locator (tree reads only). Stops after `maxTrees` - * trees or `maxFiles` files (`truncated`, also when one tree alone holds more than the bound), so - * Go to file and the language bar can list a large repo without walking all of it. + * Tree reads a walk keeps in flight (QW-028): a level of dashpay/dash's tree is dozens of + * directories, and one read at a time made Go to file and the language bar wait ~14 s for ~100 + * round trips. Reads of the same pack's neighbouring trees share their queries (the read-ahead + * walker's blocks, the chunk queries gathered per turn). + */ +export const WALK_POOL = 16 + +/** + * Every blob (and symlink) under a tree, gitlinks and unsafe names skipped, breadth first with up + * to `pool` trees read at once, sorted by path, each with its stored size from the locator (tree + * reads only). Stops after `maxTrees` trees or `maxFiles` files (`truncated`, also when one tree + * alone holds more than the bound), so Go to file and the language bar can list a large repo + * without walking all of it. */ export async function walkFiles( reader: ObjectReader, treeOid: string, - { maxTrees = Infinity, maxFiles = Infinity }: { readonly maxTrees?: number; readonly maxFiles?: number } = {}, + { maxTrees = Infinity, maxFiles = Infinity, pool = WALK_POOL }: { readonly maxTrees?: number; readonly maxFiles?: number; readonly pool?: number } = {}, ): Promise { const files: ZipFile[] = [] const queue: [string, string][] = [[treeOid, '']] let trees = 0 let dropped = false - while (queue.length > 0 && trees < maxTrees && files.length < maxFiles) { - const [oid, prefix] = queue.shift() as [string, string] - trees += 1 - for (const e of await readTree(reader, oid)) { + const take = (entries: readonly TreeEntry[], prefix: string): void => { + for (const e of entries) { // A tree is hash-checked, not sane: a hostile pusher can name an entry `..` (zip-slip). if (!isSafeName(e.name)) continue const path = prefix ? `${prefix}/${e.name}` : e.name @@ -101,6 +108,40 @@ export async function walkFiles( else dropped = true } } + // Reads land in any order, but are taken in the order they were started: children are queued, + // and files counted against `maxFiles`, as a one-at-a-time walk would, so a truncated walk keeps + // the same set on every visit (the language bar must not vary with network timing). + const started: { readonly prefix: string; entries?: readonly TreeEntry[] }[] = [] + await new Promise((resolve, reject) => { + let active = 0 + let failed = false + const pump = (): void => { + while (!failed && active < pool && queue.length > 0 && trees < maxTrees && files.length < maxFiles) { + const [oid, prefix] = queue.shift() as [string, string] + trees += 1 + active += 1 + const read: (typeof started)[number] = { prefix } + started.push(read) + readTree(reader, oid) + .then((entries) => { + active -= 1 + if (failed) return + read.entries = entries + for (let head = started[0]; head?.entries !== undefined; head = started[0]) { + started.shift() + take(head.entries, head.prefix) + } + pump() + }) + .catch((e: unknown) => { + failed = true + reject(e) + }) + } + if (active === 0 && !failed) resolve() + } + pump() + }) files.sort((a, b) => (a.path < b.path ? -1 : 1)) return { files, truncated: dropped || queue.length > 0 } }