From 2b2faf2be419aed7aaff3b232085be03868a1837 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 09:27:11 -0500 Subject: [PATCH 1/2] fix(web): build the .zip as git archive does: modes, commit time, export-subst and export-ignore QW-026: an executable is recorded 0100755 and a symlink 0120777 (made by Unix), every entry and directory carries the commit's time (DOS date and the UT timestamp), the archive comment is the commit id, and the tree's .gitattributes apply: export-ignore leaves paths out, export-subst expands $Format:...$ including %(describe) worked out as git describe does. Checked against git archive and git describe in the tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/components/repo/clone-box.tsx | 28 +- forge-web/lib/browse/reader.ts | 5 + forge-web/lib/view/archive.test.ts | 286 ++++++++++++ forge-web/lib/view/archive.ts | 575 ++++++++++++++++++++++++ forge-web/lib/view/zip-entries.ts | 79 ++++ forge-web/lib/view/zip.ts | 165 ++++++- forge-web/lib/view/zip.worker.ts | 10 +- 7 files changed, 1136 insertions(+), 12 deletions(-) create mode 100644 forge-web/lib/view/archive.test.ts create mode 100644 forge-web/lib/view/archive.ts create mode 100644 forge-web/lib/view/zip-entries.ts diff --git a/forge-web/components/repo/clone-box.tsx b/forge-web/components/repo/clone-box.tsx index 7e98a3b59..9c518a519 100644 --- a/forge-web/components/repo/clone-box.tsx +++ b/forge-web/components/repo/clone-box.tsx @@ -18,14 +18,16 @@ import type { RepoAddress } from '@/hooks/use-query-param' import { ACTIVE_NETWORK } from '@/lib/constants' import { errorMessage } from '@/lib/utils' import { saveBytes } from '@/lib/view/release-download' -import { formatBytes, plural, selectedTip, type RepoHome, type SelectedRef } from '@/lib/view' +import { formatBytes, plural, selectedTip, tipOidOf, type RepoHome, type SelectedRef } from '@/lib/view' import { repoCommands } from '@/lib/view/repo-commands' import { resolveTip } from '@/lib/view/tip' import { repoKey } from '@/lib/repo' import { compressInWorker, - listFiles, + planArchive, readZipFiles, + substituteFiles, + type ArchiveRefs, storedSize, ZIP_MAX_BYTES, zipFileName, @@ -78,6 +80,16 @@ export function CloneBox({ home, addr, selected }: { home: RepoHome; addr: RepoA ) } +/** The repo's tags and branches, as `%(describe)` and `%d` in an export-subst file read them. */ +function archiveRefs(home: RepoHome): ArchiveRefs { + const named = (refs: RepoHome['tags'], prefix: string) => + refs.flatMap((r) => { + const oid = tipOidOf(r) + return oid === null ? [] : [{ name: r.refName.replace(prefix, ''), oid }] + }) + return { tags: named(home.tags, 'refs/tags/'), heads: named(home.branches, 'refs/heads/') } +} + function ZipDownload({ home, addr, selected }: { home: RepoHome; addr: RepoAddress; selected: SelectedRef }): JSX.Element | null { const state = useBrowseReader(home.repo) const [progress, setProgress] = useState(null) @@ -97,14 +109,20 @@ function ZipDownload({ home, addr, selected }: { home: RepoHome; addr: RepoAddre setProgress({ phase: 'listing', files: 0, filesTotal: 0, bytes: 0 }) try { // A short pinned id resolved to its commit, and a tag to what it names (L-01, L-32). - const files = await listFiles(reader, (await resolveTip(reader, tip, { repoKey: repoKey(home.repo), pinned: selected.pinned !== undefined })).oid) + // As `git archive` (QW-026): the tree's export-ignore and export-subst, modes, the commit's time. + const plan = await planArchive(reader, (await resolveTip(reader, tip, { repoKey: repoKey(home.repo), pinned: selected.pinned !== undefined })).oid) + const { files } = plan const stored = storedSize(files) if (stored > ZIP_MAX_BYTES) throw new ZipTooLargeError(stored) const entries = await readZipFiles(reader, files, setProgress, cancel.current.signal) + await substituteFiles(reader, plan, entries, archiveRefs(home)) const name = zipFileName(addr.name, selected.name) + const prefix = `${name.replace(/\.zip$/, '')}/` const rooted: Record = {} - for (const [path, bytes] of Object.entries(entries)) rooted[`${name.replace(/\.zip$/, '')}/${path}`] = bytes - const zip = await compressInWorker(rooted, setProgress, cancel.current.signal) + const modes: Record = {} + for (const f of files) modes[prefix + f.path] = f.mode + for (const [path, bytes] of Object.entries(entries)) rooted[prefix + path] = bytes + const zip = await compressInWorker(rooted, setProgress, cancel.current.signal, { modes, mtime: plan.mtime, comment: plan.commit?.oid ?? null }) saveBytes(zip, name, 'application/zip') setMessage(`Saved ${name} (${formatBytes(zip.length)}, ${plural(files.length, 'file')}, each hash-checked).`) } catch (e) { diff --git a/forge-web/lib/browse/reader.ts b/forge-web/lib/browse/reader.ts index 573a7120a..e29d09aa8 100644 --- a/forge-web/lib/browse/reader.ts +++ b/forge-web/lib/browse/reader.ts @@ -403,6 +403,11 @@ export class BrowseReader { return Object.assign(walker, { flush: () => verdicts?.flush() }) } + /** How many objects the locator indexes (git's automatic abbreviation length grows with it). */ + get objectCount(): number { + return this.locator.count + } + /** OIDs starting with a hex `prefix` ({@link ObjectLocator.findByPrefix}). */ findByPrefix(prefix: string, limit?: number): string[] { return this.locator.findByPrefix(prefix, limit) diff --git a/forge-web/lib/view/archive.test.ts b/forge-web/lib/view/archive.test.ts new file mode 100644 index 000000000..c3e90252d --- /dev/null +++ b/forge-web/lib/view/archive.test.ts @@ -0,0 +1,286 @@ +/** + * The browser .zip against `git archive` (QW-026): `.gitattributes` (export-ignore, + * export-subst), `$Format:` placeholders, `git describe`, Unix modes, the commit's time and the + * archive comment. The parts that can be are checked against real git; skipped without git. + */ + +import { spawnSync } from 'node:child_process' + +import { unzipSync, zipSync } from 'fflate' +import { describe, expect, it } from 'vitest' + +import { gitOidHex, MODE_TREE } from '../browse' +import { HAVE_GIT, scratchRepo, writeLiterally } from '../merge/git-oracle' +import { + autoAbbrevLength, + describeCommit, + exportAttributes, + formatCommit, + parseArchiveCommit, + taggedCommits, + uniqueAbbrev, + type DescribeTag, + type FormatContext, +} from './archive' +import { Store } from './diff-fixtures' +import { planArchive, readZipFiles, substituteFiles } from './zip' +import { withArchiveComment, zipEntries } from './zip-entries' + +const enc = new TextEncoder() + +/** Put a raw object of `type` into the store; its oid. */ +function raw(s: Store, type: 'commit' | 'tag', text: string): string { + const bytes = enc.encode(text) + const oid = gitOidHex(type, bytes) + s.objects.set(oid, { type, bytes }) + return oid +} + +function git(dir: string, args: string[], input?: Uint8Array): { out: string; bytes: Buffer; status: number | null } { + const r = spawnSync('git', args, { + cwd: dir, + input, + env: { ...process.env, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null', TZ: process.env['TZ'] ?? '' }, + maxBuffer: 1 << 26, + }) + return { out: r.stdout.toString().trim(), bytes: r.stdout, status: r.status } +} + +const findIn = (s: Store) => (prefix: string, limit = 2): string[] => [...s.objects.keys()].filter((o) => o.startsWith(prefix)).slice(0, limit) + +describe('.gitattributes for an archive', () => { + it('matches basenames and anchored paths, lets later lines and deeper files win, and ignores whole directories', () => { + const attrs = exportAttributes( + new Map([ + ['.gitattributes', 'src/version.c export-subst\n*.md export-ignore\nREADME.md -export-ignore\n/ci export-ignore\ndocs/**/*.png export-ignore\n# a comment\n"odd name.txt" export-ignore\n'], + ['lib/.gitattributes', 'keep.md -export-ignore\n*.c export-subst\n'], + ]), + ) + expect(attrs('src/version.c')).toEqual({ ignore: false, subst: true }) + expect(attrs('notes.md').ignore).toBe(true) + expect(attrs('deep/dir/notes.md').ignore).toBe(true) + expect(attrs('README.md').ignore).toBe(false) + expect(attrs('lib/keep.md').ignore).toBe(false) + expect(attrs('lib/x.c').subst).toBe(true) + expect(attrs('src/other.c').subst).toBe(false) + // An anchored directory: everything under it. + expect(attrs('ci/build.sh').ignore).toBe(true) + expect(attrs('src/ci/build.sh').ignore).toBe(false) + expect(attrs('docs/a/b/c.png').ignore).toBe(true) + expect(attrs('docs/c.png').ignore).toBe(true) + expect(attrs('odd name.txt').ignore).toBe(true) + }) +}) + +describe('$Format: placeholders', () => { + const s = new Store() + const tree = s.files({ a: 'a\n' }) + const parent = s.commit(tree) + const oid = raw( + s, + 'commit', + `tree ${tree}\nparent ${parent}\nauthor Ada Lovelace 1700000000 +0200\ncommitter Bob 1700003600 -0530\n\nFix the thing\nacross two lines\n\nThe body.\n`, + ) + const commit = parseArchiveCommit(oid, s.objects.get(oid)!.bytes) + const ctx: FormatContext = { commit, abbrev: (o, min = 7) => o.slice(0, min), describe: () => 'v1.0-3-gabcdef', decorations: ['main', 'tag: v1.1'] } + + it('expands what git log --format would', () => { + expect(formatCommit('%H|%h|%T|%P|%p', ctx)).toBe(`${oid}|${oid.slice(0, 7)}|${tree}|${parent}|${parent.slice(0, 7)}`) + expect(formatCommit('%an <%ae> %al', ctx)).toBe('Ada Lovelace ada') + expect(formatCommit('%ad|%aD|%ai|%aI|%at|%as', ctx)).toBe( + // 1700000000 is 2023-11-14 22:13:20 UTC: in the author's +0200, past midnight. + 'Wed Nov 15 00:13:20 2023 +0200|Wed, 15 Nov 2023 00:13:20 +0200|2023-11-15 00:13:20 +0200|2023-11-15T00:13:20+02:00|1700000000|2023-11-15', + ) + expect(formatCommit('%cn %cI', ctx)).toBe('Bob 2023-11-14T17:43:20-05:30') + expect(formatCommit('%s', ctx)).toBe('Fix the thing across two lines') + expect(formatCommit('%b', ctx)).toBe('The body.\n') + expect(formatCommit('a%nb %% %x41 %d %D %(describe) %(describe:abbrev=12)', ctx)).toBe('a\nb % A (main, tag: v1.1) main, tag: v1.1 v1.0-3-gabcdef v1.0-3-gabcdef') + // Unknown placeholders stay as written, as git leaves them. + expect(formatCommit('%Q %(foo) %(describe:bogus)', ctx)).toBe('%Q %(foo) %(describe:bogus)') + }) + + it('abbreviates as git does', () => { + expect(autoAbbrevLength(0)).toBe(7) + expect(autoAbbrevLength(5_000)).toBe(7) + // dashpay/dash: ~500k objects is 10 digits, git's `--short` there. + expect(autoAbbrevLength(500_000)).toBe(10) + expect(uniqueAbbrev('abcdef1234', 4, (p) => (p.length < 6 ? ['abcdef1234', 'abcde99999'] : ['abcdef1234']))).toBe('abcdef') + }) +}) + +/** A history with merges, annotated and lightweight tags, some tags on one commit. */ +function taggedHistory(s: Store): { commits: string[]; tags: DescribeTag[] } { + let x = 7 + const rand = (): number => ((x = (x * 1103515245 + 12345) >>> 0), x / 2 ** 32) + const commits: string[] = [] + let a = s.commit(s.files({ f: '0\n' }), [], 'root') + let b = a + commits.push(a) + const tags: DescribeTag[] = [] + let time = 1_600_000_000 + const tag = (target: string, name: string, annotated: boolean): void => { + if (!annotated) { + tags.push({ name, oid: target }) + return + } + time += 10 + tags.push({ name, oid: raw(s, 'tag', `object ${target}\ntype commit\ntag ${name}\ntagger T ${time} +0000\n\n${name}\n`) }) + } + for (let i = 1; i < 60; i++) { + const onA = rand() < 0.6 + const merge = i % 7 === 0 + const parents = onA ? (merge ? [a, b] : [a]) : merge ? [b, a] : [b] + const c = s.commit(s.files({ f: `${i}\n` }), parents, `c${i}`) + commits.push(c) + if (onA) a = c + else b = c + if (i % 9 === 0) tag(c, `v${i}`, true) + if (i % 13 === 0) tag(c, `light${i}`, false) + if (i === 27) tag(c, `v${i}-again`, true) + } + return { commits, tags } +} + +describe.skipIf(!HAVE_GIT)('git describe, against git', () => { + it('names the same tag and depth for every commit, annotated only and with --tags', async () => { + const s = new Store() + const { commits, tags } = taggedHistory(s) + const { dir, done } = scratchRepo() + try { + writeLiterally(dir, s.objects.values()) + for (const t of tags) git(dir, ['update-ref', `refs/tags/${t.name}`, t.oid]) + const reader = s.reader() + const abbrevOf = (oid: string, min: number): string => uniqueAbbrev(oid, min, findIn(s)) + for (const opts of [{ tags: false, abbrev: 12 }, { tags: true, abbrev: 7 }, { tags: false, abbrev: 0 }]) { + const named = await taggedCommits(reader, tags, { match: [], exclude: [] }) + for (const c of commits) { + const want = git(dir, ['describe', `--abbrev=${opts.abbrev}`, ...(opts.tags ? ['--tags'] : []), c]) + const got = await describeCommit(reader, c, named, opts, abbrevOf) + expect(got ?? '', `${c} ${JSON.stringify(opts)}`).toBe(want.status === 0 ? want.out : '') + } + } + } finally { + done() + } + }) + + it('takes match= and exclude= as git does', async () => { + const s = new Store() + const { commits, tags } = taggedHistory(s) + const { dir, done } = scratchRepo() + try { + writeLiterally(dir, s.objects.values()) + for (const t of tags) git(dir, ['update-ref', `refs/tags/${t.name}`, t.oid]) + const reader = s.reader() + const named = await taggedCommits(reader, tags, { match: ['v*'], exclude: ['*-again'] }) + const tip = commits[commits.length - 1] as string + const got = await describeCommit(reader, tip, named, { tags: false, abbrev: 7 }, (o, m) => uniqueAbbrev(o, m, findIn(s))) + expect(got).toBe(git(dir, ['describe', '--abbrev=7', '--match', 'v*', '--exclude', '*-again', tip]).out) + } finally { + done() + } + }) +}) + +/** The central directory's entries: name, "made by" OS byte, external attributes, and the UT time. */ +function centralDirectory(zip: Uint8Array): Map { + const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength) + const out = new Map() + for (let at = 0; at + 46 <= zip.length; ) { + if (view.getUint32(at, true) !== 0x02014b50) { + at++ + continue + } + const nameLen = view.getUint16(at + 28, true) + const extraLen = view.getUint16(at + 30, true) + const commentLen = view.getUint16(at + 32, true) + const name = new TextDecoder().decode(zip.subarray(at + 46, at + 46 + nameLen)) + let ut: number | null = null + for (let e = at + 46 + nameLen; e + 4 <= at + 46 + nameLen + extraLen; ) { + const id = view.getUint16(e, true) + const len = view.getUint16(e + 2, true) + if (id === 0x5455 && len >= 5) ut = view.getUint32(e + 5, true) + e += 4 + len + } + out.set(name, { os: view.getUint8(at + 5), attrs: view.getUint32(at + 38, true), ut }) + at += 46 + nameLen + extraLen + commentLen + } + return out +} + +/** The archive comment (end-of-central-directory record's). */ +function archiveComment(zip: Uint8Array): string { + for (let at = zip.length - 22; at >= 0; at--) { + const view = new DataView(zip.buffer, zip.byteOffset + at) + if (view.getUint32(0, true) === 0x06054b50) return new TextDecoder().decode(zip.subarray(at + 22, at + 22 + view.getUint16(20, true))) + } + return '' +} + +describe.skipIf(!HAVE_GIT)('the browser zip against git archive --format=zip', () => { + it('the same files, contents (export-subst expanded, export-ignore left out), modes, times and comment', async () => { + const s = new Store() + const script = s.blob('#!/bin/sh\necho hi\n') + const version = s.blob('const char* v = "$Format:%H %h %an %ad %cI %s$";\n#define DESC "$Format:%(describe:abbrev=12)$"\n') + const src = s.tree([ + { name: 'version.c', oid: version }, + { name: 'main.c', oid: s.blob('int main(void) { return 0; }\n') }, + ]) + const ci = s.files({ 'build.yml': 'on: push\n' }) + const tree = s.tree([ + { name: '.gitattributes', oid: s.blob('src/version.c export-subst\n/ci export-ignore\n*.log export-ignore\n') }, + { name: 'autogen.sh', oid: script, mode: 0o100755 }, + { name: 'ci', oid: ci, mode: MODE_TREE }, + { name: 'debug.log', oid: s.blob('noise\n') }, + { name: 'link', oid: s.blob('src/main.c'), mode: 0o120000 }, + { name: 'README.md', oid: s.blob('# hi\n') }, + { name: 'src', oid: src, mode: MODE_TREE }, + ]) + const root = s.commit(s.files({ README: 'x\n' }), [], 'root') + const tagObj = raw(s, 'tag', `object ${root}\ntype commit\ntag v0.1\ntagger T 1600000000 +0000\n\nv0.1\n`) + const tip = raw( + s, + 'commit', + `tree ${tree}\nparent ${root}\nauthor Ada 1700000000 +0200\ncommitter Ada 1700000500 +0200\n\nRelease prep\n`, + ) + const { dir, done } = scratchRepo() + try { + writeLiterally(dir, s.objects.values()) + git(dir, ['update-ref', 'refs/tags/v0.1', tagObj]) + const want = new Uint8Array(git(dir, ['archive', '--format=zip', '--prefix=p/', tip]).bytes) + + // The browser's pipeline, as the clone box runs it. + const reader = Object.assign(s.reader(), { findByPrefix: findIn(s), objectCount: s.objects.size }) + const plan = await planArchive(reader, tip) + const entries = await readZipFiles(reader, plan.files, () => undefined) + await substituteFiles(reader, plan, entries, { tags: [{ name: 'v0.1', oid: tagObj }], heads: [] }) + const rooted: Record = {} + const modes: Record = {} + for (const f of plan.files) modes[`p/${f.path}`] = f.mode + for (const [path, bytes] of Object.entries(entries)) rooted[`p/${path}`] = bytes + const meta = { modes, mtime: plan.mtime, comment: tip } + const got = withArchiveComment(zipSync(zipEntries(rooted, meta), { level: 6 }), meta.comment) + + const wantFiles = unzipSync(want) + const gotFiles = unzipSync(got) + expect(Object.keys(gotFiles).sort()).toEqual(Object.keys(wantFiles).sort()) + for (const [name, bytes] of Object.entries(wantFiles)) expect(new TextDecoder().decode(gotFiles[name]), name).toBe(new TextDecoder().decode(bytes)) + // `$Format:` really was expanded (not merely equal because both left it). + expect(new TextDecoder().decode(gotFiles['p/src/version.c'])).toContain(`${tip} ${tip.slice(0, 7)} Ada Wed Nov 15 00:13:20 2023 +0200`) + expect(new TextDecoder().decode(gotFiles['p/src/version.c'])).toContain('v0.1-1-g') + + const wantDir = centralDirectory(want) + const gotDir = centralDirectory(got) + for (const [name, w] of wantDir) { + const g = gotDir.get(name) + expect(g, name).toBeDefined() + expect({ name, os: g?.os, attrs: g?.attrs, ut: g?.ut }).toEqual({ name, os: w.os, attrs: w.attrs, ut: w.ut }) + } + expect(gotDir.get('p/autogen.sh')?.attrs).toBe((0o100755 << 16) >>> 0) + expect(archiveComment(got)).toBe(archiveComment(want)) + expect(archiveComment(got)).toBe(tip) + } finally { + done() + } + }) +}) diff --git a/forge-web/lib/view/archive.ts b/forge-web/lib/view/archive.ts new file mode 100644 index 000000000..386b16692 --- /dev/null +++ b/forge-web/lib/view/archive.ts @@ -0,0 +1,575 @@ +/** + * What `git archive` does to a tree besides copying its files (QW-026), for the browser's .zip: + * + * - `.gitattributes` of the tree archived (never the viewer's): `export-ignore` leaves a path out, + * `export-subst` expands `$Format:…$` in a file ({@link expandExportSubst}); + * - the placeholders `git log --format` takes (the ones a commit alone answers), `%(describe)` + * included, computed as `git describe` does ({@link describeCommit}); + * - each entry's Unix mode and the commit's time, which the zip writer records. + * + * `.gitattributes` patterns are matched as git's attr.c matches them: a pattern with no `/` is + * matched against the basename, one with a `/` against the path from its file's directory with + * `*` not crossing `/` and `**` crossing any number of directories. Later lines, and files deeper + * in the tree, win. + */ + +import { OID_HEX, parseTag } from './git-objects' +import type { ObjectReader } from './tree-nav' +import { mapPooled } from './pool' + +// --------------------------------------------------------------------------- +// .gitattributes +// --------------------------------------------------------------------------- + +/** One `.gitattributes` line, reduced to the attributes an archive reads. */ +interface AttrRule { + readonly match: (path: string) => boolean + /** true: set; false: unset (`-attr` or `!attr`); absent: this line says nothing. */ + readonly attrs: ReadonlyMap<'export-ignore' | 'export-subst', boolean> +} + +/** A git wildmatch pattern (`WM_PATHNAME`) as a regular expression over a whole path. */ +function wildmatchRegex(pattern: string): RegExp { + let re = '' + for (let i = 0; i < pattern.length; i++) { + const c = pattern[i] as string + if (c === '*') { + if (pattern[i + 1] === '*') { + // `**/` at the start or after `/` matches any number of directories; a trailing `/**` everything below. + const atStart = i === 0 || pattern[i - 1] === '/' + if (atStart && pattern[i + 2] === '/') { + re += '(?:.*/)?' + i += 2 + continue + } + if (atStart && i + 2 === pattern.length) { + re += '.*' + i += 1 + continue + } + re += '[^/]*' + i += 1 + continue + } + re += '[^/]*' + } else if (c === '?') { + re += '[^/]' + } else if (c === '[') { + const close = pattern.indexOf(']', i + 2) + if (close === -1) { + re += '\\[' + continue + } + let body = pattern.slice(i + 1, close) + const negate = body.startsWith('!') || body.startsWith('^') + if (negate) body = body.slice(1) + re += `[${negate ? '^' : ''}${body.replace(/\\/g, '\\\\').replace(/]/g, '\\]')}]` + i = close + } else if (c === '\\' && i + 1 < pattern.length) { + re += (pattern[i + 1] as string).replace(/[.*+?^${}()|[\]\\/]/g, '\\$&') + i += 1 + } else { + re += c.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&') + } + } + return new RegExp(`^${re}$`) +} + +/** A `.gitattributes` pattern, from its file's directory (`''` = the root). */ +function patternMatcher(pattern: string, dir: string): ((path: string) => boolean) | null { + // A trailing `/` (a directory-only pattern) is not supported by git's attributes: it matches nothing. + if (pattern === '' || pattern.endsWith('/') || pattern.startsWith('!')) return null + const anchored = pattern.includes('/') + const re = wildmatchRegex(anchored ? pattern.replace(/^\//, '') : pattern) + const prefix = dir === '' ? '' : `${dir}/` + return (path) => { + if (!path.startsWith(prefix)) return false + const rel = path.slice(prefix.length) + return re.test(anchored ? rel : rel.slice(rel.lastIndexOf('/') + 1)) + } +} + +/** The pattern of a line, unquoting a C-style quoted one. */ +function splitLine(line: string): [string, string] { + if (!line.startsWith('"')) { + const at = line.search(/\s/) + return at === -1 ? [line, ''] : [line.slice(0, at), line.slice(at)] + } + let out = '' + let i = 1 + for (; i < line.length && line[i] !== '"'; i++) { + if (line[i] === '\\' && i + 1 < line.length) { + const n = line[++i] as string + out += n === 'n' ? '\n' : n === 't' ? '\t' : n + } else out += line[i] + } + return [out, line.slice(i + 1)] +} + +/** The archive-relevant rules of one `.gitattributes` file at `dir`. */ +export function parseGitAttributes(text: string, dir: string): AttrRule[] { + const rules: AttrRule[] = [] + for (const raw of text.split(/\r?\n/)) { + const line = raw.trim() + if (line === '' || line.startsWith('#')) continue + const [pattern, rest] = splitLine(line) + const attrs = new Map<'export-ignore' | 'export-subst', boolean>() + for (const word of rest.trim().split(/\s+/)) { + const unset = word.startsWith('-') || word.startsWith('!') + const name = (unset ? word.slice(1) : word).split('=')[0] + if (name === 'export-ignore' || name === 'export-subst') attrs.set(name, !unset) + } + if (attrs.size === 0) continue + const match = patternMatcher(pattern, dir) + if (match !== null) rules.push({ match, attrs }) + } + return rules +} + +/** How the archive treats a path. */ +export interface ExportAttrs { + readonly ignore: boolean + readonly subst: boolean +} + +/** + * The archive's attributes from every `.gitattributes` of the tree: `files` maps each one's path + * to its text. A path is ignored when it, or a directory above it, is `export-ignore`. + */ +export function exportAttributes(files: ReadonlyMap): (path: string) => ExportAttrs { + // Root first, then deeper: a deeper file's lines come later, so they win. + const ordered = [...files.keys()].sort((a, b) => a.split('/').length - b.split('/').length || (a < b ? -1 : 1)) + const rules = ordered.flatMap((p) => { + const slash = p.lastIndexOf('/') + return parseGitAttributes(files.get(p) as string, slash === -1 ? '' : p.slice(0, slash)) + }) + const attr = (path: string, name: 'export-ignore' | 'export-subst'): boolean => { + let value = false + for (const r of rules) { + const v = r.attrs.get(name) + if (v !== undefined && r.match(path)) value = v + } + return value + } + return (path) => { + let ignore = attr(path, 'export-ignore') + for (let at = path.indexOf('/'); !ignore && at !== -1; at = path.indexOf('/', at + 1)) ignore = attr(path.slice(0, at), 'export-ignore') + return { ignore, subst: !ignore && attr(path, 'export-subst') } + } +} + +// --------------------------------------------------------------------------- +// The commit, as `--format` sees it +// --------------------------------------------------------------------------- + +/** An author or committer line, with the time zone the pretty dates are printed in. */ +export interface Person { + readonly name: string + readonly email: string + /** Seconds since the epoch. */ + readonly time: number + /** The zone as written (`+0200`), and its offset in minutes. */ + readonly tz: string + readonly offset: number +} + +/** A commit as `$Format:` expands it. */ +export interface ArchiveCommit { + readonly oid: string + readonly tree: string + readonly parents: readonly string[] + readonly author: Person + readonly committer: Person + readonly message: string +} + +function parsePerson(value: string): Person { + const m = /^(.*?) <([^>]*)> (\d+) ([+-]\d{4})$/.exec(value.trim()) + if (m === null) return { name: value.trim(), email: '', time: 0, tz: '+0000', offset: 0 } + const tz = m[4] as string + const sign = tz.startsWith('-') ? -1 : 1 + const offset = sign * (Number(tz.slice(1, 3)) * 60 + Number(tz.slice(3, 5))) + return { name: m[1] as string, email: m[2] as string, time: Number(m[3]), tz, offset } +} + +/** Parse a commit object's bytes, keeping what `--format` prints. */ +export function parseArchiveCommit(oid: string, bytes: Uint8Array): ArchiveCommit { + const text = new TextDecoder('utf-8', { fatal: false, ignoreBOM: true }).decode(bytes) + const sep = text.indexOf('\n\n') + const lines = (sep === -1 ? text : text.slice(0, sep)).split('\n') + const field = (key: string): string => lines.find((l) => l.startsWith(`${key} `))?.slice(key.length + 1) ?? '' + return { + oid, + tree: field('tree'), + parents: lines.filter((l) => l.startsWith('parent ')).map((l) => l.slice(7)), + author: parsePerson(field('author')), + committer: parsePerson(field('committer')), + message: sep === -1 ? '' : text.slice(sep + 2), + } +} + +const DAYS = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'] +const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] +const pad = (n: number, w = 2): string => String(n).padStart(w, '0') + +/** A person's time in their own zone, as git prints each date format. */ +function dateOf(p: Person, format: 'default' | 'rfc' | 'iso' | 'strict' | 'short'): string { + const d = new Date((p.time + p.offset * 60) * 1000) + const Y = d.getUTCFullYear() + const M = d.getUTCMonth() + const D = d.getUTCDate() + const hms = `${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}` + switch (format) { + case 'default': + return `${DAYS[d.getUTCDay()]} ${MONTHS[M]} ${D} ${hms} ${Y} ${p.tz}` + case 'rfc': + return `${DAYS[d.getUTCDay()]}, ${D} ${MONTHS[M]} ${Y} ${hms} ${p.tz}` + case 'iso': + return `${Y}-${pad(M + 1)}-${pad(D)} ${hms} ${p.tz}` + case 'strict': { + const zone = p.offset === 0 ? 'Z' : `${p.tz.slice(0, 3)}:${p.tz.slice(3)}` + return `${Y}-${pad(M + 1)}-${pad(D)}T${hms}${zone}` + } + case 'short': + return `${Y}-${pad(M + 1)}-${pad(D)}` + } +} + +/** The message's subject (its first paragraph, lines joined by spaces) and body. */ +function subjectAndBody(message: string): { subject: string; body: string } { + const lines = message.split('\n') + let i = 0 + while (i < lines.length && (lines[i] as string).trim() === '') i++ + const subject: string[] = [] + for (; i < lines.length && (lines[i] as string).trim() !== ''; i++) subject.push((lines[i] as string).trim()) + while (i < lines.length && (lines[i] as string).trim() === '') i++ + return { subject: subject.join(' '), body: lines.slice(i).join('\n') } +} + +/** What `$Format:` placeholders need besides the commit. */ +export interface FormatContext { + readonly commit: ArchiveCommit + /** An oid's shortest unique abbreviation of at least `min` digits (`%h`, and `%(describe)`'s suffix). */ + readonly abbrev: (oid: string, min?: number) => string + /** `%(describe…)`: the expansion, or null to leave the placeholder as written. */ + readonly describe: (options: DescribeOptions) => string | null + /** The refs pointing at the commit, as `%D` prints them (`tag: v1`, `main`). */ + readonly decorations: readonly string[] +} + +/** `%(describe)`'s options. */ +export interface DescribeOptions { + readonly tags: boolean + readonly abbrev: number | null + readonly match: readonly string[] + readonly exclude: readonly string[] +} + +/** Parse `%(describe:opt,opt=…)`'s option list; null when an option is not one git knows. */ +export function parseDescribeOptions(spec: string): DescribeOptions | null { + const out = { tags: false, abbrev: null as number | null, match: [] as string[], exclude: [] as string[] } + if (spec === '') return out + for (const opt of spec.split(',')) { + const [key, value] = [opt.split('=')[0] as string, opt.includes('=') ? opt.slice(opt.indexOf('=') + 1) : null] + if (key === 'tags') out.tags = value === null || !/^(false|no|off|0)$/i.test(value) + else if (key === 'abbrev' && value !== null && /^\d+$/.test(value)) out.abbrev = Number(value) + else if (key === 'match' && value !== null) out.match.push(value) + else if (key === 'exclude' && value !== null) out.exclude.push(value) + else return null + } + return out +} + +/** + * Expand a pretty format (`git log --format=`) for one commit: the placeholders a commit + * alone answers. One git would print from a mailmap, a reflog, notes or GPG checks, and anything + * unknown, is left as written, as git leaves an unknown `%` sequence. + */ +export function formatCommit(fmt: string, ctx: FormatContext): string { + const { commit } = ctx + const { subject, body } = subjectAndBody(commit.message) + const person = (who: 'a' | 'c', key: string): string | null => { + const p = who === 'a' ? commit.author : commit.committer + switch (key) { + case 'n': + case 'N': + return p.name + case 'e': + case 'E': + return p.email + case 'l': + case 'L': + return p.email.split('@')[0] ?? '' + case 'd': + return dateOf(p, 'default') + case 'D': + return dateOf(p, 'rfc') + case 'i': + return dateOf(p, 'iso') + case 'I': + return dateOf(p, 'strict') + case 's': + return dateOf(p, 'short') + case 't': + return String(p.time) + default: + return null + } + } + let out = '' + for (let i = 0; i < fmt.length; i++) { + const c = fmt[i] as string + if (c !== '%' || i + 1 >= fmt.length) { + out += c + continue + } + const n = fmt[i + 1] as string + let value: string | null = null + let used = 2 + if (n === '(') { + const close = fmt.indexOf(')', i + 2) + const inner = close === -1 ? null : fmt.slice(i + 2, close) + if (inner !== null && (inner === 'describe' || inner.startsWith('describe:'))) { + const opts = parseDescribeOptions(inner === 'describe' ? '' : inner.slice('describe:'.length)) + value = opts === null ? null : ctx.describe(opts) + used = close - i + 1 + } + } else if (n === 'x' && /^[0-9a-fA-F]{2}$/.test(fmt.slice(i + 2, i + 4))) { + value = String.fromCharCode(parseInt(fmt.slice(i + 2, i + 4), 16)) + used = 4 + } else if ((n === 'a' || n === 'c') && i + 2 < fmt.length) { + value = person(n, fmt[i + 2] as string) + used = 3 + } else { + const decorated = ctx.decorations.join(', ') + const simple: Record = { + H: commit.oid, + h: ctx.abbrev(commit.oid), + T: commit.tree, + t: ctx.abbrev(commit.tree), + P: commit.parents.join(' '), + p: commit.parents.map((p) => ctx.abbrev(p)).join(' '), + s: subject, + f: subject.replace(/[^A-Za-z0-9.]+/g, '-').replace(/^[-.]+|[-.]+$/g, ''), + b: body, + B: commit.message, + n: '\n', + '%': '%', + D: decorated, + d: decorated === '' ? '' : ` (${decorated})`, + } + value = simple[n] ?? null + } + if (value === null) { + out += c + continue + } + out += value + i += used - 1 + } + return out +} + +/** + * `export-subst` (gitattributes(5)): each `$Format:$` in the file replaced by the commit's + * `` expansion. The format runs to the next `$`. + */ +export function expandExportSubst(text: string, ctx: FormatContext): string { + return text.replace(/\$Format:([^$]*)\$/g, (_, fmt: string) => formatCommit(fmt, ctx)) +} + +/** The shortest unique abbreviation of `oid` of at least `min` digits, as git finds one. */ +export function uniqueAbbrev(oid: string, min: number, findByPrefix?: (prefix: string, limit?: number) => string[]): string { + let len = Math.max(4, min) + if (findByPrefix === undefined) return oid.slice(0, len) + while (len < oid.length && findByPrefix(oid.slice(0, len), 2).length > 1) len++ + return oid.slice(0, len) +} + +/** git's automatic abbreviation for a repository of `objects` objects (`core.abbrev` unset): at least 7. */ +export function autoAbbrevLength(objects: number): number { + if (!(objects > 0)) return 7 + const msb = Math.floor(Math.log2(objects)) + return Math.max(7, Math.floor((msb + 1 + 1) / 2)) +} + +// --------------------------------------------------------------------------- +// git describe +// --------------------------------------------------------------------------- + +/** A tag `git describe` may name: its short name, and what it points at. */ +export interface DescribeTag { + readonly name: string + /** The tag ref's object: an annotated tag object, or the commit a lightweight tag names. */ + readonly oid: string +} + +/** Commits {@link describeCommit} walks at most before it gives up (it then says so, and nothing is written). */ +export const DESCRIBE_COMMIT_CAP = 100_000 +/** git describe's `--candidates` default. */ +const MAX_CANDIDATES = 10 + +interface Named { + readonly name: string + /** 2 annotated, 1 lightweight (git's `prio`). */ + readonly prio: number + /** The tagger time (s), to prefer the newer of two annotated tags of one commit. */ + readonly date: number +} + +/** Tag objects are read this many at a time. */ +const TAG_READ_POOL = 16 + +/** A shell glob (`match=`, `exclude=`) as git's `wildmatch` without `WM_PATHNAME`. */ +function globRegex(glob: string): RegExp { + return new RegExp(`^${wildmatchRegex(glob).source.slice(1, -1).replace(/\[\^\/\]\*/g, '.*').replace(/\[\^\/\]/g, '.')}$`) +} + +/** + * The commits the tags name, as `git describe` keys them (every tag peeled to its commit): for each + * commit, the best name (annotated over lightweight, then the newer tagger date). + */ +export async function taggedCommits(reader: ObjectReader, tags: readonly DescribeTag[], options: Pick): Promise> { + const match = options.match.map(globRegex) + const exclude = options.exclude.map(globRegex) + const wanted = tags.filter((t) => (match.length === 0 || match.some((r) => r.test(t.name))) && !exclude.some((r) => r.test(t.name))) + const named = new Map() + await mapPooled(wanted, TAG_READ_POOL, async (t) => { + let at = t.oid + let prio = 1 + let date = 0 + // Peel through nested tags; the outermost tag's date is the one that counts. + for (let depth = 0; depth < 10; depth++) { + const obj = await reader.readObject(at) + if (obj.type !== 'tag') { + if (obj.type !== 'commit') return + break + } + const tag = parseTag(obj.bytes) + if (tag === null || !OID_HEX.test(tag.object)) return + if (depth === 0) { + prio = 2 + const tagger = /\ntagger .* (\d+) [+-]\d{4}\n/.exec(`\n${new TextDecoder().decode(obj.bytes)}`) + date = tagger === null ? 0 : Number(tagger[1]) + } + at = tag.object + } + const had = named.get(at) + if (had === undefined || prio > had.prio || (prio === 2 && had.prio === 2 && date > had.date)) named.set(at, { name: t.name, prio, date }) + }) + return named +} + +/** `git describe --abbrev= [--tags] ` could not be worked out. */ +export class DescribeError extends Error { + constructor(message: string) { + super(message) + this.name = 'DescribeError' + } +} + +/** + * `git describe` of `commitOid` (builtin/describe.c): the nearest tag by the number of commits it + * does not contain, with `--g` after it unless it names the commit itself, and + * `abbrev` 0 for the tag alone. By default only annotated tags count (`tags` for all). Walks the + * history newest first as git does (up to 10 candidates, then the depth of the best one); null + * when no tag is reachable. + */ +export async function describeCommit( + reader: ObjectReader, + commitOid: string, + named: ReadonlyMap, + options: Pick, + abbrevOf: (oid: string, min: number) => string, + cap = DESCRIBE_COMMIT_CAP, +): Promise { + const eligible = (n: Named | undefined): n is Named => n !== undefined && (options.tags || n.prio === 2) + const suffix = (depth: number): string => (options.abbrev === 0 ? '' : `-${depth}-g${abbrevOf(commitOid, options.abbrev ?? 7)}`) + const exact = named.get(commitOid) + if (eligible(exact)) return exact.name + + interface Node { + when: number + parents: readonly string[] + flags: number + } + const SEEN = 1 + const nodes = new Map() + const load = async (oid: string): Promise => { + const known = nodes.get(oid) + if (known !== undefined) return known + if (nodes.size >= cap) throw new DescribeError(`gave up after ${cap} commits`) + const obj = await reader.readObject(oid) + if (obj.type !== 'commit') throw new DescribeError(`${oid.slice(0, 12)} is not a commit`) + const c = parseArchiveCommit(oid, obj.bytes) + const node = { when: c.committer.time, parents: c.parents, flags: 0 } + nodes.set(oid, node) + return node + } + // git's prio_queue by commit date, newest first; ties in insertion order. + const queue: { oid: string; when: number; seq: number }[] = [] + let seq = 0 + const put = (oid: string, when: number): void => { + queue.push({ oid, when, seq: seq++ }) + } + const get = (): string => { + let best = 0 + for (let i = 1; i < queue.length; i++) { + const q = queue[i] as (typeof queue)[number] + const b = queue[best] as (typeof queue)[number] + if (q.when > b.when || (q.when === b.when && q.seq < b.seq)) best = i + } + return (queue.splice(best, 1)[0] as (typeof queue)[number]).oid + } + const visitParents = async (c: Node): Promise => { + const parents = await Promise.all(c.parents.map(load)) + parents.forEach((p, i) => { + if ((p.flags & SEEN) === 0) put(c.parents[i] as string, p.when) + p.flags |= c.flags + }) + } + + const matches: { name: string; depth: number; flag: number; order: number }[] = [] + let annotated = 0 + let seen = 0 + let gaveUpOn: string | null = null + const start = await load(commitOid) + start.flags = SEEN + put(commitOid, start.when) + while (queue.length > 0) { + const oid = get() + const c = nodes.get(oid) as Node + seen++ + const n = named.get(oid) + if (n !== undefined) { + if (!options.tags && n.prio < 2) { + // git counts it, and goes on. + } else if (matches.length < MAX_CANDIDATES) { + const flag = 1 << (matches.length + 1) + matches.push({ name: n.name, depth: seen - 1, flag, order: matches.length + 1 }) + c.flags |= flag + if (n.prio === 2) annotated++ + } else { + gaveUpOn = oid + break + } + } + for (const m of matches) if ((c.flags & m.flag) === 0) m.depth++ + if (annotated > 0 && queue.length === 0) break + await visitParents(c) + } + if (matches.length === 0) return null + matches.sort((a, b) => a.depth - b.depth || a.order - b.order) + const best = matches[0] as (typeof matches)[number] + if (gaveUpOn !== null) put(gaveUpOn, (nodes.get(gaveUpOn) as Node).when) + // finish_depth_computation: walk on until every queued commit is within the best tag. + while (queue.length > 0) { + const oid = get() + const c = nodes.get(oid) as Node + if ((c.flags & best.flag) !== 0) { + if (queue.every((q) => ((nodes.get(q.oid) as Node).flags & best.flag) !== 0)) break + } else best.depth++ + await visitParents(c) + } + return `${best.name}${suffix(best.depth)}` +} diff --git a/forge-web/lib/view/zip-entries.ts b/forge-web/lib/view/zip-entries.ts new file mode 100644 index 000000000..feb0cd1a3 --- /dev/null +++ b/forge-web/lib/view/zip-entries.ts @@ -0,0 +1,79 @@ +/** + * Zip entries as `git archive --format=zip` writes them (QW-026; git's archive-zip.c), for the + * zip worker: kept apart from it so the tests run the same code on fflate directly. + * + * - An executable is recorded with its Unix mode (`0100755`, "made by" Unix) so it unpacks + * executable; a symlink as `0120777` with its target as its content, so it unpacks as a link; + * a regular file with no Unix mode, as git writes it (unzip then applies the umask). + * - Every directory gets its own entry, as git archive writes one. + * - Every entry carries the commit's time: as the DOS date (local time, as git's `localtime`) and + * as the `UT` extended timestamp (UTC seconds), which unzip prefers. + * - The archive comment is the commit id, as git writes it. + */ + +import type { Zippable, ZipOptions } from 'fflate' + +/** The message the zip worker takes. */ +export interface ZipMessage { + readonly entries: Record + readonly meta: { + readonly modes: Readonly> + readonly mtime: number + readonly comment: string | null + } | null +} + +const S_IFMT = 0o170000 +const S_IFLNK = 0o120000 +/** Zip's "version made by" high byte for Unix, and the DOS directory attribute. */ +const OS_UNIX = 3 +const DOS_DIRECTORY = 0x10 +/** The `UT` extended timestamp's header id (APPNOTE 4.6.1 / Info-ZIP). */ +const EXTENDED_TIMESTAMP = 0x5455 + +/** The `UT` field: flags (modification time present) and the time in UTC seconds. */ +function extendedTimestamp(mtime: number): Uint8Array { + const field = new Uint8Array(5) + field[0] = 1 + new DataView(field.buffer).setUint32(1, Math.floor(mtime / 1000) >>> 0, true) + return field +} + +/** The attributes git archive records for a file of git mode `mode`. */ +export function entryAttributes(mode: number): Pick { + if ((mode & S_IFMT) === S_IFLNK) return { os: OS_UNIX, attrs: ((mode | 0o777) << 16) >>> 0 } + if ((mode & 0o111) !== 0) return { os: OS_UNIX, attrs: (mode << 16) >>> 0 } + return { os: 0, attrs: 0 } +} + +/** `entries` with each file's attributes, every directory's entry, and the commit's time. */ +export function zipEntries(entries: Record, meta: NonNullable): Zippable { + const mtime = new Date(meta.mtime) + const extra = { [EXTENDED_TIMESTAMP]: extendedTimestamp(meta.mtime) } + const out: Zippable = {} + const dirs = new Set() + for (const path of Object.keys(entries)) { + for (let at = path.indexOf('/'); at !== -1; at = path.indexOf('/', at + 1)) dirs.add(path.slice(0, at + 1)) + } + // Directories first, in path order, then the files (git writes each directory before its contents). + for (const dir of [...dirs].sort()) out[dir] = [new Uint8Array(0), { os: 0, attrs: DOS_DIRECTORY, mtime, extra, level: 0 }] + for (const [path, bytes] of Object.entries(entries)) { + out[path] = [bytes, { ...entryAttributes(meta.modes[path] ?? 0o100644), mtime, extra }] + } + return out +} + +/** `zip` with `comment` as its archive comment (the end-of-central-directory record's). */ +export function withArchiveComment(zip: Uint8Array, comment: string | null): Uint8Array { + if (comment === null || comment === '' || zip.length < 22) return zip + const bytes = new TextEncoder().encode(comment).subarray(0, 0xffff) + const eocd = zip.length - 22 + const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength) + // Only a zip that ends in a comment-less end record (fflate writes one). + if (view.getUint32(eocd, true) !== 0x06054b50 || view.getUint16(eocd + 20, true) !== 0) return zip + const out = new Uint8Array(zip.length + bytes.length) + out.set(zip) + out.set(bytes, zip.length) + new DataView(out.buffer).setUint16(eocd + 20, bytes.length, true) + return out +} diff --git a/forge-web/lib/view/zip.ts b/forge-web/lib/view/zip.ts index 4d4f31d87..8fb5f6ad4 100644 --- a/forge-web/lib/view/zip.ts +++ b/forge-web/lib/view/zip.ts @@ -8,7 +8,25 @@ import { MODE_GITLINK, MODE_TREE } from '../browse' import type { ObjectReader } from './tree-nav' -import { ObjectTypeError, peel, readCommit, readTree } from './tree-nav' +import { ObjectTypeError, peel, readBlob, readTree } from './tree-nav' +import { decodeTextBlob } from './git-objects' +import { historyWalker } from './commit-log' +import { + autoAbbrevLength, + describeCommit, + DescribeError, + expandExportSubst, + exportAttributes, + parseArchiveCommit, + parseDescribeOptions, + taggedCommits, + uniqueAbbrev, + type ArchiveCommit, + type DescribeOptions, + type DescribeTag, + type FormatContext, +} from './archive' +import type { ZipMessage } from './zip-entries' import { mapPooled } from './pool' /** The largest ref the browser zips (uncompressed bytes). Above it: clone instead. */ @@ -92,10 +110,140 @@ export async function walkFiles( * nested tags (L-01): the commit's tree, or a tagged tree itself. A tagged blob has no tree. */ export async function listFiles(reader: ObjectReader, tipOid: string): Promise { + return (await planArchive(reader, tipOid)).files +} + +/** What `git archive` writes for a ref (QW-026): the files, and what it does to them. */ +export interface ArchivePlan { + /** The files, those `export-ignore` leaves out already left out. */ + readonly files: ZipFile[] + /** The files whose `$Format:…$` are expanded (`export-subst`). */ + readonly subst: ReadonlySet + /** The commit archived, or null for a tag of a tree (git archive then has no commit to describe). */ + readonly commit: ArchiveCommit | null + /** Every entry's time (ms): the committer time, as git archive stamps it; now for a tree. */ + readonly mtime: number +} + +/** + * The files of a ref's tip, as `git archive` takes them: the tree's own `.gitattributes` decide + * which are left out (`export-ignore`) and which are expanded (`export-subst`), and the commit's + * time is every entry's. + */ +export async function planArchive(reader: ObjectReader, tipOid: string): Promise { const tip = await peel(reader, tipOid) - const tree = tip.type === 'commit' ? (await readCommit(reader, tip.oid)).tree : tip.type === 'tree' ? tip.oid : null + let commit: ArchiveCommit | null = null + let tree: string | null = tip.type === 'tree' ? tip.oid : null + if (tip.type === 'commit') { + const obj = await reader.readObject(tip.oid) + if (obj.type !== 'commit') throw new ObjectTypeError(tip.oid, obj.type, 'commit') + commit = parseArchiveCommit(tip.oid, obj.bytes) + tree = commit.tree + } if (tree === null) throw new ObjectTypeError(tip.oid, tip.type, 'tree') - return (await walkFiles(reader, tree)).files + const all = (await walkFiles(reader, tree)).files + // The tree's attributes, never the viewer's: every `.gitattributes` it holds (a few KiB). + const attrFiles = new Map() + await mapPooled( + all.filter((f) => f.path === '.gitattributes' || f.path.endsWith('/.gitattributes')), + 6, + async (f) => { + const text = decodeTextBlob(await readBlob(reader, f.oid, ATTRIBUTES_MAX_BYTES).catch(() => new Uint8Array())) + if (text !== null) attrFiles.set(f.path, text) + }, + ) + const attrs = exportAttributes(attrFiles) + const files: ZipFile[] = [] + const subst = new Set() + for (const f of all) { + const a = attrs(f.path) + if (a.ignore) continue + files.push(f) + // A symlink's target is not a file's text: git expands regular files only. + if (a.subst && commit !== null && (f.mode & 0o170000) === 0o100000) subst.add(f.path) + } + return { files, subst, commit, mtime: commit === null ? Date.now() : commit.committer.time * 1000 } +} + +/** A `.gitattributes` larger than this is not read (git's own limit is 100 MB; real ones are a few KiB). */ +const ATTRIBUTES_MAX_BYTES = 1024 * 1024 + +/** The refs `%(describe)` and `%d` read. */ +export interface ArchiveRefs { + readonly tags: readonly DescribeTag[] + /** Branch short names and tips. */ + readonly heads: readonly { readonly name: string; readonly oid: string }[] +} + +/** + * Expand the `export-subst` files of `entries` in place (`$Format:…$`, gitattributes(5)), for the + * plan's commit. `%(describe)` walks the history as `git describe` does, once per option set; + * `%d`/`%D` name the branches and tags at the commit. A file that is not UTF-8 is left as it is. + */ +export async function substituteFiles(reader: ObjectReader & { findByPrefix?(prefix: string, limit?: number): string[]; readonly objectCount?: number }, plan: ArchivePlan, entries: Record, refs: ArchiveRefs): Promise { + const commit = plan.commit + if (commit === null || plan.subst.size === 0) return + const texts = new Map() + for (const path of plan.subst) { + const bytes = entries[path] + if (bytes === undefined) continue + try { + texts.set(path, new TextDecoder('utf-8', { fatal: true }).decode(bytes)) + } catch { + // Not UTF-8: left as stored. + } + } + const all = [...texts.values()].join('\n') + if (!all.includes('$Format:')) return + const findByPrefix = reader.findByPrefix?.bind(reader) + const auto = autoAbbrevLength(reader.objectCount ?? 0) + const abbrev = (oid: string, min = auto): string => uniqueAbbrev(oid, min, findByPrefix) + + // `%(describe…)` is worked out before the (synchronous) expansion, once per option set. + const described = new Map() + for (const [, spec] of all.matchAll(/%\(describe(?::([^)]*))?\)/g)) { + const key = spec ?? '' + if (described.has(key)) continue + const opts = parseDescribeOptions(key) + if (opts === null) { + described.set(key, null) + continue + } + const named = await taggedCommits(reader, refs.tags, opts) + const walker = historyWalker(reader) + try { + described.set(key, await describeCommit(walker, commit.oid, named, opts, abbrev)) + } catch (e) { + if (!(e instanceof DescribeError)) throw e + described.set(key, null) + } finally { + walker.flush?.() + } + } + // `%d` / `%D`: the branches, then the tags, at the commit. + let decorations: string[] = [] + if (/%[dD]/.test(all)) { + const tagged = await taggedCommits(reader, refs.tags, { match: [], exclude: [] }) + const tagNames = refs.tags.filter((t) => t.oid === commit.oid || tagged.get(commit.oid)?.name === t.name).map((t) => `tag: ${t.name}`) + decorations = [...refs.heads.filter((h) => h.oid === commit.oid).map((h) => h.name), ...tagNames] + } + const ctx: FormatContext = { + commit, + abbrev, + decorations, + describe: (opts) => described.get(describeKey(opts)) ?? null, + } + for (const [path, text] of texts) entries[path] = new TextEncoder().encode(expandExportSubst(text, ctx)) +} + +/** The `%(describe:…)` spec an option set came from, to find its worked-out value. */ +function describeKey(opts: DescribeOptions): string { + const parts: string[] = [] + if (opts.tags) parts.push('tags') + if (opts.abbrev !== null) parts.push(`abbrev=${opts.abbrev}`) + for (const m of opts.match) parts.push(`match=${m}`) + for (const x of opts.exclude) parts.push(`exclude=${x}`) + return parts.join(',') } /** Stored (compressed-on-disk) sizes from the locator: a lower bound, cheap to sum. */ @@ -126,11 +274,20 @@ export async function readZipFiles( return entries } +/** + * How each entry is recorded, as `git archive --format=zip` records it (archive-zip.c): a file's + * git mode (`0100755` for an executable, `0120000` for a symlink; a regular file carries no Unix + * mode, as git writes it), every entry at `mtime` (DOS time and the `UT` extended timestamp), and + * the commit id as the archive comment. + */ +export type ZipMeta = NonNullable + /** Compress in a worker (fflate); resolves with the zip bytes. */ export function compressInWorker( entries: Record, onProgress?: (p: ZipProgress) => void, signal?: AbortSignal, + meta?: ZipMeta, ): Promise { const count = Object.keys(entries).length const bytes = Object.values(entries).reduce((n, b) => n + b.length, 0) @@ -150,7 +307,7 @@ export function compressInWorker( worker.terminate() reject(new Error('cancelled')) }) - worker.postMessage(entries, Object.values(entries).map((b) => b.buffer as ArrayBuffer)) + worker.postMessage({ entries, meta: meta ?? null }, Object.values(entries).map((b) => b.buffer as ArrayBuffer)) }) } diff --git a/forge-web/lib/view/zip.worker.ts b/forge-web/lib/view/zip.worker.ts index 882c340a6..cefe9ce25 100644 --- a/forge-web/lib/view/zip.worker.ts +++ b/forge-web/lib/view/zip.worker.ts @@ -2,16 +2,20 @@ /** * The zip worker: compress `{ path: bytes }` with fflate off the main thread. Every file is - * placed under one top-level directory by the caller's paths, as `git archive` would. + * placed under one top-level directory by the caller's paths, as `git archive --prefix` would. + * With `meta`, each entry is recorded as git archive records it ({@link zipEntries}). */ import { zipSync } from 'fflate' +import { withArchiveComment, zipEntries, type ZipMessage } from './zip-entries' + const scope = self as unknown as DedicatedWorkerGlobalScope -scope.onmessage = (ev: MessageEvent>): void => { +scope.onmessage = (ev: MessageEvent): void => { try { - const zip = zipSync(ev.data, { level: 6 }) + const { entries, meta } = ev.data + const zip = withArchiveComment(zipSync(meta === null ? entries : zipEntries(entries, meta), { level: 6 }), meta?.comment ?? null) scope.postMessage({ ok: true, zip }, [zip.buffer]) } catch (e) { scope.postMessage({ ok: false, error: e instanceof Error ? e.message : String(e) }) From 41360b78be6c4f4f9ed713108d61158a8c3e0a37 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 09:49:24 -0500 Subject: [PATCH 2/2] fix(web): follow git archive and git describe more closely in the zip Review: one %(describe) per archive (the first, whatever its option spelling), empty when no tag describes the commit; the describe walk stops as current git does (every tag found, or ten candidates) and its suffix takes git's automatic abbreviation; tag names resolve in ref-name order (newer tagger date first for two annotated tags) and print the tag object's own name; %d names every tag at the commit; a directory-only pattern (/tests/) is honoured for export-ignore; a pre-1980 commit time no longer fails the zip; the describe walk can be cancelled and shows its progress. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/components/repo/clone-box.tsx | 6 +- forge-web/lib/view/archive.test.ts | 76 +++++++- forge-web/lib/view/archive.ts | 226 +++++++++++++++--------- forge-web/lib/view/zip-entries.ts | 13 +- forge-web/lib/view/zip.ts | 79 ++++----- 5 files changed, 261 insertions(+), 139 deletions(-) diff --git a/forge-web/components/repo/clone-box.tsx b/forge-web/components/repo/clone-box.tsx index 9c518a519..2c6d99a50 100644 --- a/forge-web/components/repo/clone-box.tsx +++ b/forge-web/components/repo/clone-box.tsx @@ -115,7 +115,7 @@ function ZipDownload({ home, addr, selected }: { home: RepoHome; addr: RepoAddre const stored = storedSize(files) if (stored > ZIP_MAX_BYTES) throw new ZipTooLargeError(stored) const entries = await readZipFiles(reader, files, setProgress, cancel.current.signal) - await substituteFiles(reader, plan, entries, archiveRefs(home)) + await substituteFiles(reader, plan, entries, archiveRefs(home), { signal: cancel.current.signal, onProgress: setProgress }) const name = zipFileName(addr.name, selected.name) const prefix = `${name.replace(/\.zip$/, '')}/` const rooted: Record = {} @@ -141,7 +141,9 @@ function ZipDownload({ home, addr, selected }: { home: RepoHome; addr: RepoAddre ? 'Listing files…' : progress.phase === 'reading' ? `Reading ${progress.files} of ${plural(progress.filesTotal, 'file')} (${formatBytes(progress.bytes)})` - : `Compressing ${formatBytes(progress.bytes)}…` + : progress.phase === 'describing' + ? 'Describing the commit for export-subst (git describe)…' + : `Compressing ${formatBytes(progress.bytes)}…` return (
diff --git a/forge-web/lib/view/archive.test.ts b/forge-web/lib/view/archive.test.ts index c3e90252d..95276b1d6 100644 --- a/forge-web/lib/view/archive.test.ts +++ b/forge-web/lib/view/archive.test.ts @@ -16,8 +16,9 @@ import { describeCommit, exportAttributes, formatCommit, + describeNames, parseArchiveCommit, - taggedCommits, + peelTags, uniqueAbbrev, type DescribeTag, type FormatContext, @@ -70,6 +71,15 @@ describe('.gitattributes for an archive', () => { expect(attrs('docs/c.png').ignore).toBe(true) expect(attrs('odd name.txt').ignore).toBe(true) }) + + it('takes a directory-only pattern for the directory and everything in it, never for a file of that name', () => { + const attrs = exportAttributes(new Map([['.gitattributes', '/tests/ export-ignore\nfixtures/ export-ignore\n']])) + expect(attrs('tests/a_test.c').ignore).toBe(true) + expect(attrs('src/tests/a.c').ignore).toBe(false) + expect(attrs('lib/fixtures/data.json').ignore).toBe(true) + // A file named like the directory pattern is not a directory. + expect(attrs('fixtures').ignore).toBe(false) + }) }) describe('$Format: placeholders', () => { @@ -82,7 +92,7 @@ describe('$Format: placeholders', () => { `tree ${tree}\nparent ${parent}\nauthor Ada Lovelace 1700000000 +0200\ncommitter Bob 1700003600 -0530\n\nFix the thing\nacross two lines\n\nThe body.\n`, ) const commit = parseArchiveCommit(oid, s.objects.get(oid)!.bytes) - const ctx: FormatContext = { commit, abbrev: (o, min = 7) => o.slice(0, min), describe: () => 'v1.0-3-gabcdef', decorations: ['main', 'tag: v1.1'] } + const ctx: FormatContext = { commit, abbrev: (o, min = 7) => o.slice(0, min), describe: (spec) => (spec === 'bogus' ? null : 'v1.0-3-gabcdef'), decorations: ['main', 'tag: v1.1'] } it('expands what git log --format would', () => { expect(formatCommit('%H|%h|%T|%P|%p', ctx)).toBe(`${oid}|${oid.slice(0, 7)}|${tree}|${parent}|${parent.slice(0, 7)}`) @@ -108,6 +118,55 @@ describe('$Format: placeholders', () => { }) }) +describe('substituteFiles, as git archive expands', () => { + /** A two-commit repo whose tip has `files`, an annotated tag on the root and two tags on the tip. */ + function repo(files: Record, { tagged = true, when = 1_700_000_000 } = {}) { + const s = new Store() + const root = s.commit(s.files({ a: '1\n' }), [], 'root') + const tree = s.files({ '.gitattributes': '*.txt export-subst\n', ...files }) + const tip = raw(s, 'commit', `tree ${tree}\nparent ${root}\nauthor A ${when} +0000\ncommitter A ${when} +0000\n\ntip\n`) + const tag = (target: string, name: string, time: number): DescribeTag => ({ + name, + oid: raw(s, 'tag', `object ${target}\ntype commit\ntag ${name}\ntagger T ${time} +0000\n\n${name}\n`), + }) + const tags = tagged ? [tag(root, 'v1.0', 1), tag(tip, 'v2.0', 5), tag(tip, 'v2.0-final', 9)] : [] + return { s, tip, tags } + } + async function expand(files: Record, opts: { tagged?: boolean; when?: number } = {}): Promise> { + const { s, tip, tags } = repo(files, opts) + const reader = Object.assign(s.reader(), { findByPrefix: findIn(s), objectCount: s.objects.size }) + const plan = await planArchive(reader, tip) + const entries = await readZipFiles(reader, plan.files, () => undefined) + await substituteFiles(reader, plan, entries, { tags, heads: [{ name: 'main', oid: tip }] }) + return Object.fromEntries(Object.entries(entries).map(([k, v]) => [k, new TextDecoder().decode(v)])) + } + + it('expands the first %(describe) whatever its option spelling, and leaves the ones after it', async () => { + const got = await expand({ 'a.txt': '$Format:%(describe:tags=true)$ $Format:%(describe)$\n', 'b.txt': '$Format:%(describe:abbrev=0)$\n' }) + // Both tags are on the tip: the newer tagger date wins, as describe's replace_name picks. + expect(got['a.txt']).toBe('v2.0-final %(describe)\n') + expect(got['b.txt']).toBe('%(describe:abbrev=0)\n') + }) + + it('writes nothing for %(describe) when no tag describes the commit, as git does', async () => { + expect((await expand({ 'a.txt': 'v="$Format:%(describe)$"\n' }, { tagged: false }))['a.txt']).toBe('v=""\n') + }) + + it('names every tag at the commit in %d', async () => { + expect((await expand({ 'a.txt': '$Format:%d$\n' }))['a.txt']).toBe(' (main, tag: v2.0, tag: v2.0-final)\n') + }) + + it('zips a commit dated before 1980 (the DOS date is clamped; the UT field keeps the time)', async () => { + const { s, tip } = repo({ 'a.txt': 'x\n' }, { tagged: false, when: 86_400 }) + const reader = s.reader() + const plan = await planArchive(reader, tip) + const entries = await readZipFiles(reader, plan.files, () => undefined) + const zip = zipSync(zipEntries(entries, { modes: {}, mtime: plan.mtime, comment: tip }), { level: 6 }) + expect(Object.keys(unzipSync(zip))).toContain('a.txt') + expect(centralDirectory(zip).get('a.txt')?.ut).toBe(86_400) + }) +}) + /** A history with merges, annotated and lightweight tags, some tags on one commit. */ function taggedHistory(s: Store): { commits: string[]; tags: DescribeTag[] } { let x = 7 @@ -150,11 +209,12 @@ describe.skipIf(!HAVE_GIT)('git describe, against git', () => { writeLiterally(dir, s.objects.values()) for (const t of tags) git(dir, ['update-ref', `refs/tags/${t.name}`, t.oid]) const reader = s.reader() - const abbrevOf = (oid: string, min: number): string => uniqueAbbrev(oid, min, findIn(s)) - for (const opts of [{ tags: false, abbrev: 12 }, { tags: true, abbrev: 7 }, { tags: false, abbrev: 0 }]) { - const named = await taggedCommits(reader, tags, { match: [], exclude: [] }) + const abbrevOf = (oid: string, min = 7): string => uniqueAbbrev(oid, min, findIn(s)) + const peeled = await peelTags(reader, tags) + for (const opts of [{ tags: false, abbrev: 12 }, { tags: true, abbrev: 7 }, { tags: false, abbrev: 0 }, { tags: false, abbrev: null }]) { + const named = describeNames(peeled, { match: [], exclude: [] }) for (const c of commits) { - const want = git(dir, ['describe', `--abbrev=${opts.abbrev}`, ...(opts.tags ? ['--tags'] : []), c]) + const want = git(dir, ['describe', ...(opts.abbrev === null ? [] : [`--abbrev=${opts.abbrev}`]), ...(opts.tags ? ['--tags'] : []), c]) const got = await describeCommit(reader, c, named, opts, abbrevOf) expect(got ?? '', `${c} ${JSON.stringify(opts)}`).toBe(want.status === 0 ? want.out : '') } @@ -172,9 +232,9 @@ describe.skipIf(!HAVE_GIT)('git describe, against git', () => { writeLiterally(dir, s.objects.values()) for (const t of tags) git(dir, ['update-ref', `refs/tags/${t.name}`, t.oid]) const reader = s.reader() - const named = await taggedCommits(reader, tags, { match: ['v*'], exclude: ['*-again'] }) + const named = describeNames(await peelTags(reader, tags), { match: ['v*'], exclude: ['*-again'] }) const tip = commits[commits.length - 1] as string - const got = await describeCommit(reader, tip, named, { tags: false, abbrev: 7 }, (o, m) => uniqueAbbrev(o, m, findIn(s))) + const got = await describeCommit(reader, tip, named, { tags: false, abbrev: 7 }, (o, m = 7) => uniqueAbbrev(o, m, findIn(s))) expect(got).toBe(git(dir, ['describe', '--abbrev=7', '--match', 'v*', '--exclude', '*-again', tip]).out) } finally { done() diff --git a/forge-web/lib/view/archive.ts b/forge-web/lib/view/archive.ts index 386b16692..d9cf086f5 100644 --- a/forge-web/lib/view/archive.ts +++ b/forge-web/lib/view/archive.ts @@ -23,7 +23,7 @@ import { mapPooled } from './pool' /** One `.gitattributes` line, reduced to the attributes an archive reads. */ interface AttrRule { - readonly match: (path: string) => boolean + readonly match: (path: string, isDir: boolean) => boolean /** true: set; false: unset (`-attr` or `!attr`); absent: this line says nothing. */ readonly attrs: ReadonlyMap<'export-ignore' | 'export-subst', boolean> } @@ -75,15 +75,22 @@ function wildmatchRegex(pattern: string): RegExp { return new RegExp(`^${re}$`) } -/** A `.gitattributes` pattern, from its file's directory (`''` = the root). */ -function patternMatcher(pattern: string, dir: string): ((path: string) => boolean) | null { - // A trailing `/` (a directory-only pattern) is not supported by git's attributes: it matches nothing. - if (pattern === '' || pattern.endsWith('/') || pattern.startsWith('!')) return null - const anchored = pattern.includes('/') - const re = wildmatchRegex(anchored ? pattern.replace(/^\//, '') : pattern) +/** + * A `.gitattributes` pattern, from its file's directory (`''` = the root), as attr.c's + * `path_matches`: a trailing `/` makes it match directories only (git archive checks a directory + * as `dir/`), a pattern with no other `/` is matched against the basename, else against the path + * from the file's directory. + */ +function patternMatcher(pattern: string, dir: string): ((path: string, isDir: boolean) => boolean) | null { + if (pattern === '' || pattern.startsWith('!')) return null + const dirOnly = pattern.endsWith('/') + const body = dirOnly ? pattern.replace(/\/+$/, '') : pattern + if (body === '') return null + const anchored = body.includes('/') + const re = wildmatchRegex(anchored ? body.replace(/^\//, '') : body) const prefix = dir === '' ? '' : `${dir}/` - return (path) => { - if (!path.startsWith(prefix)) return false + return (path, isDir) => { + if ((dirOnly && !isDir) || !path.startsWith(prefix)) return false const rel = path.slice(prefix.length) return re.test(anchored ? rel : rel.slice(rel.lastIndexOf('/') + 1)) } @@ -143,18 +150,20 @@ export function exportAttributes(files: ReadonlyMap): (path: str const slash = p.lastIndexOf('/') return parseGitAttributes(files.get(p) as string, slash === -1 ? '' : p.slice(0, slash)) }) - const attr = (path: string, name: 'export-ignore' | 'export-subst'): boolean => { + const attr = (path: string, isDir: boolean, name: 'export-ignore' | 'export-subst'): boolean => { let value = false for (const r of rules) { const v = r.attrs.get(name) - if (v !== undefined && r.match(path)) value = v + if (v !== undefined && r.match(path, isDir)) value = v } return value } return (path) => { - let ignore = attr(path, 'export-ignore') - for (let at = path.indexOf('/'); !ignore && at !== -1; at = path.indexOf('/', at + 1)) ignore = attr(path.slice(0, at), 'export-ignore') - return { ignore, subst: !ignore && attr(path, 'export-subst') } + // A directory left out leaves out everything under it (git archive does not descend into it). + let ignore = false + for (let at = path.indexOf('/'); !ignore && at !== -1; at = path.indexOf('/', at + 1)) ignore = attr(path.slice(0, at), true, 'export-ignore') + ignore ||= attr(path, false, 'export-ignore') + return { ignore, subst: !ignore && attr(path, false, 'export-subst') } } } @@ -251,8 +260,12 @@ export interface FormatContext { readonly commit: ArchiveCommit /** An oid's shortest unique abbreviation of at least `min` digits (`%h`, and `%(describe)`'s suffix). */ readonly abbrev: (oid: string, min?: number) => string - /** `%(describe…)`: the expansion, or null to leave the placeholder as written. */ - readonly describe: (options: DescribeOptions) => string | null + /** + * `%(describe…)` by its option text (`abbrev=12`, `''` for none): the expansion (`''` when no + * tag describes the commit, as git's is then empty), or null to leave the placeholder as + * written (git archive expands one `%(describe)` per archive: the ones after it stay). + */ + readonly describe: (spec: string) => string | null /** The refs pointing at the commit, as `%D` prints them (`tag: v1`, `main`). */ readonly decorations: readonly string[] } @@ -330,8 +343,7 @@ export function formatCommit(fmt: string, ctx: FormatContext): string { const close = fmt.indexOf(')', i + 2) const inner = close === -1 ? null : fmt.slice(i + 2, close) if (inner !== null && (inner === 'describe' || inner.startsWith('describe:'))) { - const opts = parseDescribeOptions(inner === 'describe' ? '' : inner.slice('describe:'.length)) - value = opts === null ? null : ctx.describe(opts) + value = ctx.describe(inner === 'describe' ? '' : inner.slice('describe:'.length)) used = close - i + 1 } } else if (n === 'x' && /^[0-9a-fA-F]{2}$/.test(fmt.slice(i + 2, i + 4))) { @@ -404,19 +416,23 @@ export interface DescribeTag { readonly oid: string } -/** Commits {@link describeCommit} walks at most before it gives up (it then says so, and nothing is written). */ -export const DESCRIBE_COMMIT_CAP = 100_000 -/** git describe's `--candidates` default. */ -const MAX_CANDIDATES = 10 - -interface Named { - readonly name: string +/** A tag peeled to its commit, as `git describe` knows it (builtin/describe.c `commit_name`). */ +export interface PeeledTag { + /** The ref's short name (`v1.0` for `refs/tags/v1.0`). */ + readonly ref: string + readonly commit: string /** 2 annotated, 1 lightweight (git's `prio`). */ - readonly prio: number - /** The tagger time (s), to prefer the newer of two annotated tags of one commit. */ + readonly prio: 1 | 2 + /** The outermost tag object's tagger time (s), 0 for a lightweight tag. */ readonly date: number + /** What describe prints: the tag object's own name for an annotated tag, else the ref's. */ + readonly display: string } +/** Commits {@link describeCommit} walks at most before it gives up. */ +export const DESCRIBE_COMMIT_CAP = 100_000 +/** git describe's `--candidates` default. */ +const MAX_CANDIDATES = 10 /** Tag objects are read this many at a time. */ const TAG_READ_POOL = 16 @@ -426,41 +442,53 @@ function globRegex(glob: string): RegExp { } /** - * The commits the tags name, as `git describe` keys them (every tag peeled to its commit): for each - * commit, the best name (annotated over lightweight, then the newer tagger date). + * Every tag peeled to the commit it names (a tag of a tree or a blob is left out, as describe + * leaves it), in ref-name order. The tag objects are read side by side. */ -export async function taggedCommits(reader: ObjectReader, tags: readonly DescribeTag[], options: Pick): Promise> { - const match = options.match.map(globRegex) - const exclude = options.exclude.map(globRegex) - const wanted = tags.filter((t) => (match.length === 0 || match.some((r) => r.test(t.name))) && !exclude.some((r) => r.test(t.name))) - const named = new Map() - await mapPooled(wanted, TAG_READ_POOL, async (t) => { +export async function peelTags(reader: ObjectReader, tags: readonly DescribeTag[], signal?: AbortSignal): Promise { + const sorted = [...tags].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) + const peeled = await mapPooled(sorted, TAG_READ_POOL, async (t): Promise => { + signal?.throwIfAborted() let at = t.oid - let prio = 1 + let prio: 1 | 2 = 1 let date = 0 - // Peel through nested tags; the outermost tag's date is the one that counts. + let display = t.name for (let depth = 0; depth < 10; depth++) { const obj = await reader.readObject(at) - if (obj.type !== 'tag') { - if (obj.type !== 'commit') return - break - } + if (obj.type !== 'tag') return obj.type === 'commit' ? { ref: t.name, commit: at, prio, date, display } : null const tag = parseTag(obj.bytes) - if (tag === null || !OID_HEX.test(tag.object)) return + if (tag === null || !OID_HEX.test(tag.object)) return null if (depth === 0) { prio = 2 + display = tag.tag || t.name const tagger = /\ntagger .* (\d+) [+-]\d{4}\n/.exec(`\n${new TextDecoder().decode(obj.bytes)}`) date = tagger === null ? 0 : Number(tagger[1]) } at = tag.object } - const had = named.get(at) - if (had === undefined || prio > had.prio || (prio === 2 && had.prio === 2 && date > had.date)) named.set(at, { name: t.name, prio, date }) + return null }) + return peeled.filter((t): t is PeeledTag => t !== null) +} + +/** + * The name `git describe` gives each tagged commit (its `names` map): the tags that pass `match` + * and `exclude`, taken in ref-name order, an annotated tag over a lightweight one and, of two + * annotated ones, the newer tagger date; otherwise the first stays (describe.c `replace_name`). + */ +export function describeNames(peeled: readonly PeeledTag[], options: Pick): Map { + const match = options.match.map(globRegex) + const exclude = options.exclude.map(globRegex) + const named = new Map() + for (const t of peeled) { + if ((match.length > 0 && !match.some((r) => r.test(t.ref))) || exclude.some((r) => r.test(t.ref))) continue + const had = named.get(t.commit) + if (had === undefined || had.prio < t.prio || (had.prio === 2 && t.prio === 2 && had.date < t.date)) named.set(t.commit, t) + } return named } -/** `git describe --abbrev= [--tags] ` could not be worked out. */ +/** The description could not be worked out (the history is past its bound). */ export class DescribeError extends Error { constructor(message: string) { super(message) @@ -469,28 +497,32 @@ export class DescribeError extends Error { } /** - * `git describe` of `commitOid` (builtin/describe.c): the nearest tag by the number of commits it - * does not contain, with `--g` after it unless it names the commit itself, and - * `abbrev` 0 for the tag alone. By default only annotated tags count (`tags` for all). Walks the - * history newest first as git does (up to 10 candidates, then the depth of the best one); null - * when no tag is reachable. + * `git describe [--tags] [--abbrev=] ` (builtin/describe.c `describe_commit`): the + * nearest tag by the number of commits it does not contain, with `--g` after it unless + * it names the commit itself (`abbrev` 0: the tag alone). Only annotated tags count unless `tags`. + * The walk is git's: newest first, up to 10 candidates (or every tag there is), stopping early + * when the last path is covered, then the best candidate's depth finished. `abbrevOf(oid, min)` + * gives an oid's unique abbreviation (`min` undefined: git's automatic length). Null when no tag + * describes the commit (git then fails, and `%(describe)` is empty). */ export async function describeCommit( reader: ObjectReader, commitOid: string, - named: ReadonlyMap, + named: ReadonlyMap, options: Pick, - abbrevOf: (oid: string, min: number) => string, - cap = DESCRIBE_COMMIT_CAP, + abbrevOf: (oid: string, min?: number) => string, + { signal, cap = DESCRIBE_COMMIT_CAP }: { readonly signal?: AbortSignal; readonly cap?: number } = {}, ): Promise { - const eligible = (n: Named | undefined): n is Named => n !== undefined && (options.tags || n.prio === 2) - const suffix = (depth: number): string => (options.abbrev === 0 ? '' : `-${depth}-g${abbrevOf(commitOid, options.abbrev ?? 7)}`) + const suffix = (depth: number): string => `-${depth}-g${abbrevOf(commitOid, options.abbrev ?? undefined)}` const exact = named.get(commitOid) - if (eligible(exact)) return exact.name + if (exact !== undefined && (options.tags || exact.prio === 2)) { + const misnamed = exact.prio === 2 && exact.display !== exact.ref + return misnamed ? `${exact.display}${suffix(0)}` : exact.display + } interface Node { - when: number - parents: readonly string[] + readonly when: number + readonly parents: readonly string[] flags: number } const SEEN = 1 @@ -498,19 +530,20 @@ export async function describeCommit( const load = async (oid: string): Promise => { const known = nodes.get(oid) if (known !== undefined) return known + signal?.throwIfAborted() if (nodes.size >= cap) throw new DescribeError(`gave up after ${cap} commits`) const obj = await reader.readObject(oid) if (obj.type !== 'commit') throw new DescribeError(`${oid.slice(0, 12)} is not a commit`) const c = parseArchiveCommit(oid, obj.bytes) - const node = { when: c.committer.time, parents: c.parents, flags: 0 } + const node: Node = { when: c.committer.time, parents: c.parents, flags: 0 } nodes.set(oid, node) return node } // git's prio_queue by commit date, newest first; ties in insertion order. const queue: { oid: string; when: number; seq: number }[] = [] let seq = 0 - const put = (oid: string, when: number): void => { - queue.push({ oid, when, seq: seq++ }) + const put = (oid: string): void => { + queue.push({ oid, when: (nodes.get(oid) as Node).when, seq: seq++ }) } const get = (): string => { let best = 0 @@ -521,55 +554,76 @@ export async function describeCommit( } return (queue.splice(best, 1)[0] as (typeof queue)[number]).oid } - const visitParents = async (c: Node): Promise => { + const flagsOf = (oid: string): number => (nodes.get(oid) as Node).flags + /** Queue `c`'s unseen parents and pass them its flags; `onParent` sees each (was it seen, its flags before). */ + const visitParents = async (c: Node, onParent?: (oid: string, seen: boolean, before: number) => void): Promise => { const parents = await Promise.all(c.parents.map(load)) parents.forEach((p, i) => { - if ((p.flags & SEEN) === 0) put(c.parents[i] as string, p.when) + const oid = c.parents[i] as string + const seen = (p.flags & SEEN) !== 0 + if (!seen) put(oid) + const before = p.flags p.flags |= c.flags + onParent?.(oid, seen, before) }) } const matches: { name: string; depth: number; flag: number; order: number }[] = [] let annotated = 0 - let seen = 0 let gaveUpOn: string | null = null - const start = await load(commitOid) - start.flags = SEEN - put(commitOid, start.when) + let seen = 0 + ;(await load(commitOid)).flags = SEEN + put(commitOid) while (queue.length > 0) { const oid = get() const c = nodes.get(oid) as Node seen++ + if (matches.length === MAX_CANDIDATES || matches.length === named.size) { + gaveUpOn = oid + break + } const n = named.get(oid) - if (n !== undefined) { - if (!options.tags && n.prio < 2) { - // git counts it, and goes on. - } else if (matches.length < MAX_CANDIDATES) { - const flag = 1 << (matches.length + 1) - matches.push({ name: n.name, depth: seen - 1, flag, order: matches.length + 1 }) - c.flags |= flag - if (n.prio === 2) annotated++ - } else { - gaveUpOn = oid - break - } + if (n !== undefined && (options.tags || n.prio === 2)) { + const flag = 1 << (matches.length + 1) + matches.push({ name: n.display, depth: seen - 1, flag, order: matches.length + 1 }) + c.flags |= flag + if (n.prio === 2) annotated++ } for (const m of matches) if ((c.flags & m.flag) === 0) m.depth++ - if (annotated > 0 && queue.length === 0) break + // Stop if the last remaining path is already covered by the best candidate(s). + if (annotated > 0 && queue.length === 0) { + let bestDepth = Infinity + let bestWithin = 0 + for (const m of matches) { + if (m.depth < bestDepth) { + bestDepth = m.depth + bestWithin = m.flag + } else if (m.depth === bestDepth) bestWithin |= m.flag + } + if ((c.flags & bestWithin) === bestWithin) break + } await visitParents(c) } if (matches.length === 0) return null matches.sort((a, b) => a.depth - b.depth || a.order - b.order) const best = matches[0] as (typeof matches)[number] - if (gaveUpOn !== null) put(gaveUpOn, (nodes.get(gaveUpOn) as Node).when) - // finish_depth_computation: walk on until every queued commit is within the best tag. + if (gaveUpOn !== null) put(gaveUpOn) + // finish_depth_computation: on until no queued commit is outside the best candidate. + const unflagged = new Set(queue.filter((q) => (flagsOf(q.oid) & best.flag) === 0).map((q) => q.oid)) while (queue.length > 0) { const oid = get() const c = nodes.get(oid) as Node if ((c.flags & best.flag) !== 0) { - if (queue.every((q) => ((nodes.get(q.oid) as Node).flags & best.flag) !== 0)) break - } else best.depth++ - await visitParents(c) + if (unflagged.size === 0) break + } else { + unflagged.delete(oid) + best.depth++ + } + await visitParents(c, (p, wasSeen, before) => { + const after = flagsOf(p) & best.flag + if (!wasSeen && after === 0) unflagged.add(p) + if (wasSeen && (before & best.flag) === 0 && after !== 0) unflagged.delete(p) + }) } - return `${best.name}${suffix(best.depth)}` + return options.abbrev === 0 ? best.name : `${best.name}${suffix(best.depth)}` } diff --git a/forge-web/lib/view/zip-entries.ts b/forge-web/lib/view/zip-entries.ts index feb0cd1a3..c2f4e2093 100644 --- a/forge-web/lib/view/zip-entries.ts +++ b/forge-web/lib/view/zip-entries.ts @@ -46,9 +46,20 @@ export function entryAttributes(mode: number): Pick return { os: 0, attrs: 0 } } +/** + * The DOS date an entry carries: zip's DOS date holds 1980 to 2107 and fflate takes 1980 to 2099, + * so a commit time outside that (an imported history's 1970 dates) is clamped to its edge rather + * than failing the whole zip. The `UT` field keeps the exact time. + */ +function dosDate(mtime: number): Date { + const earliest = new Date(1980, 0, 1, 0, 0, 0).getTime() + const latest = new Date(2099, 11, 31, 23, 59, 58).getTime() + return new Date(Math.min(Math.max(mtime, earliest), latest)) +} + /** `entries` with each file's attributes, every directory's entry, and the commit's time. */ export function zipEntries(entries: Record, meta: NonNullable): Zippable { - const mtime = new Date(meta.mtime) + const mtime = dosDate(meta.mtime) const extra = { [EXTENDED_TIMESTAMP]: extendedTimestamp(meta.mtime) } const out: Zippable = {} const dirs = new Set() diff --git a/forge-web/lib/view/zip.ts b/forge-web/lib/view/zip.ts index 8fb5f6ad4..1f963a90a 100644 --- a/forge-web/lib/view/zip.ts +++ b/forge-web/lib/view/zip.ts @@ -15,14 +15,14 @@ import { autoAbbrevLength, describeCommit, DescribeError, + describeNames, expandExportSubst, exportAttributes, parseArchiveCommit, parseDescribeOptions, - taggedCommits, + peelTags, uniqueAbbrev, type ArchiveCommit, - type DescribeOptions, type DescribeTag, type FormatContext, } from './archive' @@ -54,7 +54,7 @@ export interface FileWalk { } export interface ZipProgress { - readonly phase: 'listing' | 'reading' | 'compressing' + readonly phase: 'listing' | 'reading' | 'describing' | 'compressing' readonly files: number readonly filesTotal: number readonly bytes: number @@ -177,14 +177,22 @@ export interface ArchiveRefs { /** * Expand the `export-subst` files of `entries` in place (`$Format:…$`, gitattributes(5)), for the - * plan's commit. `%(describe)` walks the history as `git describe` does, once per option set; - * `%d`/`%D` name the branches and tags at the commit. A file that is not UTF-8 is left as it is. + * plan's commit, as git archive does: files in archive order, and one `%(describe)` per archive + * (the first; it is empty when no tag describes the commit, and the ones after it stay as + * written), worked out as `git describe` does; `%d`/`%D` name the branches and tags at the + * commit. A file that is not UTF-8 is left as it is. `signal` stops the describe walk. */ -export async function substituteFiles(reader: ObjectReader & { findByPrefix?(prefix: string, limit?: number): string[]; readonly objectCount?: number }, plan: ArchivePlan, entries: Record, refs: ArchiveRefs): Promise { +export async function substituteFiles( + reader: ObjectReader & { findByPrefix?(prefix: string, limit?: number): string[]; readonly objectCount?: number }, + plan: ArchivePlan, + entries: Record, + refs: ArchiveRefs, + { signal, onProgress }: { readonly signal?: AbortSignal; readonly onProgress?: (p: ZipProgress) => void } = {}, +): Promise { const commit = plan.commit if (commit === null || plan.subst.size === 0) return const texts = new Map() - for (const path of plan.subst) { + for (const path of [...plan.subst].sort()) { const bytes = entries[path] if (bytes === undefined) continue try { @@ -193,59 +201,46 @@ export async function substituteFiles(reader: ObjectReader & { findByPrefix?(pre // Not UTF-8: left as stored. } } - const all = [...texts.values()].join('\n') - if (!all.includes('$Format:')) return + const formats = [...texts.values()].flatMap((t) => [...t.matchAll(/\$Format:([^$]*)\$/g)].map((m) => m[1] as string)) + if (formats.length === 0) return const findByPrefix = reader.findByPrefix?.bind(reader) const auto = autoAbbrevLength(reader.objectCount ?? 0) - const abbrev = (oid: string, min = auto): string => uniqueAbbrev(oid, min, findByPrefix) + const abbrev = (oid: string, min: number = auto): string => uniqueAbbrev(oid, min, findByPrefix) - // `%(describe…)` is worked out before the (synchronous) expansion, once per option set. - const described = new Map() - for (const [, spec] of all.matchAll(/%\(describe(?::([^)]*))?\)/g)) { - const key = spec ?? '' - if (described.has(key)) continue - const opts = parseDescribeOptions(key) - if (opts === null) { - described.set(key, null) - continue - } - const named = await taggedCommits(reader, refs.tags, opts) + // The one `%(describe…)` git archive expands: the first, worked out before the (synchronous) expansion. + const first = formats.map((f) => /%\(describe(?::([^)]*))?\)/.exec(f)).find((m) => m !== null) + const firstSpec = first == null ? null : (first[1] ?? '') + const firstOptions = firstSpec === null ? null : parseDescribeOptions(firstSpec) + const needTags = firstOptions !== null || formats.some((f) => /%[dD]/.test(f)) + const peeled = needTags ? await peelTags(reader, refs.tags, signal) : [] + let described = '' + if (firstOptions !== null) { + onProgress?.({ phase: 'describing', files: 0, filesTotal: 0, bytes: 0 }) const walker = historyWalker(reader) try { - described.set(key, await describeCommit(walker, commit.oid, named, opts, abbrev)) + described = (await describeCommit(walker, commit.oid, describeNames(peeled, firstOptions), firstOptions, abbrev, { ...(signal ? { signal } : {}) })) ?? '' } catch (e) { + // Past its bound: empty, as git's describe would fail. if (!(e instanceof DescribeError)) throw e - described.set(key, null) } finally { walker.flush?.() } } - // `%d` / `%D`: the branches, then the tags, at the commit. - let decorations: string[] = [] - if (/%[dD]/.test(all)) { - const tagged = await taggedCommits(reader, refs.tags, { match: [], exclude: [] }) - const tagNames = refs.tags.filter((t) => t.oid === commit.oid || tagged.get(commit.oid)?.name === t.name).map((t) => `tag: ${t.name}`) - decorations = [...refs.heads.filter((h) => h.oid === commit.oid).map((h) => h.name), ...tagNames] - } + let invoked = false const ctx: FormatContext = { commit, abbrev, - decorations, - describe: (opts) => described.get(describeKey(opts)) ?? null, + // `%d` / `%D`: the branches, then every tag, at the commit. + decorations: [...refs.heads.filter((h) => h.oid === commit.oid).map((h) => h.name), ...peeled.filter((t) => t.commit === commit.oid).map((t) => `tag: ${t.ref}`)], + describe: (spec) => { + if (invoked) return null + invoked = true + return parseDescribeOptions(spec) === null ? null : spec === firstSpec ? described : null + }, } for (const [path, text] of texts) entries[path] = new TextEncoder().encode(expandExportSubst(text, ctx)) } -/** The `%(describe:…)` spec an option set came from, to find its worked-out value. */ -function describeKey(opts: DescribeOptions): string { - const parts: string[] = [] - if (opts.tags) parts.push('tags') - if (opts.abbrev !== null) parts.push(`abbrev=${opts.abbrev}`) - for (const m of opts.match) parts.push(`match=${m}`) - for (const x of opts.exclude) parts.push(`exclude=${x}`) - return parts.join(',') -} - /** Stored (compressed-on-disk) sizes from the locator: a lower bound, cheap to sum. */ export function storedSize(files: readonly ZipFile[]): number { return files.reduce((total, f) => total + f.size, 0)