diff --git a/forge-web/components/repo/clone-box.tsx b/forge-web/components/repo/clone-box.tsx index 7e98a3b59..2c6d99a50 100644 --- a/forge-web/components/repo/clone-box.tsx +++ b/forge-web/components/repo/clone-box.tsx @@ -18,14 +18,16 @@ import type { RepoAddress } from '@/hooks/use-query-param' import { ACTIVE_NETWORK } from '@/lib/constants' import { errorMessage } from '@/lib/utils' import { saveBytes } from '@/lib/view/release-download' -import { formatBytes, plural, selectedTip, type RepoHome, type SelectedRef } from '@/lib/view' +import { formatBytes, plural, selectedTip, tipOidOf, type RepoHome, type SelectedRef } from '@/lib/view' import { repoCommands } from '@/lib/view/repo-commands' import { resolveTip } from '@/lib/view/tip' import { repoKey } from '@/lib/repo' import { compressInWorker, - listFiles, + planArchive, readZipFiles, + substituteFiles, + type ArchiveRefs, storedSize, ZIP_MAX_BYTES, zipFileName, @@ -78,6 +80,16 @@ export function CloneBox({ home, addr, selected }: { home: RepoHome; addr: RepoA ) } +/** The repo's tags and branches, as `%(describe)` and `%d` in an export-subst file read them. */ +function archiveRefs(home: RepoHome): ArchiveRefs { + const named = (refs: RepoHome['tags'], prefix: string) => + refs.flatMap((r) => { + const oid = tipOidOf(r) + return oid === null ? [] : [{ name: r.refName.replace(prefix, ''), oid }] + }) + return { tags: named(home.tags, 'refs/tags/'), heads: named(home.branches, 'refs/heads/') } +} + function ZipDownload({ home, addr, selected }: { home: RepoHome; addr: RepoAddress; selected: SelectedRef }): JSX.Element | null { const state = useBrowseReader(home.repo) const [progress, setProgress] = useState(null) @@ -97,14 +109,20 @@ function ZipDownload({ home, addr, selected }: { home: RepoHome; addr: RepoAddre setProgress({ phase: 'listing', files: 0, filesTotal: 0, bytes: 0 }) try { // A short pinned id resolved to its commit, and a tag to what it names (L-01, L-32). - const files = await listFiles(reader, (await resolveTip(reader, tip, { repoKey: repoKey(home.repo), pinned: selected.pinned !== undefined })).oid) + // As `git archive` (QW-026): the tree's export-ignore and export-subst, modes, the commit's time. + const plan = await planArchive(reader, (await resolveTip(reader, tip, { repoKey: repoKey(home.repo), pinned: selected.pinned !== undefined })).oid) + const { files } = plan const stored = storedSize(files) if (stored > ZIP_MAX_BYTES) throw new ZipTooLargeError(stored) const entries = await readZipFiles(reader, files, setProgress, cancel.current.signal) + await substituteFiles(reader, plan, entries, archiveRefs(home), { signal: cancel.current.signal, onProgress: setProgress }) const name = zipFileName(addr.name, selected.name) + const prefix = `${name.replace(/\.zip$/, '')}/` const rooted: Record = {} - for (const [path, bytes] of Object.entries(entries)) rooted[`${name.replace(/\.zip$/, '')}/${path}`] = bytes - const zip = await compressInWorker(rooted, setProgress, cancel.current.signal) + const modes: Record = {} + for (const f of files) modes[prefix + f.path] = f.mode + for (const [path, bytes] of Object.entries(entries)) rooted[prefix + path] = bytes + const zip = await compressInWorker(rooted, setProgress, cancel.current.signal, { modes, mtime: plan.mtime, comment: plan.commit?.oid ?? null }) saveBytes(zip, name, 'application/zip') setMessage(`Saved ${name} (${formatBytes(zip.length)}, ${plural(files.length, 'file')}, each hash-checked).`) } catch (e) { @@ -123,7 +141,9 @@ function ZipDownload({ home, addr, selected }: { home: RepoHome; addr: RepoAddre ? 'Listing files…' : progress.phase === 'reading' ? `Reading ${progress.files} of ${plural(progress.filesTotal, 'file')} (${formatBytes(progress.bytes)})` - : `Compressing ${formatBytes(progress.bytes)}…` + : progress.phase === 'describing' + ? 'Describing the commit for export-subst (git describe)…' + : `Compressing ${formatBytes(progress.bytes)}…` return (
diff --git a/forge-web/lib/browse/reader.ts b/forge-web/lib/browse/reader.ts index 573a7120a..e29d09aa8 100644 --- a/forge-web/lib/browse/reader.ts +++ b/forge-web/lib/browse/reader.ts @@ -403,6 +403,11 @@ export class BrowseReader { return Object.assign(walker, { flush: () => verdicts?.flush() }) } + /** How many objects the locator indexes (git's automatic abbreviation length grows with it). */ + get objectCount(): number { + return this.locator.count + } + /** OIDs starting with a hex `prefix` ({@link ObjectLocator.findByPrefix}). */ findByPrefix(prefix: string, limit?: number): string[] { return this.locator.findByPrefix(prefix, limit) diff --git a/forge-web/lib/view/archive.test.ts b/forge-web/lib/view/archive.test.ts new file mode 100644 index 000000000..95276b1d6 --- /dev/null +++ b/forge-web/lib/view/archive.test.ts @@ -0,0 +1,346 @@ +/** + * The browser .zip against `git archive` (QW-026): `.gitattributes` (export-ignore, + * export-subst), `$Format:` placeholders, `git describe`, Unix modes, the commit's time and the + * archive comment. The parts that can be are checked against real git; skipped without git. + */ + +import { spawnSync } from 'node:child_process' + +import { unzipSync, zipSync } from 'fflate' +import { describe, expect, it } from 'vitest' + +import { gitOidHex, MODE_TREE } from '../browse' +import { HAVE_GIT, scratchRepo, writeLiterally } from '../merge/git-oracle' +import { + autoAbbrevLength, + describeCommit, + exportAttributes, + formatCommit, + describeNames, + parseArchiveCommit, + peelTags, + uniqueAbbrev, + type DescribeTag, + type FormatContext, +} from './archive' +import { Store } from './diff-fixtures' +import { planArchive, readZipFiles, substituteFiles } from './zip' +import { withArchiveComment, zipEntries } from './zip-entries' + +const enc = new TextEncoder() + +/** Put a raw object of `type` into the store; its oid. */ +function raw(s: Store, type: 'commit' | 'tag', text: string): string { + const bytes = enc.encode(text) + const oid = gitOidHex(type, bytes) + s.objects.set(oid, { type, bytes }) + return oid +} + +function git(dir: string, args: string[], input?: Uint8Array): { out: string; bytes: Buffer; status: number | null } { + const r = spawnSync('git', args, { + cwd: dir, + input, + env: { ...process.env, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null', TZ: process.env['TZ'] ?? '' }, + maxBuffer: 1 << 26, + }) + return { out: r.stdout.toString().trim(), bytes: r.stdout, status: r.status } +} + +const findIn = (s: Store) => (prefix: string, limit = 2): string[] => [...s.objects.keys()].filter((o) => o.startsWith(prefix)).slice(0, limit) + +describe('.gitattributes for an archive', () => { + it('matches basenames and anchored paths, lets later lines and deeper files win, and ignores whole directories', () => { + const attrs = exportAttributes( + new Map([ + ['.gitattributes', 'src/version.c export-subst\n*.md export-ignore\nREADME.md -export-ignore\n/ci export-ignore\ndocs/**/*.png export-ignore\n# a comment\n"odd name.txt" export-ignore\n'], + ['lib/.gitattributes', 'keep.md -export-ignore\n*.c export-subst\n'], + ]), + ) + expect(attrs('src/version.c')).toEqual({ ignore: false, subst: true }) + expect(attrs('notes.md').ignore).toBe(true) + expect(attrs('deep/dir/notes.md').ignore).toBe(true) + expect(attrs('README.md').ignore).toBe(false) + expect(attrs('lib/keep.md').ignore).toBe(false) + expect(attrs('lib/x.c').subst).toBe(true) + expect(attrs('src/other.c').subst).toBe(false) + // An anchored directory: everything under it. + expect(attrs('ci/build.sh').ignore).toBe(true) + expect(attrs('src/ci/build.sh').ignore).toBe(false) + expect(attrs('docs/a/b/c.png').ignore).toBe(true) + expect(attrs('docs/c.png').ignore).toBe(true) + expect(attrs('odd name.txt').ignore).toBe(true) + }) + + it('takes a directory-only pattern for the directory and everything in it, never for a file of that name', () => { + const attrs = exportAttributes(new Map([['.gitattributes', '/tests/ export-ignore\nfixtures/ export-ignore\n']])) + expect(attrs('tests/a_test.c').ignore).toBe(true) + expect(attrs('src/tests/a.c').ignore).toBe(false) + expect(attrs('lib/fixtures/data.json').ignore).toBe(true) + // A file named like the directory pattern is not a directory. + expect(attrs('fixtures').ignore).toBe(false) + }) +}) + +describe('$Format: placeholders', () => { + const s = new Store() + const tree = s.files({ a: 'a\n' }) + const parent = s.commit(tree) + const oid = raw( + s, + 'commit', + `tree ${tree}\nparent ${parent}\nauthor Ada Lovelace 1700000000 +0200\ncommitter Bob 1700003600 -0530\n\nFix the thing\nacross two lines\n\nThe body.\n`, + ) + const commit = parseArchiveCommit(oid, s.objects.get(oid)!.bytes) + const ctx: FormatContext = { commit, abbrev: (o, min = 7) => o.slice(0, min), describe: (spec) => (spec === 'bogus' ? null : 'v1.0-3-gabcdef'), decorations: ['main', 'tag: v1.1'] } + + it('expands what git log --format would', () => { + expect(formatCommit('%H|%h|%T|%P|%p', ctx)).toBe(`${oid}|${oid.slice(0, 7)}|${tree}|${parent}|${parent.slice(0, 7)}`) + expect(formatCommit('%an <%ae> %al', ctx)).toBe('Ada Lovelace ada') + expect(formatCommit('%ad|%aD|%ai|%aI|%at|%as', ctx)).toBe( + // 1700000000 is 2023-11-14 22:13:20 UTC: in the author's +0200, past midnight. + 'Wed Nov 15 00:13:20 2023 +0200|Wed, 15 Nov 2023 00:13:20 +0200|2023-11-15 00:13:20 +0200|2023-11-15T00:13:20+02:00|1700000000|2023-11-15', + ) + expect(formatCommit('%cn %cI', ctx)).toBe('Bob 2023-11-14T17:43:20-05:30') + expect(formatCommit('%s', ctx)).toBe('Fix the thing across two lines') + expect(formatCommit('%b', ctx)).toBe('The body.\n') + expect(formatCommit('a%nb %% %x41 %d %D %(describe) %(describe:abbrev=12)', ctx)).toBe('a\nb % A (main, tag: v1.1) main, tag: v1.1 v1.0-3-gabcdef v1.0-3-gabcdef') + // Unknown placeholders stay as written, as git leaves them. + expect(formatCommit('%Q %(foo) %(describe:bogus)', ctx)).toBe('%Q %(foo) %(describe:bogus)') + }) + + it('abbreviates as git does', () => { + expect(autoAbbrevLength(0)).toBe(7) + expect(autoAbbrevLength(5_000)).toBe(7) + // dashpay/dash: ~500k objects is 10 digits, git's `--short` there. + expect(autoAbbrevLength(500_000)).toBe(10) + expect(uniqueAbbrev('abcdef1234', 4, (p) => (p.length < 6 ? ['abcdef1234', 'abcde99999'] : ['abcdef1234']))).toBe('abcdef') + }) +}) + +describe('substituteFiles, as git archive expands', () => { + /** A two-commit repo whose tip has `files`, an annotated tag on the root and two tags on the tip. */ + function repo(files: Record, { tagged = true, when = 1_700_000_000 } = {}) { + const s = new Store() + const root = s.commit(s.files({ a: '1\n' }), [], 'root') + const tree = s.files({ '.gitattributes': '*.txt export-subst\n', ...files }) + const tip = raw(s, 'commit', `tree ${tree}\nparent ${root}\nauthor A ${when} +0000\ncommitter A ${when} +0000\n\ntip\n`) + const tag = (target: string, name: string, time: number): DescribeTag => ({ + name, + oid: raw(s, 'tag', `object ${target}\ntype commit\ntag ${name}\ntagger T ${time} +0000\n\n${name}\n`), + }) + const tags = tagged ? [tag(root, 'v1.0', 1), tag(tip, 'v2.0', 5), tag(tip, 'v2.0-final', 9)] : [] + return { s, tip, tags } + } + async function expand(files: Record, opts: { tagged?: boolean; when?: number } = {}): Promise> { + const { s, tip, tags } = repo(files, opts) + const reader = Object.assign(s.reader(), { findByPrefix: findIn(s), objectCount: s.objects.size }) + const plan = await planArchive(reader, tip) + const entries = await readZipFiles(reader, plan.files, () => undefined) + await substituteFiles(reader, plan, entries, { tags, heads: [{ name: 'main', oid: tip }] }) + return Object.fromEntries(Object.entries(entries).map(([k, v]) => [k, new TextDecoder().decode(v)])) + } + + it('expands the first %(describe) whatever its option spelling, and leaves the ones after it', async () => { + const got = await expand({ 'a.txt': '$Format:%(describe:tags=true)$ $Format:%(describe)$\n', 'b.txt': '$Format:%(describe:abbrev=0)$\n' }) + // Both tags are on the tip: the newer tagger date wins, as describe's replace_name picks. + expect(got['a.txt']).toBe('v2.0-final %(describe)\n') + expect(got['b.txt']).toBe('%(describe:abbrev=0)\n') + }) + + it('writes nothing for %(describe) when no tag describes the commit, as git does', async () => { + expect((await expand({ 'a.txt': 'v="$Format:%(describe)$"\n' }, { tagged: false }))['a.txt']).toBe('v=""\n') + }) + + it('names every tag at the commit in %d', async () => { + expect((await expand({ 'a.txt': '$Format:%d$\n' }))['a.txt']).toBe(' (main, tag: v2.0, tag: v2.0-final)\n') + }) + + it('zips a commit dated before 1980 (the DOS date is clamped; the UT field keeps the time)', async () => { + const { s, tip } = repo({ 'a.txt': 'x\n' }, { tagged: false, when: 86_400 }) + const reader = s.reader() + const plan = await planArchive(reader, tip) + const entries = await readZipFiles(reader, plan.files, () => undefined) + const zip = zipSync(zipEntries(entries, { modes: {}, mtime: plan.mtime, comment: tip }), { level: 6 }) + expect(Object.keys(unzipSync(zip))).toContain('a.txt') + expect(centralDirectory(zip).get('a.txt')?.ut).toBe(86_400) + }) +}) + +/** A history with merges, annotated and lightweight tags, some tags on one commit. */ +function taggedHistory(s: Store): { commits: string[]; tags: DescribeTag[] } { + let x = 7 + const rand = (): number => ((x = (x * 1103515245 + 12345) >>> 0), x / 2 ** 32) + const commits: string[] = [] + let a = s.commit(s.files({ f: '0\n' }), [], 'root') + let b = a + commits.push(a) + const tags: DescribeTag[] = [] + let time = 1_600_000_000 + const tag = (target: string, name: string, annotated: boolean): void => { + if (!annotated) { + tags.push({ name, oid: target }) + return + } + time += 10 + tags.push({ name, oid: raw(s, 'tag', `object ${target}\ntype commit\ntag ${name}\ntagger T ${time} +0000\n\n${name}\n`) }) + } + for (let i = 1; i < 60; i++) { + const onA = rand() < 0.6 + const merge = i % 7 === 0 + const parents = onA ? (merge ? [a, b] : [a]) : merge ? [b, a] : [b] + const c = s.commit(s.files({ f: `${i}\n` }), parents, `c${i}`) + commits.push(c) + if (onA) a = c + else b = c + if (i % 9 === 0) tag(c, `v${i}`, true) + if (i % 13 === 0) tag(c, `light${i}`, false) + if (i === 27) tag(c, `v${i}-again`, true) + } + return { commits, tags } +} + +describe.skipIf(!HAVE_GIT)('git describe, against git', () => { + it('names the same tag and depth for every commit, annotated only and with --tags', async () => { + const s = new Store() + const { commits, tags } = taggedHistory(s) + const { dir, done } = scratchRepo() + try { + writeLiterally(dir, s.objects.values()) + for (const t of tags) git(dir, ['update-ref', `refs/tags/${t.name}`, t.oid]) + const reader = s.reader() + const abbrevOf = (oid: string, min = 7): string => uniqueAbbrev(oid, min, findIn(s)) + const peeled = await peelTags(reader, tags) + for (const opts of [{ tags: false, abbrev: 12 }, { tags: true, abbrev: 7 }, { tags: false, abbrev: 0 }, { tags: false, abbrev: null }]) { + const named = describeNames(peeled, { match: [], exclude: [] }) + for (const c of commits) { + const want = git(dir, ['describe', ...(opts.abbrev === null ? [] : [`--abbrev=${opts.abbrev}`]), ...(opts.tags ? ['--tags'] : []), c]) + const got = await describeCommit(reader, c, named, opts, abbrevOf) + expect(got ?? '', `${c} ${JSON.stringify(opts)}`).toBe(want.status === 0 ? want.out : '') + } + } + } finally { + done() + } + }) + + it('takes match= and exclude= as git does', async () => { + const s = new Store() + const { commits, tags } = taggedHistory(s) + const { dir, done } = scratchRepo() + try { + writeLiterally(dir, s.objects.values()) + for (const t of tags) git(dir, ['update-ref', `refs/tags/${t.name}`, t.oid]) + const reader = s.reader() + const named = describeNames(await peelTags(reader, tags), { match: ['v*'], exclude: ['*-again'] }) + const tip = commits[commits.length - 1] as string + const got = await describeCommit(reader, tip, named, { tags: false, abbrev: 7 }, (o, m = 7) => uniqueAbbrev(o, m, findIn(s))) + expect(got).toBe(git(dir, ['describe', '--abbrev=7', '--match', 'v*', '--exclude', '*-again', tip]).out) + } finally { + done() + } + }) +}) + +/** The central directory's entries: name, "made by" OS byte, external attributes, and the UT time. */ +function centralDirectory(zip: Uint8Array): Map { + const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength) + const out = new Map() + for (let at = 0; at + 46 <= zip.length; ) { + if (view.getUint32(at, true) !== 0x02014b50) { + at++ + continue + } + const nameLen = view.getUint16(at + 28, true) + const extraLen = view.getUint16(at + 30, true) + const commentLen = view.getUint16(at + 32, true) + const name = new TextDecoder().decode(zip.subarray(at + 46, at + 46 + nameLen)) + let ut: number | null = null + for (let e = at + 46 + nameLen; e + 4 <= at + 46 + nameLen + extraLen; ) { + const id = view.getUint16(e, true) + const len = view.getUint16(e + 2, true) + if (id === 0x5455 && len >= 5) ut = view.getUint32(e + 5, true) + e += 4 + len + } + out.set(name, { os: view.getUint8(at + 5), attrs: view.getUint32(at + 38, true), ut }) + at += 46 + nameLen + extraLen + commentLen + } + return out +} + +/** The archive comment (end-of-central-directory record's). */ +function archiveComment(zip: Uint8Array): string { + for (let at = zip.length - 22; at >= 0; at--) { + const view = new DataView(zip.buffer, zip.byteOffset + at) + if (view.getUint32(0, true) === 0x06054b50) return new TextDecoder().decode(zip.subarray(at + 22, at + 22 + view.getUint16(20, true))) + } + return '' +} + +describe.skipIf(!HAVE_GIT)('the browser zip against git archive --format=zip', () => { + it('the same files, contents (export-subst expanded, export-ignore left out), modes, times and comment', async () => { + const s = new Store() + const script = s.blob('#!/bin/sh\necho hi\n') + const version = s.blob('const char* v = "$Format:%H %h %an %ad %cI %s$";\n#define DESC "$Format:%(describe:abbrev=12)$"\n') + const src = s.tree([ + { name: 'version.c', oid: version }, + { name: 'main.c', oid: s.blob('int main(void) { return 0; }\n') }, + ]) + const ci = s.files({ 'build.yml': 'on: push\n' }) + const tree = s.tree([ + { name: '.gitattributes', oid: s.blob('src/version.c export-subst\n/ci export-ignore\n*.log export-ignore\n') }, + { name: 'autogen.sh', oid: script, mode: 0o100755 }, + { name: 'ci', oid: ci, mode: MODE_TREE }, + { name: 'debug.log', oid: s.blob('noise\n') }, + { name: 'link', oid: s.blob('src/main.c'), mode: 0o120000 }, + { name: 'README.md', oid: s.blob('# hi\n') }, + { name: 'src', oid: src, mode: MODE_TREE }, + ]) + const root = s.commit(s.files({ README: 'x\n' }), [], 'root') + const tagObj = raw(s, 'tag', `object ${root}\ntype commit\ntag v0.1\ntagger T 1600000000 +0000\n\nv0.1\n`) + const tip = raw( + s, + 'commit', + `tree ${tree}\nparent ${root}\nauthor Ada 1700000000 +0200\ncommitter Ada 1700000500 +0200\n\nRelease prep\n`, + ) + const { dir, done } = scratchRepo() + try { + writeLiterally(dir, s.objects.values()) + git(dir, ['update-ref', 'refs/tags/v0.1', tagObj]) + const want = new Uint8Array(git(dir, ['archive', '--format=zip', '--prefix=p/', tip]).bytes) + + // The browser's pipeline, as the clone box runs it. + const reader = Object.assign(s.reader(), { findByPrefix: findIn(s), objectCount: s.objects.size }) + const plan = await planArchive(reader, tip) + const entries = await readZipFiles(reader, plan.files, () => undefined) + await substituteFiles(reader, plan, entries, { tags: [{ name: 'v0.1', oid: tagObj }], heads: [] }) + const rooted: Record = {} + const modes: Record = {} + for (const f of plan.files) modes[`p/${f.path}`] = f.mode + for (const [path, bytes] of Object.entries(entries)) rooted[`p/${path}`] = bytes + const meta = { modes, mtime: plan.mtime, comment: tip } + const got = withArchiveComment(zipSync(zipEntries(rooted, meta), { level: 6 }), meta.comment) + + const wantFiles = unzipSync(want) + const gotFiles = unzipSync(got) + expect(Object.keys(gotFiles).sort()).toEqual(Object.keys(wantFiles).sort()) + for (const [name, bytes] of Object.entries(wantFiles)) expect(new TextDecoder().decode(gotFiles[name]), name).toBe(new TextDecoder().decode(bytes)) + // `$Format:` really was expanded (not merely equal because both left it). + expect(new TextDecoder().decode(gotFiles['p/src/version.c'])).toContain(`${tip} ${tip.slice(0, 7)} Ada Wed Nov 15 00:13:20 2023 +0200`) + expect(new TextDecoder().decode(gotFiles['p/src/version.c'])).toContain('v0.1-1-g') + + const wantDir = centralDirectory(want) + const gotDir = centralDirectory(got) + for (const [name, w] of wantDir) { + const g = gotDir.get(name) + expect(g, name).toBeDefined() + expect({ name, os: g?.os, attrs: g?.attrs, ut: g?.ut }).toEqual({ name, os: w.os, attrs: w.attrs, ut: w.ut }) + } + expect(gotDir.get('p/autogen.sh')?.attrs).toBe((0o100755 << 16) >>> 0) + expect(archiveComment(got)).toBe(archiveComment(want)) + expect(archiveComment(got)).toBe(tip) + } finally { + done() + } + }) +}) diff --git a/forge-web/lib/view/archive.ts b/forge-web/lib/view/archive.ts new file mode 100644 index 000000000..d9cf086f5 --- /dev/null +++ b/forge-web/lib/view/archive.ts @@ -0,0 +1,629 @@ +/** + * What `git archive` does to a tree besides copying its files (QW-026), for the browser's .zip: + * + * - `.gitattributes` of the tree archived (never the viewer's): `export-ignore` leaves a path out, + * `export-subst` expands `$Format:…$` in a file ({@link expandExportSubst}); + * - the placeholders `git log --format` takes (the ones a commit alone answers), `%(describe)` + * included, computed as `git describe` does ({@link describeCommit}); + * - each entry's Unix mode and the commit's time, which the zip writer records. + * + * `.gitattributes` patterns are matched as git's attr.c matches them: a pattern with no `/` is + * matched against the basename, one with a `/` against the path from its file's directory with + * `*` not crossing `/` and `**` crossing any number of directories. Later lines, and files deeper + * in the tree, win. + */ + +import { OID_HEX, parseTag } from './git-objects' +import type { ObjectReader } from './tree-nav' +import { mapPooled } from './pool' + +// --------------------------------------------------------------------------- +// .gitattributes +// --------------------------------------------------------------------------- + +/** One `.gitattributes` line, reduced to the attributes an archive reads. */ +interface AttrRule { + readonly match: (path: string, isDir: boolean) => boolean + /** true: set; false: unset (`-attr` or `!attr`); absent: this line says nothing. */ + readonly attrs: ReadonlyMap<'export-ignore' | 'export-subst', boolean> +} + +/** A git wildmatch pattern (`WM_PATHNAME`) as a regular expression over a whole path. */ +function wildmatchRegex(pattern: string): RegExp { + let re = '' + for (let i = 0; i < pattern.length; i++) { + const c = pattern[i] as string + if (c === '*') { + if (pattern[i + 1] === '*') { + // `**/` at the start or after `/` matches any number of directories; a trailing `/**` everything below. + const atStart = i === 0 || pattern[i - 1] === '/' + if (atStart && pattern[i + 2] === '/') { + re += '(?:.*/)?' + i += 2 + continue + } + if (atStart && i + 2 === pattern.length) { + re += '.*' + i += 1 + continue + } + re += '[^/]*' + i += 1 + continue + } + re += '[^/]*' + } else if (c === '?') { + re += '[^/]' + } else if (c === '[') { + const close = pattern.indexOf(']', i + 2) + if (close === -1) { + re += '\\[' + continue + } + let body = pattern.slice(i + 1, close) + const negate = body.startsWith('!') || body.startsWith('^') + if (negate) body = body.slice(1) + re += `[${negate ? '^' : ''}${body.replace(/\\/g, '\\\\').replace(/]/g, '\\]')}]` + i = close + } else if (c === '\\' && i + 1 < pattern.length) { + re += (pattern[i + 1] as string).replace(/[.*+?^${}()|[\]\\/]/g, '\\$&') + i += 1 + } else { + re += c.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&') + } + } + return new RegExp(`^${re}$`) +} + +/** + * A `.gitattributes` pattern, from its file's directory (`''` = the root), as attr.c's + * `path_matches`: a trailing `/` makes it match directories only (git archive checks a directory + * as `dir/`), a pattern with no other `/` is matched against the basename, else against the path + * from the file's directory. + */ +function patternMatcher(pattern: string, dir: string): ((path: string, isDir: boolean) => boolean) | null { + if (pattern === '' || pattern.startsWith('!')) return null + const dirOnly = pattern.endsWith('/') + const body = dirOnly ? pattern.replace(/\/+$/, '') : pattern + if (body === '') return null + const anchored = body.includes('/') + const re = wildmatchRegex(anchored ? body.replace(/^\//, '') : body) + const prefix = dir === '' ? '' : `${dir}/` + return (path, isDir) => { + if ((dirOnly && !isDir) || !path.startsWith(prefix)) return false + const rel = path.slice(prefix.length) + return re.test(anchored ? rel : rel.slice(rel.lastIndexOf('/') + 1)) + } +} + +/** The pattern of a line, unquoting a C-style quoted one. */ +function splitLine(line: string): [string, string] { + if (!line.startsWith('"')) { + const at = line.search(/\s/) + return at === -1 ? [line, ''] : [line.slice(0, at), line.slice(at)] + } + let out = '' + let i = 1 + for (; i < line.length && line[i] !== '"'; i++) { + if (line[i] === '\\' && i + 1 < line.length) { + const n = line[++i] as string + out += n === 'n' ? '\n' : n === 't' ? '\t' : n + } else out += line[i] + } + return [out, line.slice(i + 1)] +} + +/** The archive-relevant rules of one `.gitattributes` file at `dir`. */ +export function parseGitAttributes(text: string, dir: string): AttrRule[] { + const rules: AttrRule[] = [] + for (const raw of text.split(/\r?\n/)) { + const line = raw.trim() + if (line === '' || line.startsWith('#')) continue + const [pattern, rest] = splitLine(line) + const attrs = new Map<'export-ignore' | 'export-subst', boolean>() + for (const word of rest.trim().split(/\s+/)) { + const unset = word.startsWith('-') || word.startsWith('!') + const name = (unset ? word.slice(1) : word).split('=')[0] + if (name === 'export-ignore' || name === 'export-subst') attrs.set(name, !unset) + } + if (attrs.size === 0) continue + const match = patternMatcher(pattern, dir) + if (match !== null) rules.push({ match, attrs }) + } + return rules +} + +/** How the archive treats a path. */ +export interface ExportAttrs { + readonly ignore: boolean + readonly subst: boolean +} + +/** + * The archive's attributes from every `.gitattributes` of the tree: `files` maps each one's path + * to its text. A path is ignored when it, or a directory above it, is `export-ignore`. + */ +export function exportAttributes(files: ReadonlyMap): (path: string) => ExportAttrs { + // Root first, then deeper: a deeper file's lines come later, so they win. + const ordered = [...files.keys()].sort((a, b) => a.split('/').length - b.split('/').length || (a < b ? -1 : 1)) + const rules = ordered.flatMap((p) => { + const slash = p.lastIndexOf('/') + return parseGitAttributes(files.get(p) as string, slash === -1 ? '' : p.slice(0, slash)) + }) + const attr = (path: string, isDir: boolean, name: 'export-ignore' | 'export-subst'): boolean => { + let value = false + for (const r of rules) { + const v = r.attrs.get(name) + if (v !== undefined && r.match(path, isDir)) value = v + } + return value + } + return (path) => { + // A directory left out leaves out everything under it (git archive does not descend into it). + let ignore = false + for (let at = path.indexOf('/'); !ignore && at !== -1; at = path.indexOf('/', at + 1)) ignore = attr(path.slice(0, at), true, 'export-ignore') + ignore ||= attr(path, false, 'export-ignore') + return { ignore, subst: !ignore && attr(path, false, 'export-subst') } + } +} + +// --------------------------------------------------------------------------- +// The commit, as `--format` sees it +// --------------------------------------------------------------------------- + +/** An author or committer line, with the time zone the pretty dates are printed in. */ +export interface Person { + readonly name: string + readonly email: string + /** Seconds since the epoch. */ + readonly time: number + /** The zone as written (`+0200`), and its offset in minutes. */ + readonly tz: string + readonly offset: number +} + +/** A commit as `$Format:` expands it. */ +export interface ArchiveCommit { + readonly oid: string + readonly tree: string + readonly parents: readonly string[] + readonly author: Person + readonly committer: Person + readonly message: string +} + +function parsePerson(value: string): Person { + const m = /^(.*?) <([^>]*)> (\d+) ([+-]\d{4})$/.exec(value.trim()) + if (m === null) return { name: value.trim(), email: '', time: 0, tz: '+0000', offset: 0 } + const tz = m[4] as string + const sign = tz.startsWith('-') ? -1 : 1 + const offset = sign * (Number(tz.slice(1, 3)) * 60 + Number(tz.slice(3, 5))) + return { name: m[1] as string, email: m[2] as string, time: Number(m[3]), tz, offset } +} + +/** Parse a commit object's bytes, keeping what `--format` prints. */ +export function parseArchiveCommit(oid: string, bytes: Uint8Array): ArchiveCommit { + const text = new TextDecoder('utf-8', { fatal: false, ignoreBOM: true }).decode(bytes) + const sep = text.indexOf('\n\n') + const lines = (sep === -1 ? text : text.slice(0, sep)).split('\n') + const field = (key: string): string => lines.find((l) => l.startsWith(`${key} `))?.slice(key.length + 1) ?? '' + return { + oid, + tree: field('tree'), + parents: lines.filter((l) => l.startsWith('parent ')).map((l) => l.slice(7)), + author: parsePerson(field('author')), + committer: parsePerson(field('committer')), + message: sep === -1 ? '' : text.slice(sep + 2), + } +} + +const DAYS = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'] +const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] +const pad = (n: number, w = 2): string => String(n).padStart(w, '0') + +/** A person's time in their own zone, as git prints each date format. */ +function dateOf(p: Person, format: 'default' | 'rfc' | 'iso' | 'strict' | 'short'): string { + const d = new Date((p.time + p.offset * 60) * 1000) + const Y = d.getUTCFullYear() + const M = d.getUTCMonth() + const D = d.getUTCDate() + const hms = `${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}` + switch (format) { + case 'default': + return `${DAYS[d.getUTCDay()]} ${MONTHS[M]} ${D} ${hms} ${Y} ${p.tz}` + case 'rfc': + return `${DAYS[d.getUTCDay()]}, ${D} ${MONTHS[M]} ${Y} ${hms} ${p.tz}` + case 'iso': + return `${Y}-${pad(M + 1)}-${pad(D)} ${hms} ${p.tz}` + case 'strict': { + const zone = p.offset === 0 ? 'Z' : `${p.tz.slice(0, 3)}:${p.tz.slice(3)}` + return `${Y}-${pad(M + 1)}-${pad(D)}T${hms}${zone}` + } + case 'short': + return `${Y}-${pad(M + 1)}-${pad(D)}` + } +} + +/** The message's subject (its first paragraph, lines joined by spaces) and body. */ +function subjectAndBody(message: string): { subject: string; body: string } { + const lines = message.split('\n') + let i = 0 + while (i < lines.length && (lines[i] as string).trim() === '') i++ + const subject: string[] = [] + for (; i < lines.length && (lines[i] as string).trim() !== ''; i++) subject.push((lines[i] as string).trim()) + while (i < lines.length && (lines[i] as string).trim() === '') i++ + return { subject: subject.join(' '), body: lines.slice(i).join('\n') } +} + +/** What `$Format:` placeholders need besides the commit. */ +export interface FormatContext { + readonly commit: ArchiveCommit + /** An oid's shortest unique abbreviation of at least `min` digits (`%h`, and `%(describe)`'s suffix). */ + readonly abbrev: (oid: string, min?: number) => string + /** + * `%(describe…)` by its option text (`abbrev=12`, `''` for none): the expansion (`''` when no + * tag describes the commit, as git's is then empty), or null to leave the placeholder as + * written (git archive expands one `%(describe)` per archive: the ones after it stay). + */ + readonly describe: (spec: string) => string | null + /** The refs pointing at the commit, as `%D` prints them (`tag: v1`, `main`). */ + readonly decorations: readonly string[] +} + +/** `%(describe)`'s options. */ +export interface DescribeOptions { + readonly tags: boolean + readonly abbrev: number | null + readonly match: readonly string[] + readonly exclude: readonly string[] +} + +/** Parse `%(describe:opt,opt=…)`'s option list; null when an option is not one git knows. */ +export function parseDescribeOptions(spec: string): DescribeOptions | null { + const out = { tags: false, abbrev: null as number | null, match: [] as string[], exclude: [] as string[] } + if (spec === '') return out + for (const opt of spec.split(',')) { + const [key, value] = [opt.split('=')[0] as string, opt.includes('=') ? opt.slice(opt.indexOf('=') + 1) : null] + if (key === 'tags') out.tags = value === null || !/^(false|no|off|0)$/i.test(value) + else if (key === 'abbrev' && value !== null && /^\d+$/.test(value)) out.abbrev = Number(value) + else if (key === 'match' && value !== null) out.match.push(value) + else if (key === 'exclude' && value !== null) out.exclude.push(value) + else return null + } + return out +} + +/** + * Expand a pretty format (`git log --format=`) for one commit: the placeholders a commit + * alone answers. One git would print from a mailmap, a reflog, notes or GPG checks, and anything + * unknown, is left as written, as git leaves an unknown `%` sequence. + */ +export function formatCommit(fmt: string, ctx: FormatContext): string { + const { commit } = ctx + const { subject, body } = subjectAndBody(commit.message) + const person = (who: 'a' | 'c', key: string): string | null => { + const p = who === 'a' ? commit.author : commit.committer + switch (key) { + case 'n': + case 'N': + return p.name + case 'e': + case 'E': + return p.email + case 'l': + case 'L': + return p.email.split('@')[0] ?? '' + case 'd': + return dateOf(p, 'default') + case 'D': + return dateOf(p, 'rfc') + case 'i': + return dateOf(p, 'iso') + case 'I': + return dateOf(p, 'strict') + case 's': + return dateOf(p, 'short') + case 't': + return String(p.time) + default: + return null + } + } + let out = '' + for (let i = 0; i < fmt.length; i++) { + const c = fmt[i] as string + if (c !== '%' || i + 1 >= fmt.length) { + out += c + continue + } + const n = fmt[i + 1] as string + let value: string | null = null + let used = 2 + if (n === '(') { + const close = fmt.indexOf(')', i + 2) + const inner = close === -1 ? null : fmt.slice(i + 2, close) + if (inner !== null && (inner === 'describe' || inner.startsWith('describe:'))) { + value = ctx.describe(inner === 'describe' ? '' : inner.slice('describe:'.length)) + used = close - i + 1 + } + } else if (n === 'x' && /^[0-9a-fA-F]{2}$/.test(fmt.slice(i + 2, i + 4))) { + value = String.fromCharCode(parseInt(fmt.slice(i + 2, i + 4), 16)) + used = 4 + } else if ((n === 'a' || n === 'c') && i + 2 < fmt.length) { + value = person(n, fmt[i + 2] as string) + used = 3 + } else { + const decorated = ctx.decorations.join(', ') + const simple: Record = { + H: commit.oid, + h: ctx.abbrev(commit.oid), + T: commit.tree, + t: ctx.abbrev(commit.tree), + P: commit.parents.join(' '), + p: commit.parents.map((p) => ctx.abbrev(p)).join(' '), + s: subject, + f: subject.replace(/[^A-Za-z0-9.]+/g, '-').replace(/^[-.]+|[-.]+$/g, ''), + b: body, + B: commit.message, + n: '\n', + '%': '%', + D: decorated, + d: decorated === '' ? '' : ` (${decorated})`, + } + value = simple[n] ?? null + } + if (value === null) { + out += c + continue + } + out += value + i += used - 1 + } + return out +} + +/** + * `export-subst` (gitattributes(5)): each `$Format:$` in the file replaced by the commit's + * `` expansion. The format runs to the next `$`. + */ +export function expandExportSubst(text: string, ctx: FormatContext): string { + return text.replace(/\$Format:([^$]*)\$/g, (_, fmt: string) => formatCommit(fmt, ctx)) +} + +/** The shortest unique abbreviation of `oid` of at least `min` digits, as git finds one. */ +export function uniqueAbbrev(oid: string, min: number, findByPrefix?: (prefix: string, limit?: number) => string[]): string { + let len = Math.max(4, min) + if (findByPrefix === undefined) return oid.slice(0, len) + while (len < oid.length && findByPrefix(oid.slice(0, len), 2).length > 1) len++ + return oid.slice(0, len) +} + +/** git's automatic abbreviation for a repository of `objects` objects (`core.abbrev` unset): at least 7. */ +export function autoAbbrevLength(objects: number): number { + if (!(objects > 0)) return 7 + const msb = Math.floor(Math.log2(objects)) + return Math.max(7, Math.floor((msb + 1 + 1) / 2)) +} + +// --------------------------------------------------------------------------- +// git describe +// --------------------------------------------------------------------------- + +/** A tag `git describe` may name: its short name, and what it points at. */ +export interface DescribeTag { + readonly name: string + /** The tag ref's object: an annotated tag object, or the commit a lightweight tag names. */ + readonly oid: string +} + +/** A tag peeled to its commit, as `git describe` knows it (builtin/describe.c `commit_name`). */ +export interface PeeledTag { + /** The ref's short name (`v1.0` for `refs/tags/v1.0`). */ + readonly ref: string + readonly commit: string + /** 2 annotated, 1 lightweight (git's `prio`). */ + readonly prio: 1 | 2 + /** The outermost tag object's tagger time (s), 0 for a lightweight tag. */ + readonly date: number + /** What describe prints: the tag object's own name for an annotated tag, else the ref's. */ + readonly display: string +} + +/** Commits {@link describeCommit} walks at most before it gives up. */ +export const DESCRIBE_COMMIT_CAP = 100_000 +/** git describe's `--candidates` default. */ +const MAX_CANDIDATES = 10 +/** Tag objects are read this many at a time. */ +const TAG_READ_POOL = 16 + +/** A shell glob (`match=`, `exclude=`) as git's `wildmatch` without `WM_PATHNAME`. */ +function globRegex(glob: string): RegExp { + return new RegExp(`^${wildmatchRegex(glob).source.slice(1, -1).replace(/\[\^\/\]\*/g, '.*').replace(/\[\^\/\]/g, '.')}$`) +} + +/** + * Every tag peeled to the commit it names (a tag of a tree or a blob is left out, as describe + * leaves it), in ref-name order. The tag objects are read side by side. + */ +export async function peelTags(reader: ObjectReader, tags: readonly DescribeTag[], signal?: AbortSignal): Promise { + const sorted = [...tags].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) + const peeled = await mapPooled(sorted, TAG_READ_POOL, async (t): Promise => { + signal?.throwIfAborted() + let at = t.oid + let prio: 1 | 2 = 1 + let date = 0 + let display = t.name + for (let depth = 0; depth < 10; depth++) { + const obj = await reader.readObject(at) + if (obj.type !== 'tag') return obj.type === 'commit' ? { ref: t.name, commit: at, prio, date, display } : null + const tag = parseTag(obj.bytes) + if (tag === null || !OID_HEX.test(tag.object)) return null + if (depth === 0) { + prio = 2 + display = tag.tag || t.name + const tagger = /\ntagger .* (\d+) [+-]\d{4}\n/.exec(`\n${new TextDecoder().decode(obj.bytes)}`) + date = tagger === null ? 0 : Number(tagger[1]) + } + at = tag.object + } + return null + }) + return peeled.filter((t): t is PeeledTag => t !== null) +} + +/** + * The name `git describe` gives each tagged commit (its `names` map): the tags that pass `match` + * and `exclude`, taken in ref-name order, an annotated tag over a lightweight one and, of two + * annotated ones, the newer tagger date; otherwise the first stays (describe.c `replace_name`). + */ +export function describeNames(peeled: readonly PeeledTag[], options: Pick): Map { + const match = options.match.map(globRegex) + const exclude = options.exclude.map(globRegex) + const named = new Map() + for (const t of peeled) { + if ((match.length > 0 && !match.some((r) => r.test(t.ref))) || exclude.some((r) => r.test(t.ref))) continue + const had = named.get(t.commit) + if (had === undefined || had.prio < t.prio || (had.prio === 2 && t.prio === 2 && had.date < t.date)) named.set(t.commit, t) + } + return named +} + +/** The description could not be worked out (the history is past its bound). */ +export class DescribeError extends Error { + constructor(message: string) { + super(message) + this.name = 'DescribeError' + } +} + +/** + * `git describe [--tags] [--abbrev=] ` (builtin/describe.c `describe_commit`): the + * nearest tag by the number of commits it does not contain, with `--g` after it unless + * it names the commit itself (`abbrev` 0: the tag alone). Only annotated tags count unless `tags`. + * The walk is git's: newest first, up to 10 candidates (or every tag there is), stopping early + * when the last path is covered, then the best candidate's depth finished. `abbrevOf(oid, min)` + * gives an oid's unique abbreviation (`min` undefined: git's automatic length). Null when no tag + * describes the commit (git then fails, and `%(describe)` is empty). + */ +export async function describeCommit( + reader: ObjectReader, + commitOid: string, + named: ReadonlyMap, + options: Pick, + abbrevOf: (oid: string, min?: number) => string, + { signal, cap = DESCRIBE_COMMIT_CAP }: { readonly signal?: AbortSignal; readonly cap?: number } = {}, +): Promise { + const suffix = (depth: number): string => `-${depth}-g${abbrevOf(commitOid, options.abbrev ?? undefined)}` + const exact = named.get(commitOid) + if (exact !== undefined && (options.tags || exact.prio === 2)) { + const misnamed = exact.prio === 2 && exact.display !== exact.ref + return misnamed ? `${exact.display}${suffix(0)}` : exact.display + } + + interface Node { + readonly when: number + readonly parents: readonly string[] + flags: number + } + const SEEN = 1 + const nodes = new Map() + const load = async (oid: string): Promise => { + const known = nodes.get(oid) + if (known !== undefined) return known + signal?.throwIfAborted() + if (nodes.size >= cap) throw new DescribeError(`gave up after ${cap} commits`) + const obj = await reader.readObject(oid) + if (obj.type !== 'commit') throw new DescribeError(`${oid.slice(0, 12)} is not a commit`) + const c = parseArchiveCommit(oid, obj.bytes) + const node: Node = { when: c.committer.time, parents: c.parents, flags: 0 } + nodes.set(oid, node) + return node + } + // git's prio_queue by commit date, newest first; ties in insertion order. + const queue: { oid: string; when: number; seq: number }[] = [] + let seq = 0 + const put = (oid: string): void => { + queue.push({ oid, when: (nodes.get(oid) as Node).when, seq: seq++ }) + } + const get = (): string => { + let best = 0 + for (let i = 1; i < queue.length; i++) { + const q = queue[i] as (typeof queue)[number] + const b = queue[best] as (typeof queue)[number] + if (q.when > b.when || (q.when === b.when && q.seq < b.seq)) best = i + } + return (queue.splice(best, 1)[0] as (typeof queue)[number]).oid + } + const flagsOf = (oid: string): number => (nodes.get(oid) as Node).flags + /** Queue `c`'s unseen parents and pass them its flags; `onParent` sees each (was it seen, its flags before). */ + const visitParents = async (c: Node, onParent?: (oid: string, seen: boolean, before: number) => void): Promise => { + const parents = await Promise.all(c.parents.map(load)) + parents.forEach((p, i) => { + const oid = c.parents[i] as string + const seen = (p.flags & SEEN) !== 0 + if (!seen) put(oid) + const before = p.flags + p.flags |= c.flags + onParent?.(oid, seen, before) + }) + } + + const matches: { name: string; depth: number; flag: number; order: number }[] = [] + let annotated = 0 + let gaveUpOn: string | null = null + let seen = 0 + ;(await load(commitOid)).flags = SEEN + put(commitOid) + while (queue.length > 0) { + const oid = get() + const c = nodes.get(oid) as Node + seen++ + if (matches.length === MAX_CANDIDATES || matches.length === named.size) { + gaveUpOn = oid + break + } + const n = named.get(oid) + if (n !== undefined && (options.tags || n.prio === 2)) { + const flag = 1 << (matches.length + 1) + matches.push({ name: n.display, depth: seen - 1, flag, order: matches.length + 1 }) + c.flags |= flag + if (n.prio === 2) annotated++ + } + for (const m of matches) if ((c.flags & m.flag) === 0) m.depth++ + // Stop if the last remaining path is already covered by the best candidate(s). + if (annotated > 0 && queue.length === 0) { + let bestDepth = Infinity + let bestWithin = 0 + for (const m of matches) { + if (m.depth < bestDepth) { + bestDepth = m.depth + bestWithin = m.flag + } else if (m.depth === bestDepth) bestWithin |= m.flag + } + if ((c.flags & bestWithin) === bestWithin) break + } + await visitParents(c) + } + if (matches.length === 0) return null + matches.sort((a, b) => a.depth - b.depth || a.order - b.order) + const best = matches[0] as (typeof matches)[number] + if (gaveUpOn !== null) put(gaveUpOn) + // finish_depth_computation: on until no queued commit is outside the best candidate. + const unflagged = new Set(queue.filter((q) => (flagsOf(q.oid) & best.flag) === 0).map((q) => q.oid)) + while (queue.length > 0) { + const oid = get() + const c = nodes.get(oid) as Node + if ((c.flags & best.flag) !== 0) { + if (unflagged.size === 0) break + } else { + unflagged.delete(oid) + best.depth++ + } + await visitParents(c, (p, wasSeen, before) => { + const after = flagsOf(p) & best.flag + if (!wasSeen && after === 0) unflagged.add(p) + if (wasSeen && (before & best.flag) === 0 && after !== 0) unflagged.delete(p) + }) + } + return options.abbrev === 0 ? best.name : `${best.name}${suffix(best.depth)}` +} diff --git a/forge-web/lib/view/zip-entries.ts b/forge-web/lib/view/zip-entries.ts new file mode 100644 index 000000000..c2f4e2093 --- /dev/null +++ b/forge-web/lib/view/zip-entries.ts @@ -0,0 +1,90 @@ +/** + * Zip entries as `git archive --format=zip` writes them (QW-026; git's archive-zip.c), for the + * zip worker: kept apart from it so the tests run the same code on fflate directly. + * + * - An executable is recorded with its Unix mode (`0100755`, "made by" Unix) so it unpacks + * executable; a symlink as `0120777` with its target as its content, so it unpacks as a link; + * a regular file with no Unix mode, as git writes it (unzip then applies the umask). + * - Every directory gets its own entry, as git archive writes one. + * - Every entry carries the commit's time: as the DOS date (local time, as git's `localtime`) and + * as the `UT` extended timestamp (UTC seconds), which unzip prefers. + * - The archive comment is the commit id, as git writes it. + */ + +import type { Zippable, ZipOptions } from 'fflate' + +/** The message the zip worker takes. */ +export interface ZipMessage { + readonly entries: Record + readonly meta: { + readonly modes: Readonly> + readonly mtime: number + readonly comment: string | null + } | null +} + +const S_IFMT = 0o170000 +const S_IFLNK = 0o120000 +/** Zip's "version made by" high byte for Unix, and the DOS directory attribute. */ +const OS_UNIX = 3 +const DOS_DIRECTORY = 0x10 +/** The `UT` extended timestamp's header id (APPNOTE 4.6.1 / Info-ZIP). */ +const EXTENDED_TIMESTAMP = 0x5455 + +/** The `UT` field: flags (modification time present) and the time in UTC seconds. */ +function extendedTimestamp(mtime: number): Uint8Array { + const field = new Uint8Array(5) + field[0] = 1 + new DataView(field.buffer).setUint32(1, Math.floor(mtime / 1000) >>> 0, true) + return field +} + +/** The attributes git archive records for a file of git mode `mode`. */ +export function entryAttributes(mode: number): Pick { + if ((mode & S_IFMT) === S_IFLNK) return { os: OS_UNIX, attrs: ((mode | 0o777) << 16) >>> 0 } + if ((mode & 0o111) !== 0) return { os: OS_UNIX, attrs: (mode << 16) >>> 0 } + return { os: 0, attrs: 0 } +} + +/** + * The DOS date an entry carries: zip's DOS date holds 1980 to 2107 and fflate takes 1980 to 2099, + * so a commit time outside that (an imported history's 1970 dates) is clamped to its edge rather + * than failing the whole zip. The `UT` field keeps the exact time. + */ +function dosDate(mtime: number): Date { + const earliest = new Date(1980, 0, 1, 0, 0, 0).getTime() + const latest = new Date(2099, 11, 31, 23, 59, 58).getTime() + return new Date(Math.min(Math.max(mtime, earliest), latest)) +} + +/** `entries` with each file's attributes, every directory's entry, and the commit's time. */ +export function zipEntries(entries: Record, meta: NonNullable): Zippable { + const mtime = dosDate(meta.mtime) + const extra = { [EXTENDED_TIMESTAMP]: extendedTimestamp(meta.mtime) } + const out: Zippable = {} + const dirs = new Set() + for (const path of Object.keys(entries)) { + for (let at = path.indexOf('/'); at !== -1; at = path.indexOf('/', at + 1)) dirs.add(path.slice(0, at + 1)) + } + // Directories first, in path order, then the files (git writes each directory before its contents). + for (const dir of [...dirs].sort()) out[dir] = [new Uint8Array(0), { os: 0, attrs: DOS_DIRECTORY, mtime, extra, level: 0 }] + for (const [path, bytes] of Object.entries(entries)) { + out[path] = [bytes, { ...entryAttributes(meta.modes[path] ?? 0o100644), mtime, extra }] + } + return out +} + +/** `zip` with `comment` as its archive comment (the end-of-central-directory record's). */ +export function withArchiveComment(zip: Uint8Array, comment: string | null): Uint8Array { + if (comment === null || comment === '' || zip.length < 22) return zip + const bytes = new TextEncoder().encode(comment).subarray(0, 0xffff) + const eocd = zip.length - 22 + const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength) + // Only a zip that ends in a comment-less end record (fflate writes one). + if (view.getUint32(eocd, true) !== 0x06054b50 || view.getUint16(eocd + 20, true) !== 0) return zip + const out = new Uint8Array(zip.length + bytes.length) + out.set(zip) + out.set(bytes, zip.length) + new DataView(out.buffer).setUint16(eocd + 20, bytes.length, true) + return out +} diff --git a/forge-web/lib/view/zip.ts b/forge-web/lib/view/zip.ts index 4d4f31d87..1f963a90a 100644 --- a/forge-web/lib/view/zip.ts +++ b/forge-web/lib/view/zip.ts @@ -8,7 +8,25 @@ import { MODE_GITLINK, MODE_TREE } from '../browse' import type { ObjectReader } from './tree-nav' -import { ObjectTypeError, peel, readCommit, readTree } from './tree-nav' +import { ObjectTypeError, peel, readBlob, readTree } from './tree-nav' +import { decodeTextBlob } from './git-objects' +import { historyWalker } from './commit-log' +import { + autoAbbrevLength, + describeCommit, + DescribeError, + describeNames, + expandExportSubst, + exportAttributes, + parseArchiveCommit, + parseDescribeOptions, + peelTags, + uniqueAbbrev, + type ArchiveCommit, + type DescribeTag, + type FormatContext, +} from './archive' +import type { ZipMessage } from './zip-entries' import { mapPooled } from './pool' /** The largest ref the browser zips (uncompressed bytes). Above it: clone instead. */ @@ -36,7 +54,7 @@ export interface FileWalk { } export interface ZipProgress { - readonly phase: 'listing' | 'reading' | 'compressing' + readonly phase: 'listing' | 'reading' | 'describing' | 'compressing' readonly files: number readonly filesTotal: number readonly bytes: number @@ -92,10 +110,135 @@ export async function walkFiles( * nested tags (L-01): the commit's tree, or a tagged tree itself. A tagged blob has no tree. */ export async function listFiles(reader: ObjectReader, tipOid: string): Promise { + return (await planArchive(reader, tipOid)).files +} + +/** What `git archive` writes for a ref (QW-026): the files, and what it does to them. */ +export interface ArchivePlan { + /** The files, those `export-ignore` leaves out already left out. */ + readonly files: ZipFile[] + /** The files whose `$Format:…$` are expanded (`export-subst`). */ + readonly subst: ReadonlySet + /** The commit archived, or null for a tag of a tree (git archive then has no commit to describe). */ + readonly commit: ArchiveCommit | null + /** Every entry's time (ms): the committer time, as git archive stamps it; now for a tree. */ + readonly mtime: number +} + +/** + * The files of a ref's tip, as `git archive` takes them: the tree's own `.gitattributes` decide + * which are left out (`export-ignore`) and which are expanded (`export-subst`), and the commit's + * time is every entry's. + */ +export async function planArchive(reader: ObjectReader, tipOid: string): Promise { const tip = await peel(reader, tipOid) - const tree = tip.type === 'commit' ? (await readCommit(reader, tip.oid)).tree : tip.type === 'tree' ? tip.oid : null + let commit: ArchiveCommit | null = null + let tree: string | null = tip.type === 'tree' ? tip.oid : null + if (tip.type === 'commit') { + const obj = await reader.readObject(tip.oid) + if (obj.type !== 'commit') throw new ObjectTypeError(tip.oid, obj.type, 'commit') + commit = parseArchiveCommit(tip.oid, obj.bytes) + tree = commit.tree + } if (tree === null) throw new ObjectTypeError(tip.oid, tip.type, 'tree') - return (await walkFiles(reader, tree)).files + const all = (await walkFiles(reader, tree)).files + // The tree's attributes, never the viewer's: every `.gitattributes` it holds (a few KiB). + const attrFiles = new Map() + await mapPooled( + all.filter((f) => f.path === '.gitattributes' || f.path.endsWith('/.gitattributes')), + 6, + async (f) => { + const text = decodeTextBlob(await readBlob(reader, f.oid, ATTRIBUTES_MAX_BYTES).catch(() => new Uint8Array())) + if (text !== null) attrFiles.set(f.path, text) + }, + ) + const attrs = exportAttributes(attrFiles) + const files: ZipFile[] = [] + const subst = new Set() + for (const f of all) { + const a = attrs(f.path) + if (a.ignore) continue + files.push(f) + // A symlink's target is not a file's text: git expands regular files only. + if (a.subst && commit !== null && (f.mode & 0o170000) === 0o100000) subst.add(f.path) + } + return { files, subst, commit, mtime: commit === null ? Date.now() : commit.committer.time * 1000 } +} + +/** A `.gitattributes` larger than this is not read (git's own limit is 100 MB; real ones are a few KiB). */ +const ATTRIBUTES_MAX_BYTES = 1024 * 1024 + +/** The refs `%(describe)` and `%d` read. */ +export interface ArchiveRefs { + readonly tags: readonly DescribeTag[] + /** Branch short names and tips. */ + readonly heads: readonly { readonly name: string; readonly oid: string }[] +} + +/** + * Expand the `export-subst` files of `entries` in place (`$Format:…$`, gitattributes(5)), for the + * plan's commit, as git archive does: files in archive order, and one `%(describe)` per archive + * (the first; it is empty when no tag describes the commit, and the ones after it stay as + * written), worked out as `git describe` does; `%d`/`%D` name the branches and tags at the + * commit. A file that is not UTF-8 is left as it is. `signal` stops the describe walk. + */ +export async function substituteFiles( + reader: ObjectReader & { findByPrefix?(prefix: string, limit?: number): string[]; readonly objectCount?: number }, + plan: ArchivePlan, + entries: Record, + refs: ArchiveRefs, + { signal, onProgress }: { readonly signal?: AbortSignal; readonly onProgress?: (p: ZipProgress) => void } = {}, +): Promise { + const commit = plan.commit + if (commit === null || plan.subst.size === 0) return + const texts = new Map() + for (const path of [...plan.subst].sort()) { + const bytes = entries[path] + if (bytes === undefined) continue + try { + texts.set(path, new TextDecoder('utf-8', { fatal: true }).decode(bytes)) + } catch { + // Not UTF-8: left as stored. + } + } + const formats = [...texts.values()].flatMap((t) => [...t.matchAll(/\$Format:([^$]*)\$/g)].map((m) => m[1] as string)) + if (formats.length === 0) return + const findByPrefix = reader.findByPrefix?.bind(reader) + const auto = autoAbbrevLength(reader.objectCount ?? 0) + const abbrev = (oid: string, min: number = auto): string => uniqueAbbrev(oid, min, findByPrefix) + + // The one `%(describe…)` git archive expands: the first, worked out before the (synchronous) expansion. + const first = formats.map((f) => /%\(describe(?::([^)]*))?\)/.exec(f)).find((m) => m !== null) + const firstSpec = first == null ? null : (first[1] ?? '') + const firstOptions = firstSpec === null ? null : parseDescribeOptions(firstSpec) + const needTags = firstOptions !== null || formats.some((f) => /%[dD]/.test(f)) + const peeled = needTags ? await peelTags(reader, refs.tags, signal) : [] + let described = '' + if (firstOptions !== null) { + onProgress?.({ phase: 'describing', files: 0, filesTotal: 0, bytes: 0 }) + const walker = historyWalker(reader) + try { + described = (await describeCommit(walker, commit.oid, describeNames(peeled, firstOptions), firstOptions, abbrev, { ...(signal ? { signal } : {}) })) ?? '' + } catch (e) { + // Past its bound: empty, as git's describe would fail. + if (!(e instanceof DescribeError)) throw e + } finally { + walker.flush?.() + } + } + let invoked = false + const ctx: FormatContext = { + commit, + abbrev, + // `%d` / `%D`: the branches, then every tag, at the commit. + decorations: [...refs.heads.filter((h) => h.oid === commit.oid).map((h) => h.name), ...peeled.filter((t) => t.commit === commit.oid).map((t) => `tag: ${t.ref}`)], + describe: (spec) => { + if (invoked) return null + invoked = true + return parseDescribeOptions(spec) === null ? null : spec === firstSpec ? described : null + }, + } + for (const [path, text] of texts) entries[path] = new TextEncoder().encode(expandExportSubst(text, ctx)) } /** Stored (compressed-on-disk) sizes from the locator: a lower bound, cheap to sum. */ @@ -126,11 +269,20 @@ export async function readZipFiles( return entries } +/** + * How each entry is recorded, as `git archive --format=zip` records it (archive-zip.c): a file's + * git mode (`0100755` for an executable, `0120000` for a symlink; a regular file carries no Unix + * mode, as git writes it), every entry at `mtime` (DOS time and the `UT` extended timestamp), and + * the commit id as the archive comment. + */ +export type ZipMeta = NonNullable + /** Compress in a worker (fflate); resolves with the zip bytes. */ export function compressInWorker( entries: Record, onProgress?: (p: ZipProgress) => void, signal?: AbortSignal, + meta?: ZipMeta, ): Promise { const count = Object.keys(entries).length const bytes = Object.values(entries).reduce((n, b) => n + b.length, 0) @@ -150,7 +302,7 @@ export function compressInWorker( worker.terminate() reject(new Error('cancelled')) }) - worker.postMessage(entries, Object.values(entries).map((b) => b.buffer as ArrayBuffer)) + worker.postMessage({ entries, meta: meta ?? null }, Object.values(entries).map((b) => b.buffer as ArrayBuffer)) }) } diff --git a/forge-web/lib/view/zip.worker.ts b/forge-web/lib/view/zip.worker.ts index 882c340a6..cefe9ce25 100644 --- a/forge-web/lib/view/zip.worker.ts +++ b/forge-web/lib/view/zip.worker.ts @@ -2,16 +2,20 @@ /** * The zip worker: compress `{ path: bytes }` with fflate off the main thread. Every file is - * placed under one top-level directory by the caller's paths, as `git archive` would. + * placed under one top-level directory by the caller's paths, as `git archive --prefix` would. + * With `meta`, each entry is recorded as git archive records it ({@link zipEntries}). */ import { zipSync } from 'fflate' +import { withArchiveComment, zipEntries, type ZipMessage } from './zip-entries' + const scope = self as unknown as DedicatedWorkerGlobalScope -scope.onmessage = (ev: MessageEvent>): void => { +scope.onmessage = (ev: MessageEvent): void => { try { - const zip = zipSync(ev.data, { level: 6 }) + const { entries, meta } = ev.data + const zip = withArchiveComment(zipSync(meta === null ? entries : zipEntries(entries, meta), { level: 6 }), meta?.comment ?? null) scope.postMessage({ ok: true, zip }, [zip.buffer]) } catch (e) { scope.postMessage({ ok: false, error: e instanceof Error ? e.message : String(e) })