From d6b9de8174825c4661412aa1bc9afef086d15f6b Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sat, 26 Sep 2026 01:45:04 +0000 Subject: [PATCH 01/35] feat(token_history): replay every token of one path Replay the history of one path of a cregit repository and give each token a birth commit and a death commit. A merge tries its real parents in their real order, and gives all lines to a parent with an identical file, as git blame does. On Linux, the tokens alive at the tip match git blame on every token of kernel/fork.c, mm/memory.c and include/uapi/linux/io_uring.h. --- token_history/replay.py | 261 +++++++++++++++++++++++++++++++++++ token_history/test_replay.py | 180 ++++++++++++++++++++++++ 2 files changed, 441 insertions(+) create mode 100644 token_history/replay.py create mode 100644 token_history/test_replay.py diff --git a/token_history/replay.py b/token_history/replay.py new file mode 100644 index 00000000..531c0dc6 --- /dev/null +++ b/token_history/replay.py @@ -0,0 +1,261 @@ +"""Replay the history of one path of a cregit repository, token by token. + +Each line of a cregit file is one token. A token is born in the commit whose +diff adds its line, and it keeps its identity while later diffs leave the line +unchanged. At a merge, a line takes its identity from the first parent that +has it unchanged, as `git blame` does; a line that no parent has is born in +the merge. +""" +import re +import subprocess +from dataclasses import dataclass, field + +HEADER = "\x01" +NULL_BLOB = "0" * 40 +HUNK = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@") +RAW = re.compile(r"^:\d+ \d+ ([0-9a-f]{40}) ([0-9a-f]{40}) ") + + +@dataclass +class Hunk: + old_start: int + old_len: int + added: list = field(default_factory=list) + + +@dataclass +class Diff: + old_blob: str = NULL_BLOB + hunks: list = field(default_factory=list) + + +@dataclass +class Commit: + sha: str + parents: list + ct: int + blob: str = None + diffs: list = field(default_factory=list) + + +def path_log(repo, path): + return subprocess.run( + ["git", "-C", repo, "log", "--full-history", "--simplify-merges", + "--reverse", "--topo-order", "--parents", "-m", "-p", "-U0", "--raw", + "--no-abbrev", "--no-renames", "--no-color", + f"--format={HEADER}%H %P%x09%ct", "--", path], + check=True, capture_output=True, text=True, errors="replace").stdout + + +def parse_log(text): + """Commits in topological order; a merge holds one diff per parent.""" + commits, by_sha, cur, diff = [], {}, None, None + for line in text.split("\n"): + if line.startswith(HEADER): + head, ct = line[1:].split("\t") + sha, *parents = head.split() + cur = by_sha.get(sha) + if cur is None: + cur = by_sha[sha] = Commit(sha, parents, int(ct)) + commits.append(cur) + diff = Diff() + cur.diffs.append(diff) + elif cur is None: + continue + elif (m := RAW.match(line)): + diff.old_blob, cur.blob = m.group(1), m.group(2) + elif (m := HUNK.match(line)): + old_len = 1 if m.group(2) is None else int(m.group(2)) + diff.hunks.append(Hunk(int(m.group(1)), old_len)) + elif diff.hunks and line.startswith("+") and not line.startswith("+++"): + diff.hunks[-1].added.append(line[1:]) + return commits + + +def git_out(repo, *args): + return subprocess.run(["git", "-C", repo, *args], capture_output=True, + text=True, errors="replace").stdout + + +def parse_hunks(diff_text): + return parse_log(f"{HEADER}x\t0\n{diff_text}")[0].diffs[0].hunks + + +def real_parent_sources(repo, path): + """Real parents, blobs and blob diffs straight from git, one call each.""" + def blob_at(sha): + return git_out(repo, "rev-parse", "-q", "--verify", + f"{sha}:{path}").strip() or NULL_BLOB + + def diff_blobs(a, b): + return parse_hunks(git_out(repo, "diff", "-U0", "--no-color", a, b)) + + return dict(real_parents=lambda sha: git_out( + repo, "log", "-1", "--format=%P", sha).split(), + blob_at=blob_at, diff_blobs=diff_blobs) + + +def align(parent_len, hunks): + """For each child line: the parent line index it keeps, or its new text.""" + child, pos = [], 0 + for h in hunks: + start = h.old_start - 1 if h.old_len else h.old_start + child.extend(range(pos, start)) + child.extend(("+", text) for text in h.added) + pos = start + h.old_len + child.extend(range(pos, parent_len)) + return child + + +@dataclass +class Token: + text: str + born: str + born_in_merge: bool + died: str = None + + +def unique(items): + return list(dict.fromkeys(items)) + + +class PathReplay: + """Token identities along the history of one path. + + real_parents, blob_at and diff_blobs let a merge try its real parents in + their real order, as `git blame` does, when the simplified history dropped + one of them. + """ + + def __init__(self, commits, real_parents=None, blob_at=None, + diff_blobs=None): + self.commits = commits + self.tokens = [] + self.state = {} + self.blob_nodes = {} + self.order = {c.sha: i for i, c in enumerate(commits)} + self.real_parents = real_parents + self.blob_at = blob_at + self.diff_blobs = diff_blobs + self.missing_parents = 0 + self.unaligned_merges = 0 + + def born(self, text, commit, in_merge): + self.tokens.append(Token(text, commit.sha, in_merge)) + return len(self.tokens) - 1 + + def parent_ids(self, sha): + if sha not in self.state: + self.missing_parents += 1 + return [] + return self.state[sha] + + def blob_of(self, sha): + i = self.order.get(sha) + return NULL_BLOB if i is None else self.commits[i].blob + + def parents_of(self, c): + return self.real_parents(c.sha) if self.real_parents else c.parents + + def merge_blobs(self, c, real): + blobs = unique(self.blob_of(p) for p in c.parents) + if real == c.parents: + return blobs + real_blobs = unique(self.blob_at(p) for p in real) + return blobs if real_blobs == blobs else real_blobs + + def ids_of(self, blob, c): + for p in c.parents: + if self.blob_of(p) == blob: + return self.parent_ids(p) + return self.state_of_blob(blob, c.ct) + + def sources(self, c): + """(parent ids, alignment) per parent, in the order blame tries them.""" + real = self.parents_of(c) + if len(real) <= 1 or not c.parents: + ids = self.parent_ids(c.parents[0]) if c.parents else [] + return [(ids, align(len(ids), c.diffs[0].hunks))] + blobs = self.merge_blobs(c, real) + if c.blob in blobs: + ids = self.ids_of(c.blob, c) + if ids is not None: + return [(ids, align(len(ids), []))] + by_blob = {d.old_blob: d.hunks for d in c.diffs} + out = [] + for blob in blobs: + ids = self.ids_of(blob, c) + if blob in by_blob: + hunks = by_blob[blob] + elif blob == c.blob: + hunks = [] + elif self.diff_blobs and ids is not None: + hunks = self.diff_blobs(blob, c.blob) + else: + ids = None + if ids is None: + self.unaligned_merges += 1 + continue + out.append((ids, align(len(ids), hunks))) + if not out: + ids = self.parent_ids(c.parents[0]) + out = [(ids, align(len(ids), c.diffs[0].hunks))] + return out + + def step(self, c): + if c.blob is None: + c.blob = self.blob_of(c.parents[0]) if c.parents else NULL_BLOB + sources = self.sources(c) + in_merge = len(sources) > 1 or len(c.parents) > 1 + base_ids, base = sources[0] + ids = [] + for i, line in enumerate(base): + if not isinstance(line, tuple): + ids.append(base_ids[line]) + continue + kept = next((s_ids[m[i]] for s_ids, m in sources[1:] + if i < len(m) and not isinstance(m[i], tuple)), None) + ids.append(kept if kept is not None + else self.born(line[1], c, in_merge)) + if not in_merge: + alive = set(ids) + for t in base_ids: + if t not in alive and self.tokens[t].died is None: + self.tokens[t].died = c.sha + self.state[c.sha] = ids + self.blob_nodes.setdefault(c.blob, []).append(c.sha) + + def run(self): + for c in self.commits: + self.step(c) + return self + + def state_of_blob(self, blob, before_ct): + """The latest node with this blob, committed at or before a time.""" + if blob == NULL_BLOB: + return [] + nodes = [s for s in self.blob_nodes.get(blob, ()) + if self.commits[self.order[s]].ct <= before_ct] + nodes = nodes or self.blob_nodes.get(blob, ()) + return self.state[max(nodes, key=self.order.get)] if nodes else None + + def mainline_intervals(self, changes): + """Intervals on the mainline: {token: [(in_sha, in_ct, out_sha, out_ct)]}. + + changes: [(sha, ct, blob)] of the first-parent commits that changed + this path, oldest first. + """ + intervals, alive, unmapped = {}, set(), 0 + for sha, ct, blob in changes: + ids = self.state_of_blob(blob, ct) + if ids is None: + unmapped += 1 + continue + now = set(ids) + for t in now - alive: + intervals.setdefault(t, []).append([sha, ct, None, None]) + for t in alive - now: + intervals[t][-1][2:] = [sha, ct] + alive = now + self.unmapped_mainline = unmapped + return intervals diff --git a/token_history/test_replay.py b/token_history/test_replay.py new file mode 100644 index 00000000..505e7b14 --- /dev/null +++ b/token_history/test_replay.py @@ -0,0 +1,180 @@ +import os +import re +import subprocess + +import pytest + +from replay import (NULL_BLOB, PathReplay, align, Hunk, parse_log, path_log, + real_parent_sources) + + +def git(repo, *args, date=None): + env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@x", + GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@x") + if date: + env["GIT_AUTHOR_DATE"] = env["GIT_COMMITTER_DATE"] = f"{date} +0000" + return subprocess.run(["git", "-C", str(repo), *args], check=True, + capture_output=True, text=True, env=env).stdout + + +def write(repo, path, lines): + (repo / path).write_text("".join(f"{x}\n" for x in lines)) + + +def commit(repo, msg, date): + git(repo, "add", "-A") + git(repo, "commit", "-q", "-m", msg, date=date) + return git(repo, "rev-parse", "HEAD").strip() + + +def blame(repo, path): + out = git(repo, "blame", "--line-porcelain", path) + return [line[:40] for line in out.splitlines() + if re.match(r"^[0-9a-f]{40} \d+ \d+", line)] + + +@pytest.fixture +def repo(tmp_path): + r = tmp_path / "r" + r.mkdir() + git(r, "init", "-q", "-b", "main") + write(r, "a.c", ["int", "a", ";", "int", "b", ";"]) + commit(r, "c1", "2020-01-01T00:00:00") + write(r, "a.c", ["int", "a", ";", "long", "b", ";", "x"]) + commit(r, "c2", "2020-02-01T00:00:00") + git(r, "checkout", "-q", "-b", "side") + write(r, "a.c", ["int", "a", ";", "long", "b", ";", "x", "side1"]) + commit(r, "s1", "2020-03-01T00:00:00") + git(r, "checkout", "-q", "main") + write(r, "a.c", ["main0", "int", "a", ";", "long", "b", ";", "x"]) + commit(r, "m1", "2020-04-01T00:00:00") + git(r, "merge", "-q", "--no-commit", "side") + write(r, "a.c", ["main0", "int", "a", ";", "long", "b", ";", "x", + "side1", "fix"]) + commit(r, "merge", "2020-05-01T00:00:00") + write(r, "a.c", ["main0", "a", ";", "long", "b", ";", "x", "side1", "fix"]) + commit(r, "c3", "2020-06-01T00:00:00") + return r + + +def replay(repo, path): + return PathReplay(parse_log(path_log(str(repo), path))).run() + + +def tip_born(rp, repo, path): + tip = git(repo, "rev-parse", f"HEAD:{path}").strip() + ids = rp.state_of_blob(tip, 2**62) + return [rp.tokens[t].born for t in ids], [rp.tokens[t].text for t in ids] + + +def test_align_insert_delete(): + assert align(3, [Hunk(2, 1, ["y"])]) == [0, ("+", "y"), 2] + assert align(2, [Hunk(1, 0, ["z"])]) == [0, ("+", "z"), 1] + assert align(2, [Hunk(0, 0, ["z"])]) == [("+", "z"), 0, 1] + + +def test_tip_matches_git_blame(repo): + rp = replay(repo, "a.c") + born, texts = tip_born(rp, repo, "a.c") + assert texts == (repo / "a.c").read_text().split() + assert born == blame(repo, "a.c") + assert rp.missing_parents == 0 and rp.unaligned_merges == 0 + + +def test_merge_resolution_line_is_born_in_merge(repo): + rp = replay(repo, "a.c") + merge = git(repo, "rev-parse", "HEAD~1").strip() + fix = next(t for t in rp.tokens if t.text == "fix") + assert fix.born == merge and fix.born_in_merge + + +def test_deleted_token_keeps_its_death(repo): + rp = replay(repo, "a.c") + c3 = git(repo, "rev-parse", "HEAD").strip() + first_int = next(t for t in rp.tokens if t.text == "int") + assert first_int.died == c3 + + +def test_deleted_file_and_recreated(repo): + git(repo, "rm", "-q", "a.c") + commit(repo, "del", "2020-07-01T00:00:00") + write(repo, "a.c", ["int", "new"]) + recreate = commit(repo, "re", "2020-08-01T00:00:00") + rp = replay(repo, "a.c") + born, _ = tip_born(rp, repo, "a.c") + assert born == [recreate, recreate] == blame(repo, "a.c") + assert all(t.died for t in rp.tokens if t.born != recreate) + + +def test_mainline_intervals(repo): + rp = replay(repo, "a.c") + fp = git(repo, "log", "--first-parent", "--reverse", "-m", "--raw", + "--no-abbrev", "--format=\x01%H %ct", "--", "a.c") + changes, sha = [], None + for line in fp.splitlines(): + if line.startswith("\x01"): + sha, ct = line[1:].split() + elif line.startswith(":"): + changes.append((sha, int(ct), line.split()[3])) + iv = rp.mainline_intervals(changes) + side1 = next(i for i, t in enumerate(rp.tokens) if t.text == "side1") + merge = git(repo, "rev-parse", "HEAD~1").strip() + assert iv[side1][0][0] == merge and iv[side1][0][2] is None + first_int = next(i for i, t in enumerate(rp.tokens) if t.text == "int") + assert iv[first_int][0][2] == git(repo, "rev-parse", "HEAD").strip() + assert NULL_BLOB not in {c[2] for c in changes} + + +def test_merge_of_equal_parents_keeps_the_blob(repo): + git(repo, "checkout", "-q", "-b", "twin") + write(repo, "b.c", ["other"]) + commit(repo, "twin only touches b.c", "2020-07-01T00:00:00") + git(repo, "checkout", "-q", "main") + write(repo, "c.c", ["main only touches c.c"]) + commit(repo, "main only touches c.c", "2020-07-02T00:00:00") + git(repo, "merge", "-q", "--no-edit", "twin", date="2020-07-03T00:00:00") + write(repo, "a.c", ["main0", "a", ";", "long", "b", ";", "x", "side1", + "fix", "after"]) + commit(repo, "after merge", "2020-07-04T00:00:00") + rp = replay(repo, "a.c") + born, _ = tip_born(rp, repo, "a.c") + assert born == blame(repo, "a.c") + assert rp.unaligned_merges == 0 + + +def test_merge_follows_real_parent_order(repo): + base = ["main0", "a", ";", "long", "b", ";", "x", "side1", "fix"] + git(repo, "checkout", "-q", "-b", "redo") + write(repo, "a.c", [t for t in base if t != "long"]) + commit(repo, "drop long", "2020-07-01T00:00:00") + write(repo, "a.c", base + ["tail"]) + commit(repo, "re-add long", "2020-07-02T00:00:00") + git(repo, "checkout", "-q", "main") + git(repo, "merge", "-q", "--no-ff", "--no-commit", "redo") + write(repo, "a.c", base + ["tail", "merged"]) + commit(repo, "merge redo", "2020-07-03T00:00:00") + rp = PathReplay(parse_log(path_log(str(repo), "a.c")), + **real_parent_sources(str(repo), "a.c")).run() + born, _ = tip_born(rp, repo, "a.c") + assert born == blame(repo, "a.c") + + +def test_merge_equal_to_second_parent_takes_it_whole(repo): + base = ["main0", "a", ";", "long", "b", ";", "x", "side1", "fix"] + git(repo, "checkout", "-q", "-b", "same") + write(repo, "a.c", base + ["s"]) + commit(repo, "side adds s", "2020-07-01T00:00:00") + git(repo, "checkout", "-q", "main") + write(repo, "a.c", [t for t in base if t != "long"]) + commit(repo, "main drops long", "2020-07-02T00:00:00") + write(repo, "a.c", base) + commit(repo, "main re-adds long", "2020-07-03T00:00:00") + write(repo, "c.c", ["other"]) + commit(repo, "main touches c.c", "2020-07-04T00:00:00") + git(repo, "merge", "-q", "--no-ff", "--no-commit", "same") + write(repo, "a.c", base + ["s"]) + commit(repo, "merge same", "2020-07-05T00:00:00") + rp = PathReplay(parse_log(path_log(str(repo), "a.c")), + **real_parent_sources(str(repo), "a.c")).run() + born, _ = tip_born(rp, repo, "a.c") + assert born == blame(repo, "a.c") From 6d71ec6127eba029d728c9c0675e97c86393306e Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sat, 26 Sep 2026 02:13:43 +0000 Subject: [PATCH 02/35] feat(token_history): add the runner, move pass and build token_history.py replays every path in parallel and writes one row per token and mainline interval. It pings a progress line and history-progress.json every --ping seconds; history_status.sh prints them. A stopped run resumes from paths-done.txt. build.py links moved tokens in one of three modes (off, renames, moves) and writes history-.parquet, so the mode can change without a new replay. validate.py compares the tip with the dataset. A merge that keeps one token twice now gives the second line a copy token with the same birth, so every line has its own identity. --- token_history/build.py | 129 ++++++++++++++++++ token_history/history_status.sh | 20 +++ token_history/moves.py | 113 ++++++++++++++++ token_history/replay.py | 124 ++++++++++++++--- token_history/test_moves.py | 72 ++++++++++ token_history/test_replay.py | 65 +++++++++ token_history/token_history.py | 232 ++++++++++++++++++++++++++++++++ token_history/validate.py | 55 ++++++++ 8 files changed, 793 insertions(+), 17 deletions(-) create mode 100644 token_history/build.py create mode 100755 token_history/history_status.sh create mode 100644 token_history/moves.py create mode 100644 token_history/test_moves.py create mode 100755 token_history/token_history.py create mode 100644 token_history/validate.py diff --git a/token_history/build.py b/token_history/build.py new file mode 100644 index 00000000..d93450d8 --- /dev/null +++ b/token_history/build.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Build history-.parquet from the parts that token_history.py wrote. + +The move mode is chosen here, not in the replay, so a run can switch between +off, renames and moves without replaying the history again. + +Usage: build.py --out DIR [--mode moves] [--min-alnum 100] +""" +import argparse +import glob +import os +import sys +import time + +import duckdb + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from moves import MODES, link, origins # noqa: E402 +from token_history import COLUMNS, RUN_COLUMNS, SEP, log # noqa: E402 + +TYPES = dict(token_id="BIGINT", born_in_merge="INTEGER", copy_of="BIGINT", + tip_index="BIGINT", origin_token_id="BIGINT", + mainline_in_ts="BIGINT", mainline_out_ts="BIGINT", + whole_file="INTEGER") + + +def read_tsv(con, name, pattern, columns): + cols = ", ".join(f"'{c}': '{TYPES.get(c, 'VARCHAR')}'" for c in columns) + con.execute(f""" + CREATE TABLE {name} AS SELECT DISTINCT * FROM read_csv( + '{pattern}', delim='{SEP}', header=false, quote='', escape='', + nullstr='', columns={{{cols}}})""") + + +def commit_runs(con): + """Yield (sha, runs, texts) for commits that have both kinds of run.""" + cur = con.execute(""" + WITH r AS ( + SELECT sha, kind, whole_file, file_path, token_ids, + row_number() OVER () AS run_no + FROM runs + WHERE sha IN (SELECT sha FROM runs GROUP BY sha + HAVING count(DISTINCT kind) = 2)), + t AS ( + SELECT r.*, unnest(string_split(token_ids, ','))::BIGINT AS tid, + generate_subscripts(string_split(token_ids, ','), 1) AS pos + FROM r) + SELECT t.sha, t.run_no, t.kind, t.whole_file, t.file_path, t.tid, + tok.token + FROM t JOIN tokens tok + ON tok.file_path = t.file_path AND tok.token_id = t.tid + ORDER BY t.sha, t.run_no, t.pos""") + sha, runs, texts, cur_run = None, [], {}, None + while batch := cur.fetchmany(100_000): + for s, run_no, kind, whole, path, tid, token in batch: + if s != sha: + if sha is not None: + yield sha, runs, texts + sha, runs, texts, cur_run = s, [], {}, None + if run_no != cur_run: + runs.append((kind, bool(whole), path, [])) + cur_run = run_no + runs[-1][3].append(tid) + texts[(path, tid)] = token + if sha is not None: + yield sha, runs, texts + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[1]) + ap.add_argument("--out", required=True) + ap.add_argument("--mode", choices=MODES, default="moves") + ap.add_argument("--min-alnum", type=int, default=100) + ap.add_argument("--memory", default="6GB") + args = ap.parse_args() + t0 = time.time() + con = duckdb.connect(os.path.join(args.out, "build.duckdb")) + con.execute(f"SET memory_limit='{args.memory}'") + con.execute(f"SET temp_directory='{os.path.join(args.out, 'tmp')}'") + for name in ("tokens", "runs", "links"): + con.execute(f"DROP TABLE IF EXISTS {name}") + read_tsv(con, "tokens", os.path.join(args.out, "part-*.tsv"), COLUMNS) + if glob.glob(os.path.join(args.out, "runs-*.tsv")): + read_tsv(con, "runs", os.path.join(args.out, "runs-*.tsv"), + RUN_COLUMNS) + else: + con.execute("CREATE TABLE runs (sha VARCHAR, kind VARCHAR, " + "whole_file INTEGER, file_path VARCHAR, token_ids VARCHAR)") + log(f"build: {con.execute('SELECT count(*) FROM tokens').fetchone()[0]:,}" + f" token rows, {con.execute('SELECT count(*) FROM runs').fetchone()[0]:,}" + " runs") + + links, commits = {}, 0 + if args.mode != "off": + for sha, runs, texts in commit_runs(con): + links.update(link(runs, texts.__getitem__, args.mode, + args.min_alnum)) + commits += 1 + root = origins(links) + log(f"build: mode {args.mode}, {len(links):,} tokens linked in " + f"{commits:,} commits") + links_file = os.path.join(args.out, f"links-{args.mode}.tsv") + with open(links_file, "w", encoding="utf-8") as f: + f.write(f"{SEP}{SEP}{SEP}\n" if not root else "") + for (p, t), (op, ot) in root.items(): + f.write(f"{p}{SEP}{t}{SEP}{op}{SEP}{ot}\n") + read_tsv(con, "links", links_file, + ["file_path", "token_id", "origin_path", "origin_token_id"]) + + target = os.path.join(args.out, f"history-{args.mode}.parquet") + con.execute(f""" + COPY ( + SELECT t.*, + coalesce(l.origin_path, t.file_path) AS origin_path, + coalesce(l.origin_token_id, t.token_id) AS origin_token_id, + coalesce(o.born_sha, t.born_sha) AS origin_born_sha, + '{args.mode}' AS move_mode + FROM tokens t + LEFT JOIN links l USING (file_path, token_id) + LEFT JOIN (SELECT DISTINCT file_path, token_id, born_sha + FROM tokens) o + ON o.file_path = l.origin_path AND o.token_id = l.origin_token_id + ORDER BY t.file_path, t.token_id, t.mainline_in_ts + ) TO '{target}' (FORMAT parquet, COMPRESSION zstd)""") + log(f"build: wrote {target} in {time.time() - t0:.0f} s") + + +if __name__ == "__main__": + main() diff --git a/token_history/history_status.sh b/token_history/history_status.sh new file mode 100755 index 00000000..f26fbdfe --- /dev/null +++ b/token_history/history_status.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Print the progress of a token_history run: history_status.sh +set -euo pipefail +f="${1:?usage: history_status.sh }/history-progress.json" +[ -f "$f" ] || { echo "no progress file at $f"; exit 1; } +python3 - "$f" <<'PY' +import json, sys, time +p = json.load(open(sys.argv[1])) +pct = 100 * p["paths_done"] / max(p["paths_total"], 1) +eta = p.get("eta_s") +print(f"state {p['state']} (updated {p['updated']})") +print(f"paths {p['paths_done']:,} / {p['paths_total']:,} ({pct:.1f}%)") +print(f"tokens {p['tokens']:,} in {p['rows']:,} rows") +print(f"rate {p['paths_per_min']} paths/min, elapsed {p['elapsed_s'] / 3600:.1f} h") +print(f"eta {'?' if eta is None else f'{eta / 3600:.1f} h'}") +print(f"flags missing parents {p['missing_parents']}, unaligned merges " + f"{p['unaligned_merges']}, unmapped mainline {p['unmapped_mainline']}") +for s, path in p["slowest"]: + print(f"slow {s:7.1f} s {path}") +PY diff --git a/token_history/moves.py b/token_history/moves.py new file mode 100644 index 00000000..5a972ca8 --- /dev/null +++ b/token_history/moves.py @@ -0,0 +1,113 @@ +"""Link tokens that moved between files, so they keep their first identity. + +Within one commit, a run of born tokens that equals a run of died tokens is a +move. Modes: + off no links: every path keeps its own history + renames only from a file the commit deleted to a file it created, as + plain `git blame` follows a renamed file + moves any run of at least --min-alnum alphanumeric characters, as + `git blame -C` does for code moved between files +""" +from collections import defaultdict + +from replay import alnum + +MODES = ("off", "renames", "moves") +ANCHOR = 4 +MAX_CANDIDATES = 64 + + +def matching_blocks(a, b): + """Maximal equal blocks (i, j, size) of b in a, found from k-token anchors. + + Each anchor of b is looked up in an index of a, preferring the candidate + that continues the previous block, and grown forward while equal. Linear + in practice, where difflib is quadratic on runs full of repeated tokens. + """ + k = min(ANCHOR, len(a), len(b)) + if not k: + return [] + index = defaultdict(list) + for i in range(len(a) - k + 1): + cands = index[tuple(a[i:i + k])] + if len(cands) < MAX_CANDIDATES: + cands.append(i) + blocks, j, next_i = [], 0, None + while j <= len(b) - k: + cands = index.get(tuple(b[j:j + k])) + if not cands: + j += 1 + continue + i = next_i if next_i in cands else cands[0] + n = k + while i + n < len(a) and j + n < len(b) and a[i + n] == b[j + n]: + n += 1 + blocks.append((i, j, n)) + j, next_i = j + n, i + n + return blocks + + +SEPARATOR = object() + + +def link(runs, text_of, mode, min_alnum): + """{(path, born id): (path, died id)} for one commit's runs. + + runs: [(kind, whole_file, path, ids)]; text_of((path, id)) -> token text. + """ + if mode == "off": + return {} + eligible = [r for r in runs if mode == "moves" or r[1]] + died_text, died_key = [], [] + for kind, _, path, ids in eligible: + if kind == "died": + for t in ids: + died_text.append(text_of((path, t))) + died_key.append((path, t)) + died_text.append(SEPARATOR) + died_key.append(None) + blocks = [] + for kind, _, b_path, b_ids in eligible: + if kind != "born" or not died_text: + continue + b_text = [text_of((b_path, t)) for t in b_ids] + for i, j, n in matching_blocks(died_text, b_text): + if mode == "renames" and died_key[i][0] == b_path: + continue + if mode == "moves" and alnum(died_text[i:i + n]) < min_alnum: + continue + blocks.append((n, died_key[i:i + n], + [(b_path, t) for t in b_ids[j:j + n]])) + links, used = {}, set() + for _, died, born in sorted(blocks, key=lambda x: -x[0]): + for d, b in zip(died, born): + if d not in used and b not in links: + links[b] = d + used.add(d) + return links + + +def origins(links): + """Follow each chain of moves back to the token's first identity.""" + root = {} + + def find(key): + chain = [] + while key in links and key not in root: + chain.append(key) + key = links[key] + top = root.get(key, key) + for k in chain: + root[k] = top + return top + + for key in links: + find(key) + return root + + +def by_commit(run_rows): + grouped = defaultdict(list) + for sha, kind, whole, path, ids in run_rows: + grouped[sha].append((kind, whole, path, ids)) + return grouped diff --git a/token_history/replay.py b/token_history/replay.py index 531c0dc6..82691e2d 100644 --- a/token_history/replay.py +++ b/token_history/replay.py @@ -113,6 +113,11 @@ class Token: born: str born_in_merge: bool died: str = None + copy_of: int = None + + +def alnum(texts): + return sum(ch.isalnum() for text in texts for ch in text) def unique(items): @@ -128,22 +133,41 @@ class PathReplay: """ def __init__(self, commits, real_parents=None, blob_at=None, - diff_blobs=None): + diff_blobs=None, run_min_alnum=None): self.commits = commits self.tokens = [] self.state = {} self.blob_nodes = {} self.order = {c.sha: i for i, c in enumerate(commits)} self.real_parents = real_parents - self.blob_at = blob_at + self.blob_at = blob_at and self.cached(blob_at) self.diff_blobs = diff_blobs + self.run_min_alnum = run_min_alnum + self.runs = [] self.missing_parents = 0 self.unaligned_merges = 0 + @staticmethod + def cached(fn): + cache = {} + + def lookup(sha): + if sha not in cache: + cache[sha] = fn(sha) + return cache[sha] + return lookup + def born(self, text, commit, in_merge): self.tokens.append(Token(text, commit.sha, in_merge)) return len(self.tokens) - 1 + def copy(self, t): + """A second line with the same identity: a new token, same birth.""" + orig = self.tokens[t] + self.tokens.append(Token(orig.text, orig.born, orig.born_in_merge, + copy_of=t)) + return len(self.tokens) - 1 + def parent_ids(self, sha): if sha not in self.state: self.missing_parents += 1 @@ -203,41 +227,107 @@ def sources(self, c): return out def step(self, c): - if c.blob is None: - c.blob = self.blob_of(c.parents[0]) if c.parents else NULL_BLOB sources = self.sources(c) in_merge = len(sources) > 1 or len(c.parents) > 1 base_ids, base = sources[0] - ids = [] + ids, used = [], set() for i, line in enumerate(base): if not isinstance(line, tuple): - ids.append(base_ids[line]) - continue - kept = next((s_ids[m[i]] for s_ids, m in sources[1:] - if i < len(m) and not isinstance(m[i], tuple)), None) - ids.append(kept if kept is not None - else self.born(line[1], c, in_merge)) + kept = base_ids[line] + else: + kept = next((s_ids[m[i]] for s_ids, m in sources[1:] + if i < len(m) and not isinstance(m[i], tuple)), + None) + if kept is None: + kept = self.born(line[1], c, in_merge) + elif kept in used: + kept = self.copy(kept) + used.add(kept) + ids.append(kept) if not in_merge: alive = set(ids) for t in base_ids: if t not in alive and self.tokens[t].died is None: self.tokens[t].died = c.sha + self.add_runs(c, "died", [(j, t) for j, t in enumerate(base_ids) + if t not in alive]) + self.add_runs(c, "born", [(i, t) for i, t in enumerate(ids) + if self.tokens[t].born == c.sha]) self.state[c.sha] = ids - self.blob_nodes.setdefault(c.blob, []).append(c.sha) - def run(self): + def resolve_blobs(self): + for c in self.commits: + if c.blob is None: + c.blob = self.blob_of(c.parents[0]) if c.parents else NULL_BLOB + + def latest_node(self, blob, before_ct): + nodes = [s for s in self.blob_nodes.get(blob, ()) + if self.commits[self.order[s]].ct <= before_ct] + nodes = nodes or self.blob_nodes.get(blob, ()) + return max(nodes, key=self.order.get) if nodes else None + + def add_runs(self, c, kind, positioned): + """Record runs of adjacent tokens that are long enough to move. + + positioned: [(position in the file, token id)], in file order. + """ + if self.run_min_alnum is None: + return + if kind == "died": + whole = c.blob == NULL_BLOB + else: + whole = all(self.blob_of(p) == NULL_BLOB for p in c.parents) + run, last = [], None + for pos, t in positioned + [(None, None)]: + if run and (pos is None or pos != last + 1): + if alnum(self.tokens[i].text for i in run) >= self.run_min_alnum: + self.runs.append((c.sha, kind, whole, run)) + run = [] + if t is not None: + run.append(t) + last = pos + + def run(self, mainline=None): + """Replay every commit. With mainline changes, free unneeded states.""" + self.resolve_blobs() + for c in self.commits: + self.blob_nodes.setdefault(c.blob, []).append(c.sha) + if mainline is None: + for c in self.commits: + self.step(c) + return self + keep = {self.latest_node(b, ct) for _, ct, b in mainline} + for c in self.commits: + real = self.parents_of(c) + if len(real) > 1 and real != c.parents: + keep |= {self.latest_node(self.blob_at(p), c.ct) + for p in real} + pending = {} + for c in self.commits: + for p in c.parents: + pending[p] = pending.get(p, 0) + 1 for c in self.commits: self.step(c) + for p in c.parents: + pending[p] -= 1 + if not pending[p] and p not in keep: + self.state.pop(p, None) return self def state_of_blob(self, blob, before_ct): """The latest node with this blob, committed at or before a time.""" if blob == NULL_BLOB: return [] - nodes = [s for s in self.blob_nodes.get(blob, ()) - if self.commits[self.order[s]].ct <= before_ct] - nodes = nodes or self.blob_nodes.get(blob, ()) - return self.state[max(nodes, key=self.order.get)] if nodes else None + node = self.latest_node(blob, before_ct) + return None if node is None else self.state.get(node) + + def tip_positions(self, changes): + """{token: position} in the path's file at the last mainline change.""" + if not changes: + return {} + sha, ct, blob = changes[-1] + ids = self.state_of_blob(blob, ct) or [] + return {t: i for i, t in enumerate(ids)} def mainline_intervals(self, changes): """Intervals on the mainline: {token: [(in_sha, in_ct, out_sha, out_ct)]}. diff --git a/token_history/test_moves.py b/token_history/test_moves.py new file mode 100644 index 00000000..efa90ebf --- /dev/null +++ b/token_history/test_moves.py @@ -0,0 +1,72 @@ +from moves import link, origins + + +def texts(d): + return lambda key: d[key] + + +RUN = ["int", "alpha_long_identifier", ";", "beta_long_identifier"] + + +def runs_and_text(moved_whole=True): + tx = {} + for i, t in enumerate(RUN): + tx[("old.c", 10 + i)] = t + tx[("new.c", i)] = t + runs = [("died", moved_whole, "old.c", [10, 11, 12, 13]), + ("born", moved_whole, "new.c", [0, 1, 2, 3])] + return runs, texts(tx) + + +def test_off_links_nothing(): + runs, tx = runs_and_text() + assert link(runs, tx, "off", 10) == {} + + +def test_moves_link_each_token(): + runs, tx = runs_and_text(moved_whole=False) + got = link(runs, tx, "moves", 10) + assert got == {("new.c", i): ("old.c", 10 + i) for i in range(4)} + + +def test_renames_need_a_deleted_and_a_created_file(): + runs, tx = runs_and_text(moved_whole=False) + assert link(runs, tx, "renames", 10) == {} + runs, tx = runs_and_text(moved_whole=True) + assert len(link(runs, tx, "renames", 10)) == 4 + + +def test_short_runs_do_not_move(): + runs, tx = runs_and_text(moved_whole=False) + assert link(runs, tx, "moves", 10_000) == {} + + +def test_origins_follow_chains(): + links = {("c", 1): ("b", 1), ("b", 1): ("a", 1)} + assert origins(links) == {("c", 1): ("a", 1), ("b", 1): ("a", 1)} + + +def test_matching_blocks_on_a_long_repetitive_run(): + import time + from moves import matching_blocks + a = [";", "}", "x", ";"] * 20000 + ["unique_tail_token"] * 10 + b = ["new"] + a + t = time.time() + blocks = matching_blocks(a, b) + assert time.time() - t < 5 + assert sum(n for _, _, n in blocks) == len(a) + assert blocks[0][:2] == (0, 1) + + +def test_a_block_never_spans_two_died_runs(): + a = [f"a_token_{i}" for i in range(4)] + b = [f"b_token_{i}" for i in range(4)] + tx = {("a.c", i): t for i, t in enumerate(a)} + tx |= {("b.c", i): t for i, t in enumerate(b)} + tx |= {("n.c", i): t for i, t in enumerate(a + b)} + runs = [("died", False, "a.c", [0, 1, 2, 3]), + ("died", False, "b.c", [0, 1, 2, 3]), + ("born", False, "n.c", list(range(8)))] + got = link(runs, tx.__getitem__, "moves", 5) + assert [got[("n.c", i)] for i in range(8)] == ( + [("a.c", i) for i in range(4)] + [("b.c", i) for i in range(4)]) diff --git a/token_history/test_replay.py b/token_history/test_replay.py index 505e7b14..d9a4ad73 100644 --- a/token_history/test_replay.py +++ b/token_history/test_replay.py @@ -178,3 +178,68 @@ def test_merge_equal_to_second_parent_takes_it_whole(repo): **real_parent_sources(str(repo), "a.c")).run() born, _ = tip_born(rp, repo, "a.c") assert born == blame(repo, "a.c") + + +def first_parent_changes(repo, path): + fp = git(repo, "log", "--first-parent", "--reverse", "-m", "--raw", + "--no-abbrev", "--format=\x01%H %ct", "--", path) + changes, sha, ct = [], None, 0 + for line in fp.splitlines(): + if line.startswith("\x01"): + sha, ct = line[1:].split() + elif line.startswith(":"): + changes.append((sha, int(ct), line.split()[3])) + return changes + + +def test_freeing_states_changes_nothing(repo): + test_merge_follows_real_parent_order(repo) + changes = first_parent_changes(repo, "a.c") + src = real_parent_sources(str(repo), "a.c") + full = PathReplay(parse_log(path_log(str(repo), "a.c")), **src).run() + lean = PathReplay(parse_log(path_log(str(repo), "a.c")), **src).run(changes) + assert len(lean.state) < len(full.state) + assert lean.tokens == full.tokens + assert lean.mainline_intervals(changes) == full.mainline_intervals(changes) + + +def test_runs_mark_a_whole_file_move(repo): + body = ["int", "alpha_long_identifier", ";", "int", "beta_long_identifier", + ";", "return", "gamma_long_identifier", ";"] + write(repo, "m.c", body) + commit(repo, "add m.c", "2020-07-01T00:00:00") + git(repo, "mv", "m.c", "n.c") + moved = commit(repo, "move m.c to n.c", "2020-07-02T00:00:00") + runs = {} + for path in ("m.c", "n.c"): + rp = PathReplay(parse_log(path_log(str(repo), path)), + run_min_alnum=20).run() + runs[path] = [(sha, kind, whole, [rp.tokens[t].text for t in ids]) + for sha, kind, whole, ids in rp.runs if sha == moved] + assert runs["m.c"] == [(moved, "died", True, body)] + assert runs["n.c"] == [(moved, "born", True, body)] + + +def test_a_line_kept_from_two_parents_becomes_a_copy(tmp_path): + r = tmp_path / "dup" + r.mkdir() + git(r, "init", "-q", "-b", "main") + write(r, "d.c", ["A"]) + first = commit(r, "base", "2021-01-01T00:00:00") + git(r, "checkout", "-q", "-b", "keep") + write(r, "d.c", ["Q", "A"]) + commit(r, "keep adds Q before A", "2021-01-02T00:00:00") + git(r, "checkout", "-q", "main") + write(r, "d.c", ["A", "P"]) + commit(r, "main adds P after A", "2021-01-03T00:00:00") + subprocess.run(["git", "-C", str(r), "merge", "-q", "--no-ff", + "--no-commit", "keep"], capture_output=True) + write(r, "d.c", ["A", "P", "Q", "A"]) + commit(r, "merge keeps both", "2021-01-04T00:00:00") + rp = PathReplay(parse_log(path_log(str(r), "d.c")), + **real_parent_sources(str(r), "d.c")).run() + for ids in rp.state.values(): + assert len(ids) == len(set(ids)) + born, _ = tip_born(rp, r, "d.c") + assert born == blame(r, "d.c") + assert any(t.copy_of is not None and t.born == first for t in rp.tokens) diff --git a/token_history/token_history.py b/token_history/token_history.py new file mode 100755 index 00000000..5d325a72 --- /dev/null +++ b/token_history/token_history.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Extract every token that ever existed in a cregit repository. + +For each path, replay its history (replay.py), then write one row per token +and mainline interval: when the token was born, when a commit first removed +it, and when it entered and left the mainline (the first-parent history of +HEAD). Progress goes to a JSON file and to a log line every --ping seconds. + +Usage: + token_history.py --repo X-cregit.git --out DIR [--paths FILE | -- PATHSPEC] + [--jobs 8] [--ping 60] +""" +import argparse +import json +import multiprocessing as mp +import os +import re +import subprocess +import sys +import time +from datetime import datetime + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from replay import (HEADER, NULL_BLOB, PathReplay, git_out, # noqa: E402 + parse_hunks, parse_log, path_log) + +MASK = re.compile(r"\.[ch]$") +SEP = "\x1f" +COLUMNS = ["file_path", "token_id", "token", "born_sha", "born_in_merge", + "copy_of", "died_sha", "tip_index", "mainline_in_sha", "mainline_in_ts", + "mainline_out_sha", "mainline_out_ts"] +RUN_COLUMNS = ["sha", "kind", "whole_file", "file_path", "token_ids"] + + +def log(msg): + print(f"[{datetime.now():%Y-%m-%d %H:%M:%S}] {msg}", flush=True) + + +def merge_parents(repo): + out = git_out(repo, "rev-list", "--merges", "--parents", "HEAD") + return {sha: parents for sha, *parents in map(str.split, out.splitlines())} + + +def mainline_changes(repo, pathspec): + """{path: [(sha, ct, new blob)]} along the first-parent history.""" + proc = subprocess.Popen( + ["git", "-C", repo, "log", "--first-parent", "--reverse", "-m", + "--raw", "--no-abbrev", "--no-renames", f"--format={HEADER}%H %ct", + "HEAD", "--", *pathspec], + stdout=subprocess.PIPE, text=True, errors="replace") + changes, sha, ct = {}, None, 0 + for line in proc.stdout: + if line.startswith(HEADER): + sha, ct = line[1:].split() + ct = int(ct) + elif line.startswith(":"): + meta, path = line.rstrip("\n").split("\t", 1) + if MASK.search(path): + changes.setdefault(path, []).append((sha, ct, meta.split()[3])) + proc.wait() + return changes + + +class GitSource: + """Real parents from a shared map, blobs from one cat-file per worker.""" + + def __init__(self, repo, merges): + self.repo, self.merges = repo, merges + self.cat = subprocess.Popen( + ["git", "-C", repo, "cat-file", "--batch-check=%(objectname)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + self.path = None + + def real_parents(self, sha): + return self.merges.get(sha) or self.rewritten[sha] + + def blob_at(self, sha): + self.cat.stdin.write(f"{sha}:{self.path}\n") + self.cat.stdin.flush() + out = self.cat.stdout.readline().split()[0] + return out if len(out) == 40 else NULL_BLOB + + def diff_blobs(self, a, b): + return parse_hunks(git_out(self.repo, "diff", "-U0", "--no-color", + a, b)) + + def replay(self, path, mainline, run_min_alnum): + self.path = path + commits = parse_log(path_log(self.repo, path)) + self.rewritten = {c.sha: c.parents for c in commits} + return PathReplay(commits, real_parents=self.real_parents, + blob_at=self.blob_at, diff_blobs=self.diff_blobs, + run_min_alnum=run_min_alnum).run(mainline) + + +def rows(path, rp, intervals, tip): + for i, t in enumerate(rp.tokens): + base = [path, i, t.text, t.born, int(t.born_in_merge), + "" if t.copy_of is None else t.copy_of, t.died or "", + tip.get(i, "")] + for span in intervals.get(i) or [["", "", "", ""]]: + yield base + ["" if v is None else v for v in span] + + +WORKER = {} + + +def init_worker(repo, merges, changes, out_dir, run_min_alnum): + pid = os.getpid() + WORKER.update( + src=GitSource(repo, merges), changes=changes, + run_min_alnum=run_min_alnum, + out=open(os.path.join(out_dir, f"part-{pid}.tsv"), "a", + encoding="utf-8"), + runs=open(os.path.join(out_dir, f"runs-{pid}.tsv"), "a", + encoding="utf-8")) + + +def clean(value): + return str(value).replace(SEP, " ").replace("\n", " ") + + +def work(path): + t0 = time.time() + mainline = WORKER["changes"].get(path, []) + rp = WORKER["src"].replay(path, mainline, WORKER["run_min_alnum"]) + lines = [SEP.join(map(clean, r)) + "\n" + for r in rows(path, rp, rp.mainline_intervals(mainline), + rp.tip_positions(mainline))] + runs = [SEP.join([sha, kind, str(int(whole)), clean(path), + ",".join(map(str, ids))]) + "\n" + for sha, kind, whole, ids in rp.runs] + WORKER["out"].write("".join(lines)) + WORKER["runs"].write("".join(runs)) + WORKER["out"].flush() + WORKER["runs"].flush() + n = len(lines) + return dict(path=path, tokens=len(rp.tokens), rows=n, + commits=len(rp.commits), seconds=round(time.time() - t0, 2), + missing_parents=rp.missing_parents, + unaligned_merges=rp.unaligned_merges, + unmapped_mainline=rp.unmapped_mainline) + + +class Progress: + def __init__(self, path, total, ping): + self.path, self.total, self.ping = path, total, ping + self.start = self.last = time.time() + self.done, self.tokens, self.rows = 0, 0, 0 + self.flags = dict(missing_parents=0, unaligned_merges=0, + unmapped_mainline=0) + self.slowest = [] + + def add(self, r): + self.done += 1 + self.tokens += r["tokens"] + self.rows += r["rows"] + for k in self.flags: + self.flags[k] += r[k] + self.slowest = sorted(self.slowest + [(r["seconds"], r["path"])])[-5:] + if time.time() - self.last >= self.ping or self.done == self.total: + self.write() + + def write(self, state="running"): + self.last = time.time() + elapsed = self.last - self.start + rate = self.done / elapsed if elapsed else 0 + eta = (self.total - self.done) / rate if rate else None + snap = dict(state=state, paths_done=self.done, + paths_total=self.total, tokens=self.tokens, + rows=self.rows, elapsed_s=round(elapsed), + paths_per_min=round(60 * rate, 1), + eta_s=None if eta is None else round(eta), + updated=datetime.now().isoformat(timespec="seconds"), + **self.flags, slowest=self.slowest[::-1]) + tmp = self.path + ".tmp" + with open(tmp, "w") as f: + json.dump(snap, f, indent=1) + os.replace(tmp, self.path) + eta_txt = "?" if eta is None else f"{eta / 3600:.1f} h" + log(f"token_history: {self.done}/{self.total} paths, " + f"{self.tokens:,} tokens, {60 * rate:.1f} paths/min, ETA {eta_txt}") + + +def list_paths(args): + if args.paths: + return [p for p in open(args.paths).read().split("\n") if p] + out = git_out(args.repo, "log", "--diff-filter=A", "--name-only", + "--no-renames", "-m", "--format=", "HEAD", "--", + *args.pathspec) + return sorted({p for p in out.splitlines() if MASK.search(p)}) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[1]) + ap.add_argument("--repo", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--paths") + ap.add_argument("--jobs", type=int, default=8) + ap.add_argument("--ping", type=int, default=60) + ap.add_argument("--run-min-alnum", type=int, default=100, + help="shortest run that the move pass may link") + ap.add_argument("pathspec", nargs="*") + args = ap.parse_args() + os.makedirs(args.out, exist_ok=True) + progress_file = os.path.join(args.out, "history-progress.json") + done_file = os.path.join(args.out, "paths-done.txt") + done = set(open(done_file).read().split("\n")) if os.path.exists( + done_file) else set() + + paths = [p for p in list_paths(args) if p not in done] + log(f"token_history: {len(paths)} paths to replay ({len(done)} done)") + merges = merge_parents(args.repo) + changes = mainline_changes(args.repo, args.pathspec) + log(f"token_history: {len(merges):,} merges, mainline changes for " + f"{len(changes):,} paths") + + progress = Progress(progress_file, len(paths), args.ping) + progress.write("starting") + with open(done_file, "a") as done_out, mp.Pool( + args.jobs, init_worker, + (args.repo, merges, changes, args.out, + args.run_min_alnum)) as pool: + for r in pool.imap_unordered(work, paths): + done_out.write(r["path"] + "\n") + done_out.flush() + progress.add(r) + progress.write("done") + + +if __name__ == "__main__": + main() diff --git a/token_history/validate.py b/token_history/validate.py new file mode 100644 index 00000000..748cd77a --- /dev/null +++ b/token_history/validate.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Compare the tip of a history parquet with the tip dataset parquet. + +Every token alive at the tip must be a row of the dataset at the same file and +token_index, with the same text; the share whose origin commit equals the +dataset's cregit_commit_sha is the agreement with its blame. + +Usage: validate.py --history history-moves.parquet --dataset X-dataset.parquet +""" +import argparse +import json + +import duckdb + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[1]) + ap.add_argument("--history", required=True) + ap.add_argument("--dataset", required=True) + args = ap.parse_args() + con = duckdb.connect() + con.execute("SET memory_limit='6GB'") + con.execute(f""" + CREATE VIEW h AS SELECT * FROM read_parquet('{args.history}') + WHERE tip_index IS NOT NULL AND mainline_out_sha IS NULL""") + con.execute(f""" + CREATE VIEW d AS SELECT file_path, token_index, cregit_commit_sha, + token_type, token_value + FROM read_parquet('{args.dataset}') + WHERE file_path IN (SELECT DISTINCT file_path FROM h)""") + row = con.execute(""" + SELECT + (SELECT count(*) FROM h) AS history_tip, + (SELECT count(*) FROM d) AS dataset_rows, + count(*) AS joined, + count(*) FILTER (WHERE h.token IN (d.token_type, d.token_value, + d.token_type || '|' || d.token_value)) + AS same_text, + count(*) FILTER (WHERE h.origin_born_sha = d.cregit_commit_sha) + AS same_origin, + count(*) FILTER (WHERE h.born_sha = d.cregit_commit_sha) AS same_born + FROM h JOIN d ON d.file_path = h.file_path + AND d.token_index = h.tip_index""").fetchone() + names = ["history_tip", "dataset_rows", "joined", "same_text", + "same_origin", "same_born"] + out = dict(zip(names, row)) + out["origin_agreement_pct"] = round(100 * out["same_origin"] + / max(out["joined"], 1), 2) + out["files"] = con.execute( + "SELECT count(DISTINCT file_path) FROM h").fetchone()[0] + print(json.dumps(out, indent=1)) + + +if __name__ == "__main__": + main() From 58c443311bffa1d27d7aca1b716fb4e6bca0a0ad Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sat, 26 Sep 2026 02:30:39 +0000 Subject: [PATCH 03/35] fix(token_history): pass the io_uring pilot validation A merge no longer takes lines from a parent that lacks the file. A failing path is logged to errors.txt and the run continues. The parts use a delimiter that tokens never hold, and empty tokens stay empty strings. The move pass matches from 4-token anchors, one index per commit, instead of difflib, which was quadratic. On the io_uring pilot (96 paths), the tip holds all 148,189 dataset tokens with the same text, and the renames mode matches plain git blame on 99.90% of them. --- token_history/build.py | 4 +++- token_history/history_status.sh | 3 ++- token_history/replay.py | 2 ++ token_history/test_replay.py | 23 +++++++++++++++++++++++ token_history/token_history.py | 24 ++++++++++++++++++++---- token_history/validate.py | 0 6 files changed, 50 insertions(+), 6 deletions(-) mode change 100644 => 100755 token_history/build.py mode change 100644 => 100755 token_history/validate.py diff --git a/token_history/build.py b/token_history/build.py old mode 100644 new mode 100755 index d93450d8..36fae75a --- a/token_history/build.py +++ b/token_history/build.py @@ -26,10 +26,12 @@ def read_tsv(con, name, pattern, columns): cols = ", ".join(f"'{c}': '{TYPES.get(c, 'VARCHAR')}'" for c in columns) + keep_text = ", force_not_null=['token']" if "token" in columns else "" con.execute(f""" CREATE TABLE {name} AS SELECT DISTINCT * FROM read_csv( '{pattern}', delim='{SEP}', header=false, quote='', escape='', - nullstr='', columns={{{cols}}})""") + nullstr='', auto_detect=false{keep_text}, + columns={{{cols}}})""") def commit_runs(con): diff --git a/token_history/history_status.sh b/token_history/history_status.sh index f26fbdfe..ab170a49 100755 --- a/token_history/history_status.sh +++ b/token_history/history_status.sh @@ -14,7 +14,8 @@ print(f"tokens {p['tokens']:,} in {p['rows']:,} rows") print(f"rate {p['paths_per_min']} paths/min, elapsed {p['elapsed_s'] / 3600:.1f} h") print(f"eta {'?' if eta is None else f'{eta / 3600:.1f} h'}") print(f"flags missing parents {p['missing_parents']}, unaligned merges " - f"{p['unaligned_merges']}, unmapped mainline {p['unmapped_mainline']}") + f"{p['unaligned_merges']}, unmapped mainline {p['unmapped_mainline']}, " + f"errors {p.get('errors', 0)}") for s, path in p["slowest"]: print(f"slow {s:7.1f} s {path}") PY diff --git a/token_history/replay.py b/token_history/replay.py index 82691e2d..7293092c 100644 --- a/token_history/replay.py +++ b/token_history/replay.py @@ -208,6 +208,8 @@ def sources(self, c): by_blob = {d.old_blob: d.hunks for d in c.diffs} out = [] for blob in blobs: + if blob == NULL_BLOB: + continue ids = self.ids_of(blob, c) if blob in by_blob: hunks = by_blob[blob] diff --git a/token_history/test_replay.py b/token_history/test_replay.py index d9a4ad73..f34bccd5 100644 --- a/token_history/test_replay.py +++ b/token_history/test_replay.py @@ -243,3 +243,26 @@ def test_a_line_kept_from_two_parents_becomes_a_copy(tmp_path): born, _ = tip_born(rp, r, "d.c") assert born == blame(r, "d.c") assert any(t.copy_of is not None and t.born == first for t in rp.tokens) + + +def test_merge_that_brings_a_new_file_to_main(tmp_path): + r = tmp_path / "newfile" + r.mkdir() + git(r, "init", "-q", "-b", "main") + write(r, "other.c", ["o"]) + commit(r, "base", "2022-01-01T00:00:00") + git(r, "checkout", "-q", "-b", "topic") + write(r, "n.c", ["a", "b", "c"]) + commit(r, "topic adds n.c", "2022-01-02T00:00:00") + git(r, "checkout", "-q", "main") + write(r, "other.c", ["o", "p"]) + commit(r, "main changes other.c", "2022-01-03T00:00:00") + git(r, "merge", "-q", "--no-ff", "--no-commit", "topic") + write(r, "n.c", ["a", "b", "c", "fixed in merge"]) + commit(r, "merge topic with a fix", "2022-01-04T00:00:00") + write(r, "n.c", ["a", "b", "c", "fixed in merge", "later"]) + commit(r, "later", "2022-01-05T00:00:00") + rp = PathReplay(parse_log(path_log(str(r), "n.c")), + **real_parent_sources(str(r), "n.c")).run() + born, _ = tip_born(rp, r, "n.c") + assert born == blame(r, "n.c") diff --git a/token_history/token_history.py b/token_history/token_history.py index 5d325a72..eb647eba 100755 --- a/token_history/token_history.py +++ b/token_history/token_history.py @@ -18,6 +18,7 @@ import subprocess import sys import time +import traceback from datetime import datetime sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -121,6 +122,13 @@ def clean(value): def work(path): + try: + return replay_path(path) + except Exception: + return dict(path=path, error=traceback.format_exc()) + + +def replay_path(path): t0 = time.time() mainline = WORKER["changes"].get(path, []) rp = WORKER["src"].replay(path, mainline, WORKER["run_min_alnum"]) @@ -148,15 +156,19 @@ def __init__(self, path, total, ping): self.start = self.last = time.time() self.done, self.tokens, self.rows = 0, 0, 0 self.flags = dict(missing_parents=0, unaligned_merges=0, - unmapped_mainline=0) + unmapped_mainline=0, errors=0) self.slowest = [] def add(self, r): self.done += 1 + if "error" in r: + self.flags["errors"] += 1 + r = dict(r, tokens=0, rows=0, seconds=0, missing_parents=0, + unaligned_merges=0, unmapped_mainline=0) self.tokens += r["tokens"] self.rows += r["rows"] for k in self.flags: - self.flags[k] += r[k] + self.flags[k] += r.get(k, 0) self.slowest = sorted(self.slowest + [(r["seconds"], r["path"])])[-5:] if time.time() - self.last >= self.ping or self.done == self.total: self.write() @@ -222,8 +234,12 @@ def main(): (args.repo, merges, changes, args.out, args.run_min_alnum)) as pool: for r in pool.imap_unordered(work, paths): - done_out.write(r["path"] + "\n") - done_out.flush() + if "error" in r: + with open(os.path.join(args.out, "errors.txt"), "a") as err: + err.write(f"== {r['path']}\n{r['error']}\n") + else: + done_out.write(r["path"] + "\n") + done_out.flush() progress.add(r) progress.write("done") diff --git a/token_history/validate.py b/token_history/validate.py old mode 100644 new mode 100755 From 344a687e44776a8714207c9b57dc990ecddfe948 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sat, 26 Sep 2026 12:40:41 +0000 Subject: [PATCH 04/35] docs(token_history): describe the two stages and the move modes --- token_history/README.md | 68 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 token_history/README.md diff --git a/token_history/README.md b/token_history/README.md new file mode 100644 index 00000000..6adaaf64 --- /dev/null +++ b/token_history/README.md @@ -0,0 +1,68 @@ +# token_history — every token that ever existed + +The tip dataset (`generate_dataset`) holds only the tokens alive at HEAD. This step replays the tokenized history (`-cregit.git`) and writes one row for every token that ever existed: its birth commit, the first commit that removed it, and its intervals on the mainline (the first-parent history of HEAD). + +## Two stages + +| Stage | Script | Cost | Output | +| --- | --- | --- | --- | +| 1. Replay | `token_history.py` | hours; Linux about 1.5–2 days with 8 jobs | `part-*.tsv` (tokens), `runs-*.tsv` (move candidates) | +| 2. Build | `build.py --mode ` | seconds to minutes | `history-.parquet` | + +Stage 2 reads only the files of stage 1. Keep them: every mode, and any later change to the move rules, rebuilds from them without a new replay. + +## Move modes (stage 2) + +| Mode | Links | Matches | +| --- | --- | --- | +| `off` | none | the history of each path alone | +| `renames` | a deleted file to a file created in the same commit | plain `git blame` (99.90% of tokens on the io_uring pilot) | +| `moves` (default) | any exact run of ≥ `--min-alnum` alphanumeric characters that one commit removes and adds | keeps the first identity of moved code; follows more moves than `git blame -C100` | + +`moves` is not a copy of `git blame -C100`: git diffs a block against the whole old file and drops a move that its diff breaks into short pieces. On the io_uring pilot, `moves` agrees with the `-C100` tip dataset on 49% of the tip tokens, `off` on 62%. A rule that copies git needs only the repository and the stage 1 files, so it can be added to `moves.py` later. + +## Run + +```sh +# once per repository: changed-path Bloom filters (a path-limited log goes from ~21 s to ~4 s on Linux) +git -C X-cregit.git commit-graph write --reachable --changed-paths + +# stage 1 (resumable: a new run skips the paths in paths-done.txt) +python3 token_history.py --repo X-cregit.git --out OUT --jobs 8 --ping 60 [--paths FILE | -- PATHSPEC] + +# progress at any time +./history_status.sh OUT + +# stage 2, once per mode, any time later +python3 build.py --out OUT --mode moves +python3 build.py --out OUT --mode renames + +# check the tip against the tip dataset +python3 validate.py --history OUT/history-moves.parquet --dataset X-dataset.parquet +``` + +Stage 1 writes `history-progress.json` and a log line every `--ping` seconds. A path that fails goes to `errors.txt` with its traceback, and the run continues. + +## Columns of `history-.parquet` + +| Column | Meaning | +| --- | --- | +| `file_path`, `token_id` | the token; `token_id` counts per path | +| `token` | the cregit line, `type\|value` | +| `born_sha` | the cregit commit that added the line | +| `born_in_merge` | 1 when no parent of a merge had the line | +| `copy_of` | the token this line duplicates, when a merge kept one token twice | +| `died_sha` | the first non-merge commit that removed it | +| `tip_index` | its line in the file at HEAD, when alive there (joins `token_index` of the tip dataset) | +| `mainline_in_sha`, `mainline_in_ts` | the first-parent commit (and committer time) where it entered the mainline | +| `mainline_out_sha`, `mainline_out_ts` | where it left the mainline; empty while alive | +| `origin_path`, `origin_token_id`, `origin_born_sha` | its first identity after the move links of the mode | +| `move_mode` | the mode of the build | + +A token that left the mainline and came back has one row per interval. A token alive at any time T is a row with `mainline_in_ts <= T` and `T < mainline_out_ts` (or no `mainline_out_ts`). + +## Tests + +```sh +pytest token_history -q +``` From 37516a4475077355b0d064a1b2e3a18d93ca47c7 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 28 Sep 2026 12:26:32 +0000 Subject: [PATCH 05/35] fix(token_history): scale the build to the whole kernel Write move links to disk per commit and resolve move chains in DuckDB by pointer jumping. The in-memory link map needs tens of GB on Linux, with 283 M token IDs in move candidates. Number the runs in a fixed order. With repeated code, the old arbitrary order changed which copy a moved token linked to, so two builds of the same parts differed on 0.5% of pilot rows. Raise the CSV line limit for whole-file runs of large headers, and run the full-row DISTINCT only when a row key repeats. --- token_history/build.py | 95 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 18 deletions(-) diff --git a/token_history/build.py b/token_history/build.py index 36fae75a..39c49f14 100755 --- a/token_history/build.py +++ b/token_history/build.py @@ -15,7 +15,7 @@ import duckdb sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from moves import MODES, link, origins # noqa: E402 +from moves import MODES, link # noqa: E402 from token_history import COLUMNS, RUN_COLUMNS, SEP, log # noqa: E402 TYPES = dict(token_id="BIGINT", born_in_merge="INTEGER", copy_of="BIGINT", @@ -24,14 +24,58 @@ whole_file="INTEGER") -def read_tsv(con, name, pattern, columns): +# A whole-file run of a large generated header is one line of several MB. +MAX_LINE = 16_000_000 + + +def read_tsv(con, name, pattern, columns, key=None): + """Load TSV parts; drop repeated rows (a path written twice by a resume). + + key: the columns that identify a row; the full-row DISTINCT runs only + when they repeat, since it is costly on a whole kernel. + """ cols = ", ".join(f"'{c}': '{TYPES.get(c, 'VARCHAR')}'" for c in columns) keep_text = ", force_not_null=['token']" if "token" in columns else "" con.execute(f""" - CREATE TABLE {name} AS SELECT DISTINCT * FROM read_csv( + CREATE TABLE {name} AS SELECT * FROM read_csv( '{pattern}', delim='{SEP}', header=false, quote='', escape='', - nullstr='', auto_detect=false{keep_text}, + nullstr='', auto_detect=false, max_line_size={MAX_LINE}{keep_text}, columns={{{cols}}})""") + key = ", ".join(key or columns) + rows, keys = con.execute(f"SELECT count(*), count(DISTINCT ({key})) " + f"FROM {name}").fetchone() + if rows != keys: + con.execute(f"CREATE OR REPLACE TABLE {name} AS " + f"SELECT DISTINCT * FROM {name}") + + +def resolve_origins(con): + """links(file_path, token_id -> origin) to the first identity of each chain. + + Pointer jumping: each round replaces an origin by that origin's own + origin, so a chain of n moves needs about log2(n) rounds. A move always + points to an older token, so chains end. + """ + rounds = 0 + while True: + con.execute(""" + CREATE OR REPLACE TABLE next_links AS + SELECT l.file_path, l.token_id, + coalesce(u.origin_path, l.origin_path) AS origin_path, + coalesce(u.origin_token_id, l.origin_token_id) + AS origin_token_id, + u.file_path IS NOT NULL AS jumped + FROM links l LEFT JOIN links u + ON u.file_path = l.origin_path + AND u.token_id = l.origin_token_id""") + jumped = con.execute("SELECT count(*) FILTER (WHERE jumped) " + "FROM next_links").fetchone()[0] + con.execute("CREATE OR REPLACE TABLE links AS SELECT file_path, " + "token_id, origin_path, origin_token_id FROM next_links") + con.execute("DROP TABLE next_links") + rounds += 1 + if not jumped: + return rounds def commit_runs(con): @@ -39,7 +83,12 @@ def commit_runs(con): cur = con.execute(""" WITH r AS ( SELECT sha, kind, whole_file, file_path, token_ids, - row_number() OVER () AS run_no + -- a fixed order: with repeated code, the order of the + -- runs decides which copy a moved token links to + row_number() OVER ( + ORDER BY sha, kind, file_path, + string_split(token_ids, ',')[1]::BIGINT) + AS run_no FROM runs WHERE sha IN (SELECT sha FROM runs GROUP BY sha HAVING count(DISTINCT kind) = 2)), @@ -74,14 +123,18 @@ def main(): ap.add_argument("--mode", choices=MODES, default="moves") ap.add_argument("--min-alnum", type=int, default=100) ap.add_argument("--memory", default="6GB") + ap.add_argument("--threads", type=int, default=4) args = ap.parse_args() t0 = time.time() con = duckdb.connect(os.path.join(args.out, "build.duckdb")) con.execute(f"SET memory_limit='{args.memory}'") con.execute(f"SET temp_directory='{os.path.join(args.out, 'tmp')}'") + con.execute(f"SET threads={args.threads}") + con.execute("SET preserve_insertion_order=false") for name in ("tokens", "runs", "links"): con.execute(f"DROP TABLE IF EXISTS {name}") - read_tsv(con, "tokens", os.path.join(args.out, "part-*.tsv"), COLUMNS) + read_tsv(con, "tokens", os.path.join(args.out, "part-*.tsv"), COLUMNS, + key=["file_path", "token_id", "mainline_in_sha"]) if glob.glob(os.path.join(args.out, "runs-*.tsv")): read_tsv(con, "runs", os.path.join(args.out, "runs-*.tsv"), RUN_COLUMNS) @@ -92,22 +145,28 @@ def main(): f" token rows, {con.execute('SELECT count(*) FROM runs').fetchone()[0]:,}" " runs") - links, commits = {}, 0 - if args.mode != "off": - for sha, runs, texts in commit_runs(con): - links.update(link(runs, texts.__getitem__, args.mode, - args.min_alnum)) - commits += 1 - root = origins(links) - log(f"build: mode {args.mode}, {len(links):,} tokens linked in " - f"{commits:,} commits") + # Links go to disk commit by commit: on a whole kernel they do not fit + # in memory. A born token has one birth commit, so it has one link. links_file = os.path.join(args.out, f"links-{args.mode}.tsv") + linked = commits = 0 with open(links_file, "w", encoding="utf-8") as f: - f.write(f"{SEP}{SEP}{SEP}\n" if not root else "") - for (p, t), (op, ot) in root.items(): - f.write(f"{p}{SEP}{t}{SEP}{op}{SEP}{ot}\n") + f.write(f"{SEP}{SEP}{SEP}\n") + if args.mode != "off": + for sha, runs, texts in commit_runs(con): + for (p, t), (op, ot) in link(runs, texts.__getitem__, + args.mode, + args.min_alnum).items(): + f.write(f"{p}{SEP}{t}{SEP}{op}{SEP}{ot}\n") + linked += 1 + commits += 1 + if commits % 20_000 == 0: + log(f"build: {commits:,} commits, {linked:,} links") + log(f"build: mode {args.mode}, {linked:,} tokens linked in " + f"{commits:,} commits") read_tsv(con, "links", links_file, ["file_path", "token_id", "origin_path", "origin_token_id"]) + con.execute("DELETE FROM links WHERE file_path IS NULL") + log(f"build: move chains resolved in {resolve_origins(con)} rounds") target = os.path.join(args.out, f"history-{args.mode}.parquet") con.execute(f""" From fc010fbc555e304ee018edaa06a6820d3a1034ec Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 28 Sep 2026 12:29:47 +0000 Subject: [PATCH 06/35] feat(token_history): attribute every cregit commit Write one row per cregit commit with the author, person and firm columns of generate_dataset phase 2, through the same joins, so that a history token joins its birth commit to the firm the tip dataset uses. On Linux: 1,482,779 commits; the 837,522 that the tip dataset also holds have the same person, firm and original commit in all cases. --- token_history/attribute.py | 88 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 token_history/attribute.py diff --git a/token_history/attribute.py b/token_history/attribute.py new file mode 100644 index 00000000..ac4f3530 --- /dev/null +++ b/token_history/attribute.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Author, person and firm of every cregit commit, as the tip dataset has them. + +A history parquet names commits only. This writes one row per cregit commit +with the same columns and the same joins as generate_dataset phase 2, so a +history token joins its born_sha (or origin_born_sha) to the author and firm +that the tip dataset gives a surviving token of the same commit. + +Usage: attribute.py --cregit-db X-cregit.db --persons-db X-persons.db + --firm-map affiliation.merged.csv + [--firm-canonical firm_canonical.csv] + --out commits.parquet [--dataset X-dataset.parquet] + +With --dataset, every commit that the dataset also has is compared on its +person and firm, and the counts are printed. +""" +import argparse +import json +import os +import sys + +import duckdb + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), + "..", "generate_dataset")) +from generate_dataset import firm_sql, sql_literal # noqa: E402 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[1]) + ap.add_argument("--cregit-db", required=True) + ap.add_argument("--persons-db", required=True) + ap.add_argument("--firm-map", required=True) + ap.add_argument("--firm-canonical") + ap.add_argument("--out", required=True) + ap.add_argument("--dataset") + ap.add_argument("--memory", default="4GB") + args = ap.parse_args() + con = duckdb.connect() + con.execute(f"SET memory_limit='{args.memory}'") + con.execute("INSTALL sqlite_scanner; LOAD sqlite_scanner;") + con.execute(f"CALL sqlite_attach({sql_literal(args.cregit_db)})") + con.execute(f"CALL sqlite_attach({sql_literal(args.persons_db)})") + firm_select, firm_join = firm_sql(args.firm_map, args.firm_canonical) + con.execute(f""" + COPY ( + SELECT + c.cid AS cregit_commit_sha, + coalesce(m.originalcid, c.cid) AS original_commit_sha, + c.autname AS author_name, + c.autemail AS author_email, + c.autdate AS author_date, + c.comdate AS committer_date, + e.personid, + coalesce(p.personname, e.personid) AS person_name, + e.emailaddr AS person_email, + e.domain AS person_domain, +{firm_select} + coalesce(m.repo, '') AS repo_tag + FROM commits c + LEFT JOIN commitmap m ON c.cid = m.cid + LEFT JOIN emails e ON (c.autname = e.emailname + AND c.autemail = e.emailaddr) + LEFT JOIN persons p ON e.personid = p.personid +{firm_join} ORDER BY c.cid + ) TO {sql_literal(args.out)} (FORMAT parquet, COMPRESSION zstd)""") + out = dict(commits=con.execute( + f"SELECT count(*), count(DISTINCT cregit_commit_sha) " + f"FROM read_parquet({sql_literal(args.out)})").fetchone()) + if args.dataset: + out.update(zip(["dataset_commits", "same_person", "same_firm", + "same_original"], con.execute(f""" + WITH d AS ( + SELECT DISTINCT cregit_commit_sha, original_commit_sha, + personid, firm + FROM read_parquet({sql_literal(args.dataset)})) + SELECT count(*), + count(*) FILTER (WHERE a.personid IS NOT DISTINCT FROM d.personid), + count(*) FILTER (WHERE a.firm IS NOT DISTINCT FROM d.firm), + count(*) FILTER (WHERE a.original_commit_sha + = d.original_commit_sha) + FROM d LEFT JOIN read_parquet({sql_literal(args.out)}) a + USING (cregit_commit_sha)""").fetchone())) + print(json.dumps(out, indent=1)) + + +if __name__ == "__main__": + main() From 6ef3e4c21ea706c9d9f458905575ac0c8b3c6e3e Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 28 Sep 2026 15:10:00 +0000 Subject: [PATCH 07/35] fix(token_history): write the history in path chunks The final join of every token with every link ran out of memory on the whole kernel (5.5 GB limit). The build now finds the origin birth commit and writes the output in chunks of paths, then copies the parts into one parquet. It no longer sorts the rows. --reuse keeps the loaded tables and a finished links file, so a failed build restarts at the chain resolution. The link pass writes to a .part file and renames it at the end. On the io_uring pilot, all three modes give the same rows as the previous build. --- token_history/build.py | 140 +++++++++++++++++++++++++++++------------ 1 file changed, 101 insertions(+), 39 deletions(-) mode change 100755 => 100644 token_history/build.py diff --git a/token_history/build.py b/token_history/build.py old mode 100755 new mode 100644 index 39c49f14..8f6347cc --- a/token_history/build.py +++ b/token_history/build.py @@ -4,11 +4,16 @@ The move mode is chosen here, not in the replay, so a run can switch between off, renames and moves without replaying the history again. -Usage: build.py --out DIR [--mode moves] [--min-alnum 100] +Usage: build.py --out DIR [--mode moves] [--min-alnum 100] [--reuse] + +--reuse keeps the tokens and runs tables of DIR/build.duckdb and a finished +links-.tsv, so a build that failed after the link pass restarts at the +chain resolution. """ import argparse import glob import os +import shutil import sys import time @@ -78,6 +83,58 @@ def resolve_origins(con): return rounds +def in_chunk(column, k, chunks): + return f"hash({column}) % {chunks} = {k}" + + +def write_history(con, target, mode, chunks): + """Tokens with their resolved origin, written in chunks of paths. + + On a whole kernel, one join of every token with every link does not fit + in memory. Each chunk joins only the paths whose hash falls in it; the + parts are then copied into one parquet (no row order is kept). + """ + con.execute("CREATE OR REPLACE TABLE origins (file_path VARCHAR, " + "token_id BIGINT, origin_path VARCHAR, origin_token_id BIGINT, " + "origin_born_sha VARCHAR)") + for k in range(chunks): + con.execute(f""" + INSERT INTO origins + SELECT l.file_path, l.token_id, l.origin_path, l.origin_token_id, + o.born_sha + FROM (SELECT * FROM links + WHERE {in_chunk('origin_path', k, chunks)}) l + JOIN (SELECT DISTINCT file_path, token_id, born_sha FROM tokens + WHERE {in_chunk('file_path', k, chunks)}) o + ON o.file_path = l.origin_path + AND o.token_id = l.origin_token_id""") + parts = os.path.join(os.path.dirname(target), + f".{os.path.basename(target)}.parts") + shutil.rmtree(parts, ignore_errors=True) + os.makedirs(parts) + for k in range(chunks): + con.execute(f""" + COPY ( + SELECT t.*, + coalesce(o.origin_path, t.file_path) AS origin_path, + coalesce(o.origin_token_id, t.token_id) + AS origin_token_id, + coalesce(o.origin_born_sha, t.born_sha) + AS origin_born_sha, + '{mode}' AS move_mode + FROM (SELECT * FROM tokens + WHERE {in_chunk('file_path', k, chunks)}) t + LEFT JOIN (SELECT * FROM origins + WHERE {in_chunk('file_path', k, chunks)}) o + USING (file_path, token_id) + ) TO '{os.path.join(parts, f"{k:03d}.parquet")}' + (FORMAT parquet, COMPRESSION zstd)""") + con.execute(f"COPY (SELECT * FROM read_parquet('{parts}/*.parquet')) " + f"TO '{target}' (FORMAT parquet, COMPRESSION zstd)") + shutil.rmtree(parts) + con.execute("DROP TABLE origins") + + def commit_runs(con): """Yield (sha, runs, texts) for commits that have both kinds of run.""" cur = con.execute(""" @@ -117,6 +174,34 @@ def commit_runs(con): yield sha, runs, texts +def load(con, out): + read_tsv(con, "tokens", os.path.join(out, "part-*.tsv"), COLUMNS, + key=["file_path", "token_id", "mainline_in_sha"]) + if glob.glob(os.path.join(out, "runs-*.tsv")): + read_tsv(con, "runs", os.path.join(out, "runs-*.tsv"), RUN_COLUMNS) + else: + con.execute("CREATE TABLE runs (sha VARCHAR, kind VARCHAR, " + "whole_file INTEGER, file_path VARCHAR, token_ids VARCHAR)") + + +def write_links(con, links_file, mode, min_alnum): + """The link pass, to a .part file that is renamed only at the end.""" + linked = commits = 0 + with open(links_file + ".part", "w", encoding="utf-8") as f: + f.write(f"{SEP}{SEP}{SEP}\n") + if mode != "off": + for sha, runs, texts in commit_runs(con): + for (p, t), (op, ot) in link(runs, texts.__getitem__, mode, + min_alnum).items(): + f.write(f"{p}{SEP}{t}{SEP}{op}{SEP}{ot}\n") + linked += 1 + commits += 1 + if commits % 20_000 == 0: + log(f"build: {commits:,} commits, {linked:,} links") + os.replace(links_file + ".part", links_file) + log(f"build: mode {mode}, {linked:,} tokens linked in {commits:,} commits") + + def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[1]) ap.add_argument("--out", required=True) @@ -124,6 +209,10 @@ def main(): ap.add_argument("--min-alnum", type=int, default=100) ap.add_argument("--memory", default="6GB") ap.add_argument("--threads", type=int, default=4) + ap.add_argument("--chunks", type=int, default=16, + help="path chunks for the final write") + ap.add_argument("--reuse", action="store_true", + help="keep the loaded tables and a finished links file") args = ap.parse_args() t0 = time.time() con = duckdb.connect(os.path.join(args.out, "build.duckdb")) @@ -131,16 +220,13 @@ def main(): con.execute(f"SET temp_directory='{os.path.join(args.out, 'tmp')}'") con.execute(f"SET threads={args.threads}") con.execute("SET preserve_insertion_order=false") - for name in ("tokens", "runs", "links"): + tables = {n for n, in con.execute("SELECT table_name FROM " + "duckdb_tables()").fetchall()} + reuse = args.reuse and {"tokens", "runs"} <= tables + for name in ("links",) if reuse else ("tokens", "runs", "links"): con.execute(f"DROP TABLE IF EXISTS {name}") - read_tsv(con, "tokens", os.path.join(args.out, "part-*.tsv"), COLUMNS, - key=["file_path", "token_id", "mainline_in_sha"]) - if glob.glob(os.path.join(args.out, "runs-*.tsv")): - read_tsv(con, "runs", os.path.join(args.out, "runs-*.tsv"), - RUN_COLUMNS) - else: - con.execute("CREATE TABLE runs (sha VARCHAR, kind VARCHAR, " - "whole_file INTEGER, file_path VARCHAR, token_ids VARCHAR)") + if not reuse: + load(con, args.out) log(f"build: {con.execute('SELECT count(*) FROM tokens').fetchone()[0]:,}" f" token rows, {con.execute('SELECT count(*) FROM runs').fetchone()[0]:,}" " runs") @@ -148,41 +234,17 @@ def main(): # Links go to disk commit by commit: on a whole kernel they do not fit # in memory. A born token has one birth commit, so it has one link. links_file = os.path.join(args.out, f"links-{args.mode}.tsv") - linked = commits = 0 - with open(links_file, "w", encoding="utf-8") as f: - f.write(f"{SEP}{SEP}{SEP}\n") - if args.mode != "off": - for sha, runs, texts in commit_runs(con): - for (p, t), (op, ot) in link(runs, texts.__getitem__, - args.mode, - args.min_alnum).items(): - f.write(f"{p}{SEP}{t}{SEP}{op}{SEP}{ot}\n") - linked += 1 - commits += 1 - if commits % 20_000 == 0: - log(f"build: {commits:,} commits, {linked:,} links") - log(f"build: mode {args.mode}, {linked:,} tokens linked in " - f"{commits:,} commits") + if args.reuse and os.path.exists(links_file): + log(f"build: reuse {links_file}") + else: + write_links(con, links_file, args.mode, args.min_alnum) read_tsv(con, "links", links_file, ["file_path", "token_id", "origin_path", "origin_token_id"]) con.execute("DELETE FROM links WHERE file_path IS NULL") log(f"build: move chains resolved in {resolve_origins(con)} rounds") target = os.path.join(args.out, f"history-{args.mode}.parquet") - con.execute(f""" - COPY ( - SELECT t.*, - coalesce(l.origin_path, t.file_path) AS origin_path, - coalesce(l.origin_token_id, t.token_id) AS origin_token_id, - coalesce(o.born_sha, t.born_sha) AS origin_born_sha, - '{args.mode}' AS move_mode - FROM tokens t - LEFT JOIN links l USING (file_path, token_id) - LEFT JOIN (SELECT DISTINCT file_path, token_id, born_sha - FROM tokens) o - ON o.file_path = l.origin_path AND o.token_id = l.origin_token_id - ORDER BY t.file_path, t.token_id, t.mainline_in_ts - ) TO '{target}' (FORMAT parquet, COMPRESSION zstd)""") + write_history(con, target, args.mode, args.chunks) log(f"build: wrote {target} in {time.time() - t0:.0f} s") From 8aa65e6f985c1fd3d7c64accfac291eb1f15c579 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 28 Sep 2026 15:25:05 +0000 Subject: [PATCH 08/35] fix(token_history): join before the DISTINCT in the origin lookup The DISTINCT over the tokens of a chunk (about 25 M rows on the kernel) still ran out of memory. A token has one born_sha on all its rows, so the lookup now joins the links of the chunk first and removes repeats from the join output. Chunk 0 of the kernel: 6 s, 5.23 M origins. The build logs each chunk. The pilot output does not change. --- token_history/build.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/token_history/build.py b/token_history/build.py index 8f6347cc..331c0976 100644 --- a/token_history/build.py +++ b/token_history/build.py @@ -97,17 +97,20 @@ def write_history(con, target, mode, chunks): con.execute("CREATE OR REPLACE TABLE origins (file_path VARCHAR, " "token_id BIGINT, origin_path VARCHAR, origin_token_id BIGINT, " "origin_born_sha VARCHAR)") + # A token has one born_sha on all its rows, so the DISTINCT runs on the + # join output (a few rows per link), not on the tokens of the chunk. for k in range(chunks): con.execute(f""" INSERT INTO origins - SELECT l.file_path, l.token_id, l.origin_path, l.origin_token_id, - o.born_sha + SELECT DISTINCT l.file_path, l.token_id, l.origin_path, + l.origin_token_id, o.born_sha FROM (SELECT * FROM links WHERE {in_chunk('origin_path', k, chunks)}) l - JOIN (SELECT DISTINCT file_path, token_id, born_sha FROM tokens + JOIN (SELECT file_path, token_id, born_sha FROM tokens WHERE {in_chunk('file_path', k, chunks)}) o ON o.file_path = l.origin_path AND o.token_id = l.origin_token_id""") + log(f"build: origins chunk {k + 1}/{chunks}") parts = os.path.join(os.path.dirname(target), f".{os.path.basename(target)}.parts") shutil.rmtree(parts, ignore_errors=True) @@ -129,6 +132,7 @@ def write_history(con, target, mode, chunks): USING (file_path, token_id) ) TO '{os.path.join(parts, f"{k:03d}.parquet")}' (FORMAT parquet, COMPRESSION zstd)""") + log(f"build: write chunk {k + 1}/{chunks}") con.execute(f"COPY (SELECT * FROM read_parquet('{parts}/*.parquet')) " f"TO '{target}' (FORMAT parquet, COMPRESSION zstd)") shutil.rmtree(parts) From 34564604dca5682f98fa1ab9ddd34f79078d71ef Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 1 Oct 2026 14:06:46 +0000 Subject: [PATCH 09/35] fix(token_history): stop when git fails merge_parents, mainline_changes and list_paths ignored the exit status of git. A failed git call gave empty data, and the run still ended with 0 errors and no mainline intervals. git_out now raises GitError on a non-zero exit; rev-parse --verify of a missing object passes check=False. diff_hunks keeps the old result for a null blob (no hunks) so the replay output does not change. The io_uring pilot replay gives the same part and runs files as before. --- token_history/replay.py | 28 +++++++++++--- token_history/test_token_history.py | 57 +++++++++++++++++++++++++++++ token_history/token_history.py | 15 +++++--- 3 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 token_history/test_token_history.py diff --git a/token_history/replay.py b/token_history/replay.py index 7293092c..f3843062 100644 --- a/token_history/replay.py +++ b/token_history/replay.py @@ -72,9 +72,27 @@ def parse_log(text): return commits -def git_out(repo, *args): - return subprocess.run(["git", "-C", repo, *args], capture_output=True, - text=True, errors="replace").stdout +class GitError(RuntimeError): + """git exited with an error: the caller must not go on with no data.""" + + +def git_out(repo, *args, check=True): + """The output of one git command. check=False accepts a non-zero exit + (for example rev-parse --verify of a missing object).""" + proc = subprocess.run(["git", "-C", repo, *args], capture_output=True, + text=True, errors="replace") + if check and proc.returncode: + raise GitError(f"git {' '.join(args[:3])} exited {proc.returncode}: " + f"{proc.stderr.strip()[:300]}") + return proc.stdout + + +def diff_hunks(repo, a, b): + """The hunks from blob a to blob b. A null blob gives no hunks: git + cannot diff it, and the replay treats that case as no change.""" + if NULL_BLOB in (a, b): + return [] + return parse_hunks(git_out(repo, "diff", "-U0", "--no-color", a, b)) def parse_hunks(diff_text): @@ -85,10 +103,10 @@ def real_parent_sources(repo, path): """Real parents, blobs and blob diffs straight from git, one call each.""" def blob_at(sha): return git_out(repo, "rev-parse", "-q", "--verify", - f"{sha}:{path}").strip() or NULL_BLOB + f"{sha}:{path}", check=False).strip() or NULL_BLOB def diff_blobs(a, b): - return parse_hunks(git_out(repo, "diff", "-U0", "--no-color", a, b)) + return diff_hunks(repo, a, b) return dict(real_parents=lambda sha: git_out( repo, "log", "-1", "--format=%P", sha).split(), diff --git a/token_history/test_token_history.py b/token_history/test_token_history.py new file mode 100644 index 00000000..b4283648 --- /dev/null +++ b/token_history/test_token_history.py @@ -0,0 +1,57 @@ +import argparse + +import pytest + +from replay import NULL_BLOB, GitError, diff_hunks, git_out +from test_replay import commit, git, write +from token_history import list_paths, mainline_changes, merge_parents + + +@pytest.fixture +def not_a_repo(tmp_path): + d = tmp_path / "empty" + d.mkdir() + return str(d) + + +def test_git_failures_stop_the_run(not_a_repo): + # Before, each of these returned empty data and the run went on with + # no mainline intervals, no merges or no paths, and 0 errors. + with pytest.raises(GitError): + merge_parents(not_a_repo) + with pytest.raises(GitError): + mainline_changes(not_a_repo, []) + with pytest.raises(GitError): + list_paths(argparse.Namespace(paths=None, repo=not_a_repo, + pathspec=[])) + + +def test_an_expected_non_zero_exit_is_accepted(not_a_repo): + assert git_out(not_a_repo, "rev-parse", "-q", "--verify", "x:y", + check=False) == "" + + +def test_the_null_blob_diffs_to_no_hunks(tmp_path): + r = tmp_path / "r" + r.mkdir() + git(r, "init", "-q", "-b", "main") + write(r, "a.c", ["x", "y"]) + commit(r, "a", "2020-01-01T00:00:00") + blob = git(r, "rev-parse", "HEAD:a.c").strip() + assert diff_hunks(str(r), blob, NULL_BLOB) == [] + assert diff_hunks(str(r), NULL_BLOB, blob) == [] + with pytest.raises(GitError): + diff_hunks(str(r), blob, "1" * 40) + + +def test_a_good_repo_still_works(tmp_path): + r = tmp_path / "r" + r.mkdir() + git(r, "init", "-q", "-b", "main") + write(r, "a.c", ["x"]) + sha = commit(r, "a", "2020-01-01T00:00:00") + assert merge_parents(str(r)) == {} + assert list(mainline_changes(str(r), [])) == ["a.c"] + assert mainline_changes(str(r), [])["a.c"][0][0] == sha + assert list_paths(argparse.Namespace(paths=None, repo=str(r), + pathspec=[])) == ["a.c"] diff --git a/token_history/token_history.py b/token_history/token_history.py index eb647eba..1d2137ab 100755 --- a/token_history/token_history.py +++ b/token_history/token_history.py @@ -22,8 +22,8 @@ from datetime import datetime sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from replay import (HEADER, NULL_BLOB, PathReplay, git_out, # noqa: E402 - parse_hunks, parse_log, path_log) +from replay import (HEADER, NULL_BLOB, GitError, PathReplay, # noqa: E402 + diff_hunks, git_out, parse_log, path_log) MASK = re.compile(r"\.[ch]$") SEP = "\x1f" @@ -48,7 +48,8 @@ def mainline_changes(repo, pathspec): ["git", "-C", repo, "log", "--first-parent", "--reverse", "-m", "--raw", "--no-abbrev", "--no-renames", f"--format={HEADER}%H %ct", "HEAD", "--", *pathspec], - stdout=subprocess.PIPE, text=True, errors="replace") + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, + errors="replace") changes, sha, ct = {}, None, 0 for line in proc.stdout: if line.startswith(HEADER): @@ -58,7 +59,10 @@ def mainline_changes(repo, pathspec): meta, path = line.rstrip("\n").split("\t", 1) if MASK.search(path): changes.setdefault(path, []).append((sha, ct, meta.split()[3])) - proc.wait() + # a failed git log would leave every path with no mainline interval, + # and the run would still end with 0 errors + if proc.wait(): + raise GitError(f"git log --first-parent exited {proc.returncode}") return changes @@ -82,8 +86,7 @@ def blob_at(self, sha): return out if len(out) == 40 else NULL_BLOB def diff_blobs(self, a, b): - return parse_hunks(git_out(self.repo, "diff", "-U0", "--no-color", - a, b)) + return diff_hunks(self.repo, a, b) def replay(self, path, mainline, run_min_alnum): self.path = path From d015f9dde3f7f675686642cd6a2475e3ced7d504 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 1 Oct 2026 14:06:46 +0000 Subject: [PATCH 10/35] fix(token_history): one place per token in a run commit_runs joined the run token ids with tokens, which has one row per mainline interval. A token with two intervals came twice in its run, and the move pass matched the run at a shifted position. On the io_uring pilot (10 tokens with two intervals) 56 rows in 3 files get another origin, and 38 more tokens are linked (153,061). The kernel history-moves.parquet was built before this fix. --- token_history/build.py | 5 ++++- token_history/test_build.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 token_history/test_build.py diff --git a/token_history/build.py b/token_history/build.py index 331c0976..fd7af02a 100644 --- a/token_history/build.py +++ b/token_history/build.py @@ -159,7 +159,10 @@ def commit_runs(con): FROM r) SELECT t.sha, t.run_no, t.kind, t.whole_file, t.file_path, t.tid, tok.token - FROM t JOIN tokens tok + -- tokens has one row per mainline interval; a token with two + -- intervals must still be one place in its run + FROM t JOIN (SELECT DISTINCT file_path, token_id, token + FROM tokens) tok ON tok.file_path = t.file_path AND tok.token_id = t.tid ORDER BY t.sha, t.run_no, t.pos""") sha, runs, texts, cur_run = None, [], {}, None diff --git a/token_history/test_build.py b/token_history/test_build.py new file mode 100644 index 00000000..7c33fc11 --- /dev/null +++ b/token_history/test_build.py @@ -0,0 +1,22 @@ +import duckdb + +from build import commit_runs + + +def test_a_token_with_two_mainline_intervals_is_one_place_in_its_run(): + # tokens has one row per mainline interval; token 1 left the mainline + # and came back, so it has two rows + con = duckdb.connect() + con.execute("CREATE TABLE tokens (file_path VARCHAR, token_id BIGINT, " + "token VARCHAR, mainline_in_sha VARCHAR)") + con.execute("INSERT INTO tokens VALUES ('a.c', 0, 'x', 'm1'), " + "('a.c', 1, 'y', 'm1'), ('a.c', 1, 'y', 'm3'), " + "('a.c', 2, 'z', 'm1')") + con.execute("CREATE TABLE runs (sha VARCHAR, kind VARCHAR, " + "whole_file INTEGER, file_path VARCHAR, token_ids VARCHAR)") + con.execute("INSERT INTO runs VALUES ('c1', 'born', 0, 'a.c', '0,1,2'), " + "('c1', 'died', 0, 'a.c', '0,1,2')") + (sha, runs, texts), = list(commit_runs(con)) + assert sha == "c1" + assert [ids for _, _, _, ids in runs] == [[0, 1, 2], [0, 1, 2]] + assert texts[("a.c", 1)] == "y" From ba68d1157b80f40b6dcad0feacb589b0d2b8db9f Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 1 Oct 2026 14:10:58 +0000 Subject: [PATCH 11/35] test(token_history): cover the three untested replay rules A mutation of each of the seven replay rules showed three with no test that failed: --simplify-merges in the path log, a commit with no diff keeps its parent blob, and an error in one path is a record, not a stop. Add one test for each. Now each of the seven mutations fails a test. --- token_history/test_replay.py | 34 +++++++++++++++++++++++++++++ token_history/test_token_history.py | 10 +++++++++ 2 files changed, 44 insertions(+) diff --git a/token_history/test_replay.py b/token_history/test_replay.py index f34bccd5..4bedf56d 100644 --- a/token_history/test_replay.py +++ b/token_history/test_replay.py @@ -266,3 +266,37 @@ def test_merge_that_brings_a_new_file_to_main(tmp_path): **real_parent_sources(str(r), "n.c")).run() born, _ = tip_born(rp, r, "n.c") assert born == blame(r, "n.c") + + +def test_path_log_drops_a_merge_that_brings_no_change(tmp_path): + # --simplify-merges: a side branch that never touches the path adds no + # merge to its log, so no merge is replayed for nothing + r = tmp_path / "r" + r.mkdir() + git(r, "init", "-q", "-b", "main") + write(r, "a.c", ["a"]) + commit(r, "base", "2020-01-01T00:00:00") + git(r, "checkout", "-q", "-b", "side") + write(r, "o.c", ["o"]) + commit(r, "side", "2020-01-02T00:00:00") + git(r, "checkout", "-q", "main") + write(r, "a.c", ["a", "m"]) + commit(r, "main", "2020-01-03T00:00:00") + git(r, "merge", "-q", "--no-ff", "-m", "merge", "side") + assert [len(c.parents) for c in parse_log(path_log(str(r), "a.c"))] \ + == [0, 1] + + +def test_a_commit_with_no_diff_keeps_the_parent_blob(): + # git log can list a commit with no diff for the path; it has the + # parent's content, not an empty file + a, b = "a" * 40, "b" * 40 + blob = "1" * 40 + text = (f"\x01{a} \t1\n:000000 100644 {NULL_BLOB} {blob} A\ta.c\n" + "@@ -0,0 +1,2 @@\n+x\n+y\n" + f"\x01{b} {a}\t2\n") + commits = parse_log(text) + rp = PathReplay(commits) + rp.run() + assert commits[1].blob == blob + assert rp.state[b] == rp.state[a] diff --git a/token_history/test_token_history.py b/token_history/test_token_history.py index b4283648..99ffe74c 100644 --- a/token_history/test_token_history.py +++ b/token_history/test_token_history.py @@ -55,3 +55,13 @@ def test_a_good_repo_still_works(tmp_path): assert mainline_changes(str(r), [])["a.c"][0][0] == sha assert list_paths(argparse.Namespace(paths=None, repo=str(r), pathspec=[])) == ["a.c"] + + +def test_an_error_in_one_path_is_a_record_not_a_stop(monkeypatch): + import token_history + + def broken(path): + raise ValueError("bad path") + monkeypatch.setattr(token_history, "replay_path", broken) + r = token_history.work("a.c") + assert r["path"] == "a.c" and "ValueError" in r["error"] From e0df76b97662b1e5e43b3fdb63ff2199550ee60c Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 1 Oct 2026 14:12:59 +0000 Subject: [PATCH 12/35] docs(token_history): describe the kernel build State the .c/.h scope, the measured cost of each stage on Linux, the attribute step, and the chunked and resumable build. Correct three claims: moves does not follow more moves than git blame -C100 (-C100 gives the older commit in about 87% of the disagreements, because moves misses copies); the 99.90% renames figure has no committed script, so it is removed; validate.py compares only the .c/.h files. --- token_history/README.md | 42 +++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/token_history/README.md b/token_history/README.md index 6adaaf64..3c908afb 100644 --- a/token_history/README.md +++ b/token_history/README.md @@ -2,24 +2,41 @@ The tip dataset (`generate_dataset`) holds only the tokens alive at HEAD. This step replays the tokenized history (`-cregit.git`) and writes one row for every token that ever existed: its birth commit, the first commit that removed it, and its intervals on the mainline (the first-parent history of HEAD). +**Scope: `.c` and `.h` paths only** (`MASK` in `token_history.py`). The tip dataset also has other file types; on Linux, 503 files (`.rs`, `.cpp`, `.cc`, 1.05 M rows) are not in the history. + ## Two stages -| Stage | Script | Cost | Output | +| Stage | Script | Cost (Linux, Sep 2026) | Output | | --- | --- | --- | --- | -| 1. Replay | `token_history.py` | hours; Linux about 1.5–2 days with 8 jobs | `part-*.tsv` (tokens), `runs-*.tsv` (move candidates) | -| 2. Build | `build.py --mode ` | seconds to minutes | `history-.parquet` | +| 1. Replay | `token_history.py` | 50.4 h with 8 jobs; 122,910 paths, 407.9 M token rows | `part-*.tsv` (tokens), `runs-*.tsv` (move candidates) | +| 2. Build | `build.py --mode ` | load 12 min; link pass 2.3 h (`moves`) or 1.9 h (`renames`); chain resolution 5–10 min; write 5 min | `history-.parquet`, `links-.tsv` | +| 3. Attribute | `attribute.py` | not timed | `commits.parquet`: person and firm of every cregit commit | + +On the io_uring pilot (96 paths) stage 1 takes 2–3 min and stage 2 under 10 s. + +Stage 2 reads only the files of stage 1. Keep them: every mode, and any later change to the move rules, rebuilds from them without a new replay. A change to the replay rules (`replay.py`) needs a new stage 1. -Stage 2 reads only the files of stage 1. Keep them: every mode, and any later change to the move rules, rebuilds from them without a new replay. +At kernel scale, stage 2 works in parts so that it stays inside its memory limit (`--memory`, default 6 GB): + +- `--reuse` keeps the loaded tables in `OUT/build.duckdb` and a finished `links-.tsv`, so a build that stopped after the link pass restarts at the chain resolution. The link pass writes `links-.tsv.part` and renames it only at the end. +- `--chunks N` (default 16) looks up the origins and writes the parquet in N chunks of paths, with no global sort. The row order of the parquet is not defined. +- `links-.parquet` is not written here: the analysis (`msr-code/analysis/rq_history.py`) converts the TSV once for its own use. ## Move modes (stage 2) | Mode | Links | Matches | | --- | --- | --- | | `off` | none | the history of each path alone | -| `renames` | a deleted file to a file created in the same commit | plain `git blame` (99.90% of tokens on the io_uring pilot) | -| `moves` (default) | any exact run of ≥ `--min-alnum` alphanumeric characters that one commit removes and adds | keeps the first identity of moved code; follows more moves than `git blame -C100` | +| `renames` | a deleted file to a file created in the same commit | close to plain `git blame` with rename detection | +| `moves` (default) | any exact run of ≥ `--min-alnum` alphanumeric characters that one commit removes and adds | keeps the first identity of moved code | + +`moves` is not a copy of `git blame -C100`, and it does **not** follow more moves: -`moves` is not a copy of `git blame -C100`: git diffs a block against the whole old file and drops a move that its diff breaks into short pieces. On the io_uring pilot, `moves` agrees with the `-C100` tip dataset on 49% of the tip tokens, `off` on 62%. A rule that copies git needs only the repository and the stage 1 files, so it can be added to `moves.py` later. +- `moves` links only an exact run that the **same commit** removes and adds. Each removed token is used once. So it does not follow a **copy** whose source stays in place, which `git blame -C` does follow. +- `git blame -C100` diffs a block against the whole old file, so it drops a move that its diff breaks into short pieces. `moves` keeps some of those. +- Measured on Linux (a 1/32 sample of the tip, 5.49 M tokens, 30 Sep 2026): where the two disagree on the birth time, `-C100` gives the **older** commit in about 87% of the cases (981,149 tokens against 145,019). + +On the whole Linux tip, the origin commit agrees with the `-C100` tip dataset on 80.46% of the tokens for `moves` and 81.13% for `renames` (`history/validate-.json`). A rule that copies `git blame -C` needs only the repository and the stage 1 files, so it can be added to `moves.py` later. ## Run @@ -39,9 +56,16 @@ python3 build.py --out OUT --mode renames # check the tip against the tip dataset python3 validate.py --history OUT/history-moves.parquet --dataset X-dataset.parquet + +# person and firm of every cregit commit, with the joins of the tip dataset +python3 attribute.py --cregit-db X-cregit.db --persons-db X-persons.db \ + --firm-map affiliation.merged.csv --firm-canonical firm_canonical.csv \ + --out OUT/commits.parquet --dataset X-dataset.parquet ``` -Stage 1 writes `history-progress.json` and a log line every `--ping` seconds. A path that fails goes to `errors.txt` with its traceback, and the run continues. +`validate.py` compares only the dataset rows of the files that the history has (so only `.c` and `.h`). Its `same_text` counts tokens whose text matches the dataset's token type, its token value, or both joined by `|`. + +Stage 1 writes `history-progress.json` and a log line every `--ping` seconds. A path that fails goes to `errors.txt` with its traceback, and the run continues. A `git` command that fails for the whole repository (the merge list, the mainline log or the path list) stops the run with `GitError`, so a run cannot end with no data and 0 errors. ## Columns of `history-.parquet` @@ -66,3 +90,5 @@ A token that left the mainline and came back has one row per interval. A token a ```sh pytest token_history -q ``` + +The tests build small git repositories. Each of the seven replay rules (simplify-merges, real parent order, a merge equal to one parent takes it whole, a commit with no diff keeps the parent blob, a line kept twice becomes a copy, a parent without the file is no source, an error in one path is a record) has a test that fails when the rule is removed. From 3ce68befb241f7f66eda31ce8b0916347bcf1206 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 1 Oct 2026 18:17:44 +0000 Subject: [PATCH 13/35] fix(token_history): skip repeated places without a DISTINCT d015f9d made commit_runs take a DISTINCT over every token row. On the kernel (408 M rows) that ran out of memory at the start of the link pass. The query already sorts by commit, run and place, so the rows of one place are next to each other: keep the first one while reading. On the io_uring pilot the history is the same as with the DISTINCT (513,471 rows, 0 differences, the same links). --- token_history/build.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/token_history/build.py b/token_history/build.py index fd7af02a..4c377f31 100644 --- a/token_history/build.py +++ b/token_history/build.py @@ -158,16 +158,21 @@ def commit_runs(con): generate_subscripts(string_split(token_ids, ','), 1) AS pos FROM r) SELECT t.sha, t.run_no, t.kind, t.whole_file, t.file_path, t.tid, - tok.token - -- tokens has one row per mainline interval; a token with two - -- intervals must still be one place in its run - FROM t JOIN (SELECT DISTINCT file_path, token_id, token - FROM tokens) tok + tok.token, t.pos + FROM t JOIN tokens tok ON tok.file_path = t.file_path AND tok.token_id = t.tid ORDER BY t.sha, t.run_no, t.pos""") - sha, runs, texts, cur_run = None, [], {}, None + # tokens has one row per mainline interval, so a token with two + # intervals comes twice at one place of its run. The rows of one place + # are next to each other (the sort), and a token id has one text, so + # keeping the first row of each place is a DISTINCT that costs no + # memory (a DISTINCT over the 408 M kernel token rows does not fit). + sha, runs, texts, cur_run, last = None, [], {}, None, None while batch := cur.fetchmany(100_000): - for s, run_no, kind, whole, path, tid, token in batch: + for s, run_no, kind, whole, path, tid, token, pos in batch: + if (s, run_no, pos) == last: + continue + last = (s, run_no, pos) if s != sha: if sha is not None: yield sha, runs, texts From a428ae4bf6c9aa393c81f23d9b35877b3a2baa93 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:09:04 +0000 Subject: [PATCH 14/35] fix(token_history): stop the run when cat-file dies GitSource.blob_at did not check its git cat-file process. When it died, readline() gave "" and .split()[0] raised IndexError; work() caught it as an error of one path, so the run went on and wrote one errors.txt record for every later path of that worker. blob_at now raises GitError on a broken pipe or an empty read, and work() lets a GitError through, so the pool stops the run. Any other exception stays a record of its path. mainline_changes sent the stderr of git log to /dev/null, so its GitError had no reason. It now carries the first 300 characters of stderr, as git_out does. --- token_history/test_token_history.py | 11 ++++++++++- token_history/token_history.py | 19 ++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/token_history/test_token_history.py b/token_history/test_token_history.py index 99ffe74c..6476cc14 100644 --- a/token_history/test_token_history.py +++ b/token_history/test_token_history.py @@ -19,7 +19,7 @@ def test_git_failures_stop_the_run(not_a_repo): # no mainline intervals, no merges or no paths, and 0 errors. with pytest.raises(GitError): merge_parents(not_a_repo) - with pytest.raises(GitError): + with pytest.raises(GitError, match="not a git repository"): mainline_changes(not_a_repo, []) with pytest.raises(GitError): list_paths(argparse.Namespace(paths=None, repo=not_a_repo, @@ -65,3 +65,12 @@ def broken(path): monkeypatch.setattr(token_history, "replay_path", broken) r = token_history.work("a.c") assert r["path"] == "a.c" and "ValueError" in r["error"] + + +def test_a_dead_cat_file_stops_the_run(not_a_repo, monkeypatch): + import token_history + src = token_history.GitSource(not_a_repo, {}) + monkeypatch.setattr(token_history, "replay_path", + lambda path: src.blob_at("HEAD")) + with pytest.raises(GitError): # not one errors.txt record per path + token_history.work("a.c") diff --git a/token_history/token_history.py b/token_history/token_history.py index 1d2137ab..da5f8072 100755 --- a/token_history/token_history.py +++ b/token_history/token_history.py @@ -48,7 +48,7 @@ def mainline_changes(repo, pathspec): ["git", "-C", repo, "log", "--first-parent", "--reverse", "-m", "--raw", "--no-abbrev", "--no-renames", f"--format={HEADER}%H %ct", "HEAD", "--", *pathspec], - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, errors="replace") changes, sha, ct = {}, None, 0 for line in proc.stdout: @@ -62,7 +62,8 @@ def mainline_changes(repo, pathspec): # a failed git log would leave every path with no mainline interval, # and the run would still end with 0 errors if proc.wait(): - raise GitError(f"git log --first-parent exited {proc.returncode}") + raise GitError(f"git log --first-parent exited {proc.returncode}: " + f"{proc.stderr.read().strip()[:300]}") return changes @@ -80,9 +81,15 @@ def real_parents(self, sha): return self.merges.get(sha) or self.rewritten[sha] def blob_at(self, sha): - self.cat.stdin.write(f"{sha}:{self.path}\n") - self.cat.stdin.flush() - out = self.cat.stdout.readline().split()[0] + try: + self.cat.stdin.write(f"{sha}:{self.path}\n") + self.cat.stdin.flush() + line = self.cat.stdout.readline() + except BrokenPipeError: + line = "" + if not line: # cat-file died: every later path would fail too + raise GitError(f"git cat-file exited {self.cat.wait()}") + out = line.split()[0] return out if len(out) == 40 else NULL_BLOB def diff_blobs(self, a, b): @@ -127,6 +134,8 @@ def clean(value): def work(path): try: return replay_path(path) + except GitError: + raise # git failed, not the path: stop the run except Exception: return dict(path=path, error=traceback.format_exc()) From 714ef58ca94f840a93c6b730e2a730d9c9f20cc7 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:10:36 +0000 Subject: [PATCH 15/35] fix(token_history): keep a resumed path from one file A worker writes the rows of a path before the main process adds the path to paths-done.txt. A run that stops between the two replays the path again on resume, in a new part file. read_tsv then saw repeated keys and ran SELECT DISTINCT * over every token row: the 408 M row case that ran out of memory in 3ce68be. And when the replay changed between the runs, the DISTINCT kept both versions, so keys still repeated. read_tsv now reads the file name of each row and keeps each path from one file only (the one with the most rows, so a cut write loses). The check groups by path and file (about 123 k groups on the kernel), not by row. The file names now carry the start time of the worker, so a later run that reuses a pid writes a new file. Existing part-.tsv files still load. --- token_history/build.py | 29 ++++++++++++++--------------- token_history/test_build.py | 18 +++++++++++++++++- token_history/token_history.py | 7 ++++--- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/token_history/build.py b/token_history/build.py index 4c377f31..ae0ab3b6 100644 --- a/token_history/build.py +++ b/token_history/build.py @@ -33,25 +33,25 @@ MAX_LINE = 16_000_000 -def read_tsv(con, name, pattern, columns, key=None): - """Load TSV parts; drop repeated rows (a path written twice by a resume). - - key: the columns that identify a row; the full-row DISTINCT runs only - when they repeat, since it is costly on a whole kernel. - """ +def read_tsv(con, name, pattern, columns): + """Load TSV parts. A run that stopped after the rows of a path, before + paths-done.txt, writes the path again on resume, in a new part file: + keep each path from one file only, the one with the most rows.""" cols = ", ".join(f"'{c}': '{TYPES.get(c, 'VARCHAR')}'" for c in columns) keep_text = ", force_not_null=['token']" if "token" in columns else "" con.execute(f""" CREATE TABLE {name} AS SELECT * FROM read_csv( '{pattern}', delim='{SEP}', header=false, quote='', escape='', nullstr='', auto_detect=false, max_line_size={MAX_LINE}{keep_text}, - columns={{{cols}}})""") - key = ", ".join(key or columns) - rows, keys = con.execute(f"SELECT count(*), count(DISTINCT ({key})) " - f"FROM {name}").fetchone() - if rows != keys: - con.execute(f"CREATE OR REPLACE TABLE {name} AS " - f"SELECT DISTINCT * FROM {name}") + filename=true, columns={{{cols}}})""") + con.execute(f""" + DELETE FROM {name} t USING ( + SELECT file_path, arg_max(filename, n) AS keep + FROM (SELECT file_path, filename, count(*) AS n + FROM {name} GROUP BY ALL) + GROUP BY file_path HAVING count(*) > 1) d + WHERE t.file_path = d.file_path AND t.filename <> d.keep""") + con.execute(f"ALTER TABLE {name} DROP COLUMN filename") def resolve_origins(con): @@ -187,8 +187,7 @@ def commit_runs(con): def load(con, out): - read_tsv(con, "tokens", os.path.join(out, "part-*.tsv"), COLUMNS, - key=["file_path", "token_id", "mainline_in_sha"]) + read_tsv(con, "tokens", os.path.join(out, "part-*.tsv"), COLUMNS) if glob.glob(os.path.join(out, "runs-*.tsv")): read_tsv(con, "runs", os.path.join(out, "runs-*.tsv"), RUN_COLUMNS) else: diff --git a/token_history/test_build.py b/token_history/test_build.py index 7c33fc11..b4074a73 100644 --- a/token_history/test_build.py +++ b/token_history/test_build.py @@ -1,6 +1,7 @@ import duckdb -from build import commit_runs +from build import commit_runs, load +from token_history import SEP def test_a_token_with_two_mainline_intervals_is_one_place_in_its_run(): @@ -20,3 +21,18 @@ def test_a_token_with_two_mainline_intervals_is_one_place_in_its_run(): assert sha == "c1" assert [ids for _, _, _, ids in runs] == [[0, 1, 2], [0, 1, 2]] assert texts[("a.c", 1)] == "y" + + +def test_a_path_written_again_by_a_resume_is_loaded_once(tmp_path): + # part-1 has a.c from a run that stopped before paths-done.txt; the + # resume wrote a.c again in part-2, here after a change of the replay + def row(path, born): + return SEP.join([path, "0", "x", born, "0", "", "", "", "m", "1", "", + ""]) + "\n" + (tmp_path / "part-1.tsv").write_text(row("a.c", "b1") + row("b.c", "b1")) + (tmp_path / "part-2.tsv").write_text(row("a.c", "b2")) + con = duckdb.connect() + load(con, str(tmp_path)) + assert con.execute("SELECT count(*), count(DISTINCT (file_path, token_id))" + " FROM tokens").fetchone() == (2, 2) + assert len(con.execute("SELECT * FROM tokens").description) == 12 diff --git a/token_history/token_history.py b/token_history/token_history.py index da5f8072..1f83db70 100755 --- a/token_history/token_history.py +++ b/token_history/token_history.py @@ -117,13 +117,14 @@ def rows(path, rp, intervals, tip): def init_worker(repo, merges, changes, out_dir, run_min_alnum): - pid = os.getpid() + # unique per worker, even when a later run reuses a pid + tag = f"{os.getpid()}-{time.time_ns()}" WORKER.update( src=GitSource(repo, merges), changes=changes, run_min_alnum=run_min_alnum, - out=open(os.path.join(out_dir, f"part-{pid}.tsv"), "a", + out=open(os.path.join(out_dir, f"part-{tag}.tsv"), "a", encoding="utf-8"), - runs=open(os.path.join(out_dir, f"runs-{pid}.tsv"), "a", + runs=open(os.path.join(out_dir, f"runs-{tag}.tsv"), "a", encoding="utf-8")) From 8ece8926983fea8cbdade0902c539c62e47f85c2 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:10:55 +0000 Subject: [PATCH 16/35] fix(token_history): say when there are no part files load() read runs-*.tsv only after a glob, but read part-*.tsv without one. With no part files (a wrong --out, or a replay that has not started) DuckDB failed with "No files found that match the pattern". build.py now stops with a message that names the directory and the step to run. An empty runs table stays valid (a history with no move candidates), so that branch is unchanged. --- token_history/build.py | 2 ++ token_history/test_build.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/token_history/build.py b/token_history/build.py index ae0ab3b6..755384cd 100644 --- a/token_history/build.py +++ b/token_history/build.py @@ -187,6 +187,8 @@ def commit_runs(con): def load(con, out): + if not glob.glob(os.path.join(out, "part-*.tsv")): + raise SystemExit(f"build: no part-*.tsv in {out}: run token_history.py") read_tsv(con, "tokens", os.path.join(out, "part-*.tsv"), COLUMNS) if glob.glob(os.path.join(out, "runs-*.tsv")): read_tsv(con, "runs", os.path.join(out, "runs-*.tsv"), RUN_COLUMNS) diff --git a/token_history/test_build.py b/token_history/test_build.py index b4074a73..e26f3719 100644 --- a/token_history/test_build.py +++ b/token_history/test_build.py @@ -1,4 +1,5 @@ import duckdb +import pytest from build import commit_runs, load from token_history import SEP @@ -36,3 +37,8 @@ def row(path, born): assert con.execute("SELECT count(*), count(DISTINCT (file_path, token_id))" " FROM tokens").fetchone() == (2, 2) assert len(con.execute("SELECT * FROM tokens").description) == 12 + + +def test_no_part_files_is_a_clear_error(tmp_path): + with pytest.raises(SystemExit, match="no part"): + load(duckdb.connect(), str(tmp_path)) From 3563069b0ada57696c00debbaa59045d6e107330 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:11:11 +0000 Subject: [PATCH 17/35] ci(token_history): run the token_history tests The Python step ran pytest on generate_dataset only, so no CI run tried the 31 token_history tests. Add the directory to the same pytest call: its JUnit report and the count check then cover both (99 tests in devenv). The report keeps its file name, so the check and upload steps do not change. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79765e82..740a9f00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,8 +95,8 @@ jobs: # Without this step the coverage rule never reaches the dataset code. # devenv.nix already supplies pytest (see the comment beside # python3.withPackages), so only the CI step was missing. - - name: Python tests (pytest, generate_dataset) - run: devenv shell -- pytest generate_dataset -q --junitxml=junit-generate-dataset.xml + - name: Python tests (pytest, generate_dataset, token_history) + run: devenv shell -- pytest generate_dataset token_history -q --junitxml=junit-generate-dataset.xml # `devenv shell -- pytest` does not flush its summary line, so the printed # count is not trustworthy. Read the counts from the XML instead, and fail From b19ae4c1d390d049ec3903c0990b1a297f4b1761 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:11:42 +0000 Subject: [PATCH 18/35] fix(token_history): keep only .c/.h paths of --paths list_paths applied MASK to the git path list but not to a --paths file, so a file with other paths replayed them, although mainline_changes keeps .c/.h only and gives them no mainline interval. Both sources now go through one filter. A --paths list is now sorted and without repeats, as the git list was already. --- token_history/test_token_history.py | 2 ++ token_history/token_history.py | 8 +++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/token_history/test_token_history.py b/token_history/test_token_history.py index 6476cc14..b67e1d3c 100644 --- a/token_history/test_token_history.py +++ b/token_history/test_token_history.py @@ -55,6 +55,8 @@ def test_a_good_repo_still_works(tmp_path): assert mainline_changes(str(r), [])["a.c"][0][0] == sha assert list_paths(argparse.Namespace(paths=None, repo=str(r), pathspec=[])) == ["a.c"] + (tmp_path / "paths").write_text("a.c\nREADME\n") + assert list_paths(argparse.Namespace(paths=tmp_path / "paths")) == ["a.c"] def test_an_error_in_one_path_is_a_record_not_a_stop(monkeypatch): diff --git a/token_history/token_history.py b/token_history/token_history.py index 1f83db70..e90111db 100755 --- a/token_history/token_history.py +++ b/token_history/token_history.py @@ -208,11 +208,9 @@ def write(self, state="running"): def list_paths(args): - if args.paths: - return [p for p in open(args.paths).read().split("\n") if p] - out = git_out(args.repo, "log", "--diff-filter=A", "--name-only", - "--no-renames", "-m", "--format=", "HEAD", "--", - *args.pathspec) + out = open(args.paths).read() if args.paths else git_out( + args.repo, "log", "--diff-filter=A", "--name-only", "--no-renames", + "-m", "--format=", "HEAD", "--", *args.pathspec) return sorted({p for p in out.splitlines() if MASK.search(p)}) From 4e37bb5711eaded0afe9884dd25b930b1c83a12a Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:11:55 +0000 Subject: [PATCH 19/35] fix(token_history): drop stale links when tables reload With --reuse but no tokens and runs tables in build.duckdb, build.py loaded the parts again yet kept the old links-.tsv, which may come from other parts. The links file is now kept only when the tables are kept too. --- token_history/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/token_history/build.py b/token_history/build.py index 755384cd..c166c1fc 100644 --- a/token_history/build.py +++ b/token_history/build.py @@ -247,7 +247,7 @@ def main(): # Links go to disk commit by commit: on a whole kernel they do not fit # in memory. A born token has one birth commit, so it has one link. links_file = os.path.join(args.out, f"links-{args.mode}.tsv") - if args.reuse and os.path.exists(links_file): + if reuse and os.path.exists(links_file): log(f"build: reuse {links_file}") else: write_links(con, links_file, args.mode, args.min_alnum) From 4ad1bc8f81d1341341c5b26786a345ea6edc29ad Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:12:42 +0000 Subject: [PATCH 20/35] refactor(token_history): drop the unused move helpers moves.by_commit had no caller. moves.origins was a second, in-memory chain resolution that only test_moves used; the build resolves chains with build.resolve_origins, which had no test. Remove both and move the chain test onto resolve_origins. The texts() test helper was dict.__getitem__, so use that. --- token_history/moves.py | 26 -------------------------- token_history/test_build.py | 12 +++++++++++- token_history/test_moves.py | 14 ++------------ 3 files changed, 13 insertions(+), 39 deletions(-) diff --git a/token_history/moves.py b/token_history/moves.py index 5a972ca8..a9d8bffc 100644 --- a/token_history/moves.py +++ b/token_history/moves.py @@ -85,29 +85,3 @@ def link(runs, text_of, mode, min_alnum): links[b] = d used.add(d) return links - - -def origins(links): - """Follow each chain of moves back to the token's first identity.""" - root = {} - - def find(key): - chain = [] - while key in links and key not in root: - chain.append(key) - key = links[key] - top = root.get(key, key) - for k in chain: - root[k] = top - return top - - for key in links: - find(key) - return root - - -def by_commit(run_rows): - grouped = defaultdict(list) - for sha, kind, whole, path, ids in run_rows: - grouped[sha].append((kind, whole, path, ids)) - return grouped diff --git a/token_history/test_build.py b/token_history/test_build.py index e26f3719..6547666c 100644 --- a/token_history/test_build.py +++ b/token_history/test_build.py @@ -1,7 +1,7 @@ import duckdb import pytest -from build import commit_runs, load +from build import commit_runs, load, resolve_origins from token_history import SEP @@ -42,3 +42,13 @@ def row(path, born): def test_no_part_files_is_a_clear_error(tmp_path): with pytest.raises(SystemExit, match="no part"): load(duckdb.connect(), str(tmp_path)) + + +def test_origins_follow_chains(): + con = duckdb.connect() + con.execute("CREATE TABLE links (file_path VARCHAR, token_id BIGINT, " + "origin_path VARCHAR, origin_token_id BIGINT)") + con.execute("INSERT INTO links VALUES ('c', 1, 'b', 1), ('b', 1, 'a', 1)") + resolve_origins(con) + assert sorted(con.execute("FROM links").fetchall()) == [ + ("b", 1, "a", 1), ("c", 1, "a", 1)] diff --git a/token_history/test_moves.py b/token_history/test_moves.py index efa90ebf..a65e2a03 100644 --- a/token_history/test_moves.py +++ b/token_history/test_moves.py @@ -1,8 +1,4 @@ -from moves import link, origins - - -def texts(d): - return lambda key: d[key] +from moves import link, matching_blocks RUN = ["int", "alpha_long_identifier", ";", "beta_long_identifier"] @@ -15,7 +11,7 @@ def runs_and_text(moved_whole=True): tx[("new.c", i)] = t runs = [("died", moved_whole, "old.c", [10, 11, 12, 13]), ("born", moved_whole, "new.c", [0, 1, 2, 3])] - return runs, texts(tx) + return runs, tx.__getitem__ def test_off_links_nothing(): @@ -41,14 +37,8 @@ def test_short_runs_do_not_move(): assert link(runs, tx, "moves", 10_000) == {} -def test_origins_follow_chains(): - links = {("c", 1): ("b", 1), ("b", 1): ("a", 1)} - assert origins(links) == {("c", 1): ("a", 1), ("b", 1): ("a", 1)} - - def test_matching_blocks_on_a_long_repetitive_run(): import time - from moves import matching_blocks a = [";", "}", "x", ";"] * 20000 + ["unique_tail_token"] * 10 b = ["new"] + a t = time.time() From a9d4e3ce4b978755d90542353068d557d23bd483 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:13:20 +0000 Subject: [PATCH 21/35] test(token_history): replay merges through GitSource replay.real_parent_sources gave the replay its real parents and blobs with one git call each. Only the tests used it; the runner uses GitSource, which had no test of a replay. The merge tests now run GitSource itself (merge_parents and cat-file), so the shipped path is the tested path, and real_parent_sources is gone. The tests also parsed the first-parent log twice by hand, a copy of mainline_changes. They call mainline_changes now. Same tests, same assertions, 32 fewer lines. --- token_history/replay.py | 14 ---------- token_history/test_replay.py | 50 ++++++++++++------------------------ 2 files changed, 16 insertions(+), 48 deletions(-) diff --git a/token_history/replay.py b/token_history/replay.py index f3843062..9332ef84 100644 --- a/token_history/replay.py +++ b/token_history/replay.py @@ -99,20 +99,6 @@ def parse_hunks(diff_text): return parse_log(f"{HEADER}x\t0\n{diff_text}")[0].diffs[0].hunks -def real_parent_sources(repo, path): - """Real parents, blobs and blob diffs straight from git, one call each.""" - def blob_at(sha): - return git_out(repo, "rev-parse", "-q", "--verify", - f"{sha}:{path}", check=False).strip() or NULL_BLOB - - def diff_blobs(a, b): - return diff_hunks(repo, a, b) - - return dict(real_parents=lambda sha: git_out( - repo, "log", "-1", "--format=%P", sha).split(), - blob_at=blob_at, diff_blobs=diff_blobs) - - def align(parent_len, hunks): """For each child line: the parent line index it keeps, or its new text.""" child, pos = [], 0 diff --git a/token_history/test_replay.py b/token_history/test_replay.py index 4bedf56d..b97c150b 100644 --- a/token_history/test_replay.py +++ b/token_history/test_replay.py @@ -4,8 +4,8 @@ import pytest -from replay import (NULL_BLOB, PathReplay, align, Hunk, parse_log, path_log, - real_parent_sources) +from replay import NULL_BLOB, PathReplay, align, Hunk, parse_log, path_log +from token_history import GitSource, mainline_changes, merge_parents def git(repo, *args, date=None): @@ -61,6 +61,12 @@ def replay(repo, path): return PathReplay(parse_log(path_log(str(repo), path))).run() +def real_replay(repo, path, mainline=None): + """The replay of the runner, with the real parents and blobs of git.""" + return GitSource(str(repo), merge_parents(str(repo))).replay( + path, mainline, None) + + def tip_born(rp, repo, path): tip = git(repo, "rev-parse", f"HEAD:{path}").strip() ids = rp.state_of_blob(tip, 2**62) @@ -108,14 +114,7 @@ def test_deleted_file_and_recreated(repo): def test_mainline_intervals(repo): rp = replay(repo, "a.c") - fp = git(repo, "log", "--first-parent", "--reverse", "-m", "--raw", - "--no-abbrev", "--format=\x01%H %ct", "--", "a.c") - changes, sha = [], None - for line in fp.splitlines(): - if line.startswith("\x01"): - sha, ct = line[1:].split() - elif line.startswith(":"): - changes.append((sha, int(ct), line.split()[3])) + changes = mainline_changes(str(repo), [])["a.c"] iv = rp.mainline_intervals(changes) side1 = next(i for i, t in enumerate(rp.tokens) if t.text == "side1") merge = git(repo, "rev-parse", "HEAD~1").strip() @@ -153,8 +152,7 @@ def test_merge_follows_real_parent_order(repo): git(repo, "merge", "-q", "--no-ff", "--no-commit", "redo") write(repo, "a.c", base + ["tail", "merged"]) commit(repo, "merge redo", "2020-07-03T00:00:00") - rp = PathReplay(parse_log(path_log(str(repo), "a.c")), - **real_parent_sources(str(repo), "a.c")).run() + rp = real_replay(repo, "a.c") born, _ = tip_born(rp, repo, "a.c") assert born == blame(repo, "a.c") @@ -174,30 +172,16 @@ def test_merge_equal_to_second_parent_takes_it_whole(repo): git(repo, "merge", "-q", "--no-ff", "--no-commit", "same") write(repo, "a.c", base + ["s"]) commit(repo, "merge same", "2020-07-05T00:00:00") - rp = PathReplay(parse_log(path_log(str(repo), "a.c")), - **real_parent_sources(str(repo), "a.c")).run() + rp = real_replay(repo, "a.c") born, _ = tip_born(rp, repo, "a.c") assert born == blame(repo, "a.c") -def first_parent_changes(repo, path): - fp = git(repo, "log", "--first-parent", "--reverse", "-m", "--raw", - "--no-abbrev", "--format=\x01%H %ct", "--", path) - changes, sha, ct = [], None, 0 - for line in fp.splitlines(): - if line.startswith("\x01"): - sha, ct = line[1:].split() - elif line.startswith(":"): - changes.append((sha, int(ct), line.split()[3])) - return changes - - def test_freeing_states_changes_nothing(repo): test_merge_follows_real_parent_order(repo) - changes = first_parent_changes(repo, "a.c") - src = real_parent_sources(str(repo), "a.c") - full = PathReplay(parse_log(path_log(str(repo), "a.c")), **src).run() - lean = PathReplay(parse_log(path_log(str(repo), "a.c")), **src).run(changes) + changes = mainline_changes(str(repo), [])["a.c"] + full = real_replay(repo, "a.c") + lean = real_replay(repo, "a.c", changes) assert len(lean.state) < len(full.state) assert lean.tokens == full.tokens assert lean.mainline_intervals(changes) == full.mainline_intervals(changes) @@ -236,8 +220,7 @@ def test_a_line_kept_from_two_parents_becomes_a_copy(tmp_path): "--no-commit", "keep"], capture_output=True) write(r, "d.c", ["A", "P", "Q", "A"]) commit(r, "merge keeps both", "2021-01-04T00:00:00") - rp = PathReplay(parse_log(path_log(str(r), "d.c")), - **real_parent_sources(str(r), "d.c")).run() + rp = real_replay(r, "d.c") for ids in rp.state.values(): assert len(ids) == len(set(ids)) born, _ = tip_born(rp, r, "d.c") @@ -262,8 +245,7 @@ def test_merge_that_brings_a_new_file_to_main(tmp_path): commit(r, "merge topic with a fix", "2022-01-04T00:00:00") write(r, "n.c", ["a", "b", "c", "fixed in merge", "later"]) commit(r, "later", "2022-01-05T00:00:00") - rp = PathReplay(parse_log(path_log(str(r), "n.c")), - **real_parent_sources(str(r), "n.c")).run() + rp = real_replay(r, "n.c") born, _ = tip_born(rp, r, "n.c") assert born == blame(r, "n.c") From 4f1201e63a4122abbe32c310df15b5ec054edccb Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:13:59 +0000 Subject: [PATCH 22/35] refactor(token_history): drop the sys.path inserts build.py and token_history.py put their own directory first on sys.path before the local imports. Python already does that for a script, and pytest does it for the tests, so the inserts and their noqa: E402 marks did nothing. attribute.py keeps its insert: it imports from ../generate_dataset. Checked: both scripts run from /tmp with 2 workers on a small repo. --- token_history/build.py | 6 ++---- token_history/token_history.py | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/token_history/build.py b/token_history/build.py index c166c1fc..19588934 100644 --- a/token_history/build.py +++ b/token_history/build.py @@ -14,14 +14,12 @@ import glob import os import shutil -import sys import time import duckdb -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from moves import MODES, link # noqa: E402 -from token_history import COLUMNS, RUN_COLUMNS, SEP, log # noqa: E402 +from moves import MODES, link +from token_history import COLUMNS, RUN_COLUMNS, SEP, log TYPES = dict(token_id="BIGINT", born_in_merge="INTEGER", copy_of="BIGINT", tip_index="BIGINT", origin_token_id="BIGINT", diff --git a/token_history/token_history.py b/token_history/token_history.py index e90111db..5d830a0b 100755 --- a/token_history/token_history.py +++ b/token_history/token_history.py @@ -16,14 +16,12 @@ import os import re import subprocess -import sys import time import traceback from datetime import datetime -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from replay import (HEADER, NULL_BLOB, GitError, PathReplay, # noqa: E402 - diff_hunks, git_out, parse_log, path_log) +from replay import (HEADER, NULL_BLOB, GitError, PathReplay, diff_hunks, + git_out, parse_log, path_log) MASK = re.compile(r"\.[ch]$") SEP = "\x1f" From efbf65d36672b9a1852190fdb69e9209ea2148f6 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:14:31 +0000 Subject: [PATCH 23/35] refactor(token_history): shorten three small helpers PathReplay.cached was functools.cache. GitSource.diff_blobs only bound the repository to diff_hunks. Progress.add zeroed three flags of a failed path that r.get(k, 0) already reads as 0. No change in behaviour; 13 fewer lines. --- token_history/replay.py | 13 ++----------- token_history/token_history.py | 12 ++++-------- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/token_history/replay.py b/token_history/replay.py index 9332ef84..b311613e 100644 --- a/token_history/replay.py +++ b/token_history/replay.py @@ -6,6 +6,7 @@ has it unchanged, as `git blame` does; a line that no parent has is born in the merge. """ +import functools import re import subprocess from dataclasses import dataclass, field @@ -144,23 +145,13 @@ def __init__(self, commits, real_parents=None, blob_at=None, self.blob_nodes = {} self.order = {c.sha: i for i, c in enumerate(commits)} self.real_parents = real_parents - self.blob_at = blob_at and self.cached(blob_at) + self.blob_at = blob_at and functools.cache(blob_at) self.diff_blobs = diff_blobs self.run_min_alnum = run_min_alnum self.runs = [] self.missing_parents = 0 self.unaligned_merges = 0 - @staticmethod - def cached(fn): - cache = {} - - def lookup(sha): - if sha not in cache: - cache[sha] = fn(sha) - return cache[sha] - return lookup - def born(self, text, commit, in_merge): self.tokens.append(Token(text, commit.sha, in_merge)) return len(self.tokens) - 1 diff --git a/token_history/token_history.py b/token_history/token_history.py index 5d830a0b..391aa1a1 100755 --- a/token_history/token_history.py +++ b/token_history/token_history.py @@ -90,15 +90,13 @@ def blob_at(self, sha): out = line.split()[0] return out if len(out) == 40 else NULL_BLOB - def diff_blobs(self, a, b): - return diff_hunks(self.repo, a, b) - def replay(self, path, mainline, run_min_alnum): self.path = path commits = parse_log(path_log(self.repo, path)) self.rewritten = {c.sha: c.parents for c in commits} return PathReplay(commits, real_parents=self.real_parents, - blob_at=self.blob_at, diff_blobs=self.diff_blobs, + blob_at=self.blob_at, + diff_blobs=lambda a, b: diff_hunks(self.repo, a, b), run_min_alnum=run_min_alnum).run(mainline) @@ -153,8 +151,7 @@ def replay_path(path): WORKER["runs"].write("".join(runs)) WORKER["out"].flush() WORKER["runs"].flush() - n = len(lines) - return dict(path=path, tokens=len(rp.tokens), rows=n, + return dict(path=path, tokens=len(rp.tokens), rows=len(lines), commits=len(rp.commits), seconds=round(time.time() - t0, 2), missing_parents=rp.missing_parents, unaligned_merges=rp.unaligned_merges, @@ -174,8 +171,7 @@ def add(self, r): self.done += 1 if "error" in r: self.flags["errors"] += 1 - r = dict(r, tokens=0, rows=0, seconds=0, missing_parents=0, - unaligned_merges=0, unmapped_mainline=0) + r = dict(r, tokens=0, rows=0, seconds=0) self.tokens += r["tokens"] self.rows += r["rows"] for k in self.flags: From 3e998f68e9b510d95dbe9115d36b0ef5caaec432 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:15:07 +0000 Subject: [PATCH 24/35] docs(token_history): correct four README claims - A change to the move rules does not always rebuild without a new replay: stage 1 records only runs of at least --run-min-alnum (100) alphanumeric characters, so a lower --min-alnum cannot link a shorter run, and renames never links a file with fewer than 100. - history/validate-.json is not a file that validate.py writes: it prints its JSON. Cite the script instead. - Every GitError now stops stage 1, not only the three whole-repository calls; other errors of a path still go to errors.txt. - The tests need the devenv shell (plain python has no pytest). --- token_history/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/token_history/README.md b/token_history/README.md index 3c908afb..ef882865 100644 --- a/token_history/README.md +++ b/token_history/README.md @@ -14,7 +14,7 @@ The tip dataset (`generate_dataset`) holds only the tokens alive at HEAD. This s On the io_uring pilot (96 paths) stage 1 takes 2–3 min and stage 2 under 10 s. -Stage 2 reads only the files of stage 1. Keep them: every mode, and any later change to the move rules, rebuilds from them without a new replay. A change to the replay rules (`replay.py`) needs a new stage 1. +Stage 2 reads only the files of stage 1. Keep them: every mode, and a change to `moves.py`, rebuilds from them without a new replay, within one limit: stage 1 keeps only runs of at least `--run-min-alnum` (100) alphanumeric characters. So with a `--min-alnum` below 100, a removed or added run shorter than 100 still never links, and `renames` never links a file with fewer than 100. A change to `replay.py` or to `--run-min-alnum` needs a new stage 1. At kernel scale, stage 2 works in parts so that it stays inside its memory limit (`--memory`, default 6 GB): @@ -36,7 +36,7 @@ At kernel scale, stage 2 works in parts so that it stays inside its memory limit - `git blame -C100` diffs a block against the whole old file, so it drops a move that its diff breaks into short pieces. `moves` keeps some of those. - Measured on Linux (a 1/32 sample of the tip, 5.49 M tokens, 30 Sep 2026): where the two disagree on the birth time, `-C100` gives the **older** commit in about 87% of the cases (981,149 tokens against 145,019). -On the whole Linux tip, the origin commit agrees with the `-C100` tip dataset on 80.46% of the tokens for `moves` and 81.13% for `renames` (`history/validate-.json`). A rule that copies `git blame -C` needs only the repository and the stage 1 files, so it can be added to `moves.py` later. +On the whole Linux tip, the origin commit agrees with the `-C100` tip dataset on 80.46% of the tokens for `moves` and 81.13% for `renames` (`validate.py`). A rule that copies `git blame -C` needs only the repository and the stage 1 files, so it can be added to `moves.py` later. ## Run @@ -65,7 +65,7 @@ python3 attribute.py --cregit-db X-cregit.db --persons-db X-persons.db \ `validate.py` compares only the dataset rows of the files that the history has (so only `.c` and `.h`). Its `same_text` counts tokens whose text matches the dataset's token type, its token value, or both joined by `|`. -Stage 1 writes `history-progress.json` and a log line every `--ping` seconds. A path that fails goes to `errors.txt` with its traceback, and the run continues. A `git` command that fails for the whole repository (the merge list, the mainline log or the path list) stops the run with `GitError`, so a run cannot end with no data and 0 errors. +Stage 1 writes `history-progress.json` and a log line every `--ping` seconds. A `git` command that fails stops the run with `GitError`, so a run cannot end with no data and 0 errors. Any other error of a path goes to `errors.txt` with its traceback, and the run continues. ## Columns of `history-.parquet` @@ -88,7 +88,7 @@ A token that left the mainline and came back has one row per interval. A token a ## Tests ```sh -pytest token_history -q +devenv shell -- pytest token_history -q ``` The tests build small git repositories. Each of the seven replay rules (simplify-merges, real parent order, a merge equal to one parent takes it whole, a commit with no diff keeps the parent blob, a line kept twice becomes a copy, a parent without the file is no source, an error in one path is a record) has a test that fails when the rule is removed. From 76091b8f0707e12906ffd12d0a709919f1335277 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:15:51 +0000 Subject: [PATCH 25/35] docs(token_history): shorten the README Fold the three stage 2 bullets and the four -C100 paragraphs into one paragraph each, and drop what the code or the commits already say: the .part rename of the links file, the same_text rule of validate.py, a future -C copy rule, and the list of the seven replay rules that have a test (ba68d11). No claim changes. --- token_history/README.md | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/token_history/README.md b/token_history/README.md index ef882865..13bcde44 100644 --- a/token_history/README.md +++ b/token_history/README.md @@ -4,7 +4,7 @@ The tip dataset (`generate_dataset`) holds only the tokens alive at HEAD. This s **Scope: `.c` and `.h` paths only** (`MASK` in `token_history.py`). The tip dataset also has other file types; on Linux, 503 files (`.rs`, `.cpp`, `.cc`, 1.05 M rows) are not in the history. -## Two stages +## Stages | Stage | Script | Cost (Linux, Sep 2026) | Output | | --- | --- | --- | --- | @@ -16,11 +16,7 @@ On the io_uring pilot (96 paths) stage 1 takes 2–3 min and stage 2 under 10 s. Stage 2 reads only the files of stage 1. Keep them: every mode, and a change to `moves.py`, rebuilds from them without a new replay, within one limit: stage 1 keeps only runs of at least `--run-min-alnum` (100) alphanumeric characters. So with a `--min-alnum` below 100, a removed or added run shorter than 100 still never links, and `renames` never links a file with fewer than 100. A change to `replay.py` or to `--run-min-alnum` needs a new stage 1. -At kernel scale, stage 2 works in parts so that it stays inside its memory limit (`--memory`, default 6 GB): - -- `--reuse` keeps the loaded tables in `OUT/build.duckdb` and a finished `links-.tsv`, so a build that stopped after the link pass restarts at the chain resolution. The link pass writes `links-.tsv.part` and renames it only at the end. -- `--chunks N` (default 16) looks up the origins and writes the parquet in N chunks of paths, with no global sort. The row order of the parquet is not defined. -- `links-.parquet` is not written here: the analysis (`msr-code/analysis/rq_history.py`) converts the TSV once for its own use. +On the kernel, stage 2 stays inside `--memory` (default 6 GB): the links go to disk, and the parquet is written in `--chunks` (16) chunks of paths, so its row order is not defined. `--reuse` keeps the tables of `OUT/build.duckdb` and a finished `links-.tsv`, so a build that stopped after the link pass restarts at the chain resolution. The analysis (`msr-code/analysis/rq_history.py`) converts the links TSV to parquet itself. ## Move modes (stage 2) @@ -30,13 +26,7 @@ At kernel scale, stage 2 works in parts so that it stays inside its memory limit | `renames` | a deleted file to a file created in the same commit | close to plain `git blame` with rename detection | | `moves` (default) | any exact run of ≥ `--min-alnum` alphanumeric characters that one commit removes and adds | keeps the first identity of moved code | -`moves` is not a copy of `git blame -C100`, and it does **not** follow more moves: - -- `moves` links only an exact run that the **same commit** removes and adds. Each removed token is used once. So it does not follow a **copy** whose source stays in place, which `git blame -C` does follow. -- `git blame -C100` diffs a block against the whole old file, so it drops a move that its diff breaks into short pieces. `moves` keeps some of those. -- Measured on Linux (a 1/32 sample of the tip, 5.49 M tokens, 30 Sep 2026): where the two disagree on the birth time, `-C100` gives the **older** commit in about 87% of the cases (981,149 tokens against 145,019). - -On the whole Linux tip, the origin commit agrees with the `-C100` tip dataset on 80.46% of the tokens for `moves` and 81.13% for `renames` (`validate.py`). A rule that copies `git blame -C` needs only the repository and the stage 1 files, so it can be added to `moves.py` later. +`moves` is not `git blame -C100`, and it does **not** follow more moves. It links only an exact run that the **same commit** removes and adds, each removed token once, so it misses a **copy** whose source stays in place; `-C100` follows copies, but drops a move that its diff breaks into short pieces. On a 1/32 sample of the Linux tip (5.49 M tokens, 30 Sep 2026), where the two disagree on the birth time, `-C100` gives the **older** commit in about 87% of the cases (981,149 tokens against 145,019). On the whole Linux tip, the origin commit agrees with the `-C100` tip dataset on 80.46% of the tokens for `moves` and 81.13% for `renames` (`validate.py`). ## Run @@ -63,7 +53,7 @@ python3 attribute.py --cregit-db X-cregit.db --persons-db X-persons.db \ --out OUT/commits.parquet --dataset X-dataset.parquet ``` -`validate.py` compares only the dataset rows of the files that the history has (so only `.c` and `.h`). Its `same_text` counts tokens whose text matches the dataset's token type, its token value, or both joined by `|`. +`validate.py` compares only the files that the history has (`.c` and `.h`). Stage 1 writes `history-progress.json` and a log line every `--ping` seconds. A `git` command that fails stops the run with `GitError`, so a run cannot end with no data and 0 errors. Any other error of a path goes to `errors.txt` with its traceback, and the run continues. @@ -90,5 +80,3 @@ A token that left the mainline and came back has one row per interval. A token a ```sh devenv shell -- pytest token_history -q ``` - -The tests build small git repositories. Each of the seven replay rules (simplify-merges, real parent order, a merge equal to one parent takes it whole, a commit with no diff keeps the parent blob, a line kept twice becomes a copy, a parent without the file is no source, an error in one path is a record) has a test that fails when the rule is removed. From 03a2bc3be9f0d915d0d42d911fe53935e7bd6881 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:16:34 +0000 Subject: [PATCH 26/35] refactor(token_history): drop the check flag of git_out git_out(check=False) accepted a non-zero exit for rev-parse --verify of a missing object. Its only caller was real_parent_sources, gone since GitSource serves the tests, so only its own test was left. Every git_out call now raises GitError on a non-zero exit. --- token_history/replay.py | 7 +++---- token_history/test_token_history.py | 7 +------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/token_history/replay.py b/token_history/replay.py index b311613e..9b373578 100644 --- a/token_history/replay.py +++ b/token_history/replay.py @@ -77,12 +77,11 @@ class GitError(RuntimeError): """git exited with an error: the caller must not go on with no data.""" -def git_out(repo, *args, check=True): - """The output of one git command. check=False accepts a non-zero exit - (for example rev-parse --verify of a missing object).""" +def git_out(repo, *args): + """The output of one git command; GitError on a non-zero exit.""" proc = subprocess.run(["git", "-C", repo, *args], capture_output=True, text=True, errors="replace") - if check and proc.returncode: + if proc.returncode: raise GitError(f"git {' '.join(args[:3])} exited {proc.returncode}: " f"{proc.stderr.strip()[:300]}") return proc.stdout diff --git a/token_history/test_token_history.py b/token_history/test_token_history.py index b67e1d3c..b623657b 100644 --- a/token_history/test_token_history.py +++ b/token_history/test_token_history.py @@ -2,7 +2,7 @@ import pytest -from replay import NULL_BLOB, GitError, diff_hunks, git_out +from replay import NULL_BLOB, GitError, diff_hunks from test_replay import commit, git, write from token_history import list_paths, mainline_changes, merge_parents @@ -26,11 +26,6 @@ def test_git_failures_stop_the_run(not_a_repo): pathspec=[])) -def test_an_expected_non_zero_exit_is_accepted(not_a_repo): - assert git_out(not_a_repo, "rev-parse", "-q", "--verify", "x:y", - check=False) == "" - - def test_the_null_blob_diffs_to_no_hunks(tmp_path): r = tmp_path / "r" r.mkdir() From 435fe6060ffafabdbe505197cf8b7a6cf4ec2521 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 04:16:54 +0000 Subject: [PATCH 27/35] docs(token_history): trim comments that repeat the README The build.py docstring restated the README on the move modes and --reuse (also in --help). The comment of commit_runs retold the 408 M row story of 3ce68be in five lines; two say what the loop does. The comment above the mainline_changes exit check restated the GitError it raises. --- token_history/build.py | 14 ++------------ token_history/token_history.py | 2 -- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/token_history/build.py b/token_history/build.py index 19588934..88ad9f4a 100644 --- a/token_history/build.py +++ b/token_history/build.py @@ -1,14 +1,7 @@ #!/usr/bin/env python3 """Build history-.parquet from the parts that token_history.py wrote. -The move mode is chosen here, not in the replay, so a run can switch between -off, renames and moves without replaying the history again. - Usage: build.py --out DIR [--mode moves] [--min-alnum 100] [--reuse] - ---reuse keeps the tokens and runs tables of DIR/build.duckdb and a finished -links-.tsv, so a build that failed after the link pass restarts at the -chain resolution. """ import argparse import glob @@ -160,11 +153,8 @@ def commit_runs(con): FROM t JOIN tokens tok ON tok.file_path = t.file_path AND tok.token_id = t.tid ORDER BY t.sha, t.run_no, t.pos""") - # tokens has one row per mainline interval, so a token with two - # intervals comes twice at one place of its run. The rows of one place - # are next to each other (the sort), and a token id has one text, so - # keeping the first row of each place is a DISTINCT that costs no - # memory (a DISTINCT over the 408 M kernel token rows does not fit). + # tokens has a row per mainline interval: keep the first row of each + # place (the sort puts them together), a DISTINCT that needs no memory sha, runs, texts, cur_run, last = None, [], {}, None, None while batch := cur.fetchmany(100_000): for s, run_no, kind, whole, path, tid, token, pos in batch: diff --git a/token_history/token_history.py b/token_history/token_history.py index 391aa1a1..bbe6bc62 100755 --- a/token_history/token_history.py +++ b/token_history/token_history.py @@ -57,8 +57,6 @@ def mainline_changes(repo, pathspec): meta, path = line.rstrip("\n").split("\t", 1) if MASK.search(path): changes.setdefault(path, []).append((sha, ct, meta.split()[3])) - # a failed git log would leave every path with no mainline interval, - # and the run would still end with 0 errors if proc.wait(): raise GitError(f"git log --first-parent exited {proc.returncode}: " f"{proc.stderr.read().strip()[:300]}") From 3445ce88a858f37bf484b7a4470e01b8ae49e947 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 14:11:06 +0000 Subject: [PATCH 28/35] docs(token_history): keep each comment to three lines Cut the module docstrings to what the code cannot say: the usage blocks repeated --help and the README, and the move modes are in the README table. Fold the other long docstrings to three lines, move the links comment of build.main onto write_links in place of a docstring that restated it, and drop two test comments that retold history or restated the test name. The scripts took their --help description from line 1 of __doc__, which is the blank line after the summary, so --help showed none. They now take line 0; the long docstrings were what kept line 1 in range. --- token_history/attribute.py | 18 +++-------------- token_history/build.py | 30 ++++++++++------------------- token_history/moves.py | 26 +++++++------------------ token_history/replay.py | 30 +++++++++-------------------- token_history/test_build.py | 5 +---- token_history/test_token_history.py | 4 +--- token_history/token_history.py | 12 ++---------- token_history/validate.py | 11 +++-------- 8 files changed, 36 insertions(+), 100 deletions(-) diff --git a/token_history/attribute.py b/token_history/attribute.py index ac4f3530..41f13d6c 100644 --- a/token_history/attribute.py +++ b/token_history/attribute.py @@ -1,19 +1,7 @@ #!/usr/bin/env python3 """Author, person and firm of every cregit commit, as the tip dataset has them. - -A history parquet names commits only. This writes one row per cregit commit -with the same columns and the same joins as generate_dataset phase 2, so a -history token joins its born_sha (or origin_born_sha) to the author and firm -that the tip dataset gives a surviving token of the same commit. - -Usage: attribute.py --cregit-db X-cregit.db --persons-db X-persons.db - --firm-map affiliation.merged.csv - [--firm-canonical firm_canonical.csv] - --out commits.parquet [--dataset X-dataset.parquet] - -With --dataset, every commit that the dataset also has is compared on its -person and firm, and the counts are printed. -""" +Same columns and joins as generate_dataset phase 2, so a history token joins +its born_sha to them; --dataset prints how many commits agree with it.""" import argparse import json import os @@ -27,7 +15,7 @@ def main(): - ap = argparse.ArgumentParser(description=__doc__.split("\n")[1]) + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--cregit-db", required=True) ap.add_argument("--persons-db", required=True) ap.add_argument("--firm-map", required=True) diff --git a/token_history/build.py b/token_history/build.py index 88ad9f4a..b7481f05 100644 --- a/token_history/build.py +++ b/token_history/build.py @@ -1,8 +1,5 @@ #!/usr/bin/env python3 -"""Build history-.parquet from the parts that token_history.py wrote. - -Usage: build.py --out DIR [--mode moves] [--min-alnum 100] [--reuse] -""" +"""Build history-.parquet from the parts that token_history.py wrote.""" import argparse import glob import os @@ -46,12 +43,9 @@ def read_tsv(con, name, pattern, columns): def resolve_origins(con): - """links(file_path, token_id -> origin) to the first identity of each chain. - - Pointer jumping: each round replaces an origin by that origin's own - origin, so a chain of n moves needs about log2(n) rounds. A move always - points to an older token, so chains end. - """ + """Point each link to the first identity of its chain by pointer jumping: + about log2(n) rounds for a chain of n moves. A move always points to an + older token, so chains end.""" rounds = 0 while True: con.execute(""" @@ -79,12 +73,9 @@ def in_chunk(column, k, chunks): def write_history(con, target, mode, chunks): - """Tokens with their resolved origin, written in chunks of paths. - - On a whole kernel, one join of every token with every link does not fit - in memory. Each chunk joins only the paths whose hash falls in it; the - parts are then copied into one parquet (no row order is kept). - """ + """Tokens with their resolved origin, in chunks of paths: on a whole kernel + one join of every token with every link does not fit in memory. The chunk + parquets are then copied into one (no row order is kept).""" con.execute("CREATE OR REPLACE TABLE origins (file_path VARCHAR, " "token_id BIGINT, origin_path VARCHAR, origin_token_id BIGINT, " "origin_born_sha VARCHAR)") @@ -186,7 +177,8 @@ def load(con, out): def write_links(con, links_file, mode, min_alnum): - """The link pass, to a .part file that is renamed only at the end.""" + """Links go to disk commit by commit: on a whole kernel they do not fit + in memory. A born token has one birth commit, so it has one link.""" linked = commits = 0 with open(links_file + ".part", "w", encoding="utf-8") as f: f.write(f"{SEP}{SEP}{SEP}\n") @@ -204,7 +196,7 @@ def write_links(con, links_file, mode, min_alnum): def main(): - ap = argparse.ArgumentParser(description=__doc__.split("\n")[1]) + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--out", required=True) ap.add_argument("--mode", choices=MODES, default="moves") ap.add_argument("--min-alnum", type=int, default=100) @@ -232,8 +224,6 @@ def main(): f" token rows, {con.execute('SELECT count(*) FROM runs').fetchone()[0]:,}" " runs") - # Links go to disk commit by commit: on a whole kernel they do not fit - # in memory. A born token has one birth commit, so it has one link. links_file = os.path.join(args.out, f"links-{args.mode}.tsv") if reuse and os.path.exists(links_file): log(f"build: reuse {links_file}") diff --git a/token_history/moves.py b/token_history/moves.py index a9d8bffc..10f0ddce 100644 --- a/token_history/moves.py +++ b/token_history/moves.py @@ -1,13 +1,6 @@ -"""Link tokens that moved between files, so they keep their first identity. - -Within one commit, a run of born tokens that equals a run of died tokens is a -move. Modes: - off no links: every path keeps its own history - renames only from a file the commit deleted to a file it created, as - plain `git blame` follows a renamed file - moves any run of at least --min-alnum alphanumeric characters, as - `git blame -C` does for code moved between files -""" +"""Link tokens that moved between files, so they keep their first identity: +within one commit, a run of born tokens that equals a run of died tokens is a +move. The README describes the modes.""" from collections import defaultdict from replay import alnum @@ -18,12 +11,9 @@ def matching_blocks(a, b): - """Maximal equal blocks (i, j, size) of b in a, found from k-token anchors. - - Each anchor of b is looked up in an index of a, preferring the candidate - that continues the previous block, and grown forward while equal. Linear - in practice, where difflib is quadratic on runs full of repeated tokens. - """ + """Maximal equal blocks (i, j, size) of b in a, grown from k-token anchors + that prefer to continue the previous block. Linear in practice, where + difflib is quadratic on runs full of repeated tokens.""" k = min(ANCHOR, len(a), len(b)) if not k: return [] @@ -52,9 +42,7 @@ def matching_blocks(a, b): def link(runs, text_of, mode, min_alnum): """{(path, born id): (path, died id)} for one commit's runs. - - runs: [(kind, whole_file, path, ids)]; text_of((path, id)) -> token text. - """ + runs: [(kind, whole_file, path, ids)]; text_of((path, id)): its text.""" if mode == "off": return {} eligible = [r for r in runs if mode == "moves" or r[1]] diff --git a/token_history/replay.py b/token_history/replay.py index 9b373578..b7757a81 100644 --- a/token_history/replay.py +++ b/token_history/replay.py @@ -1,11 +1,6 @@ -"""Replay the history of one path of a cregit repository, token by token. - -Each line of a cregit file is one token. A token is born in the commit whose -diff adds its line, and it keeps its identity while later diffs leave the line -unchanged. At a merge, a line takes its identity from the first parent that -has it unchanged, as `git blame` does; a line that no parent has is born in -the merge. -""" +"""Replay one path of a cregit repository, one token per line. A line keeps +its identity while diffs leave it unchanged; at a merge it takes the identity +of the first parent that has it unchanged, as `git blame` does.""" import functools import re import subprocess @@ -129,12 +124,9 @@ def unique(items): class PathReplay: - """Token identities along the history of one path. - - real_parents, blob_at and diff_blobs let a merge try its real parents in - their real order, as `git blame` does, when the simplified history dropped - one of them. - """ + """Token identities along the history of one path. real_parents, blob_at + and diff_blobs let a merge try its real parents in their real order, as + `git blame` does, when the simplified history dropped one of them.""" def __init__(self, commits, real_parents=None, blob_at=None, diff_blobs=None, run_min_alnum=None): @@ -264,9 +256,7 @@ def latest_node(self, blob, before_ct): def add_runs(self, c, kind, positioned): """Record runs of adjacent tokens that are long enough to move. - - positioned: [(position in the file, token id)], in file order. - """ + positioned: [(position in the file, token id)], in file order.""" if self.run_min_alnum is None: return if kind == "died": @@ -326,11 +316,9 @@ def tip_positions(self, changes): return {t: i for i, t in enumerate(ids)} def mainline_intervals(self, changes): - """Intervals on the mainline: {token: [(in_sha, in_ct, out_sha, out_ct)]}. - + """{token: [(in_sha, in_ct, out_sha, out_ct)]} on the mainline. changes: [(sha, ct, blob)] of the first-parent commits that changed - this path, oldest first. - """ + this path, oldest first.""" intervals, alive, unmapped = {}, set(), 0 for sha, ct, blob in changes: ids = self.state_of_blob(blob, ct) diff --git a/token_history/test_build.py b/token_history/test_build.py index 6547666c..e2b2ef5b 100644 --- a/token_history/test_build.py +++ b/token_history/test_build.py @@ -6,8 +6,6 @@ def test_a_token_with_two_mainline_intervals_is_one_place_in_its_run(): - # tokens has one row per mainline interval; token 1 left the mainline - # and came back, so it has two rows con = duckdb.connect() con.execute("CREATE TABLE tokens (file_path VARCHAR, token_id BIGINT, " "token VARCHAR, mainline_in_sha VARCHAR)") @@ -25,8 +23,7 @@ def test_a_token_with_two_mainline_intervals_is_one_place_in_its_run(): def test_a_path_written_again_by_a_resume_is_loaded_once(tmp_path): - # part-1 has a.c from a run that stopped before paths-done.txt; the - # resume wrote a.c again in part-2, here after a change of the replay + # the resume wrote a.c again in part-2, after a change of the replay def row(path, born): return SEP.join([path, "0", "x", born, "0", "", "", "", "m", "1", "", ""]) + "\n" diff --git a/token_history/test_token_history.py b/token_history/test_token_history.py index b623657b..491aab78 100644 --- a/token_history/test_token_history.py +++ b/token_history/test_token_history.py @@ -15,8 +15,6 @@ def not_a_repo(tmp_path): def test_git_failures_stop_the_run(not_a_repo): - # Before, each of these returned empty data and the run went on with - # no mainline intervals, no merges or no paths, and 0 errors. with pytest.raises(GitError): merge_parents(not_a_repo) with pytest.raises(GitError, match="not a git repository"): @@ -69,5 +67,5 @@ def test_a_dead_cat_file_stops_the_run(not_a_repo, monkeypatch): src = token_history.GitSource(not_a_repo, {}) monkeypatch.setattr(token_history, "replay_path", lambda path: src.blob_at("HEAD")) - with pytest.raises(GitError): # not one errors.txt record per path + with pytest.raises(GitError): token_history.work("a.c") diff --git a/token_history/token_history.py b/token_history/token_history.py index bbe6bc62..a9a964d5 100755 --- a/token_history/token_history.py +++ b/token_history/token_history.py @@ -1,15 +1,7 @@ #!/usr/bin/env python3 """Extract every token that ever existed in a cregit repository. -For each path, replay its history (replay.py), then write one row per token -and mainline interval: when the token was born, when a commit first removed -it, and when it entered and left the mainline (the first-parent history of -HEAD). Progress goes to a JSON file and to a log line every --ping seconds. - -Usage: - token_history.py --repo X-cregit.git --out DIR [--paths FILE | -- PATHSPEC] - [--jobs 8] [--ping 60] -""" +One row per token and mainline interval; the README describes the columns.""" import argparse import json import multiprocessing as mp @@ -207,7 +199,7 @@ def list_paths(args): def main(): - ap = argparse.ArgumentParser(description=__doc__.split("\n")[1]) + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--repo", required=True) ap.add_argument("--out", required=True) ap.add_argument("--paths") diff --git a/token_history/validate.py b/token_history/validate.py index 748cd77a..03362204 100755 --- a/token_history/validate.py +++ b/token_history/validate.py @@ -1,12 +1,7 @@ #!/usr/bin/env python3 """Compare the tip of a history parquet with the tip dataset parquet. - -Every token alive at the tip must be a row of the dataset at the same file and -token_index, with the same text; the share whose origin commit equals the -dataset's cregit_commit_sha is the agreement with its blame. - -Usage: validate.py --history history-moves.parquet --dataset X-dataset.parquet -""" +Every tip token must be a dataset row at the same file, token_index and text; +the share whose origin commit is its cregit_commit_sha is the agreement.""" import argparse import json @@ -14,7 +9,7 @@ def main(): - ap = argparse.ArgumentParser(description=__doc__.split("\n")[1]) + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--history", required=True) ap.add_argument("--dataset", required=True) args = ap.parse_args() From bbaafa7bd99c3a83ebcfaa003b8eb578ce72d338 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 14:13:17 +0000 Subject: [PATCH 29/35] refactor(token_history): split the replay of a commit PathReplay.step, sources, run and add_runs were each graded C (15 to 17) by radon, and parse_log C (11). Give each step a name: first_parent_source, merge_sources, merge_source and hunks_to_child for the alignment of a merge, kept_id and mark_died for step, states_to_keep and free_parents for run, adjacent_runs and is_whole_file for add_runs, parse_diff_line for parse_log. parse_log keeps its commits in the insertion order of by_sha, and run counts the pending children with a Counter. Every function is now B or better. No change in behaviour: on three generated repositories with merges, renames and moves, the old and new code write the same part and runs files, flags and history parquets in all three modes. --- token_history/replay.py | 182 ++++++++++++++++++++++------------------ 1 file changed, 101 insertions(+), 81 deletions(-) diff --git a/token_history/replay.py b/token_history/replay.py index b7757a81..6b4b63b9 100644 --- a/token_history/replay.py +++ b/token_history/replay.py @@ -4,6 +4,7 @@ import functools import re import subprocess +from collections import Counter from dataclasses import dataclass, field HEADER = "\x01" @@ -45,27 +46,27 @@ def path_log(repo, path): def parse_log(text): """Commits in topological order; a merge holds one diff per parent.""" - commits, by_sha, cur, diff = [], {}, None, None + by_sha, cur, diff = {}, None, None for line in text.split("\n"): if line.startswith(HEADER): head, ct = line[1:].split("\t") sha, *parents = head.split() - cur = by_sha.get(sha) - if cur is None: - cur = by_sha[sha] = Commit(sha, parents, int(ct)) - commits.append(cur) + cur = by_sha.setdefault(sha, Commit(sha, parents, int(ct))) diff = Diff() cur.diffs.append(diff) - elif cur is None: - continue - elif (m := RAW.match(line)): - diff.old_blob, cur.blob = m.group(1), m.group(2) - elif (m := HUNK.match(line)): - old_len = 1 if m.group(2) is None else int(m.group(2)) - diff.hunks.append(Hunk(int(m.group(1)), old_len)) - elif diff.hunks and line.startswith("+") and not line.startswith("+++"): - diff.hunks[-1].added.append(line[1:]) - return commits + elif cur is not None: + parse_diff_line(line, cur, diff) + return list(by_sha.values()) + + +def parse_diff_line(line, commit, diff): + if (m := RAW.match(line)): + diff.old_blob, commit.blob = m.group(1), m.group(2) + elif (m := HUNK.match(line)): + old_len = 1 if m.group(2) is None else int(m.group(2)) + diff.hunks.append(Hunk(int(m.group(1)), old_len)) + elif diff.hunks and line.startswith("+") and not line.startswith("+++"): + diff.hunks[-1].added.append(line[1:]) class GitError(RuntimeError): @@ -123,6 +124,28 @@ def unique(items): return list(dict.fromkeys(items)) +def adjacent_runs(positioned): + """Split [(position, token)] into runs of consecutive positions.""" + run, last = [], None + for pos, t in positioned: + if run and pos != last + 1: + yield run + run = [] + run.append(t) + last = pos + if run: + yield run + + +def kept_id(i, line, sources): + """The id that child line i keeps: from the base parent, else from the + first other parent that has the line unchanged; None for a new line.""" + if not isinstance(line, tuple): + return sources[0][0][line] + return next((ids[m[i]] for ids, m in sources[1:] + if i < len(m) and not isinstance(m[i], tuple)), None) + + class PathReplay: """Token identities along the history of one path. real_parents, blob_at and diff_blobs let a merge try its real parents in their real order, as @@ -180,39 +203,41 @@ def ids_of(self, blob, c): return self.parent_ids(p) return self.state_of_blob(blob, c.ct) + def first_parent_source(self, c): + ids = self.parent_ids(c.parents[0]) if c.parents else [] + return [(ids, align(len(ids), c.diffs[0].hunks))] + def sources(self, c): """(parent ids, alignment) per parent, in the order blame tries them.""" real = self.parents_of(c) if len(real) <= 1 or not c.parents: - ids = self.parent_ids(c.parents[0]) if c.parents else [] - return [(ids, align(len(ids), c.diffs[0].hunks))] + return self.first_parent_source(c) blobs = self.merge_blobs(c, real) - if c.blob in blobs: - ids = self.ids_of(c.blob, c) - if ids is not None: - return [(ids, align(len(ids), []))] + ids = self.ids_of(c.blob, c) if c.blob in blobs else None + if ids is not None: + return [(ids, align(len(ids), []))] + return self.merge_sources(c, blobs) or self.first_parent_source(c) + + def merge_sources(self, c, blobs): by_blob = {d.old_blob: d.hunks for d in c.diffs} - out = [] - for blob in blobs: - if blob == NULL_BLOB: - continue - ids = self.ids_of(blob, c) - if blob in by_blob: - hunks = by_blob[blob] - elif blob == c.blob: - hunks = [] - elif self.diff_blobs and ids is not None: - hunks = self.diff_blobs(blob, c.blob) - else: - ids = None - if ids is None: - self.unaligned_merges += 1 - continue - out.append((ids, align(len(ids), hunks))) - if not out: - ids = self.parent_ids(c.parents[0]) - out = [(ids, align(len(ids), c.diffs[0].hunks))] - return out + out = [self.merge_source(c, blob, by_blob) + for blob in blobs if blob != NULL_BLOB] + return [s for s in out if s] + + def merge_source(self, c, blob, by_blob): + ids = self.ids_of(blob, c) + hunks = None if ids is None else self.hunks_to_child(c, blob, by_blob) + if hunks is None: + self.unaligned_merges += 1 + return None + return ids, align(len(ids), hunks) + + def hunks_to_child(self, c, blob, by_blob): + if blob in by_blob: + return by_blob[blob] + if blob == c.blob: + return [] + return self.diff_blobs(blob, c.blob) if self.diff_blobs else None def step(self, c): sources = self.sources(c) @@ -220,12 +245,7 @@ def step(self, c): base_ids, base = sources[0] ids, used = [], set() for i, line in enumerate(base): - if not isinstance(line, tuple): - kept = base_ids[line] - else: - kept = next((s_ids[m[i]] for s_ids, m in sources[1:] - if i < len(m) and not isinstance(m[i], tuple)), - None) + kept = kept_id(i, line, sources) if kept is None: kept = self.born(line[1], c, in_merge) elif kept in used: @@ -233,16 +253,18 @@ def step(self, c): used.add(kept) ids.append(kept) if not in_merge: - alive = set(ids) - for t in base_ids: - if t not in alive and self.tokens[t].died is None: - self.tokens[t].died = c.sha - self.add_runs(c, "died", [(j, t) for j, t in enumerate(base_ids) - if t not in alive]) + self.mark_died(c, base_ids, set(ids)) self.add_runs(c, "born", [(i, t) for i, t in enumerate(ids) if self.tokens[t].born == c.sha]) self.state[c.sha] = ids + def mark_died(self, c, base_ids, alive): + gone = [(j, t) for j, t in enumerate(base_ids) if t not in alive] + for _, t in gone: + if self.tokens[t].died is None: + self.tokens[t].died = c.sha + self.add_runs(c, "died", gone) + def resolve_blobs(self): for c in self.commits: if c.blob is None: @@ -259,46 +281,44 @@ def add_runs(self, c, kind, positioned): positioned: [(position in the file, token id)], in file order.""" if self.run_min_alnum is None: return + whole = self.is_whole_file(c, kind) + for run in adjacent_runs(positioned): + if alnum(self.tokens[t].text for t in run) >= self.run_min_alnum: + self.runs.append((c.sha, kind, whole, run)) + + def is_whole_file(self, c, kind): if kind == "died": - whole = c.blob == NULL_BLOB - else: - whole = all(self.blob_of(p) == NULL_BLOB for p in c.parents) - run, last = [], None - for pos, t in positioned + [(None, None)]: - if run and (pos is None or pos != last + 1): - if alnum(self.tokens[i].text for i in run) >= self.run_min_alnum: - self.runs.append((c.sha, kind, whole, run)) - run = [] - if t is not None: - run.append(t) - last = pos + return c.blob == NULL_BLOB + return all(self.blob_of(p) == NULL_BLOB for p in c.parents) def run(self, mainline=None): """Replay every commit. With mainline changes, free unneeded states.""" self.resolve_blobs() for c in self.commits: self.blob_nodes.setdefault(c.blob, []).append(c.sha) - if mainline is None: - for c in self.commits: - self.step(c) - return self + keep = None if mainline is None else self.states_to_keep(mainline) + pending = Counter(p for c in self.commits for p in c.parents) + for c in self.commits: + self.step(c) + if keep is not None: + self.free_parents(c, pending, keep) + return self + + def free_parents(self, c, pending, keep): + for p in c.parents: + pending[p] -= 1 + if not pending[p] and p not in keep: + self.state.pop(p, None) + + def states_to_keep(self, mainline): + """The states that a mainline change or a real-parent lookup reads.""" keep = {self.latest_node(b, ct) for _, ct, b in mainline} for c in self.commits: real = self.parents_of(c) if len(real) > 1 and real != c.parents: keep |= {self.latest_node(self.blob_at(p), c.ct) for p in real} - pending = {} - for c in self.commits: - for p in c.parents: - pending[p] = pending.get(p, 0) + 1 - for c in self.commits: - self.step(c) - for p in c.parents: - pending[p] -= 1 - if not pending[p] and p not in keep: - self.state.pop(p, None) - return self + return keep def state_of_blob(self, blob, before_ct): """The latest node with this blob, committed at or before a time.""" From e280d094ee1e0d4130c25b6e49d031c345bbaad2 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 14:14:00 +0000 Subject: [PATCH 30/35] refactor(token_history): split the link pass of a commit moves.link was graded D (22) by radon: it built the died sequence, matched every born run against it, filtered the blocks by mode and assigned the links, all in one body. Name each part: died_sequence, born_blocks, is_move and longest_first. link is now B (9). No change in behaviour: on three generated repositories the old and new code write the same history parquets in all three modes. --- token_history/moves.py | 54 +++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/token_history/moves.py b/token_history/moves.py index 10f0ddce..aad3ebb7 100644 --- a/token_history/moves.py +++ b/token_history/moves.py @@ -46,26 +46,42 @@ def link(runs, text_of, mode, min_alnum): if mode == "off": return {} eligible = [r for r in runs if mode == "moves" or r[1]] - died_text, died_key = [], [] - for kind, _, path, ids in eligible: + died = died_sequence(eligible, text_of) + if not died[0]: + return {} + return longest_first([ + block for kind, _, path, ids in eligible if kind == "born" + for block in born_blocks(path, ids, died, text_of, mode, min_alnum)]) + + +def born_blocks(path, ids, died, text_of, mode, min_alnum): + """(size, died keys, born keys) of each move into one born run.""" + died_text, died_key = died + born_text = [text_of((path, t)) for t in ids] + return [(n, died_key[i:i + n], [(path, t) for t in ids[j:j + n]]) + for i, j, n in matching_blocks(died_text, born_text) + if is_move(mode, min_alnum, died_key[i][0], path, + died_text[i:i + n])] + + +def died_sequence(runs, text_of): + """The texts and keys of every died run, a SEPARATOR between two runs.""" + texts, keys = [], [] + for kind, _, path, ids in runs: if kind == "died": - for t in ids: - died_text.append(text_of((path, t))) - died_key.append((path, t)) - died_text.append(SEPARATOR) - died_key.append(None) - blocks = [] - for kind, _, b_path, b_ids in eligible: - if kind != "born" or not died_text: - continue - b_text = [text_of((b_path, t)) for t in b_ids] - for i, j, n in matching_blocks(died_text, b_text): - if mode == "renames" and died_key[i][0] == b_path: - continue - if mode == "moves" and alnum(died_text[i:i + n]) < min_alnum: - continue - blocks.append((n, died_key[i:i + n], - [(b_path, t) for t in b_ids[j:j + n]])) + texts += [text_of((path, t)) for t in ids] + [SEPARATOR] + keys += [(path, t) for t in ids] + [None] + return texts, keys + + +def is_move(mode, min_alnum, died_path, born_path, texts): + if mode == "renames" and died_path == born_path: + return False + return mode != "moves" or alnum(texts) >= min_alnum + + +def longest_first(blocks): + """Link the longest blocks first, each died and born token once.""" links, used = {}, set() for _, died, born in sorted(blocks, key=lambda x: -x[0]): for d, b in zip(died, born): From 631da04afcd8d1655c90763c26dadede6ac90700 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 14:15:02 +0000 Subject: [PATCH 31/35] refactor(token_history): split the steps of build.py build.main set up DuckDB, chose what to reuse, loaded the tables and the links, and wrote the history in one 45-line body (radon B, 8). Give each step a function: parse_args, connect, load_tables, load_links. write_history splits into fill_origins and copy_chunk. commit_runs groups its rows with itertools.groupby by place, commit and run instead of tracking the last place, commit and run by hand (B 8 to B 6). Every function of build.py is now B or better. No change in behaviour: on three generated repositories the old and new code write the same history parquets in all three modes, also on a --reuse build. --- token_history/build.py | 118 ++++++++++++++++++++++++----------------- 1 file changed, 70 insertions(+), 48 deletions(-) diff --git a/token_history/build.py b/token_history/build.py index b7481f05..01955c2c 100644 --- a/token_history/build.py +++ b/token_history/build.py @@ -5,6 +5,8 @@ import os import shutil import time +from itertools import groupby +from operator import itemgetter import duckdb @@ -76,6 +78,22 @@ def write_history(con, target, mode, chunks): """Tokens with their resolved origin, in chunks of paths: on a whole kernel one join of every token with every link does not fit in memory. The chunk parquets are then copied into one (no row order is kept).""" + fill_origins(con, chunks) + parts = os.path.join(os.path.dirname(target), + f".{os.path.basename(target)}.parts") + shutil.rmtree(parts, ignore_errors=True) + os.makedirs(parts) + for k in range(chunks): + copy_chunk(con, os.path.join(parts, f"{k:03d}.parquet"), mode, k, + chunks) + log(f"build: write chunk {k + 1}/{chunks}") + con.execute(f"COPY (SELECT * FROM read_parquet('{parts}/*.parquet')) " + f"TO '{target}' (FORMAT parquet, COMPRESSION zstd)") + shutil.rmtree(parts) + con.execute("DROP TABLE origins") + + +def fill_origins(con, chunks): con.execute("CREATE OR REPLACE TABLE origins (file_path VARCHAR, " "token_id BIGINT, origin_path VARCHAR, origin_token_id BIGINT, " "origin_born_sha VARCHAR)") @@ -93,32 +111,22 @@ def write_history(con, target, mode, chunks): ON o.file_path = l.origin_path AND o.token_id = l.origin_token_id""") log(f"build: origins chunk {k + 1}/{chunks}") - parts = os.path.join(os.path.dirname(target), - f".{os.path.basename(target)}.parts") - shutil.rmtree(parts, ignore_errors=True) - os.makedirs(parts) - for k in range(chunks): - con.execute(f""" - COPY ( - SELECT t.*, - coalesce(o.origin_path, t.file_path) AS origin_path, - coalesce(o.origin_token_id, t.token_id) - AS origin_token_id, - coalesce(o.origin_born_sha, t.born_sha) - AS origin_born_sha, - '{mode}' AS move_mode - FROM (SELECT * FROM tokens - WHERE {in_chunk('file_path', k, chunks)}) t - LEFT JOIN (SELECT * FROM origins - WHERE {in_chunk('file_path', k, chunks)}) o - USING (file_path, token_id) - ) TO '{os.path.join(parts, f"{k:03d}.parquet")}' - (FORMAT parquet, COMPRESSION zstd)""") - log(f"build: write chunk {k + 1}/{chunks}") - con.execute(f"COPY (SELECT * FROM read_parquet('{parts}/*.parquet')) " - f"TO '{target}' (FORMAT parquet, COMPRESSION zstd)") - shutil.rmtree(parts) - con.execute("DROP TABLE origins") + + +def copy_chunk(con, part, mode, k, chunks): + con.execute(f""" + COPY ( + SELECT t.*, + coalesce(o.origin_path, t.file_path) AS origin_path, + coalesce(o.origin_token_id, t.token_id) AS origin_token_id, + coalesce(o.origin_born_sha, t.born_sha) AS origin_born_sha, + '{mode}' AS move_mode + FROM (SELECT * FROM tokens + WHERE {in_chunk('file_path', k, chunks)}) t + LEFT JOIN (SELECT * FROM origins + WHERE {in_chunk('file_path', k, chunks)}) o + USING (file_path, token_id) + ) TO '{part}' (FORMAT parquet, COMPRESSION zstd)""") def commit_runs(con): @@ -146,25 +154,23 @@ def commit_runs(con): ORDER BY t.sha, t.run_no, t.pos""") # tokens has a row per mainline interval: keep the first row of each # place (the sort puts them together), a DISTINCT that needs no memory - sha, runs, texts, cur_run, last = None, [], {}, None, None - while batch := cur.fetchmany(100_000): - for s, run_no, kind, whole, path, tid, token, pos in batch: - if (s, run_no, pos) == last: - continue - last = (s, run_no, pos) - if s != sha: - if sha is not None: - yield sha, runs, texts - sha, runs, texts, cur_run = s, [], {}, None - if run_no != cur_run: - runs.append((kind, bool(whole), path, [])) - cur_run = run_no - runs[-1][3].append(tid) - texts[(path, tid)] = token - if sha is not None: + places = (next(rows) for _, rows in + groupby(fetch_rows(cur), key=itemgetter(0, 1, 7))) + for sha, rows in groupby(places, key=itemgetter(0)): + runs, texts = [], {} + for _, run in groupby(rows, key=itemgetter(1)): + run = list(run) + _, _, kind, whole, path = run[0][:5] + runs.append((kind, bool(whole), path, [r[5] for r in run])) + texts.update(((path, r[5]), r[6]) for r in run) yield sha, runs, texts +def fetch_rows(cur): + while batch := cur.fetchmany(100_000): + yield from batch + + def load(con, out): if not glob.glob(os.path.join(out, "part-*.tsv")): raise SystemExit(f"build: no part-*.tsv in {out}: run token_history.py") @@ -195,7 +201,7 @@ def write_links(con, links_file, mode, min_alnum): log(f"build: mode {mode}, {linked:,} tokens linked in {commits:,} commits") -def main(): +def parse_args(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--out", required=True) ap.add_argument("--mode", choices=MODES, default="moves") @@ -206,24 +212,34 @@ def main(): help="path chunks for the final write") ap.add_argument("--reuse", action="store_true", help="keep the loaded tables and a finished links file") - args = ap.parse_args() - t0 = time.time() + return ap.parse_args() + + +def connect(args): con = duckdb.connect(os.path.join(args.out, "build.duckdb")) con.execute(f"SET memory_limit='{args.memory}'") con.execute(f"SET temp_directory='{os.path.join(args.out, 'tmp')}'") con.execute(f"SET threads={args.threads}") con.execute("SET preserve_insertion_order=false") + return con + + +def load_tables(con, out, reuse): + """Load tokens and runs unless reuse finds both; True when it did.""" tables = {n for n, in con.execute("SELECT table_name FROM " "duckdb_tables()").fetchall()} - reuse = args.reuse and {"tokens", "runs"} <= tables + reuse = reuse and {"tokens", "runs"} <= tables for name in ("links",) if reuse else ("tokens", "runs", "links"): con.execute(f"DROP TABLE IF EXISTS {name}") if not reuse: - load(con, args.out) + load(con, out) log(f"build: {con.execute('SELECT count(*) FROM tokens').fetchone()[0]:,}" f" token rows, {con.execute('SELECT count(*) FROM runs').fetchone()[0]:,}" " runs") + return reuse + +def load_links(con, args, reuse): links_file = os.path.join(args.out, f"links-{args.mode}.tsv") if reuse and os.path.exists(links_file): log(f"build: reuse {links_file}") @@ -232,8 +248,14 @@ def main(): read_tsv(con, "links", links_file, ["file_path", "token_id", "origin_path", "origin_token_id"]) con.execute("DELETE FROM links WHERE file_path IS NULL") - log(f"build: move chains resolved in {resolve_origins(con)} rounds") + +def main(): + args = parse_args() + t0 = time.time() + con = connect(args) + load_links(con, args, load_tables(con, args.out, args.reuse)) + log(f"build: move chains resolved in {resolve_origins(con)} rounds") target = os.path.join(args.out, f"history-{args.mode}.parquet") write_history(con, target, args.mode, args.chunks) log(f"build: wrote {target} in {time.time() - t0:.0f} s") From 380513729bdb514195d8b8e6fa0baf2a27baf46f Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 14:17:05 +0000 Subject: [PATCH 32/35] refactor(token_history): split the main of the runner token_history.main parsed the arguments, read paths-done.txt, ran the pool and wrote errors.txt or paths-done.txt for each result in one 45-line body. Move the parser to parse_args, the done set to read_done and the handling of a result to record. work() stays as it is: it only decides which errors stop the run. No change in behaviour: the generated repositories give the same output, and a second run resumes with no path left to replay. --- token_history/token_history.py | 39 ++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/token_history/token_history.py b/token_history/token_history.py index a9a964d5..79e8b40e 100755 --- a/token_history/token_history.py +++ b/token_history/token_history.py @@ -198,7 +198,7 @@ def list_paths(args): return sorted({p for p in out.splitlines() if MASK.search(p)}) -def main(): +def parse_args(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--repo", required=True) ap.add_argument("--out", required=True) @@ -208,33 +208,44 @@ def main(): ap.add_argument("--run-min-alnum", type=int, default=100, help="shortest run that the move pass may link") ap.add_argument("pathspec", nargs="*") - args = ap.parse_args() + return ap.parse_args() + + +def read_done(done_file): + if not os.path.exists(done_file): + return set() + return set(open(done_file).read().split("\n")) + + +def record(r, out_dir, done_out): + if "error" in r: + with open(os.path.join(out_dir, "errors.txt"), "a") as err: + err.write(f"== {r['path']}\n{r['error']}\n") + return + done_out.write(r["path"] + "\n") + done_out.flush() + + +def main(): + args = parse_args() os.makedirs(args.out, exist_ok=True) - progress_file = os.path.join(args.out, "history-progress.json") done_file = os.path.join(args.out, "paths-done.txt") - done = set(open(done_file).read().split("\n")) if os.path.exists( - done_file) else set() - + done = read_done(done_file) paths = [p for p in list_paths(args) if p not in done] log(f"token_history: {len(paths)} paths to replay ({len(done)} done)") merges = merge_parents(args.repo) changes = mainline_changes(args.repo, args.pathspec) log(f"token_history: {len(merges):,} merges, mainline changes for " f"{len(changes):,} paths") - - progress = Progress(progress_file, len(paths), args.ping) + progress = Progress(os.path.join(args.out, "history-progress.json"), + len(paths), args.ping) progress.write("starting") with open(done_file, "a") as done_out, mp.Pool( args.jobs, init_worker, (args.repo, merges, changes, args.out, args.run_min_alnum)) as pool: for r in pool.imap_unordered(work, paths): - if "error" in r: - with open(os.path.join(args.out, "errors.txt"), "a") as err: - err.write(f"== {r['path']}\n{r['error']}\n") - else: - done_out.write(r["path"] + "\n") - done_out.flush() + record(r, args.out, done_out) progress.add(r) progress.write("done") From b4c3185c467982e5b631fbe82091382d3045756c Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 14:17:05 +0000 Subject: [PATCH 33/35] refactor(token_history): split the main of attribute.py main attached the databases, wrote the commits parquet and compared it with the dataset in one 55-line body. Name the steps: parse_args, connect, write_commits and compare_with_dataset. No change in behaviour: on small generated cregit and persons databases the old and new code write the same parquet and print the same counts, with and without --dataset. --- token_history/attribute.py | 47 ++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/token_history/attribute.py b/token_history/attribute.py index 41f13d6c..282b7aae 100644 --- a/token_history/attribute.py +++ b/token_history/attribute.py @@ -14,7 +14,7 @@ from generate_dataset import firm_sql, sql_literal # noqa: E402 -def main(): +def parse_args(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--cregit-db", required=True) ap.add_argument("--persons-db", required=True) @@ -23,12 +23,19 @@ def main(): ap.add_argument("--out", required=True) ap.add_argument("--dataset") ap.add_argument("--memory", default="4GB") - args = ap.parse_args() + return ap.parse_args() + + +def connect(args): con = duckdb.connect() con.execute(f"SET memory_limit='{args.memory}'") con.execute("INSTALL sqlite_scanner; LOAD sqlite_scanner;") con.execute(f"CALL sqlite_attach({sql_literal(args.cregit_db)})") con.execute(f"CALL sqlite_attach({sql_literal(args.persons_db)})") + return con + + +def write_commits(con, args): firm_select, firm_join = firm_sql(args.firm_map, args.firm_canonical) con.execute(f""" COPY ( @@ -52,23 +59,33 @@ def main(): LEFT JOIN persons p ON e.personid = p.personid {firm_join} ORDER BY c.cid ) TO {sql_literal(args.out)} (FORMAT parquet, COMPRESSION zstd)""") + + +def compare_with_dataset(con, commits, dataset): + return dict(zip(["dataset_commits", "same_person", "same_firm", + "same_original"], con.execute(f""" + WITH d AS ( + SELECT DISTINCT cregit_commit_sha, original_commit_sha, + personid, firm + FROM read_parquet({sql_literal(dataset)})) + SELECT count(*), + count(*) FILTER (WHERE a.personid IS NOT DISTINCT FROM d.personid), + count(*) FILTER (WHERE a.firm IS NOT DISTINCT FROM d.firm), + count(*) FILTER (WHERE a.original_commit_sha + = d.original_commit_sha) + FROM d LEFT JOIN read_parquet({sql_literal(commits)}) a + USING (cregit_commit_sha)""").fetchone())) + + +def main(): + args = parse_args() + con = connect(args) + write_commits(con, args) out = dict(commits=con.execute( f"SELECT count(*), count(DISTINCT cregit_commit_sha) " f"FROM read_parquet({sql_literal(args.out)})").fetchone()) if args.dataset: - out.update(zip(["dataset_commits", "same_person", "same_firm", - "same_original"], con.execute(f""" - WITH d AS ( - SELECT DISTINCT cregit_commit_sha, original_commit_sha, - personid, firm - FROM read_parquet({sql_literal(args.dataset)})) - SELECT count(*), - count(*) FILTER (WHERE a.personid IS NOT DISTINCT FROM d.personid), - count(*) FILTER (WHERE a.firm IS NOT DISTINCT FROM d.firm), - count(*) FILTER (WHERE a.original_commit_sha - = d.original_commit_sha) - FROM d LEFT JOIN read_parquet({sql_literal(args.out)}) a - USING (cregit_commit_sha)""").fetchone())) + out.update(compare_with_dataset(con, args.out, args.dataset)) print(json.dumps(out, indent=1)) From 4fa6b45aba95fc8f0857250bd988c4686d609c7d Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 14:17:35 +0000 Subject: [PATCH 34/35] test(token_history): parametrize the link mode tests Four tests of moves.link built the same runs and differed only in the mode, whole_file flag and minimum length. Make them one parametrized test of five cases; the whole-file rename case now checks each link, not only the count. Six tests opened a repository with the same three lines: move them to init_repo. --- token_history/test_moves.py | 34 +++++++++++++---------------- token_history/test_replay.py | 22 +++++++++---------- token_history/test_token_history.py | 10 +++------ 3 files changed, 28 insertions(+), 38 deletions(-) diff --git a/token_history/test_moves.py b/token_history/test_moves.py index a65e2a03..2599b6ad 100644 --- a/token_history/test_moves.py +++ b/token_history/test_moves.py @@ -1,3 +1,5 @@ +import pytest + from moves import link, matching_blocks @@ -14,27 +16,21 @@ def runs_and_text(moved_whole=True): return runs, tx.__getitem__ -def test_off_links_nothing(): - runs, tx = runs_and_text() - assert link(runs, tx, "off", 10) == {} - - -def test_moves_link_each_token(): - runs, tx = runs_and_text(moved_whole=False) - got = link(runs, tx, "moves", 10) - assert got == {("new.c", i): ("old.c", 10 + i) for i in range(4)} - - -def test_renames_need_a_deleted_and_a_created_file(): - runs, tx = runs_and_text(moved_whole=False) - assert link(runs, tx, "renames", 10) == {} - runs, tx = runs_and_text(moved_whole=True) - assert len(link(runs, tx, "renames", 10)) == 4 +EACH_TOKEN = {("new.c", i): ("old.c", 10 + i) for i in range(4)} -def test_short_runs_do_not_move(): - runs, tx = runs_and_text(moved_whole=False) - assert link(runs, tx, "moves", 10_000) == {} +@pytest.mark.parametrize("mode, whole, min_alnum, links", [ + ("off", True, 10, {}), + ("moves", False, 10, EACH_TOKEN), + ("moves", False, 10_000, {}), + ("renames", False, 10, {}), + ("renames", True, 10, EACH_TOKEN), +], ids=["off links nothing", "moves link each token", + "short runs do not move", "renames need a deleted file", + "renames link a whole file"]) +def test_link_modes(mode, whole, min_alnum, links): + runs, tx = runs_and_text(moved_whole=whole) + assert link(runs, tx, mode, min_alnum) == links def test_matching_blocks_on_a_long_repetitive_run(): diff --git a/token_history/test_replay.py b/token_history/test_replay.py index b97c150b..faeaf733 100644 --- a/token_history/test_replay.py +++ b/token_history/test_replay.py @@ -17,6 +17,12 @@ def git(repo, *args, date=None): capture_output=True, text=True, env=env).stdout +def init_repo(path): + path.mkdir() + git(path, "init", "-q", "-b", "main") + return path + + def write(repo, path, lines): (repo / path).write_text("".join(f"{x}\n" for x in lines)) @@ -35,9 +41,7 @@ def blame(repo, path): @pytest.fixture def repo(tmp_path): - r = tmp_path / "r" - r.mkdir() - git(r, "init", "-q", "-b", "main") + r = init_repo(tmp_path / "r") write(r, "a.c", ["int", "a", ";", "int", "b", ";"]) commit(r, "c1", "2020-01-01T00:00:00") write(r, "a.c", ["int", "a", ";", "long", "b", ";", "x"]) @@ -205,9 +209,7 @@ def test_runs_mark_a_whole_file_move(repo): def test_a_line_kept_from_two_parents_becomes_a_copy(tmp_path): - r = tmp_path / "dup" - r.mkdir() - git(r, "init", "-q", "-b", "main") + r = init_repo(tmp_path / "dup") write(r, "d.c", ["A"]) first = commit(r, "base", "2021-01-01T00:00:00") git(r, "checkout", "-q", "-b", "keep") @@ -229,9 +231,7 @@ def test_a_line_kept_from_two_parents_becomes_a_copy(tmp_path): def test_merge_that_brings_a_new_file_to_main(tmp_path): - r = tmp_path / "newfile" - r.mkdir() - git(r, "init", "-q", "-b", "main") + r = init_repo(tmp_path / "newfile") write(r, "other.c", ["o"]) commit(r, "base", "2022-01-01T00:00:00") git(r, "checkout", "-q", "-b", "topic") @@ -253,9 +253,7 @@ def test_merge_that_brings_a_new_file_to_main(tmp_path): def test_path_log_drops_a_merge_that_brings_no_change(tmp_path): # --simplify-merges: a side branch that never touches the path adds no # merge to its log, so no merge is replayed for nothing - r = tmp_path / "r" - r.mkdir() - git(r, "init", "-q", "-b", "main") + r = init_repo(tmp_path / "r") write(r, "a.c", ["a"]) commit(r, "base", "2020-01-01T00:00:00") git(r, "checkout", "-q", "-b", "side") diff --git a/token_history/test_token_history.py b/token_history/test_token_history.py index 491aab78..5bfd16f0 100644 --- a/token_history/test_token_history.py +++ b/token_history/test_token_history.py @@ -3,7 +3,7 @@ import pytest from replay import NULL_BLOB, GitError, diff_hunks -from test_replay import commit, git, write +from test_replay import commit, git, init_repo, write from token_history import list_paths, mainline_changes, merge_parents @@ -25,9 +25,7 @@ def test_git_failures_stop_the_run(not_a_repo): def test_the_null_blob_diffs_to_no_hunks(tmp_path): - r = tmp_path / "r" - r.mkdir() - git(r, "init", "-q", "-b", "main") + r = init_repo(tmp_path / "r") write(r, "a.c", ["x", "y"]) commit(r, "a", "2020-01-01T00:00:00") blob = git(r, "rev-parse", "HEAD:a.c").strip() @@ -38,9 +36,7 @@ def test_the_null_blob_diffs_to_no_hunks(tmp_path): def test_a_good_repo_still_works(tmp_path): - r = tmp_path / "r" - r.mkdir() - git(r, "init", "-q", "-b", "main") + r = init_repo(tmp_path / "r") write(r, "a.c", ["x"]) sha = commit(r, "a", "2020-01-01T00:00:00") assert merge_parents(str(r)) == {} From 111010d50b8edf39bd39f862ff51816278da9bbe Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 4 Oct 2026 19:39:58 +0000 Subject: [PATCH 35/35] test(token_history): give the merge in the copy test an identity The two-parent copy test ran `git merge` through subprocess.run, not through the git() helper, so the merge had no author or committer identity and no exit check. On a machine where git cannot detect an identity, such as the CI runner, the merge failed without a message. The next commit then had one parent, and the copy assertion failed. Run the merge through git() with check=False (the merge conflicts on purpose), and assert that MERGE_HEAD exists, so a merge that did not start fails the test at that line. --- token_history/test_replay.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/token_history/test_replay.py b/token_history/test_replay.py index faeaf733..5f699420 100644 --- a/token_history/test_replay.py +++ b/token_history/test_replay.py @@ -8,12 +8,12 @@ from token_history import GitSource, mainline_changes, merge_parents -def git(repo, *args, date=None): +def git(repo, *args, date=None, check=True): env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@x", GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@x") if date: env["GIT_AUTHOR_DATE"] = env["GIT_COMMITTER_DATE"] = f"{date} +0000" - return subprocess.run(["git", "-C", str(repo), *args], check=True, + return subprocess.run(["git", "-C", str(repo), *args], check=check, capture_output=True, text=True, env=env).stdout @@ -218,8 +218,8 @@ def test_a_line_kept_from_two_parents_becomes_a_copy(tmp_path): git(r, "checkout", "-q", "main") write(r, "d.c", ["A", "P"]) commit(r, "main adds P after A", "2021-01-03T00:00:00") - subprocess.run(["git", "-C", str(r), "merge", "-q", "--no-ff", - "--no-commit", "keep"], capture_output=True) + git(r, "merge", "-q", "--no-ff", "--no-commit", "keep", check=False) + assert (r / ".git" / "MERGE_HEAD").exists() write(r, "d.c", ["A", "P", "Q", "A"]) commit(r, "merge keeps both", "2021-01-04T00:00:00") rp = real_replay(r, "d.c")