From b22bd8b83b760d072e4fb4ba92f8a88a30ef67bc Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Tue, 6 Oct 2026 00:38:27 +0000 Subject: [PATCH] fix(dataset): keep text that srcML read right MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 10 undoes the srcML 1.1.0 mojibake: 1.1.0 reads UTF-8 as Latin-1, so a token holds "café" where the source holds "café". srcML develop (#104) reads UTF-8 right. The repair then changes correct text. Example: the literal u8"// café naïve" in srcML's test_srcml_unit_parse_encoding.cpp became "café naïve". This also moved the walk, because the token became shorter. Step 10 gets no tokenizer identity. blobExec keeps it in the blob map as an opaque digest, and sharded mode keeps none. Thus the gate uses the source file. The repair now runs only when the source does not hold the token text as is. A 1.1.0 token with non-ASCII text never matches strict UTF-8 source, so 1.1.0 data gets the same repair as before. A correct token always matches, so the gate needs no flag and works on mixed data. Measured on the 721-file sweep (/tmp/srcml-pin/out), through process_blame_file: - 1.1.0 tokens: 26 tokens in 10 files repaired, as before. All 10 files now give the same rows as develop (before: 9 of 10). - develop tokens: 0 tokens changed (before: 1). --- generate_dataset/DATASET.md | 6 ++-- generate_dataset/generate_dataset.py | 24 ++++++++++++++-- tests/test_generate_dataset.py | 43 +++++++++++++++++++++++++++- 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index 6fdaddcb..bded0b8f 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -354,8 +354,10 @@ For each `.blame` file: 1. Parse each line as `commit_sha;token_content` 2. Repair the token line: drop the Rust tokenizer's `line:col` prefix and, in a strict UTF-8, non-`.rs` file without a byte-order mark, undo srcML - 1.1.0's Latin-1 misreading (`ö` becomes `ö`). The source is read as - `utf-8-sig`, so a BOM does not shift the walk. Counts go to a `Repaired:` line. + 1.1.0's Latin-1 misreading (`ö` becomes `ö`). A token that the source + holds as is stays as is: srcML develop reads UTF-8 right. The source is + read as `utf-8-sig`, so a BOM does not shift the walk. Counts go to a + `Repaired:` line. 3. Walk through the original source file character-by-character to match tokens 4. Classify each token (structural vs content, type, value) 5. Insert into SQLite `token_map` diff --git a/generate_dataset/generate_dataset.py b/generate_dataset/generate_dataset.py index 587370f9..1712cef7 100644 --- a/generate_dataset/generate_dataset.py +++ b/generate_dataset/generate_dataset.py @@ -26,6 +26,7 @@ import argparse import io +import itertools import logging import os import re @@ -81,6 +82,15 @@ def unread_char(self, ch: str) -> None: def location(self) -> tuple[int, int]: return (self.line, self.col) + def holds_next(self, text: str) -> bool: + """True when the next non-whitespace characters are those of `text`.""" + wanted = "".join(itertools.filterfalse(is_ws, text)) + ahead = itertools.chain(self._last_char or "", ( + self.source[i] for i in range(self.pos, len(self.source)))) + upcoming = itertools.islice(itertools.filterfalse(is_ws, ahead), + len(wanted)) + return bool(wanted) and "".join(upcoming) == wanted + # =================================================================== # Skip functions — replicate prettyPrint-author.pl's Skip_* @@ -329,8 +339,15 @@ def read_source(source_path: Path) -> tuple[str, bool, bool]: return text, has_bom, repairable +def misread_by_srcml(token_content: str, reader: SourceReader) -> bool: + """srcML develop decodes UTF-8 right, so a pair the source holds as is + (as in `u8"café"`) is true text, not mojibake.""" + return (_MOJIBAKE.search(token_content) is not None + and not reader.holds_next(token_content.partition("|")[2])) + + def repair_token_line(token_content: str, repairable: bool, - stats: Counter) -> str: + stats: Counter, reader: SourceReader) -> str: """Strip the Rust position prefix and, if `repairable`, undo srcML mojibake, counting each in `stats`. Run before classify_and_skip: a wrong token length shifts every later position in the file.""" @@ -338,7 +355,7 @@ def repair_token_line(token_content: str, repairable: bool, if prefix: stats["position_prefix"] += 1 token_content = token_content[prefix.end():] - if repairable and _MOJIBAKE.search(token_content): + if repairable and misread_by_srcml(token_content, reader): fixed = undo_mojibake(token_content) if fixed != token_content: stats["mojibake_tokens"] += 1 @@ -370,7 +387,8 @@ def rows(): continue commit_sha, token_content = parsed - token_content = repair_token_line(token_content, repairable, stats) + token_content = repair_token_line(token_content, repairable, + stats, reader) info = classify_and_skip(token_content, reader) counted[0] += 1 yield ( diff --git a/tests/test_generate_dataset.py b/tests/test_generate_dataset.py index d91861e0..e7addb25 100644 --- a/tests/test_generate_dataset.py +++ b/tests/test_generate_dataset.py @@ -582,12 +582,52 @@ def test_a_correct_token_is_not_changed(tmp_path): ("a.c", b"/* H\xc3\xb6 \xff */\nint x;\n"), # The Rust tokenizer does not use srcML. ("a.rs", "/* Hö */\nint x;\n".encode()), + # The source holds the pair, so the tokenizer read it right. + ("a.c", "/* Hö */\nint x;\n".encode()), ]) def test_the_gate_keeps_a_matching_token_as_it_is(tmp_path, name, source): rows = walk(tmp_path, name, source, "comment|/* Hö */", "keyword|int") assert rows[0][1] == "/* Hö */" +@pytest.mark.parametrize("source, text, held", [ + (" /* a\n * b */ x", "/* a * b */", True), + ("/* Hö */", "/* Hö */", False), + ("ab", "abc", False), + ("ab", "", False), +]) +def test_holds_next_compares_the_non_whitespace_text(source, text, held): + assert reader(source).holds_next(text) is held + + +def test_holds_next_sees_an_unread_char(): + r = reader("xab") + r.unread_char(r.read_char()) + assert r.holds_next("xa") + + +MIXED_SOURCE = '/* café */\nchar *s = u8"café";\n' +MIXED_TOKENS = ("comment|/* café */", "name|char", "operator|*", "name|s", + "operator|=", 'literal|u8"café"', "operator|;") + + +def misread_as_latin1(token: str) -> str: + return token.encode().decode("latin-1") + + +@pytest.mark.parametrize("tokens, repaired", [ + (MIXED_TOKENS, 0), + (tuple(map(misread_as_latin1, MIXED_TOKENS)), 2), +], ids=["srcml-develop", "srcml-1.1.0"]) +def test_both_srcml_versions_give_the_source_text(tmp_path, tokens, repaired): + stats = Counter() + rows = walk(tmp_path, "a.c", MIXED_SOURCE.encode(), *tokens, stats=stats) + assert [r[1] for r in rows] == [t.split("|", 1)[1] for t in MIXED_TOKENS] + assert rows[5][3:5] == (2, 11) + assert rows[6][3:5] == (2, 20) + assert stats["mojibake_tokens"] == repaired + + def test_a_rust_line_loses_its_position_prefix(tmp_path): rows = walk(tmp_path, "m.rs", b"fn main() {}\n", "-:-\tbegin_unit|revision:0.0.1;language:Rust;cregit-version:0.0.1", @@ -656,4 +696,5 @@ def test_a_line_from_the_fixed_rust_tokenizer_is_kept(tmp_path): ("12:5|keyword|fn", False, "12:5|keyword|fn"), # --position form: a pipe ]) def test_repair_token_line(line, repairable, expected): - assert repair_token_line(line, repairable, Counter()) == expected + assert repair_token_line(line, repairable, Counter(), + reader("")) == expected