diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index 6fdaddcb..bded0b8f 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -354,8 +354,10 @@ For each `.blame` file: 1. Parse each line as `commit_sha;token_content` 2. Repair the token line: drop the Rust tokenizer's `line:col` prefix and, in a strict UTF-8, non-`.rs` file without a byte-order mark, undo srcML - 1.1.0's Latin-1 misreading (`ö` becomes `ö`). The source is read as - `utf-8-sig`, so a BOM does not shift the walk. Counts go to a `Repaired:` line. + 1.1.0's Latin-1 misreading (`ö` becomes `ö`). A token that the source + holds as is stays as is: srcML develop reads UTF-8 right. The source is + read as `utf-8-sig`, so a BOM does not shift the walk. Counts go to a + `Repaired:` line. 3. Walk through the original source file character-by-character to match tokens 4. Classify each token (structural vs content, type, value) 5. Insert into SQLite `token_map` diff --git a/generate_dataset/generate_dataset.py b/generate_dataset/generate_dataset.py index 587370f9..1712cef7 100644 --- a/generate_dataset/generate_dataset.py +++ b/generate_dataset/generate_dataset.py @@ -26,6 +26,7 @@ import argparse import io +import itertools import logging import os import re @@ -81,6 +82,15 @@ def unread_char(self, ch: str) -> None: def location(self) -> tuple[int, int]: return (self.line, self.col) + def holds_next(self, text: str) -> bool: + """True when the next non-whitespace characters are those of `text`.""" + wanted = "".join(itertools.filterfalse(is_ws, text)) + ahead = itertools.chain(self._last_char or "", ( + self.source[i] for i in range(self.pos, len(self.source)))) + upcoming = itertools.islice(itertools.filterfalse(is_ws, ahead), + len(wanted)) + return bool(wanted) and "".join(upcoming) == wanted + # =================================================================== # Skip functions — replicate prettyPrint-author.pl's Skip_* @@ -329,8 +339,15 @@ def read_source(source_path: Path) -> tuple[str, bool, bool]: return text, has_bom, repairable +def misread_by_srcml(token_content: str, reader: SourceReader) -> bool: + """srcML develop decodes UTF-8 right, so a pair the source holds as is + (as in `u8"café"`) is true text, not mojibake.""" + return (_MOJIBAKE.search(token_content) is not None + and not reader.holds_next(token_content.partition("|")[2])) + + def repair_token_line(token_content: str, repairable: bool, - stats: Counter) -> str: + stats: Counter, reader: SourceReader) -> str: """Strip the Rust position prefix and, if `repairable`, undo srcML mojibake, counting each in `stats`. Run before classify_and_skip: a wrong token length shifts every later position in the file.""" @@ -338,7 +355,7 @@ def repair_token_line(token_content: str, repairable: bool, if prefix: stats["position_prefix"] += 1 token_content = token_content[prefix.end():] - if repairable and _MOJIBAKE.search(token_content): + if repairable and misread_by_srcml(token_content, reader): fixed = undo_mojibake(token_content) if fixed != token_content: stats["mojibake_tokens"] += 1 @@ -370,7 +387,8 @@ def rows(): continue commit_sha, token_content = parsed - token_content = repair_token_line(token_content, repairable, stats) + token_content = repair_token_line(token_content, repairable, + stats, reader) info = classify_and_skip(token_content, reader) counted[0] += 1 yield ( diff --git a/tests/test_generate_dataset.py b/tests/test_generate_dataset.py index d91861e0..e7addb25 100644 --- a/tests/test_generate_dataset.py +++ b/tests/test_generate_dataset.py @@ -582,12 +582,52 @@ def test_a_correct_token_is_not_changed(tmp_path): ("a.c", b"/* H\xc3\xb6 \xff */\nint x;\n"), # The Rust tokenizer does not use srcML. ("a.rs", "/* Hö */\nint x;\n".encode()), + # The source holds the pair, so the tokenizer read it right. + ("a.c", "/* Hö */\nint x;\n".encode()), ]) def test_the_gate_keeps_a_matching_token_as_it_is(tmp_path, name, source): rows = walk(tmp_path, name, source, "comment|/* Hö */", "keyword|int") assert rows[0][1] == "/* Hö */" +@pytest.mark.parametrize("source, text, held", [ + (" /* a\n * b */ x", "/* a * b */", True), + ("/* Hö */", "/* Hö */", False), + ("ab", "abc", False), + ("ab", "", False), +]) +def test_holds_next_compares_the_non_whitespace_text(source, text, held): + assert reader(source).holds_next(text) is held + + +def test_holds_next_sees_an_unread_char(): + r = reader("xab") + r.unread_char(r.read_char()) + assert r.holds_next("xa") + + +MIXED_SOURCE = '/* café */\nchar *s = u8"café";\n' +MIXED_TOKENS = ("comment|/* café */", "name|char", "operator|*", "name|s", + "operator|=", 'literal|u8"café"', "operator|;") + + +def misread_as_latin1(token: str) -> str: + return token.encode().decode("latin-1") + + +@pytest.mark.parametrize("tokens, repaired", [ + (MIXED_TOKENS, 0), + (tuple(map(misread_as_latin1, MIXED_TOKENS)), 2), +], ids=["srcml-develop", "srcml-1.1.0"]) +def test_both_srcml_versions_give_the_source_text(tmp_path, tokens, repaired): + stats = Counter() + rows = walk(tmp_path, "a.c", MIXED_SOURCE.encode(), *tokens, stats=stats) + assert [r[1] for r in rows] == [t.split("|", 1)[1] for t in MIXED_TOKENS] + assert rows[5][3:5] == (2, 11) + assert rows[6][3:5] == (2, 20) + assert stats["mojibake_tokens"] == repaired + + def test_a_rust_line_loses_its_position_prefix(tmp_path): rows = walk(tmp_path, "m.rs", b"fn main() {}\n", "-:-\tbegin_unit|revision:0.0.1;language:Rust;cregit-version:0.0.1", @@ -656,4 +696,5 @@ def test_a_line_from_the_fixed_rust_tokenizer_is_kept(tmp_path): ("12:5|keyword|fn", False, "12:5|keyword|fn"), # --position form: a pipe ]) def test_repair_token_line(line, repairable, expected): - assert repair_token_line(line, repairable, Counter()) == expected + assert repair_token_line(line, repairable, Counter(), + reader("")) == expected