fix(dataset): the mojibake repair keeps text that srcML read right

generate_dataset/DATASET.md

@@ -354,8 +354,10 @@ For each `.blame` file:
 1. Parse each line as `commit_sha;token_content`
 2. Repair the token line: drop the Rust tokenizer's `line:col<TAB>` prefix and,
    in a strict UTF-8, non-`.rs` file without a byte-order mark, undo srcML
-   1.1.0's Latin-1 misreading (`ö` becomes `ö`). The source is read as
-   `utf-8-sig`, so a BOM does not shift the walk. Counts go to a `Repaired:` line.
+   1.1.0's Latin-1 misreading (`ö` becomes `ö`). A token that the source
+   holds as is stays as is: srcML develop reads UTF-8 right. The source is
+   read as `utf-8-sig`, so a BOM does not shift the walk. Counts go to a
+   `Repaired:` line.
 3. Walk through the original source file character-by-character to match tokens
 4. Classify each token (structural vs content, type, value)
 5. Insert into SQLite `token_map`

generate_dataset/generate_dataset.py

@@ -26,6 +26,7 @@
 
 import argparse
 import io
+import itertools
 import logging
 import os
 import re
@@ -81,6 +82,15 @@ def unread_char(self, ch: str) -> None:
     def location(self) -> tuple[int, int]:
         return (self.line, self.col)
 
+    def holds_next(self, text: str) -> bool:
+        """True when the next non-whitespace characters are those of `text`."""
+        wanted = "".join(itertools.filterfalse(is_ws, text))
+        ahead = itertools.chain(self._last_char or "", (
+            self.source[i] for i in range(self.pos, len(self.source))))
+        upcoming = itertools.islice(itertools.filterfalse(is_ws, ahead),
+                                    len(wanted))
+        return bool(wanted) and "".join(upcoming) == wanted
+
 
 # ===================================================================
 # Skip functions — replicate prettyPrint-author.pl's Skip_*
@@ -329,16 +339,23 @@ def read_source(source_path: Path) -> tuple[str, bool, bool]:
     return text, has_bom, repairable
 
 
+def misread_by_srcml(token_content: str, reader: SourceReader) -> bool:
+    """srcML develop decodes UTF-8 right, so a pair the source holds as is
+    (as in `u8"café"`) is true text, not mojibake."""
+    return (_MOJIBAKE.search(token_content) is not None
+            and not reader.holds_next(token_content.partition("|")[2]))
+
+
 def repair_token_line(token_content: str, repairable: bool,
-                      stats: Counter) -> str:
+                      stats: Counter, reader: SourceReader) -> str:
     """Strip the Rust position prefix and, if `repairable`, undo srcML mojibake,
     counting each in `stats`. Run before classify_and_skip: a wrong token
     length shifts every later position in the file."""
     prefix = _POSITION_PREFIX.match(token_content)
     if prefix:
         stats["position_prefix"] += 1
         token_content = token_content[prefix.end():]
-    if repairable and _MOJIBAKE.search(token_content):
+    if repairable and misread_by_srcml(token_content, reader):
         fixed = undo_mojibake(token_content)
         if fixed != token_content:
             stats["mojibake_tokens"] += 1
@@ -370,7 +387,8 @@ def rows():
                 continue
             commit_sha, token_content = parsed
 
-            token_content = repair_token_line(token_content, repairable, stats)
+            token_content = repair_token_line(token_content, repairable,
+                                              stats, reader)
             info = classify_and_skip(token_content, reader)
             counted[0] += 1
             yield (

tests/test_generate_dataset.py

@@ -582,12 +582,52 @@ def test_a_correct_token_is_not_changed(tmp_path):
     ("a.c", b"/* H\xc3\xb6 \xff */\nint x;\n"),
     # The Rust tokenizer does not use srcML.
     ("a.rs", "/* Hö */\nint x;\n".encode()),
+    # The source holds the pair, so the tokenizer read it right.
+    ("a.c", "/* Hö */\nint x;\n".encode()),
 ])
 def test_the_gate_keeps_a_matching_token_as_it_is(tmp_path, name, source):
     rows = walk(tmp_path, name, source, "comment|/* Hö */", "keyword|int")
     assert rows[0][1] == "/* Hö */"
 
 
+@pytest.mark.parametrize("source, text, held", [
+    ("  /* a\n * b */ x", "/* a  * b */", True),
+    ("/* Hö */", "/* Hö */", False),
+    ("ab", "abc", False),
+    ("ab", "", False),
+])
+def test_holds_next_compares_the_non_whitespace_text(source, text, held):
+    assert reader(source).holds_next(text) is held
+
+
+def test_holds_next_sees_an_unread_char():
+    r = reader("xab")
+    r.unread_char(r.read_char())
+    assert r.holds_next("xa")
+
+
+MIXED_SOURCE = '/* café */\nchar *s = u8"café";\n'
+MIXED_TOKENS = ("comment|/* café */", "name|char", "operator|*", "name|s",
+                "operator|=", 'literal|u8"café"', "operator|;")
+
+
+def misread_as_latin1(token: str) -> str:
+    return token.encode().decode("latin-1")
+
+
+@pytest.mark.parametrize("tokens, repaired", [
+    (MIXED_TOKENS, 0),
+    (tuple(map(misread_as_latin1, MIXED_TOKENS)), 2),
+], ids=["srcml-develop", "srcml-1.1.0"])
+def test_both_srcml_versions_give_the_source_text(tmp_path, tokens, repaired):
+    stats = Counter()
+    rows = walk(tmp_path, "a.c", MIXED_SOURCE.encode(), *tokens, stats=stats)
+    assert [r[1] for r in rows] == [t.split("|", 1)[1] for t in MIXED_TOKENS]
+    assert rows[5][3:5] == (2, 11)
+    assert rows[6][3:5] == (2, 20)
+    assert stats["mojibake_tokens"] == repaired
+
+
 def test_a_rust_line_loses_its_position_prefix(tmp_path):
     rows = walk(tmp_path, "m.rs", b"fn main() {}\n",
                 "-:-\tbegin_unit|revision:0.0.1;language:Rust;cregit-version:0.0.1",
@@ -656,4 +696,5 @@ def test_a_line_from_the_fixed_rust_tokenizer_is_kept(tmp_path):
     ("12:5|keyword|fn", False, "12:5|keyword|fn"),  # --position form: a pipe
 ])
 def test_repair_token_line(line, repairable, expected):
-    assert repair_token_line(line, repairable, Counter()) == expected
+    assert repair_token_line(line, repairable, Counter(),
+                             reader("")) == expected