From 915f339c4674340becffc9242f9da858a7f2e830 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 21:30:21 +0000 Subject: [PATCH 1/2] build(devenv): pin srcml to develop d38271916 srcML 1.1.0 corrupts its heap on a GCC attribute that follows an expression in parentheses, for example "F(a __attribute__((unused)));" (srcML/srcML#2425). srcml2token then dies with signal 11 or 6. The pipeline excludes the blob, and blame gives all its lines to the next version of the file. No release has the fix. The develop head d38271916 (2026-10-04) has it (srcML/srcML#2426). This change replaces #103, which put the fix as a patch on an old develop pin. The develop build needs one more source, clip v1.15, and CLI11 2.7.0. On Linux, it also needs libxcb for the srcml command. libsrcml does not link libxcb. Token impact against the 1.1.0 tag, on 721 C, C++ and Java files (706 corpus files and the fixtures): - 698 files give the same tokens and exit status. - 3 files that 1.1.0 crashed on now parse. - 20 files give different tokens: 182 of 1,069,556 token lines. - 10 files have UTF-8 text. Develop reads it as UTF-8. 1.1.0 reads it as Latin-1. - 5 files have a top-level macro call. srcML/srcML#2325 marks it as an expression. - 3 files have a GCC attribute in parentheses. The #2425 fix marks it as an attribute. - 2 files have more than one function declarator in a declaration (srcML/srcML#2400). - No file that 1.1.0 parsed fails on develop. Recovered blobs: - mariadb: all 8,061 blobs that 1.1.0 crashed on now parse. 1.1.0 with only the #2425 patch still crashes on 442 of them. - Denylist: the 8 Java blobs (srcML/srcML#2361) parse in 0.2 s. The 4 C blobs still do not stop in 600 s. The denylist does not change. The tokenizer identity changes for C, C++, Java and Rust. The Rust binary has the devenv shell environment in its RUNPATH, and srcml is in that environment. Thus each project with cached rows stops with exit 3 until you run it with --retokenize. Cold build: the srcml derivation builds in 25 s on 16 cores and in 39 s on 4 cores (nix build --rebuild). --- README.md | 4 ++-- nix/srcml.nix | 27 +++++++++++++++++++-------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 25257705..22fbbc4f 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,8 @@ HTML views land in the sibling directory `../cregit-files/html`. See ## Preliminaries - Code is written in Scala, C++, Rust and Perl. -- Platform: Linux x86_64 or macOS arm64 — `devenv` builds the pinned `srcml` - 1.1.0 parser from its release tag (`nix/srcml.nix`). +- Platform: Linux x86_64 or macOS arm64 — `devenv` builds the `srcml` parser + from a pinned develop revision (`nix/srcml.nix`). ## Prerequisites diff --git a/nix/srcml.nix b/nix/srcml.nix index 62250b5d..614c46b4 100644 --- a/nix/srcml.nix +++ b/nix/srcml.nix @@ -1,6 +1,6 @@ -# srcML 1.1.0 built from its release tag, for the libsrcml headers. Later -# revisions change the tokens of files that 1.1.0 parses (srcML/srcML#2325 marks -# up some macro calls as expressions; UTF-8 input is no longer read as Latin-1). +# srcML develop at d38271916: it fixes the heap corruption on a GCC attribute +# after an expression in parentheses (srcML/srcML#2425), which 1.1.0 dies on. +# No release carries the fix yet; move the pin to the first one that does. { lib , stdenv , fetchFromGitHub @@ -13,6 +13,7 @@ , libarchive , curl , libiconv +, libxcb }: let @@ -35,25 +36,33 @@ let hash = "sha256-O9l7k1/2fVfEcyfJmWzXmju6TRQHK+NZFoaYmdd5KRg="; }; + clip = fetchFromGitHub { + owner = "dacap"; + repo = "clip"; + rev = "f2bd226fd96f3431f4675d9c13aa7334dd06ddda"; # tag v1.15 + hash = "sha256-qviH7XwNOTA3sMnfwaJL+GqXHxeG6giIZneuTeB+Lm8="; + }; + cli11 = fetchurl { - url = "https://github.com/CLIUtils/CLI11/releases/download/v2.5.0/CLI11.hpp"; - hash = "sha256-S/CpSQqnIJF2zNpwVE+VQT5ZTSIHzKM8nNGN7RiaY6Y="; + url = "https://github.com/CLIUtils/CLI11/releases/download/v2.7.0/CLI11.hpp"; + hash = "sha256-WqjWN4eSFj0eDfZC7uMTrO7KaKU0ZFZq5yBNWcL14Wg="; }; in stdenv.mkDerivation { pname = "srcml"; - version = "1.1.0"; + version = "1.1.0-unstable-2026-10-04"; src = fetchFromGitHub { owner = "srcML"; repo = "srcML"; - rev = "af1efb78ddd6fc4a8fc75bc6dc3c2e22c5fd812d"; # tag v1.1.0 - hash = "sha256-AdPVoUKI4/EigJz34BuGIbbN0jUGZTIxOQbmXr8IImM="; + rev = "d382719161daa1d035b594fcd3639ff376b4bb83"; + hash = "sha256-SRUvvsMLenVadNOG3S49Rwe0eDEf3AidTxvo3RtZPHg="; }; nativeBuildInputs = [ cmake ninja jdk_headless ]; buildInputs = [ libxml2 libxslt libarchive curl ] + ++ lib.optionals stdenv.hostPlatform.isLinux [ libxcb ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ libiconv ]; # Without this the manpage step downloads a prebuilt page from a bare IP over @@ -71,6 +80,7 @@ stdenv.mkDerivation { tar xf ${antlr} -C "$deps" cp -r --no-preserve=mode,ownership ${tinysha1} "$deps/tinysha1" cp -r --no-preserve=mode,ownership ${ctpl} "$deps/ctpl" + cp -r --no-preserve=mode,ownership ${clip} "$deps/clip" cp --no-preserve=mode,ownership ${cli11} "$deps/cli11/CLI11.hpp" cp --no-preserve=mode,ownership \ ${lib.getDev libarchive}/include/archive.h \ @@ -81,6 +91,7 @@ stdenv.mkDerivation { "-DFETCHCONTENT_SOURCE_DIR_ANTLRSRC=$deps/antlr-2.7.7" "-DFETCHCONTENT_SOURCE_DIR_TINYSHA1=$deps/tinysha1" "-DFETCHCONTENT_SOURCE_DIR_CTPL_STL_SRC=$deps/ctpl" + "-DFETCHCONTENT_SOURCE_DIR_CLIP=$deps/clip" "-DFETCHCONTENT_SOURCE_DIR_CLI11=$deps/cli11" "-DFETCHCONTENT_SOURCE_DIR_LIBARCHIVEINCLUDE1=$deps/libarchive-include" "-DFETCHCONTENT_SOURCE_DIR_LIBARCHIVEINCLUDE=$deps/libarchive-include" From 2a7397c91f1dbb101f2e6631fa976f6cffba4fc7 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 21:30:30 +0000 Subject: [PATCH 2/2] test(tokenize): cover srcML#2425 attribute crash Add a test for "F(a __attribute__((unused)));". srcML 1.1.0 dies on this input with signal 11. The test makes sure that tokenizeSrcMl.pl exits 0 and gives the attribute and its argument as tokens. The test passes on the develop pin. It fails on 1.1.0 (3 of 27). The test comes from #103. --- tests/t/tokenizeSrcMl.t | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/t/tokenizeSrcMl.t b/tests/t/tokenizeSrcMl.t index 6f1a3d7e..ea056032 100644 --- a/tests/t/tokenizeSrcMl.t +++ b/tests/t/tokenizeSrcMl.t @@ -18,7 +18,7 @@ plan skip_all => "ctags not on PATH" plan skip_all => "srcml2token not built (cd tokenize/srcMLtoken && make)" unless -x $srcml2token; -plan tests => 24; +plan tests => 27; my $workdir = tempdir(CLEANUP => 1); @@ -107,6 +107,18 @@ sub run_tokenizer { } } +{ + my $attr = "$workdir/attribute-in-parens.c"; + open(my $fh, '>', $attr) or die $!; + print $fh "void f(int a) { F(a __attribute__((unused))); }\n"; + close $fh; + + my ($status, $out, $err) = run_tokenizer("--position", "'$attr'"); + is($status, 0, "a GCC attribute after an expression inside parentheses tokenizes (srcML/srcML#2425)"); + like($out, qr/^\d+:\d+\|attribute\|__attribute__\(\($/m, "and the attribute is in the tokenization"); + like($out, qr/^\d+:\d+\|name\|unused$/m, "with its argument"); +} + { my $dies = "$workdir/srcml2token-that-dies"; open(my $fh, '>', $dies) or die $!;