ci: support version-routed collector change detection

docs/ci/README.md

@@ -245,6 +245,13 @@ marked it N/A.
 | Prediction Regression | Collect old/new static, scheduling, and silicon-reference snapshots; fail when a previously working case stops working |
 | Release Artifact Contract | Build the wheel and crate; verify installed versions, allowed distributions, compatibility imports, dependencies, and CLI entry points on both architectures |
 
+The changed-operation manifest parses collector registries without importing framework
+code. For versioned entries, it includes every literal `VersionRoute` module and its
+hash closure, even when the manifest currently pins an older route. Changes to route
+thresholds or ordering also produce `collector_code`. This conservative family-level
+check prevents older or newly registered collectors from being missed. Unknown or
+dynamic route syntax fails explicitly; it is not treated as unchanged.
+
 ### Numerical and installed-package evidence
 
 [Native numerical checks](../../scripts/prediction_regression/check_prediction_numerics.py) exercise

python/aisimulate/tests/unit/tools/test_changed_ops.py

@@ -10,8 +10,8 @@
 and the design-§8 output schema. One test also runs against the REAL repo
 with base=HEAD head=HEAD, which must report everything unchanged.
 
-Also covers: VersionRoute-versioned registry entries failing closed
-(NotImplementedError, FIX 1), SHARED_CORE being read per-revision from
+Also covers: literal VersionRoute modules and routing changes, unknown route
+syntax failing closed, SHARED_CORE being read per-revision from
 `collector/provenance.py` rather than imported live (FIX 2), a pre-Collector-V3
 `--base` exiting loudly on a dedicated code (FIX 2), and two frameworks
 sharing one physical `data_backend` directory disambiguating `systems` by
@@ -20,6 +20,7 @@
 
 from __future__ import annotations
 
+import ast
 import importlib.util
 import shutil
 import subprocess
@@ -183,21 +184,12 @@ def __str__(self) -> str:
     - collector/vllm/utils_xpu.py
 """
 
-# A registry whose REGISTRY list contains a VersionRoute-versioned OpEntry
-# (no literal `module=`) — the shape _parse_registry_entries does not support
-# and must fail closed on (FIX 1) rather than silently skip.
-REGISTRY_PY_WITH_VERSION_ROUTE = """from collector.registry_types import OpEntry, PerfFile, VersionRoute
-
-REGISTRY: list[OpEntry] = [
-    OpEntry(
-        op="x",
-        get_func="get_x_test_cases",
-        run_func="run_x",
-        perf_filename=PerfFile.GEMM,
-        versions=(VersionRoute(min_version="1.0", module="m"),),
-    ),
-]
-"""
+# Both keyword and positional VersionRoute forms occur in authored registries.
+REGISTRY_PY_WITH_VERSION_ROUTE = REGISTRY_PY.replace(
+    'module="collector.sglang.collect_gemm",',
+    'versions=(VersionRoute("1.0", "collector.sglang.collect_gemm_v2"), '
+    'VersionRoute(min_version="0.0.0", module="collector.sglang.collect_gemm")),',
+)
 
 
 def _manifest_yaml(*, version: str = "0.5.14", digest: str = "0" * 64, family_override: str | None = None) -> str:
@@ -372,16 +364,123 @@ def test_shared_core_is_read_per_revision_not_from_live_checkout(self, mod, repo
 
 
 # --------------------------------------------------------------------------
-# registry parsing: unsupported shapes (FIX 1)
+# registry parsing: version routes
 # --------------------------------------------------------------------------
 
 
-class TestVersionRouteUnsupported:
-    def test_version_route_entry_raises_not_implemented(self, mod, repo):
-        _write_tree(repo, _default_files() | {"collector/sglang/registry.py": REGISTRY_PY_WITH_VERSION_ROUTE})
-        base_sha = _commit_all(repo, "base")
+def _versioned_files():
+    return _default_files() | {
+        "collector/sglang/registry.py": REGISTRY_PY_WITH_VERSION_ROUTE,
+        "collector/sglang/collect_gemm_v2.py": "# collect_gemm v2\n",
+        "collector/hash_closures.yaml": CLOSURES_YAML + "collector.sglang.collect_gemm_v2: []\n",
+    }
+
+
+class TestVersionRoutes:
+    def test_identical_routes_are_unchanged(self, mod, repo):
+        _write_tree(repo, _versioned_files())
+        sha = _commit_all(repo, "versioned registry")
+        changed, unchanged = mod.compute_changed_ops(repo, sha, sha)
+        assert changed == []
+        assert {d.family for d in unchanged} == {"gemm", "attention"}
+
+    @pytest.mark.parametrize("module", ["collect_gemm", "collect_gemm_v2"])
+    def test_every_route_module_contributes_even_above_manifest_pin(self, mod, repo, module):
+        files = _versioned_files()
+        _write_tree(repo, files)
+        base = _commit_all(repo, "base")
+        _write_tree(repo, {f"collector/sglang/{module}.py": "# changed route implementation\n"})
+        head = _commit_all(repo, "head")
+        changed, unchanged = mod.compute_changed_ops(repo, base, head)
+        assert _diff_for(changed, "sglang", "gemm").reasons == ("collector_code",)
+        assert _diff_for(unchanged, "sglang", "attention").reasons == ()
+
+    @pytest.mark.parametrize("change", ["threshold", "order", "remove", "unversioned"])
+    def test_routing_changes_are_collector_code(self, mod, repo, change):
+        files = _versioned_files()
+        _write_tree(repo, files)
+        base = _commit_all(repo, "base")
+        registry = REGISTRY_PY_WITH_VERSION_ROUTE
+        new = 'VersionRoute("1.0", "collector.sglang.collect_gemm_v2")'
+        old = 'VersionRoute(min_version="0.0.0", module="collector.sglang.collect_gemm")'
+        if change == "threshold":
+            registry = registry.replace('"1.0"', '"0.8"')
+        elif change == "order":
+            registry = registry.replace(f"{new}, {old}", f"{old}, {new}")
+        elif change == "remove":
+            registry = registry.replace(f"{new}, {old}", f"{old},")
+        else:
+            registry = REGISTRY_PY
+        _write_tree(repo, {"collector/sglang/registry.py": registry})
+        head = _commit_all(repo, "head")
+        changed, unchanged = mod.compute_changed_ops(repo, base, head)
+        assert "collector_code" in _diff_for(changed, "sglang", "gemm").reasons
+        assert _diff_for(unchanged, "sglang", "attention").reasons == ()
+
+    def test_new_route_requires_hash_closure(self, mod, repo):
+        files = _versioned_files() | {"collector/hash_closures.yaml": CLOSURES_YAML}
+        _write_tree(repo, files)
+        sha = _commit_all(repo, "missing closure")
+        with pytest.raises(KeyError, match="collect_gemm_v2"):
+            mod.compute_changed_ops(repo, sha, sha)
+
+    def test_route_case_inputs_contribute(self, mod, repo):
+        files = _versioned_files()
+        files["collector/hash_closures.yaml"] = CLOSURES_YAML + (
+            "collector.sglang.collect_gemm_v2:\n  - collector/cases/base_ops/new.yaml\n"
+        )
+        files["collector/cases/base_ops/new.yaml"] = "# cases v1\n"
+        _write_tree(repo, files)
+        base = _commit_all(repo, "base")
+        _write_tree(repo, {"collector/cases/base_ops/new.yaml": "# cases v2\n"})
+        head = _commit_all(repo, "head")
+        changed, _ = mod.compute_changed_ops(repo, base, head)
+        assert _diff_for(changed, "sglang", "gemm").reasons == ("collector_code", "case_plan")
+
+    @pytest.mark.parametrize(
+        "expression",
+        [
+            "ROUTES",
+            "(make_route(),)",
+            "(*ROUTES,)",
+            '(VersionRoute("1", dynamic_module),)',
+            '(VersionRoute(version, "module"),)',
+            '(VersionRoute("1"),)',
+            '(VersionRoute("1", "module", "extra"),)',
+            '(VersionRoute("1", "module", module="other"),)',
+            '(VersionRoute("1", "module", unexpected=True),)',
+            "(VersionRoute(**route),)",
+            '(VersionRoute("", "module"),)',
+        ],
+    )
+    def test_unknown_route_syntax_fails_closed(self, mod, expression):
         with pytest.raises(NotImplementedError, match="VersionRoute"):
-            mod.compute_changed_ops(repo, base_sha, base_sha)
+            mod._parse_version_routes(ast.parse(expression, mode="eval").body, "gemm")
+
+    def test_list_and_mixed_arguments(self, mod):
+        node = ast.parse('[VersionRoute("1", module="collector.test")]', mode="eval").body
+        assert mod._parse_version_routes(node, "gemm") == (("1", "collector.test"),)
+
+    @pytest.mark.parametrize("extra", ['module="collector.other",', "module=dynamic_module,", "**extra,"])
+    def test_unknown_or_conflicting_entry_arguments_fail_closed(self, mod, repo, extra):
+        files = _versioned_files()
+        files["collector/sglang/registry.py"] = REGISTRY_PY_WITH_VERSION_ROUTE.replace(
+            'op="gemm",', f'op="gemm", {extra}'
+        )
+        _write_tree(repo, files)
+        sha = _commit_all(repo, "ambiguous versioned entry")
+        with pytest.raises((ValueError, NotImplementedError), match="VersionRoute"):
+            mod.compute_changed_ops(repo, sha, sha)
+
+    def test_empty_routes_without_module_fail_closed(self, mod, repo):
+        files = _versioned_files()
+        files["collector/sglang/registry.py"] = REGISTRY_PY.replace(
+            'module="collector.sglang.collect_gemm",', "versions=(),"
+        )
+        _write_tree(repo, files)
+        sha = _commit_all(repo, "invalid empty routes")
+        with pytest.raises(NotImplementedError, match="Empty VersionRoute"):
+            mod.compute_changed_ops(repo, sha, sha)
 
 
 # --------------------------------------------------------------------------

python/aisimulate/tools/perf_database/changed_ops.py

@@ -49,12 +49,13 @@
 what that revision's manifest says it is) and independently testable with
 small fixture trees that do not need to replicate all five real registries.
 
-Known gaps
-----------
-- VersionRoute-versioned registry entries (`OpEntry(..., versions=(...))`,
-  no literal `module=`) are not parsed: `_parse_registry_entries` raises
-  `NotImplementedError` rather than silently dropping them (fail-closed, not
-    a silent gap). No registry uses this shape today.
+Versioned registries
+--------------------
+All literal VersionRoute modules contribute to the family, including routes
+for versions other than the manifest pin (as with CUDA/XPU registry siblings).
+Route thresholds and order are compared too: routing-only edits must not be
+reported unchanged. Dynamic or unknown route syntax raises NotImplementedError
+rather than silently omitting a module; no runtime resolver is executed.
 
 Exit codes
 ----------
@@ -268,6 +269,7 @@ class RegistryEntry:
     op: str
     module: str
     table_stem: str
+    versions: tuple[tuple[str, str], ...] = ()
 
 
 @cache
@@ -337,6 +339,33 @@ def _find_registry_lists(tree: ast.Module, registry_names: tuple[str, ...]) -> l
     return registry_lists
 
 
+def _parse_version_routes(node: ast.expr, op: str | None) -> tuple[tuple[str, str], ...]:
+    """Read literal VersionRoute pairs without evaluating registry source."""
+    error = f"Unsupported VersionRoute syntax (op: {op}); expected literal min_version/module pairs"
+    if not isinstance(node, (ast.List, ast.Tuple)):
+        raise NotImplementedError(error)
+    routes = []
+    fields = ("min_version", "module")
+    for route in node.elts:
+        if not (
+            isinstance(route, ast.Call)
+            and isinstance(route.func, ast.Name)
+            and route.func.id == "VersionRoute"
+            and len(route.args) <= 2
+        ):
+            raise NotImplementedError(error)
+        values = dict(zip(fields, route.args, strict=False))
+        for keyword in route.keywords:
+            if keyword.arg not in fields or keyword.arg in values:
+                raise NotImplementedError(error)
+            values[keyword.arg] = keyword.value
+        pair = tuple(_ast_str(values.get(field)) for field in fields)
+        if not all(pair):
+            raise NotImplementedError(error)
+        routes.append(pair)
+    return tuple(routes)
+
+
 def _parse_registry_entries(repo_root: Path, rev: str, manifest: dict[str, Any], framework: str) -> list[RegistryEntry]:
     spec = manifest.get("frameworks", {}).get(framework)
     if spec is None:
@@ -361,18 +390,31 @@ def _parse_registry_entries(repo_root: Path, rev: str, manifest: dict[str, Any],
             kwargs = {kw.arg: kw.value for kw in element.keywords if kw.arg}
             op = _ast_str(kwargs.get("op"))
             module = _ast_str(kwargs.get("module"))
-            if module is None and "versions" in kwargs:
-                # Versioned entries (module=None, versions=(VersionRoute(...), ...))
-                # are not parsed — fail closed rather than silently omitting the
-                # op from its family's module set (see "Known gaps" above).
-                raise NotImplementedError(
-                    "changed_ops does not support VersionRoute-versioned registry entries; "
-                    f"extend the parser (op: {op})"
-                )
+            versions = _parse_version_routes(kwargs["versions"], op) if "versions" in kwargs else ()
+            if versions:
+                module_node = kwargs.get("module")
+                if element.args or any(kw.arg is None for kw in element.keywords):
+                    raise NotImplementedError(f"VersionRoute entry requires explicit keyword arguments (op: {op})")
+                if (
+                    module_node is not None
+                    and module is None
+                    and not (isinstance(module_node, ast.Constant) and module_node.value is None)
+                ):
+                    raise NotImplementedError(f"VersionRoute entry has a dynamic module (op: {op})")
+            if versions and module:
+                raise ValueError(f"OpEntry {op}: cannot specify both module and VersionRoute entries")
+            if "versions" in kwargs and not (versions or module):
+                raise NotImplementedError(f"Empty VersionRoute entries without a literal module (op: {op})")
             perf_filename = _resolve_perf_filename(kwargs.get("perf_filename"), perf_file_map)
-            if not (op and module and perf_filename):
+            if versions and not (op and perf_filename):
+                raise NotImplementedError(f"VersionRoute entry requires literal op and perf_filename (op: {op})")
+            if not (op and perf_filename):
                 continue
-            entries.append(RegistryEntry(op=op, module=module, table_stem=Path(perf_filename).stem))
+            modules = [route_module for _, route_module in versions] if versions else [module] if module else []
+            for route_module in modules:
+                entries.append(
+                    RegistryEntry(op=op, module=route_module, table_stem=Path(perf_filename).stem, versions=versions)
+                )
     return entries
 
 
@@ -551,7 +593,16 @@ def _diff_framework(
         reasons: list[str] = []
         if _resolve_pin(base_ctx.manifest, framework, family) != _resolve_pin(head_ctx.manifest, framework, family):
             reasons.append(REASON_PIN_VERSION)
-        if _collector_code_differs(repo_root, base_ctx, head_ctx, base_modules, head_modules):
+        base_routes = {
+            entry for entry in base_entries if entry.versions and base_ctx.family_map.get(entry.table_stem) == family
+        }
+        head_routes = {
+            entry for entry in head_entries if entry.versions and head_ctx.family_map.get(entry.table_stem) == family
+        }
+        if (
+            _collector_code_differs(repo_root, base_ctx, head_ctx, base_modules, head_modules)
+            or base_routes != head_routes
+        ):
             reasons.append(REASON_COLLECTOR_CODE)
         base_case_hash = None
         if not (base_modules - base_ctx.closures.keys()):