Run daily full-fleet public validation cadence

.github/scripts/discover-validation-samples.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""Discover the full sample inventory and emit a deterministic Actions matrix."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+
+SUPPORTED_LANGUAGES = {
+    "csharp": "csharp",
+    "java": "java",
+    "python": "python",
+    "typescript": "typescript",
+    "javascript": "typescript",
+}
+
+
+def sample_id(path: str) -> str:
+    return path.removeprefix("samples/").replace("/", "-")
+
+
+def declares_l4(metadata: Path) -> bool:
+    for line in metadata.read_text(encoding="utf-8").splitlines():
+        without_comment = line.split("#", 1)[0].rstrip()
+        if re.match(r"^[ \t]*l4[ \t]*:", without_comment):
+            return True
+    return False
+
+
+def discover(root: Path) -> dict:
+    samples = []
+    for metadata in sorted(root.glob("samples/**/sample.yaml")):
+        path = metadata.parent.relative_to(root).as_posix()
+        language = path.split("/")[1]
+        validator_language = SUPPORTED_LANGUAGES.get(language)
+        declared_l4 = declares_l4(metadata)
+        sample = {
+            "id": sample_id(path),
+            "path": path,
+            "language": language,
+            "shape": "full-fleet",
+        }
+        samples.append(sample)
+        sample["validator_language"] = validator_language or ""
+        sample["eligible"] = validator_language is not None
+        sample["skip_reason"] = (
+            "" if validator_language else f"language '{language}' is not supported by L3 validation"
+        )
+        sample["l4_declared"] = declared_l4
+
+    identities = [
+        {key: sample[key] for key in ("id", "path", "language", "shape")}
+        for sample in samples
+    ]
+    return {
+        "schema_version": 1,
+        "samples": identities,
+        "validation": {
+            sample["id"]: {
+                key: sample[key]
+                for key in ("validator_language", "eligible", "skip_reason", "l4_declared")
+            }
+            for sample in samples
+        },
+        "matrix": samples,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", type=Path, default=Path("."))
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--matrix", type=Path, required=True)
+    args = parser.parse_args()
+
+    payload = discover(args.root.resolve())
+    args.manifest.write_text(
+        json.dumps(
+            {"schema_version": payload["schema_version"], "samples": payload["samples"], "validation": payload["validation"]},
+            indent=2,
+            sort_keys=True,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    args.matrix.write_text(
+        json.dumps({"include": payload["matrix"]}, separators=(",", ":")),
+        encoding="utf-8",
+    )
+    print(json.dumps({"count": len(payload["matrix"]), "matrix": payload["matrix"]}, separators=(",", ":")))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/run-validation-pilot.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Run one curated sample and write its canonical normalized result."""
+"""Run one sample and write its canonical normalized result."""
 
 from __future__ import annotations
 
@@ -27,6 +27,7 @@ def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser()
     parser.add_argument("--sample-id", required=True)
     parser.add_argument("--language", required=True)
+    parser.add_argument("--validator-language")
     parser.add_argument("--shape", required=True)
     parser.add_argument("--sample-path", required=True)
     parser.add_argument("--validator", default=".github/scripts/validate-sample.sh")
@@ -39,37 +40,46 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--ref", default=os.environ.get("GITHUB_REF", "local"))
     parser.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY", "local"))
     parser.add_argument("--workflow", default=os.environ.get("GITHUB_WORKFLOW", "validation pilot"))
+    parser.add_argument("--run-l4", action="store_true")
+    parser.add_argument("--skip-reason")
     return parser.parse_args()
 
 
 def main() -> int:
     args = parse_args()
     started_at = utc_now()
     started = time.monotonic()
-    command = [
-        args.bash,
-        args.validator,
-        "--level",
-        "3",
-        "--language",
-        args.language,
-        "--sample-dir",
-        args.sample_path,
-    ]
-    try:
-        completed = subprocess.run(command, capture_output=True, text=True)
-        diagnostic = (
-            "$ " + " ".join(command) + "\n"
-            + completed.stdout
-            + completed.stderr
-        )
-        outcome = OUTCOMES.get(completed.returncode, "infrastructure/error")
-        stage = "L3 validation" if completed.returncode in OUTCOMES else "L3 validation invocation"
-    except (OSError, subprocess.SubprocessError) as exc:
-        completed = None
-        diagnostic = "$ " + " ".join(command) + f"\nrunner error: {exc}\n"
-        outcome = "infrastructure/error"
-        stage = "L3 validation invocation"
+    validator_language = args.validator_language or args.language
+    if args.skip_reason:
+        diagnostic = f"skipped: {args.skip_reason}\n"
+        outcome = "skipped/not-completed"
+        stage = "inventory eligibility"
+    else:
+        command = [
+            args.bash,
+            args.validator,
+            "--level",
+            "3",
+            "--language",
+            validator_language,
+            "--sample-dir",
+            args.sample_path,
+        ]
+        try:
+            completed = subprocess.run(command, capture_output=True, text=True)
+            diagnostic = "$ " + " ".join(command) + "\n" + completed.stdout + completed.stderr
+            outcome = OUTCOMES.get(completed.returncode, "infrastructure/error")
+            stage = "L3 validation" if completed.returncode in OUTCOMES else "L3 validation invocation"
+            if outcome == "passed" and args.run_l4:
+                l4_command = [args.bash, args.validator, "--level", "4", "--sample-dir", args.sample_path]
+                l4 = subprocess.run(l4_command, capture_output=True, text=True)
+                diagnostic += "\n$ " + " ".join(l4_command) + "\n" + l4.stdout + l4.stderr
+                outcome = OUTCOMES.get(l4.returncode, "infrastructure/error")
+                stage = "L4 validation" if l4.returncode in OUTCOMES else "L4 validation invocation"
+        except (OSError, subprocess.SubprocessError) as exc:
+            diagnostic = "$ " + " ".join(command) + f"\nrunner error: {exc}\n"
+            outcome = "infrastructure/error"
+            stage = "L3 validation invocation"
 
     completed_at = utc_now()
     result = {

.github/scripts/test/test_validation_pilot.py

@@ -14,8 +14,8 @@
 
 ROOT = Path(__file__).resolve().parents[2]
 RUNNER = ROOT / "scripts" / "run-validation-pilot.py"
+DISCOVERY = ROOT / "scripts" / "discover-validation-samples.py"
 COMPLETENESS = ROOT / "scripts" / "validate-validation-pilot-results.py"
-MANIFEST = ROOT / "validation-pilot-matrix.json"
 WORKFLOW = ROOT / "workflows" / "validation-pilot.yml"
 
 
@@ -31,17 +31,47 @@ def test_workflow_calls_report_after_completeness(self) -> None:
             workflow,
         )
 
-    def test_manifest_is_curated_and_supported(self) -> None:
-        manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
-        self.assertEqual(manifest["schema_version"], 1)
-        self.assertEqual(len(manifest["samples"]), 4)
-        self.assertEqual(
-            {sample["language"] for sample in manifest["samples"]},
-            {"csharp", "java", "python", "typescript"},
-        )
-        for sample in manifest["samples"]:
-            self.assertTrue((ROOT.parent / sample["path"]).is_dir(), sample["path"])
-
+    def test_discovery_covers_full_inventory_and_explicitly_skips_rust(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            manifest = root / "manifest.json"
+            matrix = root / "matrix.json"
+            completed = subprocess.run(
+                [
+                    sys.executable,
+                    str(DISCOVERY),
+                    "--root",
+                    str(ROOT.parent),
+                    "--manifest",
+                    str(manifest),
+                    "--matrix",
+                    str(matrix),
+                ],
+                capture_output=True,
+                text=True,
+            )
+            self.assertEqual(completed.returncode, 0, completed.stderr)
+            payload = json.loads(manifest.read_text(encoding="utf-8"))
+            self.assertEqual(payload["schema_version"], 1)
+            sample_metadata = list((ROOT.parent / "samples").glob("**/sample.yaml"))
+            expected_unsupported = sum(
+                path.relative_to(ROOT.parent / "samples").parts[0]
+                not in {"csharp", "java", "python", "typescript", "javascript"}
+                for path in sample_metadata
+            )
+            self.assertEqual(len(payload["samples"]), len(sample_metadata))
+            self.assertEqual(
+                sum(not value["eligible"] for value in payload["validation"].values()),
+                expected_unsupported,
+            )
+            self.assertEqual(
+                {value["validator_language"] for value in payload["validation"].values() if value["validator_language"]},
+                {"csharp", "java", "python", "typescript"},
+            )
+            self.assertTrue(all(sample["shape"] == "full-fleet" for sample in payload["samples"]))
+            self.assertEqual(json.loads(matrix.read_text(encoding="utf-8"))["include"], [
+                {**sample, **payload["validation"][sample["id"]]} for sample in payload["samples"]
+            ])
     def test_sample_failure_is_a_complete_valid_result(self) -> None:
         with tempfile.TemporaryDirectory() as directory:
             root = Path(directory)

.github/validation-pilot.README.md

@@ -1,21 +1,27 @@
-# Representative public validation pilot
+# Daily public validation cadence
 
-`validation-pilot.yml` is a manual-only, non-gating workflow. It runs the
-versioned matrix in `validation-pilot-matrix.json` with `fail-fast: false`:
+`validation-pilot.yml` runs daily at 07:00 UTC and can also be dispatched
+manually. It discovers every `samples/**/sample.yaml` at the checked-out commit,
+generates a deterministic manifest and matrix, and runs with `fail-fast: false`.
+The first full-fleet run should be manually reviewed before the first scheduled
+occurrence.
 
-| Sample | Language | Shape |
-| --- | --- | --- |
-| `csharp/quickstart/chat-with-agent` | C# | quickstart |
-| `java/quickstart/create-agent` | Java | quickstart |
-| `python/quickstart/chat-with-agent` | Python | quickstart with dependencies |
-| `typescript/quickstart/chat-with-agent` | TypeScript | declared-command quickstart |
+All supported samples run L3. JavaScript uses the existing TypeScript/node
+validator mapping. Rust samples remain in the manifest and emit
+`skipped/not-completed` with an explicit unsupported-language reason.
 
 Each matrix leg persists `sample-result.json` and `diagnostics.log` in a
-versioned artifact. The result schema is owned by the producer and includes
+versioned artifact. Declared `sample.yaml` L4 commands run after L3 through the
+existing `L4-validation` OIDC environment and warm-project seam. P4.1 does not
+provision resources and does not set a cold-provisioning default.
+
+The result schema remains owned by the producer and includes
 sample identity, one of `passed`, `sample failure`, `infrastructure/error`, or
 `skipped/not-completed`, the completed stage, duration, references to the
 diagnostic and result artifacts, completion time, and GitHub run metadata.
 
-The completeness job fails the run if any matrix member is missing, duplicated,
-malformed, or missing its diagnostic. Individual sample failures remain valid
-result records and do not prevent later matrix legs from running.
+The completeness job fails the run if any discovered sample is missing,
+duplicated, malformed, or missing its diagnostic. Individual sample failures
+remain valid result records and do not prevent later matrix legs from running.
+The generated manifest is included in the normalized run artifact so the
+same-run report consumes exactly the inventory that was executed.

.github/workflows/scripts-selftest.yml

@@ -24,7 +24,7 @@ on:
       - '.github/workflows/scripts-selftest.yml'
       - '.github/workflows/validation-report.yml'
       - '.github/workflows/validation-pilot.yml'
-      - '.github/validation-pilot-matrix.json'
+      - '.github/scripts/discover-validation-samples.py'
 
 permissions:
   contents: read