diff --git a/.github/scripts/discover-validation-samples.py b/.github/scripts/discover-validation-samples.py new file mode 100644 index 000000000..26bc50874 --- /dev/null +++ b/.github/scripts/discover-validation-samples.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Discover the full sample inventory and emit a deterministic Actions matrix.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + +SUPPORTED_LANGUAGES = { + "csharp": "csharp", + "java": "java", + "python": "python", + "typescript": "typescript", + "javascript": "typescript", +} + + +def sample_id(path: str) -> str: + return path.removeprefix("samples/").replace("/", "-") + + +def declares_l4(metadata: Path) -> bool: + for line in metadata.read_text(encoding="utf-8").splitlines(): + without_comment = line.split("#", 1)[0].rstrip() + if re.match(r"^[ \t]*l4[ \t]*:", without_comment): + return True + return False + + +def discover(root: Path) -> dict: + samples = [] + for metadata in sorted(root.glob("samples/**/sample.yaml")): + path = metadata.parent.relative_to(root).as_posix() + language = path.split("/")[1] + validator_language = SUPPORTED_LANGUAGES.get(language) + declared_l4 = declares_l4(metadata) + sample = { + "id": sample_id(path), + "path": path, + "language": language, + "shape": "full-fleet", + } + samples.append(sample) + sample["validator_language"] = validator_language or "" + sample["eligible"] = validator_language is not None + sample["skip_reason"] = ( + "" if validator_language else f"language '{language}' is not supported by L3 validation" + ) + sample["l4_declared"] = declared_l4 + + identities = [ + {key: sample[key] for key in ("id", "path", "language", "shape")} + for sample in samples + ] + return { + "schema_version": 1, + "samples": identities, + "validation": { + sample["id"]: { + key: sample[key] + for key in ("validator_language", "eligible", "skip_reason", "l4_declared") + } + for sample in samples + }, + "matrix": samples, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(".")) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--matrix", type=Path, required=True) + args = parser.parse_args() + + payload = discover(args.root.resolve()) + args.manifest.write_text( + json.dumps( + {"schema_version": payload["schema_version"], "samples": payload["samples"], "validation": payload["validation"]}, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + args.matrix.write_text( + json.dumps({"include": payload["matrix"]}, separators=(",", ":")), + encoding="utf-8", + ) + print(json.dumps({"count": len(payload["matrix"]), "matrix": payload["matrix"]}, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/run-validation-pilot.py b/.github/scripts/run-validation-pilot.py index b40b95fec..92220cac7 100644 --- a/.github/scripts/run-validation-pilot.py +++ b/.github/scripts/run-validation-pilot.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Run one curated sample and write its canonical normalized result.""" +"""Run one sample and write its canonical normalized result.""" from __future__ import annotations @@ -27,6 +27,7 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--sample-id", required=True) parser.add_argument("--language", required=True) + parser.add_argument("--validator-language") parser.add_argument("--shape", required=True) parser.add_argument("--sample-path", required=True) parser.add_argument("--validator", default=".github/scripts/validate-sample.sh") @@ -39,6 +40,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--ref", default=os.environ.get("GITHUB_REF", "local")) parser.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY", "local")) parser.add_argument("--workflow", default=os.environ.get("GITHUB_WORKFLOW", "validation pilot")) + parser.add_argument("--run-l4", action="store_true") + parser.add_argument("--skip-reason") return parser.parse_args() @@ -46,30 +49,37 @@ def main() -> int: args = parse_args() started_at = utc_now() started = time.monotonic() - command = [ - args.bash, - args.validator, - "--level", - "3", - "--language", - args.language, - "--sample-dir", - args.sample_path, - ] - try: - completed = subprocess.run(command, capture_output=True, text=True) - diagnostic = ( - "$ " + " ".join(command) + "\n" - + completed.stdout - + completed.stderr - ) - outcome = OUTCOMES.get(completed.returncode, "infrastructure/error") - stage = "L3 validation" if completed.returncode in OUTCOMES else "L3 validation invocation" - except (OSError, subprocess.SubprocessError) as exc: - completed = None - diagnostic = "$ " + " ".join(command) + f"\nrunner error: {exc}\n" - outcome = "infrastructure/error" - stage = "L3 validation invocation" + validator_language = args.validator_language or args.language + if args.skip_reason: + diagnostic = f"skipped: {args.skip_reason}\n" + outcome = "skipped/not-completed" + stage = "inventory eligibility" + else: + command = [ + args.bash, + args.validator, + "--level", + "3", + "--language", + validator_language, + "--sample-dir", + args.sample_path, + ] + try: + completed = subprocess.run(command, capture_output=True, text=True) + diagnostic = "$ " + " ".join(command) + "\n" + completed.stdout + completed.stderr + outcome = OUTCOMES.get(completed.returncode, "infrastructure/error") + stage = "L3 validation" if completed.returncode in OUTCOMES else "L3 validation invocation" + if outcome == "passed" and args.run_l4: + l4_command = [args.bash, args.validator, "--level", "4", "--sample-dir", args.sample_path] + l4 = subprocess.run(l4_command, capture_output=True, text=True) + diagnostic += "\n$ " + " ".join(l4_command) + "\n" + l4.stdout + l4.stderr + outcome = OUTCOMES.get(l4.returncode, "infrastructure/error") + stage = "L4 validation" if l4.returncode in OUTCOMES else "L4 validation invocation" + except (OSError, subprocess.SubprocessError) as exc: + diagnostic = "$ " + " ".join(command) + f"\nrunner error: {exc}\n" + outcome = "infrastructure/error" + stage = "L3 validation invocation" completed_at = utc_now() result = { diff --git a/.github/scripts/test/test_validation_pilot.py b/.github/scripts/test/test_validation_pilot.py index d36116fe1..5f66f2db3 100644 --- a/.github/scripts/test/test_validation_pilot.py +++ b/.github/scripts/test/test_validation_pilot.py @@ -14,8 +14,8 @@ ROOT = Path(__file__).resolve().parents[2] RUNNER = ROOT / "scripts" / "run-validation-pilot.py" +DISCOVERY = ROOT / "scripts" / "discover-validation-samples.py" COMPLETENESS = ROOT / "scripts" / "validate-validation-pilot-results.py" -MANIFEST = ROOT / "validation-pilot-matrix.json" WORKFLOW = ROOT / "workflows" / "validation-pilot.yml" @@ -31,17 +31,47 @@ def test_workflow_calls_report_after_completeness(self) -> None: workflow, ) - def test_manifest_is_curated_and_supported(self) -> None: - manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) - self.assertEqual(manifest["schema_version"], 1) - self.assertEqual(len(manifest["samples"]), 4) - self.assertEqual( - {sample["language"] for sample in manifest["samples"]}, - {"csharp", "java", "python", "typescript"}, - ) - for sample in manifest["samples"]: - self.assertTrue((ROOT.parent / sample["path"]).is_dir(), sample["path"]) - + def test_discovery_covers_full_inventory_and_explicitly_skips_rust(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest = root / "manifest.json" + matrix = root / "matrix.json" + completed = subprocess.run( + [ + sys.executable, + str(DISCOVERY), + "--root", + str(ROOT.parent), + "--manifest", + str(manifest), + "--matrix", + str(matrix), + ], + capture_output=True, + text=True, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + payload = json.loads(manifest.read_text(encoding="utf-8")) + self.assertEqual(payload["schema_version"], 1) + sample_metadata = list((ROOT.parent / "samples").glob("**/sample.yaml")) + expected_unsupported = sum( + path.relative_to(ROOT.parent / "samples").parts[0] + not in {"csharp", "java", "python", "typescript", "javascript"} + for path in sample_metadata + ) + self.assertEqual(len(payload["samples"]), len(sample_metadata)) + self.assertEqual( + sum(not value["eligible"] for value in payload["validation"].values()), + expected_unsupported, + ) + self.assertEqual( + {value["validator_language"] for value in payload["validation"].values() if value["validator_language"]}, + {"csharp", "java", "python", "typescript"}, + ) + self.assertTrue(all(sample["shape"] == "full-fleet" for sample in payload["samples"])) + self.assertEqual(json.loads(matrix.read_text(encoding="utf-8"))["include"], [ + {**sample, **payload["validation"][sample["id"]]} for sample in payload["samples"] + ]) def test_sample_failure_is_a_complete_valid_result(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/.github/validation-pilot-matrix.json b/.github/validation-pilot-matrix.json deleted file mode 100644 index 98903d2ae..000000000 --- a/.github/validation-pilot-matrix.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "schema_version": 1, - "repository": "microsoft-foundry/foundry-samples", - "samples": [ - { - "id": "csharp-chat-with-agent", - "language": "csharp", - "shape": "quickstart", - "path": "samples/csharp/quickstart/chat-with-agent" - }, - { - "id": "java-create-agent", - "language": "java", - "shape": "quickstart", - "path": "samples/java/quickstart/create-agent" - }, - { - "id": "python-chat-with-agent", - "language": "python", - "shape": "quickstart-with-dependencies", - "path": "samples/python/quickstart/chat-with-agent" - }, - { - "id": "typescript-chat-with-agent", - "language": "typescript", - "shape": "quickstart-declared-command", - "path": "samples/typescript/quickstart/chat-with-agent" - } - ] -} diff --git a/.github/validation-pilot.README.md b/.github/validation-pilot.README.md index ea7e3d264..e7defac6a 100644 --- a/.github/validation-pilot.README.md +++ b/.github/validation-pilot.README.md @@ -1,21 +1,27 @@ -# Representative public validation pilot +# Daily public validation cadence -`validation-pilot.yml` is a manual-only, non-gating workflow. It runs the -versioned matrix in `validation-pilot-matrix.json` with `fail-fast: false`: +`validation-pilot.yml` runs daily at 07:00 UTC and can also be dispatched +manually. It discovers every `samples/**/sample.yaml` at the checked-out commit, +generates a deterministic manifest and matrix, and runs with `fail-fast: false`. +The first full-fleet run should be manually reviewed before the first scheduled +occurrence. -| Sample | Language | Shape | -| --- | --- | --- | -| `csharp/quickstart/chat-with-agent` | C# | quickstart | -| `java/quickstart/create-agent` | Java | quickstart | -| `python/quickstart/chat-with-agent` | Python | quickstart with dependencies | -| `typescript/quickstart/chat-with-agent` | TypeScript | declared-command quickstart | +All supported samples run L3. JavaScript uses the existing TypeScript/node +validator mapping. Rust samples remain in the manifest and emit +`skipped/not-completed` with an explicit unsupported-language reason. Each matrix leg persists `sample-result.json` and `diagnostics.log` in a -versioned artifact. The result schema is owned by the producer and includes +versioned artifact. Declared `sample.yaml` L4 commands run after L3 through the +existing `L4-validation` OIDC environment and warm-project seam. P4.1 does not +provision resources and does not set a cold-provisioning default. + +The result schema remains owned by the producer and includes sample identity, one of `passed`, `sample failure`, `infrastructure/error`, or `skipped/not-completed`, the completed stage, duration, references to the diagnostic and result artifacts, completion time, and GitHub run metadata. -The completeness job fails the run if any matrix member is missing, duplicated, -malformed, or missing its diagnostic. Individual sample failures remain valid -result records and do not prevent later matrix legs from running. +The completeness job fails the run if any discovered sample is missing, +duplicated, malformed, or missing its diagnostic. Individual sample failures +remain valid result records and do not prevent later matrix legs from running. +The generated manifest is included in the normalized run artifact so the +same-run report consumes exactly the inventory that was executed. diff --git a/.github/workflows/scripts-selftest.yml b/.github/workflows/scripts-selftest.yml index 86a12ec01..9b3fa3f96 100644 --- a/.github/workflows/scripts-selftest.yml +++ b/.github/workflows/scripts-selftest.yml @@ -24,7 +24,7 @@ on: - '.github/workflows/scripts-selftest.yml' - '.github/workflows/validation-report.yml' - '.github/workflows/validation-pilot.yml' - - '.github/validation-pilot-matrix.json' + - '.github/scripts/discover-validation-samples.py' permissions: contents: read diff --git a/.github/workflows/validation-pilot.yml b/.github/workflows/validation-pilot.yml index b7d4f5113..0d4b6a0ef 100644 --- a/.github/workflows/validation-pilot.yml +++ b/.github/workflows/validation-pilot.yml @@ -1,68 +1,113 @@ -name: Representative public validation pilot +name: Daily public validation cadence on: + schedule: + - cron: '0 7 * * *' workflow_dispatch: permissions: contents: read +concurrency: + group: validation-pilot-main + cancel-in-progress: true + jobs: + discover: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + matrix: ${{ steps.discovery.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - name: Discover full sample inventory + id: discovery + run: | + set -euo pipefail + python .github/scripts/discover-validation-samples.py \ + --manifest "$RUNNER_TEMP/validation-manifest.json" \ + --matrix "$RUNNER_TEMP/validation-matrix.json" > "$RUNNER_TEMP/discovery.json" + { + echo 'matrix<> "$GITHUB_OUTPUT" + cat "$RUNNER_TEMP/discovery.json" + - name: Persist discovered manifest + uses: actions/upload-artifact@v4 + with: + name: validation-pilot-manifest + path: ${{ runner.temp }}/validation-manifest.json + if-no-files-found: error + retention-days: 90 + validate: + needs: discover runs-on: ubuntu-latest timeout-minutes: 30 + environment: L4-validation + permissions: + contents: read + id-token: write + env: + # P4.1 never provisions. Declared L4 checks use the existing warm-project seam; + # P4.2 owns any future cold-provisioning caller. + SKIP_PROVISION: ${{ matrix.l4_declared && 'true' || '' }} strategy: fail-fast: false - matrix: - include: - - id: csharp-chat-with-agent - language: csharp - shape: quickstart - path: samples/csharp/quickstart/chat-with-agent - - id: java-create-agent - language: java - shape: quickstart - path: samples/java/quickstart/create-agent - - id: python-chat-with-agent - language: python - shape: quickstart-with-dependencies - path: samples/python/quickstart/chat-with-agent - - id: typescript-chat-with-agent - language: typescript - shape: quickstart-declared-command - path: samples/typescript/quickstart/chat-with-agent + max-parallel: 32 + matrix: ${{ fromJSON(needs.discover.outputs.matrix) }} steps: - uses: actions/checkout@v4 - uses: actions/setup-dotnet@v4 - if: matrix.language == 'csharp' + if: matrix.validator_language == 'csharp' with: dotnet-version: 8.0.x - uses: actions/setup-python@v5 - if: matrix.language == 'python' + if: matrix.validator_language == 'python' with: python-version: '3.12' - uses: actions/setup-node@v4 - if: matrix.language == 'typescript' + if: matrix.validator_language == 'typescript' with: node-version: '20' - uses: actions/setup-java@v4 - if: matrix.language == 'java' + if: matrix.validator_language == 'java' with: distribution: temurin java-version: '17' - name: Install yq + if: matrix.eligible run: | sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 sudo chmod +x /usr/local/bin/yq - - name: Run sample validation + - name: Azure login for declared L4 + if: matrix.l4_declared + uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + - name: Run normalized sample validation + if: always() run: | + set -uo pipefail mkdir -p "$RUNNER_TEMP/pilot-result" - python .github/scripts/run-validation-pilot.py \ - --sample-id "${{ matrix.id }}" \ - --language "${{ matrix.language }}" \ - --shape "${{ matrix.shape }}" \ - --sample-path "${{ matrix.path }}" \ - --output "$RUNNER_TEMP/pilot-result/sample-result.json" \ + args=( + --sample-id "${{ matrix.id }}" + --language "${{ matrix.language }}" + --validator-language "${{ matrix.validator_language }}" + --shape "${{ matrix.shape }}" + --sample-path "${{ matrix.path }}" + --output "$RUNNER_TEMP/pilot-result/sample-result.json" --diagnostic "$RUNNER_TEMP/pilot-result/diagnostics.log" + ) + if [ "${{ matrix.eligible }}" != "true" ]; then + args+=(--skip-reason "${{ matrix.skip_reason }}") + elif [ "${{ matrix.l4_declared }}" = "true" ]; then + args+=(--run-l4) + fi + python .github/scripts/run-validation-pilot.py "${args[@]}" - name: Persist sample result and diagnostics if: always() uses: actions/upload-artifact@v4 @@ -74,8 +119,9 @@ jobs: completeness: if: always() - needs: validate + needs: [discover, validate] runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 @@ -84,15 +130,23 @@ jobs: path: ${{ runner.temp }}/pilot-artifacts - name: Validate complete normalized result set run: | - python .github/scripts/validate-validation-pilot-results.py \ - --manifest .github/validation-pilot-matrix.json \ - --artifacts "$RUNNER_TEMP/pilot-artifacts" + set -uo pipefail + manifest="$RUNNER_TEMP/pilot-artifacts/validation-pilot-manifest/validation-manifest.json" + if python .github/scripts/validate-validation-pilot-results.py \ + --manifest "$manifest" \ + --artifacts "$RUNNER_TEMP/pilot-artifacts"; then + completeness_rc=0 + else + completeness_rc=$? + fi + cp "$manifest" "$RUNNER_TEMP/pilot-artifacts/manifest.json" + exit "$completeness_rc" - name: Persist run metadata and normalized summary if: always() run: | python -c 'import json,os; from pathlib import Path; p=Path(os.environ["RUNNER_TEMP"])/"pilot-artifacts"/"run-metadata.json"; p.write_text(json.dumps({"schema_version":1,"repository":os.environ["GITHUB_REPOSITORY"],"workflow":os.environ["GITHUB_WORKFLOW"],"run_id":os.environ["GITHUB_RUN_ID"],"run_attempt":os.environ["GITHUB_RUN_ATTEMPT"],"sha":os.environ["GITHUB_SHA"],"ref":os.environ["GITHUB_REF"],"completed_at":__import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat().replace("+00:00","Z")},indent=2)+"\n")' continue-on-error: true - - name: Upload normalized pilot run + - name: Upload normalized run if: always() uses: actions/upload-artifact@v4 with: @@ -107,3 +161,4 @@ jobs: uses: ./.github/workflows/validation-report.yml with: results-artifact: validation-pilot-run-${{ github.run_id }}-${{ github.run_attempt }} + manifest-path: manifest.json diff --git a/.github/workflows/validation-report.yml b/.github/workflows/validation-report.yml index 7b2c00554..1fd9f342f 100644 --- a/.github/workflows/validation-report.yml +++ b/.github/workflows/validation-report.yml @@ -8,6 +8,10 @@ on: results-artifact: required: true type: string + manifest-path: + required: false + default: manifest.json + type: string permissions: contents: read @@ -18,7 +22,6 @@ jobs: if: ${{ !cancelled() }} steps: - uses: actions/checkout@v4 - - name: Download normalized result artifacts uses: actions/download-artifact@v4 with: @@ -30,17 +33,25 @@ jobs: if: always() run: | set -uo pipefail - python .github/scripts/validate-validation-pilot-results.py \ - --manifest .github/validation-pilot-matrix.json \ - --artifacts "$RUNNER_TEMP/validation-results" - completeness_rc=$? - python .github/scripts/render-validation-report.py \ + if python .github/scripts/validate-validation-pilot-results.py \ + --manifest "$RUNNER_TEMP/validation-results/${{ inputs.manifest-path }}" \ + --artifacts "$RUNNER_TEMP/validation-results"; then + completeness_rc=0 + else + completeness_rc=$? + fi + if python .github/scripts/render-validation-report.py \ --results-dir "$RUNNER_TEMP/validation-results" \ - --expected-samples ".github/validation-pilot-matrix.json" \ + --expected-samples "$RUNNER_TEMP/validation-results/${{ inputs.manifest-path }}" \ --output "$RUNNER_TEMP/validation-report.md" \ - --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" - render_rc=$? - cat "$RUNNER_TEMP/validation-report.md" >> "$GITHUB_STEP_SUMMARY" + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"; then + render_rc=0 + else + render_rc=$? + fi + if [ -f "$RUNNER_TEMP/validation-report.md" ]; then + cat "$RUNNER_TEMP/validation-report.md" >> "$GITHUB_STEP_SUMMARY" + fi echo "completeness_rc=$completeness_rc" >> "$GITHUB_OUTPUT" echo "render_rc=$render_rc" >> "$GITHUB_OUTPUT" exit 0