testing.yml

# Robonix full-stack testing — real Webots sim + real components + fake VLM.
#
# Runs on the self-hosted GPU runner (label `robonix-testing`). The selected
# GPU is configurable via workflow input, repository/org variable, or runner env.
# Cheap checks (fmt/clippy/build/unit)
# live in ci.yml on GitHub-hosted runners and gate every PR; THIS workflow is the
# heavy GPU testing job and is deliberately NOT exposed to untrusted PR code.
#
# ─────────────────────────── SECURITY (read first) ───────────────────────────
# A self-hosted runner executes whatever the triggering ref contains — and
# `rbnx build` runs each package's build.sh + Dockerfiles. So a fork PR that
# auto-ran here would be arbitrary code execution on our box. Controls:
#
#  1. Triggers below are TRUSTED ONLY: `push` to dev/main (already-merged =
#     reviewed), manual `workflow_dispatch`, ChatOps `workflow_call`, and a
#     narrow `pull_request_target` label gate. For direct PR auto-runs, the
#     workflow source stays on the trusted base branch; the GPU job starts only
#     after `safe-to-test` was added by admin/maintain and the current PR head
#     has an admin/maintain APPROVED review.
#  2. Also turn ON in repo settings: Actions → Fork pull request workflows →
#     "Require approval for all external contributors".
#  3. `permissions: contents: read` — the job gets a read-only GITHUB_TOKEN.
#     The GPU testing job uses a FAKE VLM and needs NO real secrets; keep the runner
#     secret-free. Do not add repo secrets to this workflow.
#  4. Run the runner as a DEDICATED unprivileged user (not a human account with
#     SSH keys / access to other users' data), ideally with `--ephemeral`
#     registration or each job in a throwaway container, so a compromised job
#     can't persist or read neighbouring state.
#  5. Protect this file via CODEOWNERS so a PR can't weaken the gating.

name: Webots Integration Tests

on:
  push:
    branches: [dev, main, dev-ci-test]
  workflow_dispatch:
    inputs:
      gpu_id:
        description: "GPU index visible on the selected self-hosted runner; defaults to vars.ROBONIX_CI_GPU_ID or 0"
        required: false
        default: ""
  pull_request_target:
    types: [labeled, synchronize, reopened]
  # Lets the ChatOps bot (ci-bot.yml) invoke this whole pipeline on an
  # authorized `@robonix-ci test` comment and read back the result summary.
  workflow_call:
    inputs:
      authorized:
        description: "true when a trusted caller has already authorized the run"
        type: boolean
        required: false
        default: false
      ref:
        description: "git ref to test (defaults to the triggering ref)"
        type: string
        required: false
      pr_number:
        description: "pull request number when invoked by ChatOps"
        type: string
        required: false
      pr_title:
        description: "pull request title when invoked by ChatOps"
        type: string
        required: false
      pr_author:
        description: "pull request author when invoked by ChatOps"
        type: string
        required: false
      pr_url:
        description: "pull request URL when invoked by ChatOps"
        type: string
        required: false
      pr_head_ref:
        description: "pull request head branch when invoked by ChatOps"
        type: string
        required: false
      pr_base_ref:
        description: "pull request base branch when invoked by ChatOps"
        type: string
        required: false
      pr_head_repo:
        description: "pull request head repository when invoked by ChatOps"
        type: string
        required: false
      pr_head_sha:
        description: "pull request head commit when invoked by ChatOps"
        type: string
        required: false
      triggered_by:
        description: "user who requested the ChatOps run"
        type: string
        required: false
      enable_llm_analysis:
        description: "true only for trusted ChatOps runs that should spend LLM credits"
        type: boolean
        required: false
        default: false
      gpu_id:
        description: "GPU index visible on the selected self-hosted runner; defaults to vars.ROBONIX_CI_GPU_ID or 0"
        type: string
        required: false
    secrets:
      DEEPSEEK_API_KEY:
        required: false
      ROBONIX_REPORTS_DEPLOY_KEY:
        required: false
    outputs:
      result:
        description: "PASS/FAIL summary line from the harness"
        value: ${{ jobs.gpu_testing.outputs.result }}
      report_url:
        description: "Published Pages URL for the generated Webots report"
        value: ${{ jobs.publish_report.outputs.report_url }}

permissions:
  contents: read
  issues: read
  pull-requests: read

jobs:
  # ── Gate 1: cheap, GitHub-hosted, untrusted-safe ──────────────────────────
  # Scans the ref for forbidden high-risk operations BEFORE anything runs on the
  # self-hosted GPU box. gpu_testing `needs:` this, so a dirty diff never reaches the
  # 5090 — and because it runs on push(dev/main) too, it gates merged code as
  # well. No `environment:` here, so it never blocks normal contributor CI.
  security-gate:
    name: Preflight security checks
    runs-on: ubuntu-22.04
    outputs:
      safe_to_test_authorized: ${{ steps.safe_to_test.outputs.authorized }}
      safe_to_test_labeler: ${{ steps.safe_to_test.outputs.labeler }}
      safe_to_test_permission: ${{ steps.safe_to_test.outputs.permission }}
      pr_review_approved: ${{ steps.pr_review.outputs.approved }}
      pr_review_approver: ${{ steps.pr_review.outputs.approver }}
      pr_review_permission: ${{ steps.pr_review.outputs.permission }}
    steps:
      - name: Authorize safe-to-test label
        id: safe_to_test
        uses: actions/github-script@v7
        with:
          script: |
            if (context.eventName !== 'pull_request_target') {
              core.setOutput('authorized', 'true');
              core.setOutput('labeler', 'trusted-event');
              core.setOutput('permission', 'trusted-event');
              return;
            }
            const pr = context.payload.pull_request;
            const labels = (pr.labels || []).map(l => l.name);
            if (!labels.includes('safe-to-test')) {
              core.setOutput('authorized', 'false');
              core.setOutput('labeler', '');
              core.setOutput('permission', 'none');
              core.notice('safe-to-test label is absent; GPU test will be skipped');
              return;
            }
            const events = await github.paginate(github.rest.issues.listEventsForTimeline, {
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: pr.number,
              per_page: 100,
            });
            const labeled = [...events].reverse().find(e =>
              e.event === 'labeled' && e.label && e.label.name === 'safe-to-test' && e.actor && e.actor.login
            );
            const labeler = labeled?.actor?.login || '';
            let permission = 'none';
            if (labeler) {
              try {
                const r = await github.rest.repos.getCollaboratorPermissionLevel({
                  owner: context.repo.owner, repo: context.repo.repo, username: labeler,
                });
                permission = r.data.permission || 'none';
              } catch (e) {
                core.notice(`could not resolve repo permission for safe-to-test labeler @${labeler}: ${e.message}`);
              }
            }
            const authorized = ['admin', 'maintain'].includes(permission);
            core.setOutput('authorized', String(authorized));
            core.setOutput('labeler', labeler);
            core.setOutput('permission', permission);
            if (!authorized) {
              core.notice(`safe-to-test was added by @${labeler || 'unknown'} with permission '${permission}', so GPU test will be skipped`);
            }

      - name: Require maintainer approval for PR Webots testing
        id: pr_review
        uses: actions/github-script@v7
        env:
          # ChatOps (`workflow_call`) already resolved and froze the PR head at
          # its own authorization gate; reuse that value instead of re-reading
          # the live head here. Re-reading would reopen a time-of-check /
          # time-of-use window in which the PR author can push a new commit
          # between authorization and the approval match below.
          PR_NUMBER: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.number || inputs.pr_number || '' }}
          PR_HEAD_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || inputs.pr_head_sha || '' }}
        with:
          script: |
            const prNumber = process.env.PR_NUMBER;
            if (!prNumber) {
              core.setOutput('approved', 'true');
              core.setOutput('approver', 'trusted-event');
              core.setOutput('permission', 'trusted-event');
              return;
            }
            let headSha = process.env.PR_HEAD_SHA || '';
            if (!headSha) {
              const pr = await github.rest.pulls.get({
                owner: context.repo.owner,
                repo: context.repo.repo,
                pull_number: Number(prNumber),
              });
              headSha = pr.data.head.sha || '';
            }
            const reviews = await github.paginate(github.rest.pulls.listReviews, {
              owner: context.repo.owner,
              repo: context.repo.repo,
              pull_number: Number(prNumber),
              per_page: 100,
            });
            const approved = [...reviews].reverse().filter(r =>
              r.state === 'APPROVED' && (!headSha || r.commit_id === headSha) && r.user && r.user.login
            );
            for (const review of approved) {
              const username = review.user.login;
              let permission = 'none';
              try {
                const r = await github.rest.repos.getCollaboratorPermissionLevel({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  username,
                });
                permission = r.data.permission || 'none';
              } catch (e) {
                core.notice(`could not resolve repo permission for approver @${username}: ${e.message}`);
              }
              if (['admin', 'maintain'].includes(permission)) {
                core.setOutput('approved', 'true');
                core.setOutput('approver', username);
                core.setOutput('permission', permission);
                return;
              }
            }
            core.setOutput('approved', 'false');
            core.setOutput('approver', '');
            core.setOutput('permission', 'none');
            core.notice(`PR #${prNumber} has no current-head APPROVED review from an admin/maintain user; Webots GPU test will be skipped`);

      - uses: actions/checkout@v4
        with:
          # For PR tests, checkout the current base branch first; the next
          # step merges the PR head locally so stale refs/pull/*/merge refs
          # cannot make CI test an old base.
          ref: ${{ (inputs.pr_number != '' || github.event_name == 'pull_request_target') && (inputs.pr_base_ref || github.event.pull_request.base.ref || github.event.repository.default_branch) || inputs.ref }}
          fetch-depth: ${{ (inputs.pr_number != '' || github.event_name == 'pull_request_target') && '0' || '1' }}
      - name: Merge PR head into current base
        if: ${{ inputs.pr_number != '' || github.event_name == 'pull_request_target' }}
        env:
          TEST_PR_NUMBER: ${{ inputs.pr_number || github.event.pull_request.number }}
        run: |
          set -euo pipefail
          git fetch --no-tags --prune origin \
            "+refs/pull/${TEST_PR_NUMBER}/head:refs/remotes/robonix-pr/${TEST_PR_NUMBER}/head"
          git -c user.name=github-actions[bot] \
              -c user.email=41898282+github-actions[bot]@users.noreply.github.com \
              merge --no-edit --no-ff "refs/remotes/robonix-pr/${TEST_PR_NUMBER}/head"
          git log -1 --oneline
      - name: Scan executable build and runtime files
        run: |
          set -euo pipefail
          # Patterns that must never execute on the shared GPU host. We scan the
          # code that actually RUNS on the self-hosted runner — package build.sh,
          # Dockerfiles, and Python drivers — NOT the workflow YAML (it defines
          # these very patterns, would self-match, and is CODEOWNERS-protected)
          # and NOT docs/markdown (TROUBLESHOOTING.md lists them as red lines).
          # Red lines: GPU destruction, reverse shells, obfuscated shell payloads,
          # and pipe-to-shell downloads in files that can run during package build
          # or provider startup. Use tracked files only; generated rbnx-build,
          # target, and venv trees are not part of the review surface.
          PAT='nvidia-smi[^|]*(--gpu-reset|--reset|(^| )-r( |$))|rmmod[[:space:]]+nvidia|modprobe[[:space:]]+nvidia|/dev/tcp/|[^a-z]nc[[:space:]]+.*-e[[:space:]]|base64[[:space:]]+-d[^|]*\|[^|]*sh|(curl|wget)[^|]*\|[[:space:]]*(sudo[[:space:]]+)?(ba)?sh'
          git ls-files \
            'examples/webots/**' 'services/**' 'system/**' 'capabilities/**' 'tools/**' \
            | grep -E '(\.sh|\.py|(^|/)Dockerfile[^/]*)$' > "$RUNNER_TEMP/forbidden-scan-files" || true
          if [ -s "$RUNNER_TEMP/forbidden-scan-files" ] &&
             xargs -r grep -nE "$PAT" < "$RUNNER_TEMP/forbidden-scan-files"; then
            echo "::error::forbidden high-risk pattern — needs manual maintainer review"; exit 1
          fi
          echo "forbidden-operation scan clean"
      - name: Scan for committed secrets
        run: |
          set -euo pipefail
          # ChatOps PR tests need full git history for local merge-base
          # resolution. Never scan .git history here; scan only tracked files
          # from the merged working tree that will actually run.
          scan_dir="$RUNNER_TEMP/gitleaks-tree"
          rm -rf "$scan_dir"
          mkdir -p "$scan_dir"
          while IFS= read -r -d '' f; do
            mkdir -p "$scan_dir/$(dirname "$f")"
            cp -a "$f" "$scan_dir/$f"
          done < <(git ls-files -z)
          docker run --rm -v "$scan_dir:/repo:ro" zricethezav/gitleaks:latest \
            dir /repo --no-banner --redact \
            || { echo "::error::possible secret in checked-out tree"; exit 1; }

  # ── Gate 2: the GPU testing job itself ────────────────────────────────────
  gpu_testing:
    name: Webots integration suite (${{ matrix.backend }})
    needs: security-gate
    # Scene has two perception backends and both ship, so both run the full
    # suite. The legs run one after another: the concurrency group below keeps
    # a single pending job, so a second leg queued behind the first would
    # cancel it.
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        include:
          - backend: concept_graphs
            key: cg
          - backend: dualmap
            key: dm
    concurrency:
      group: robonix-webots-gpu-${{ inputs.gpu_id || vars.ROBONIX_CI_GPU_ID || '0' }}
      cancel-in-progress: false
    outputs:
      result: ${{ steps.scenarios.outputs.result }}
    # Optional per-run manual approval: configure the `gpu-testing` Environment
    # with Required reviewers in repo settings if we want GitHub to pause this
    # job before it is assigned to the self-hosted runner.
    environment: gpu-testing
    # Belt-and-suspenders: direct safe-to-test PR auto-runs require current-head
    # approval by admin/maintain. ChatOps callers are authorized separately in
    # ci-bot.yml and do not require PR approval.
    if: >-
      inputs.authorized == true ||
      github.event_name == 'push' ||
      github.event_name == 'workflow_dispatch' ||
      (github.event_name == 'pull_request_target' &&
       needs.security-gate.outputs.safe_to_test_authorized == 'true' &&
       needs.security-gate.outputs.pr_review_approved == 'true')
    runs-on: [self-hosted, robonix-testing]
    timeout-minutes: 90
    env:
      # ── Build-env fixes for this box (see examples/webots/TROUBLESHOOTING.md) ──
      # Prefer the GitHub mirror, but fall back to GitHub through the runner's
      # loopback Clash proxy when the mirror is unavailable.
      GIT_MIRROR_INSTEADOF: "true"
      # codegen grpcio-tools MUST match the runtime grpcio pinned by packages
      # (1.80.0) or RegisterService fails with a version-skew error.
      GRPCIO_PIN: "1.80.0"
      # ── Per-run isolation (coexist with interactive users on the shared box) ──
      RUN_ID: "${{ github.run_id }}"
      RUN_ATTEMPT: "${{ github.run_attempt }}"
      RUN_KEY: "${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.key }}"
      # GPU selection is runner-specific. Priority:
      # workflow_call input -> repo/org variable -> runner env -> default GPU 0.
      ROBONIX_CI_GPU_ID_INPUT: ${{ inputs.gpu_id || '' }}
      ROBONIX_CI_GPU_ID_VAR: ${{ vars.ROBONIX_CI_GPU_ID || '' }}
      ROBONIX_CI_MIN_GPU_FREE_MIB: ${{ vars.ROBONIX_CI_MIN_GPU_FREE_MIB || '4096' }}
      RMW_IMPLEMENTATION: "rmw_zenoh_cpp"
      ROBONIX_DRIVER_INIT_TIMEOUT_S: "240"
      AUDIO_MIC_DEVICE: "null"
      AUDIO_SPEAKER_DEVICE: "null"
      # CI should not depend on external Tencent/FunASR credentials or quota.
      SPEECH_BACKEND: "mock"
      # Keep build-time ECAPA verification off the GPU reserved for Webots and Scene.
      VOICEPRINT_DEVICE: "cpu"
      SCENE_GRAPH_INTERVAL_SEC: "1"
      SCENE_GRAPH_MIN_OBSERVATIONS: "1"
      ROBONIX_SIM_NETWORK: "bridge"
      ROBONIX_SIM_IPC: "private"
      WEBOTS_HEADLESS_MODE: "nvidia"
      WEBOTS_STREAM: "1"
      ROBONIX_SIM_PRIVILEGED: "true"
      # Named explicitly so a missing or stale image fails the run instead of
      # quietly falling back to the other backend.
      SCENE_PERCEPTION_BACKEND: ${{ matrix.backend }}

    steps:
      - name: Configure network egress policy
        uses: step-security/harden-runner@v2
        with:
          # Start in 'audit' to learn the real egress set from the first runs,
          # then switch to 'block' and trim allowed-endpoints to exactly these.
          egress-policy: audit
          # disable-telemetry: true
          allowed-endpoints: >
            ghfast.top:443
            github.com:443
            objects.githubusercontent.com:443
            pypi.org:443
            files.pythonhosted.org:443
            pypi.tuna.tsinghua.edu.cn:443
            www.modelscope.cn:443
            modelscope.cn:443
            registry-1.docker.io:443
            auth.docker.io:443
            production.cloudflare.docker.com:443

      - name: Initialize current-run diagnostics
        run: |
          set -euo pipefail
          case "${GITHUB_WORKSPACE:-}" in
            /home/*/*/_work/*/*) ;;
            *) echo "::error::refusing to sanitize unexpected workspace: ${GITHUB_WORKSPACE:-<unset>}"; exit 1 ;;
          esac

          # The capacity gate deliberately runs before checkout. Self-hosted
          # workspaces persist between jobs, so remove only generated evidence
          # from the previous run before a pre-checkout failure can upload it.
          for relative in \
            testing/logs \
            testing/report \
            examples/webots/rbnx-boot/logs \
            examples/webots/rbnx-boot/processes.json \
            examples/webots/logs \
            system/scene/logs; do
            target="$GITHUB_WORKSPACE/$relative"
            [ -e "$target" ] || continue
            sudo -n chown -R "$(id -u):$(id -g)" "$target" 2>/dev/null || true
            rm -rf -- "$target"
            # Containers in this suite run as root and leave root-owned
            # evidence behind (the Soma runtime bind mount, driver logs).
            # Runners without passwordless sudo cannot chown that back, which
            # jams every later run at this step. Docker is a hard dependency of
            # the suite, so reclaim ownership through a throwaway container.
            if [ -e "$target" ]; then
              docker run --rm -v "$target":/reclaim busybox \
                chown -R "$(id -u):$(id -g)" /reclaim >/dev/null 2>&1 || true
              rm -rf -- "$target"
            fi
            if [ -e "$target" ]; then
              echo "::error::could not remove stale generated evidence: $target"
              exit 1
            fi
          done

          for target in \
            "$RUNNER_TEMP/sim-logs" \
            "$RUNNER_TEMP/lifecycle-logs" \
            "$RUNNER_TEMP/report-assets" \
            "$RUNNER_TEMP/cache-provider-logs" \
            "$RUNNER_TEMP/report-metadata.json" \
            "$RUNNER_TEMP/remote-provenance.txt" \
            "$RUNNER_TEMP/fake_vlm.log" \
            "$RUNNER_TEMP/rbnx-boot.log" \
            "$RUNNER_TEMP/lifecycle.out" \
            "$RUNNER_TEMP/scenarios.out" \
            "$RUNNER_TEMP/scenarios.restart.out"; do
            [ -e "$target" ] || continue
            sudo -n chown -R "$(id -u):$(id -g)" "$target" 2>/dev/null || true
            rm -rf -- "$target"
          done
          mkdir -p "$RUNNER_TEMP/sim-logs"
          printf 'Webots suite has not started for run %s attempt %s.\n' \
            "$RUN_ID" "$RUN_ATTEMPT" \
            >"$RUNNER_TEMP/sim-logs/infrastructure.txt"
          {
            echo "ROBONIX_CI_CHECKOUT_READY=false"
            echo "ROBONIX_CI_ISOLATION_READY=false"
          } >> "$GITHUB_ENV"

      - name: Select runner GPU
        run: |
          set -euo pipefail
          gpu_id="${ROBONIX_CI_GPU_ID_INPUT:-${ROBONIX_CI_GPU_ID_VAR:-${ROBONIX_CI_GPU_ID:-${CUDA_VISIBLE_DEVICES:-${ROBONIX_GPU_ID:-0}}}}}"
          if [ -z "$gpu_id" ]; then
            gpu_id=0
          fi
          case "$gpu_id" in
            ''|*[!0-9]*) echo "::error::ROBONIX_CI_GPU_ID must be a numeric GPU index, got '$gpu_id'"; exit 1 ;;
          esac
          if ! nvidia-smi --query-gpu=index --format=csv,noheader,nounits | awk '{print $1}' | grep -Fxq "$gpu_id"; then
            echo "::error::selected GPU index $gpu_id is not present on this runner"
            nvidia-smi --query-gpu=index,name,memory.used,memory.total --format=csv,noheader,nounits || true
            exit 1
          fi
          {
            echo "ROBONIX_CI_GPU_ID=$gpu_id"
            echo "CUDA_VISIBLE_DEVICES=$gpu_id"
            echo "ROBONIX_GPU_ID=$gpu_id"
            echo "NVIDIA_VISIBLE_DEVICES=$gpu_id"
          } >> "$GITHUB_ENV"

      - name: Verify GPU health
        run: |
          # NEVER nvidia-smi --gpu-reset / rmmod / driver writes (RTX 5090 GSP
          # fragility). Read-only check; a dirty GPU is an infra failure, not a
          # test failure — bail and let a human recover.
          nvidia-smi -q >/dev/null || { echo "::error::nvidia-smi unhealthy"; exit 1; }
          if dmesg 2>/dev/null | grep -qi 'NVRM: Xid'; then
            echo "::warning::Xid in dmesg — GPU may be degraded"; fi

      - name: Remove containers left by an interrupted run
        run: |
          set -euo pipefail
          # A runner killed mid-job never reaches its own cleanup, and its
          # containers keep the GPU, so the next run fails the capacity check.
          # This runner takes one job at a time and this step runs before this
          # job starts anything, so every ci-* resource here belongs to a job
          # that is no longer running.
          docker ps -aq --filter "name=^ci-" | xargs -r docker rm -f
          docker images --format '{{.Repository}}:{{.Tag}}' | grep '^ci-' | xargs -r docker rmi || true
          docker volume ls -q | grep '^ci-' | xargs -r docker volume rm || true
          docker network ls --format '{{.Name}}' | grep '^ci-' | xargs -r docker network rm || true

      - name: Wait for runner GPU capacity
        run: |
          set -euo pipefail
          case "${ROBONIX_CI_MIN_GPU_FREE_MIB}" in
            ''|*[!0-9]*) echo "::error::ROBONIX_CI_MIN_GPU_FREE_MIB must be an integer"; exit 1 ;;
          esac
          free_mib=0
          for attempt in $(seq 1 30); do
            free_mib="$(nvidia-smi --id="${ROBONIX_CI_GPU_ID}" --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | tr -d ' ')"
            case "$free_mib" in ''|*[!0-9]*) free_mib=0 ;; esac
            if [ "$free_mib" -ge "$ROBONIX_CI_MIN_GPU_FREE_MIB" ]; then
              echo "runner GPU ${ROBONIX_CI_GPU_ID} ready: ${free_mib} MiB free"
              break
            fi
            echo "runner GPU ${ROBONIX_CI_GPU_ID} busy: ${free_mib} MiB free; waiting (${attempt}/30)"
            sleep 10
          done
          if [ "$free_mib" -lt "$ROBONIX_CI_MIN_GPU_FREE_MIB" ]; then
            echo "::error title=Runner GPU unavailable::GPU ${ROBONIX_CI_GPU_ID} has ${free_mib} MiB free; ${ROBONIX_CI_MIN_GPU_FREE_MIB} MiB is required"
            printf 'Runner GPU capacity gate failed before checkout: GPU %s has %s MiB free; %s MiB is required.\n' \
              "$ROBONIX_CI_GPU_ID" "$free_mib" "$ROBONIX_CI_MIN_GPU_FREE_MIB" \
              >"$RUNNER_TEMP/sim-logs/infrastructure.txt"
            nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv,noheader || true
            exit 1
          fi

      - name: Reset runner workspace
        run: |
          set -euo pipefail
          case "${GITHUB_WORKSPACE:-}" in
            /home/*/*/_work/*/*) ;;
            *) echo "::error::refusing to reset unexpected workspace: ${GITHUB_WORKSPACE:-<unset>}"; exit 1 ;;
          esac
          mkdir -p "$GITHUB_WORKSPACE"
          sudo -n chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE" 2>/dev/null || true
          if ! find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + 2>/dev/null; then
            if ! docker image inspect busybox:latest >/dev/null 2>&1; then
              echo "::error::workspace has non-removable residue and busybox:latest is unavailable"
              exit 1
            fi
            timeout 60s docker run --rm --network none \
              -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE:rw" \
              -w "$GITHUB_WORKSPACE" \
              busybox:latest sh -c 'find . -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +'
          fi

      - name: Configure Git mirror for checkout
        if: env.GIT_MIRROR_INSTEADOF == 'true'
        run: |
          set -euo pipefail
          # Keep all routing scoped to this Actions job. The runner is also a
          # developer workstation, so CI must not mutate ~/.gitconfig.
          git_config="$RUNNER_TEMP/robonix-ci-gitconfig"
          : > "$git_config"
          # The runner service inherited HTTP_PROXY=127.0.0.1:9981 from a
          # macOS-forwarding config that was disabled on 2026-08-10. The
          # listener is still up but forwards nothing, so every clone since has
          # died with `GnuTLS, handshake failed` — an error that names TLS and
          # hides the fact that the proxy is a dead end. Drop the inherited
          # values for this job so the routing decided below is the one that
          # applies.
          for stale in HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy; do
            printf '%s=\n' "$stale" >> "$GITHUB_ENV"
          done
          unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy
          if timeout 15s git ls-remote --exit-code \
              "https://ghfast.top/https://github.com/syswonder/robonix.git" \
              HEAD >/dev/null 2>&1; then
            git config --file "$git_config" \
              url."https://ghfast.top/https://github.com/".insteadOf \
              "https://github.com/"
            echo "Using ghfast.top for GitHub clones"
          elif proxy_ok=0
               # Clash on this runner drops out for stretches at a time. Both
               # other routes are hard-blocked here — a direct clone and
               # ghfast.top both come back "Connection reset by peer" — so the
               # proxy is the only way out and a single failed probe is worth
               # waiting on rather than treating as absent.
               for attempt in 1 2 3 4 5 6; do
                 if timeout 20s git -c 'http.https://github.com/.proxy=http://127.0.0.1:7897' \
                      ls-remote --exit-code "https://github.com/syswonder/robonix.git" \
                      HEAD >/dev/null 2>&1; then
                   proxy_ok=1; break
                 fi
                 echo "Clash probe $attempt/6 failed; retrying in 10s"
                 sleep 10
               done
               [ "$proxy_ok" = 1 ]; then
            git config --file "$git_config" \
              'http.https://github.com/.proxy' \
              'http://127.0.0.1:7897'
            echo "ghfast.top unavailable; using GitHub through Clash at 127.0.0.1:7897"
          else
            # Both routes are gone. Pointing git at a proxy that is not
            # listening is worse than not setting one: every clone dies with a
            # GnuTLS handshake failure against a closed port, which reads like
            # a TLS problem rather than a missing proxy. Leave git alone and
            # let it reach github.com directly; if that also fails the error
            # will at least name the real cause.
            echo "ghfast.top unavailable and no proxy on 127.0.0.1:7897; going direct"
          fi
          echo "GIT_CONFIG_GLOBAL=$git_config" >> "$GITHUB_ENV"

      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.pr_number != '' && (inputs.pr_base_ref || github.event.repository.default_branch) || inputs.ref }}
          fetch-depth: ${{ inputs.pr_number != '' && '0' || '1' }}

      - name: Mark checkout ready
        run: echo "ROBONIX_CI_CHECKOUT_READY=true" >> "$GITHUB_ENV"

      - name: Merge PR head into current base
        if: ${{ inputs.pr_number != '' }}
        run: |
          set -euo pipefail
          git fetch --no-tags --prune origin \
            "+refs/pull/${{ inputs.pr_number }}/head:refs/remotes/robonix-pr/${{ inputs.pr_number }}/head"
          git -c user.name=github-actions[bot] \
              -c user.email=41898282+github-actions[bot]@users.noreply.github.com \
              merge --no-edit --no-ff "refs/remotes/robonix-pr/${{ inputs.pr_number }}/head"
          git log -1 --oneline

      - name: Initialize submodules
        run: |
          set -euo pipefail
          git submodule sync --recursive
          git submodule update --init --recursive

      - name: Configure run isolation
        run: |
          set -euo pipefail
          base_suffix=$((RUN_ID % 1000))
          suffix=""
          xdisplay=""

          port_is_listening() {
            ss -ltn "sport = :$1" | grep -q LISTEN
          }

          xdisplay_has_live_server() {
            ps -eo pid=,args= | awk -v d=":$1" '$0 ~ /(Xorg|Xvfb)/ && index($0, d) {found=1} END {exit found ? 0 : 1}'
          }

          remove_stale_x_socket() {
            display="$1"
            socket="/tmp/.X11-unix/X${display}"
            [ -S "$socket" ] || return 0
            if xdisplay_has_live_server "$display"; then
              return 1
            fi
            echo "removing stale per-run X socket: $socket"
            rm -f "$socket" 2>/dev/null || sudo -n rm -f "$socket" 2>/dev/null || true
            if [ -S "$socket" ] && docker image inspect busybox:latest >/dev/null 2>&1; then
              timeout 10s docker run --rm --network none -v /tmp/.X11-unix:/tmp/.X11-unix:rw busybox:latest rm -f "$socket" >/dev/null 2>&1 || true
            fi
            [ ! -S "$socket" ]
          }

          for offset in $(seq 0 199); do
            candidate_suffix=$(((base_suffix + offset) % 1000))
            candidate_xdisplay=$((1000 + candidate_suffix))
            candidate_ports=(
              $((19000 + candidate_suffix))
              $((20000 + candidate_suffix))
              $((21000 + candidate_suffix))
              $((22000 + candidate_suffix))
              $((18000 + candidate_suffix))
              $((23000 + candidate_suffix))
              $((24000 + candidate_suffix))
              $((25000 + candidate_suffix))
              $((26000 + candidate_suffix))
              $((27000 + candidate_suffix))
              $((28000 + candidate_suffix))
            )

            busy=0
            for port in "${candidate_ports[@]}"; do
              if port_is_listening "$port"; then
                busy=1
                break
              fi
            done
            [ "$busy" -eq 0 ] || continue

            if [ -S "/tmp/.X11-unix/X${candidate_xdisplay}" ] && ! remove_stale_x_socket "$candidate_xdisplay"; then
              continue
            fi

            suffix="$candidate_suffix"
            xdisplay="$candidate_xdisplay"
            break
          done

          if [ -z "$suffix" ] || [ -z "$xdisplay" ]; then
            echo "::error::could not find an unused per-run port/display slot from base suffix $base_suffix"
            exit 1
          fi

          atlas_port=$((19000 + suffix))
          executor_port=$((20000 + suffix))
          pilot_port=$((21000 + suffix))
          liaison_port=$((22000 + suffix))
          fake_vlm_port=$((18000 + suffix))
          stream_port=$((23000 + suffix))
          viewer_port=$((24000 + suffix))
          mapping_port=$((25000 + suffix))
          nav2_port=$((26000 + suffix))
          scene_web_port=$((27000 + suffix))
          soma_port=$((28000 + suffix))
          ros_domain_id=$((90 + (suffix % 100)))
          runtime_dir="$RUNNER_TEMP/robonix-ci-${RUN_KEY}"
          rm -rf "$runtime_dir"
          mkdir -p "$runtime_dir"

          echo "run isolation slot: suffix=${suffix} xdisplay=:${xdisplay} ports=${fake_vlm_port},${atlas_port},${executor_port},${pilot_port},${liaison_port},${stream_port},${viewer_port},${mapping_port},${nav2_port},${scene_web_port},${soma_port} ros_domain_id=${ros_domain_id} runtime_dir=${runtime_dir}"

          {
            echo "ATLAS_PORT=$atlas_port"
            echo "EXECUTOR_PORT=$executor_port"
            echo "PILOT_PORT=$pilot_port"
            echo "LIAISON_PORT=$liaison_port"
            echo "SOMA_PORT=$soma_port"
            echo "FAKE_VLM_PORT=$fake_vlm_port"
            echo "MAPPING_GRPC_PORT=$mapping_port"
            echo "NAV2_DRIVER_PORT=$nav2_port"
            echo "SCENE_WEB_PORT=$scene_web_port"
            echo "ROBONIX_CI_ATLAS=127.0.0.1:$atlas_port"
            echo "ROBONIX_ATLAS=127.0.0.1:$atlas_port"
            echo "ROBONIX_SIM_STREAM_PORT=$stream_port"
            echo "ROBONIX_SIM_VIEWER_PORT=$viewer_port"
            echo "ROBONIX_SIM_CONTAINER=ci-${RUN_KEY}-sim"
            echo "ROBONIX_SIM_PROJECT=ci-${RUN_KEY}"
            echo "ROBONIX_SIM_XDISPLAY=:$xdisplay"
            echo "ROBONIX_WEBOTS_CACHE_VOLUME=robonix_ci_webots_cache"
            echo "ROBONIX_MAPPING_CONTAINER=ci-${RUN_KEY}-mapping"
            echo "ROBONIX_NAV2_CONTAINER=ci-${RUN_KEY}-nav2"
            echo "ROBONIX_EXPLORE_CONTAINER=ci-${RUN_KEY}-explore"
            echo "ROBONIX_SCENE_CONTAINER=ci-${RUN_KEY}-scene"
            echo "ROBONIX_CI_RUNTIME_DIR=$runtime_dir"
            echo "AGENT_MEMORY_DIR=$runtime_dir/agent_memory"
            echo "AGENT_MILVUS_URI=$runtime_dir/agent_milvus.db"
            echo "MAPPING_MAPS_DIR=$runtime_dir/mapping-maps"
            echo "SCENE_DATA_DIR=$runtime_dir/scene-data"
            echo "ROS_DOMAIN_ID=$ros_domain_id"
            echo "ROBONIX_DRIVER_INIT_TIMEOUT_S=${ROBONIX_DRIVER_INIT_TIMEOUT_S}"
            echo "ROBONIX_GPU_ID=${ROBONIX_GPU_ID}"
            echo "NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES}"
            echo "ROBONIX_MODEL_CACHE_DIR=${RUNNER_TOOL_CACHE:-$HOME/.cache}/robonix-models"
            echo "RBNX_MODEL_DOWNLOAD_MAX_TIME=600"
            echo "RBNX_MODEL_DOWNLOAD_CONNECT_TIMEOUT=20"
            echo "RBNX_MODEL_DOWNLOAD_RETRIES=2"
            echo "ROBONIX_CI_ISOLATION_READY=true"
          } >> "$GITHUB_ENV"
          mkdir -p "${RUNNER_TOOL_CACHE:-$HOME/.cache}/robonix-models"

      - name: Configure toolchain and workspace
        run: |
          set -euo pipefail
          python3 -m venv "$ROBONIX_CI_RUNTIME_DIR/venv"
          export ROBONIX_HOME="$ROBONIX_CI_RUNTIME_DIR/robonix-home"
          mkdir -p "$ROBONIX_HOME"
          echo "ROBONIX_HOME=$ROBONIX_HOME" >> "$GITHUB_ENV"
          export PATH="$ROBONIX_CI_RUNTIME_DIR/venv/bin:$HOME/.local/bin:$PATH"
          echo "$ROBONIX_CI_RUNTIME_DIR/venv/bin" >> "$GITHUB_PATH"
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
          command -v uv >/dev/null || curl -LsSf https://astral.sh/uv/install.sh | sh
          # codegen tools pinned to the runtime grpcio version. Install them in
          # a per-job venv so Debian/Ubuntu PEP 668 system Python remains intact.
          python3 -m pip install "grpcio-tools==${GRPCIO_PIN}" "grpcio==${GRPCIO_PIN}"
          # testing harness (fake VLM + scenario runner) parses YAML scenarios.
          python3 -m pip install pyyaml
          # Build and use binaries from THIS checkout. Do not mutate or trust the
          # runner user's ~/.cargo/bin; this is a shared Webots host.
          cargo build --workspace
          echo "$PWD/target/debug" >> "$GITHUB_PATH"
          echo "ROBONIX_CODEGEN_BIN=$PWD/target/debug/robonix-codegen" >> "$GITHUB_ENV"
          "$PWD/target/debug/rbnx" setup "$PWD"

      - name: Validate scenario harness
        run: |
          set -euo pipefail
          python3 testing/simulate_ci.py

      - name: Build Webots deployment
        working-directory: examples/webots
        run: |
          set -euo pipefail
          mkdir -p "$RUNNER_TEMP/sim-logs"
          rbnx build 2>&1 | tee "$RUNNER_TEMP/sim-logs/build-webots-deployment.log"
          # provenance: record exactly which upstream commit each remote built from
          for d in rbnx-boot/cache/*/; do
            [ -d "$d/.git" ] && echo "$(basename "$d") @ $(git -C "$d" rev-parse --short HEAD) $(git -C "$d" log -1 --format=%s)"
          done | tee "$RUNNER_TEMP/remote-provenance.txt"

      - name: Prepare scene image for ${{ matrix.backend }}
        run: |
          set -euo pipefail
          if [ "$SCENE_PERCEPTION_BACKEND" = "concept_graphs" ]; then
            echo "ROBONIX_SCENE_IMAGE=robonix-scene" >> "$GITHUB_ENV"
            exit 0
          fi
          # DualMap is a layer over the scene image. Tag it by the Dockerfile
          # and the base image it was built from, so it is rebuilt exactly
          # when either changes and reused otherwise (it downloads ~400 MB).
          df=system/scene/docker/Dockerfile.dualmap
          base_id="$(docker image inspect robonix-scene --format '{{.Id}}')"
          tag="robonix-scene-dualmap:ci-$( { cat "$df"; echo "$base_id"; } | sha256sum | cut -c1-12)"
          if docker image inspect "$tag" >/dev/null 2>&1; then
            echo "reusing $tag"
          else
            # Same mirrors the checkout uses: large GitHub and Hugging Face
            # transfers from this runner get cut off mid-stream.
            docker build --network=host -f "$df" --build-arg BASE_IMAGE=robonix-scene \
              --build-arg GITHUB_PREFIX=https://ghfast.top/ --build-arg HF_ENDPOINT=https://hf-mirror.com \
              -t "$tag" system/scene/docker
          fi
          docker run --rm --entrypoint sh "$tag" -c 'test -d /opt/dualmap/utils'
          # Drop the tags earlier Dockerfiles or base images produced. Usually
          # there are none, which must not fail the step under pipefail.
          for old in $(docker images robonix-scene-dualmap --format '{{.Tag}}' | grep '^ci-' || true); do
            [ "$old" = "${tag#*:}" ] || docker rmi "robonix-scene-dualmap:$old" >/dev/null 2>&1 || true
          done
          echo "ROBONIX_SCENE_IMAGE=$tag" >> "$GITHUB_ENV"

      - name: Validate Executor control-plane isolation
        run: |
          set -euo pipefail
          bash scripts/test_executor_cancel_isolation.sh

      - name: Start isolated Webots simulator
        working-directory: examples/webots/sim
        env:
          NVIDIA_DRIVER_CAPABILITIES: "graphics,display,utility,compute"
        run: |
          set -euo pipefail
          # Avoid Docker Hub metadata checks on every Webots sim rebuild.
          # The helper creates a local alias and pulls through configured
          # mirrors on first use.
          helper="$GITHUB_WORKSPACE/scripts/docker_base_image.sh"
          if [ -f "$helper" ]; then
            # shellcheck disable=SC1090
            source "$helper"
          else
            # The reusable workflow may test an older PR merge tree that does not
            # yet contain scripts/docker_base_image.sh. Keep this minimal fallback
            # here so CI infrastructure changes do not make old PRs untestable.
            robonix_ensure_local_base_image() {
              local dest="$1"
              local upstream="$2"
              local mirrors_raw="${ROBONIX_DOCKER_MIRRORS:-${ROBONIX_DOCKER_MIRROR:-docker.m.daocloud.io,dockerproxy.net}}"
              local image_path="$upstream"
              local candidates=()
              if docker image inspect "$dest" >/dev/null 2>&1; then return 0; fi
              if docker image inspect "$upstream" >/dev/null 2>&1; then
                echo "[docker-base] tagging cached $upstream as $dest"
                docker tag "$upstream" "$dest"
                return 0
              fi
              image_path="${image_path#docker.io/}"
              image_path="${image_path#registry-1.docker.io/}"
              if [[ "$image_path" != */* ]]; then image_path="library/$image_path"; fi
              IFS=',' read -r -a _mirror_items <<< "$mirrors_raw"
              for mirror in "${_mirror_items[@]}"; do
                mirror="${mirror#${mirror%%[![:space:]]*}}"
                mirror="${mirror%${mirror##*[![:space:]]}}"
                [[ -z "$mirror" ]] && continue
                mirror="${mirror%/}"
                candidates+=("${mirror}/${image_path}")
              done
              candidates+=("$upstream")
              for candidate in "${candidates[@]}"; do
                echo "[docker-base] pulling base candidate: $candidate"
                if docker pull "$candidate"; then docker tag "$candidate" "$dest"; return 0; fi
                echo "[docker-base] warning: failed to pull $candidate" >&2
              done
              echo "[docker-base] error: failed to prepare local base image $dest from $upstream" >&2
              return 1
            }
          fi
          export ROBONIX_SIM_ROS_BASE_IMAGE="${ROBONIX_SIM_ROS_BASE_IMAGE:-robonix-osrf-ros:humble-desktop-full}"
          robonix_ensure_local_base_image "$ROBONIX_SIM_ROS_BASE_IMAGE" "osrf/ros:humble-desktop-full"
          export ROBONIX_WEBOTS_CACHE_VOLUME="${ROBONIX_WEBOTS_CACHE_VOLUME:-robonix_ci_webots_cache}"
          docker volume create "$ROBONIX_WEBOTS_CACHE_VOLUME" >/dev/null
          cache_count=$(docker run --rm -v "$ROBONIX_WEBOTS_CACHE_VOLUME:/cache" busybox sh -lc 'find /cache -type f 2>/dev/null | wc -l')
          if [ "${cache_count:-0}" -lt 1000 ]; then
            seed_volume="${ROBONIX_WEBOTS_SEED_CACHE_VOLUME:-robonix_tiago_sim_webots_cache}"
            seed_count=0
            if docker volume inspect "$seed_volume" >/dev/null 2>&1; then
              seed_count=$(docker run --rm -v "$seed_volume:/seed:ro" busybox sh -lc 'find /seed -type f 2>/dev/null | wc -l')
            fi
            if [ "${seed_count:-0}" -lt 1000 ]; then
              echo "::error::Webots asset cache $ROBONIX_WEBOTS_CACHE_VOLUME is not pre-warmed, and seed volume $seed_volume has only ${seed_count:-0} files. Run ROBONIX_WEBOTS_DOWNLOAD_ALL_ASSETS=1 bash examples/webots/sim/start.sh once on this runner, then rerun CI."
              exit 1
            fi
            echo "[webots-cache] prewarming $ROBONIX_WEBOTS_CACHE_VOLUME from $seed_volume ($seed_count files)"
            docker run --rm -v "$seed_volume:/seed:ro" -v "$ROBONIX_WEBOTS_CACHE_VOLUME:/cache" busybox sh -lc 'cp -a /seed/. /cache/'
          else
            echo "[webots-cache] using prewarmed $ROBONIX_WEBOTS_CACHE_VOLUME ($cache_count files)"
          fi
          : > "$RUNNER_TEMP/xauth"; export ROBONIX_HOST_XAUTH="$RUNNER_TEMP/xauth" DISPLAY=:49
          source ./nvidia_host.sh
          robonix_resolve_nvidia_xorg_modules "$ROBONIX_GPU_ID"
          # sim/start.sh creates the Soma runtime bind-mount source before
          # compose runs; CI drives compose directly, so it has to do the same.
          # Docker would otherwise create the whole rbnx-boot/logs path as root,
          # and every provider that writes <name>.log into it dies with an empty
          # log and no diagnostic.
          mkdir -p ../rbnx-boot/logs/soma-runtime
          docker compose -p "$ROBONIX_SIM_PROJECT" \
            -f compose.yaml \
            -f compose.gpu.yaml \
            -f compose.nvidia-xorg.yaml \
            -f compose.stream.yaml \
            up --build -d
          # robot publishes /scanner /odom /head_front_camera/* within ~15s.
          for i in $(seq 1 40); do
            if docker exec "$ROBONIX_SIM_CONTAINER" bash -lc 'source /opt/ros/humble/setup.bash 2>/dev/null && ros2 topic list 2>/dev/null | grep -qx /scanner' 2>/dev/null; then
              echo "sim up"
              break
            fi
            if [ "$i" -eq 40 ]; then
              echo "::error::sim did not expose /scanner"
              docker logs "$ROBONIX_SIM_CONTAINER" | tail -200 || true
              exit 1
            fi
            sleep 4
          done
          sim_network_mode=$(docker inspect -f '{{.HostConfig.NetworkMode}}' "$ROBONIX_SIM_CONTAINER")
          if [ "$sim_network_mode" = "host" ]; then
            sim_gateway="127.0.0.1"
            sim_ip="127.0.0.1"
            sim_atlas_host="127.0.0.1"
            zenoh_router_host="127.0.0.1"
          else
            sim_gateway=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{if .Gateway}}{{println .Gateway}}{{end}}{{end}}' "$ROBONIX_SIM_CONTAINER" | head -n1)
            if [ -z "$sim_gateway" ] || [ "$sim_gateway" = "invalid IP" ]; then
              echo "::error::cannot determine sim bridge gateway (network_mode=$sim_network_mode, gateway=$sim_gateway)"
              exit 1
            fi
            sim_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{if .IPAddress}}{{println .IPAddress}}{{end}}{{end}}' "$ROBONIX_SIM_CONTAINER" | head -n1)
            if [ -z "$sim_ip" ] || [ "$sim_ip" = "invalid IP" ]; then
              echo "::error::cannot determine sim bridge IP (network_mode=$sim_network_mode, ip=$sim_ip)"
              exit 1
            fi
            sim_atlas_host="$sim_gateway"
            zenoh_router_host="$sim_ip"
          fi
          {
            echo "ROBONIX_SIM_ATLAS=${sim_atlas_host}:${ATLAS_PORT}"
            echo "ROBONIX_ZENOH_ROUTER=tcp/${zenoh_router_host}:7447"
            echo "ROBONIX_ZENOH_MODE=client"
            echo "ROBONIX_ZENOH_LISTEN=tcp/${sim_gateway}:0"
            echo "NO_PROXY=127.0.0.1,localhost,${sim_gateway},${sim_ip}"
            echo "no_proxy=127.0.0.1,localhost,${sim_gateway},${sim_ip}"
          } >> "$GITHUB_ENV"
          echo "sim network mode: $sim_network_mode"
          echo "sim atlas: ${sim_atlas_host}:${ATLAS_PORT}"
          echo "sim zenoh router: tcp/${zenoh_router_host}:7447"

      - name: Start deterministic VLM planner
        working-directory: examples/webots
        run: |
          set -euo pipefail
          # OpenAI-compatible endpoint that scripts pilot's planning via the RTDL
          # envelope. Bound to a per-run localhost port; pilot dials it instead
          # of a real VLM.
          python3 "$GITHUB_WORKSPACE/testing/fake_vlm/server.py" \
            --port "$FAKE_VLM_PORT" >"$RUNNER_TEMP/fake_vlm.log" 2>&1 &
          echo "FAKE_VLM_PID=$!" >> "$GITHUB_ENV"
          for i in $(seq 1 20); do
            curl -fsS --noproxy '*' "http://127.0.0.1:${FAKE_VLM_PORT}/v1/models" >/dev/null 2>&1 && break
            sleep 0.5
          done

      - name: Boot Robonix deployment
        working-directory: examples/webots
        env:
          VLM_API_KEY: "fake-ci-key"
          VLM_MODEL: "fake-vlm"
        run: |
          set -euo pipefail
          export VLM_BASE_URL="http://127.0.0.1:${FAKE_VLM_PORT}/v1"
          mkdir -p "$AGENT_MEMORY_DIR" "$MAPPING_MAPS_DIR" "$SCENE_DATA_DIR"
          python3 - <<'PY'
          import os
          from pathlib import Path

          import yaml

          env_keys = [
              "ROBONIX_SIM_ATLAS",
              "ROBONIX_SIM_CONTAINER",
              "ROBONIX_MAPPING_CONTAINER",
              "ROBONIX_NAV2_CONTAINER",
              "ROBONIX_EXPLORE_CONTAINER",
              "ROBONIX_SCENE_CONTAINER",
              "MAPPING_GRPC_PORT",
              "NAV2_DRIVER_PORT",
              "SCENE_WEB_PORT",
              "AGENT_MEMORY_DIR",
              "AGENT_MILVUS_URI",
              "MAPPING_MAPS_DIR",
              "SCENE_DATA_DIR",
              "ROS_DOMAIN_ID",
              "CUDA_VISIBLE_DEVICES",
              "ROBONIX_GPU_ID",
              "NVIDIA_VISIBLE_DEVICES",
              "AUDIO_MIC_DEVICE",
              "AUDIO_SPEAKER_DEVICE",
              "SPEECH_BACKEND",
              "SCENE_GRAPH_INTERVAL_SEC",
              "SCENE_GRAPH_MIN_OBSERVATIONS",
              "RMW_IMPLEMENTATION",
              "ROBONIX_ZENOH_ROUTER",
              "ROBONIX_ZENOH_MODE",
              "ROBONIX_ZENOH_LISTEN",
              "VLM_BASE_URL",
              "VLM_API_KEY",
              "VLM_MODEL",
          ]
          manifest = yaml.safe_load(Path("robonix_manifest.yaml").read_text())
          env = manifest.setdefault("env", {}) or {}
          manifest["env"] = env
          for key in env_keys:
              env[key] = os.environ[key]
          system = manifest.setdefault("system", {})
          system.setdefault("atlas", {})["listen"] = f"0.0.0.0:{os.environ['ATLAS_PORT']}"
          system.setdefault("executor", {})["listen"] = f"127.0.0.1:{os.environ['EXECUTOR_PORT']}"
          system.setdefault("soma", {})["listen"] = f"127.0.0.1:{os.environ['SOMA_PORT']}"
          system.setdefault("pilot", {})["listen"] = f"127.0.0.1:{os.environ['PILOT_PORT']}"
          system.setdefault("pilot", {}).setdefault("vlm", {}).update({
              "upstream": os.environ["VLM_BASE_URL"],
              "api_key": os.environ["VLM_API_KEY"],
              "model": os.environ["VLM_MODEL"],
              "api_format": "openai",
          })
          system.setdefault("liaison", {})["listen"] = f"127.0.0.1:{os.environ['LIAISON_PORT']}"
          Path("robonix_manifest.ci.generated.yaml").write_text(
              yaml.safe_dump(manifest, sort_keys=False)
          )
          PY
          # boot is the deploy supervisor (foreground); run it detached and poll
          # atlas until the expected providers register. ROS talks to the sim
          # container over the isolated bridge network from the bringup step.
          nohup rbnx boot -f robonix_manifest.ci.generated.yaml >"$RUNNER_TEMP/rbnx-boot.log" 2>&1 &
          echo "RBNX_BOOT_PID=$!" >> "$GITHUB_ENV"
          required_caps='robonix/primitive/camera/snapshot robonix/primitive/lidar/snapshot robonix/primitive/audio/list_devices robonix/primitive/audio/select_device robonix/primitive/audio/mic robonix/primitive/audio/speaker robonix/system/scene/list_objects robonix/system/scene/goal_near robonix/service/navigation/navigate robonix/service/navigation/navigate/status robonix/service/navigation/navigate/cancel robonix/skill/explore/explore robonix/skill/explore/explore/status robonix/skill/explore/explore/cancel robonix/service/map/save_map robonix/service/memory/save robonix/service/memory/search robonix/service/speech/speak robonix/service/voiceprint/list'
          required_active='scene tiago_camera tiago_lidar audio_driver mapping nav2 memory speech voiceprint'
          for i in $(seq 1 120); do
            if rbnx caps -v --server "$ROBONIX_CI_ATLAS" >"$RUNNER_TEMP/caps.wait.txt" 2>&1; then
              missing=0
              for cap in $required_caps; do
                if ! grep -q "$cap" "$RUNNER_TEMP/caps.wait.txt"; then
                  missing=1
                  break
                fi
              done
              if [ "$missing" -eq 0 ]; then
                for provider in $required_active; do
                  if ! grep -q "^● ${provider} \[ACTIVE\]" "$RUNNER_TEMP/caps.wait.txt"; then
                    missing=1
                    break
                  fi
                done
              fi
              # A skill lists its contracts as soon as it registers, before
              # CMD_INIT makes it INACTIVE; activating it in between fails.
              # The suite's first step activates explore, so wait for that.
              if [ "$missing" -eq 0 ] && ! grep -qE "^● explore \[(INACTIVE|ACTIVE)\]" "$RUNNER_TEMP/caps.wait.txt"; then
                missing=1
              fi
              if [ "$missing" -eq 0 ]; then
                echo "required providers active"
                break
              fi
            fi
            if [ "$i" -eq 120 ]; then
              echo "::error::required providers did not register"
              cat "$RUNNER_TEMP/caps.wait.txt" || true
              tail -240 "$RUNNER_TEMP/rbnx-boot.log" || true
              exit 1
            fi
            sleep 5
          done
          rbnx caps -v --server "$ROBONIX_CI_ATLAS" || true

      - name: Run interface checks
        env:
          ROBONIX_REQUIRE_AUDIO: "1"
        run: |
          export ROBONIX_ATLAS="$ROBONIX_CI_ATLAS"
          python3 "$GITHUB_WORKSPACE/testing/run_interfaces.py" --server "$ROBONIX_CI_ATLAS"

      - name: Wait for mapping occupancy grid
        run: |
          set -euo pipefail
          mkdir -p "$RUNNER_TEMP/sim-logs"
          echo "waiting for a real /map OccupancyGrid publisher before scenario execution"
          if docker exec "$ROBONIX_SIM_CONTAINER" bash -lc '
            set -eo pipefail
            source /opt/ros/humble/setup.bash
            set -u
            export ROS_DOMAIN_ID='"${ROS_DOMAIN_ID}"'
            export RMW_IMPLEMENTATION="${RMW_IMPLEMENTATION:-rmw_zenoh_cpp}"
            for _ in $(seq 1 90); do
              if ros2 topic info -v /map > /tmp/map-info.txt 2>&1 &&
                 grep -q "Type: nav_msgs/msg/OccupancyGrid" /tmp/map-info.txt &&
                 grep -Eq "Publisher count: [1-9]" /tmp/map-info.txt; then
                cat /tmp/map-info.txt
                exit 0
              fi
              sleep 2
            done
            cat /tmp/map-info.txt 2>/dev/null || true
            exit 1
          ' >"$RUNNER_TEMP/sim-logs/map-readiness.txt" 2>&1; then
            echo "map readiness confirmed"
            exit 0
          fi

          echo "::error::mapping did not expose a /map publisher before scenario execution"
          {
            echo "=== /map readiness failure ==="
            cat "$RUNNER_TEMP/sim-logs/map-readiness.txt" || true
            echo
            echo "=== ROS topic list from simulator container ==="
            docker exec "$ROBONIX_SIM_CONTAINER" bash -lc '
              set -eo pipefail
              source /opt/ros/humble/setup.bash
              set -u
              export ROS_DOMAIN_ID='"${ROS_DOMAIN_ID}"'
              ros2 topic list || true
              echo
              ros2 topic info -v /map || true
            ' || true
            echo
            echo "=== final caps ==="
            rbnx caps -v --server "$ROBONIX_CI_ATLAS" || true
            echo
            echo "=== rbnx boot tail ==="
            tail -240 "$RUNNER_TEMP/rbnx-boot.log" || true
          } | tee -a "$RUNNER_TEMP/sim-logs/map-readiness.txt"
          exit 1

      - name: Wait for scene object detection
        run: |
          set -euo pipefail
          mkdir -p "$RUNNER_TEMP/sim-logs"
          scene_url="http://127.0.0.1:${SCENE_WEB_PORT}/api/state"
          state_file="$RUNNER_TEMP/sim-logs/scene-state-readiness.json"
          echo "waiting for a real non-robot scene object before scenario execution: ${scene_url}"
          for _ in $(seq 1 180); do
            if curl -fsS --max-time 2 "$scene_url" -o "$state_file"; then
              if python3 - "$state_file" <<'SCENE_READY_PY'
          import json, sys
          path = sys.argv[1]
          with open(path, "r", encoding="utf-8") as f:
              state = json.load(f)
          objects = state.get("objects") or []
          real = []
          for obj in objects:
              cls = str(obj.get("cls") or obj.get("label") or "").strip().lower()
              oid = str(obj.get("id") or "")
              # A missing object is one restored from a saved map and not seen
              # since; list_objects leaves those out, so they cannot count.
              if cls and cls != "robot" and oid.startswith("scene.object.") and not obj.get("missing"):
                  real.append(obj)
          if not real:
              sys.exit(1)
          print(json.dumps([
              {
                  "id": o.get("id"),
                  "cls": o.get("cls") or o.get("label"),
                  "confidence": o.get("confidence"),
                  "observation_count": o.get("observation_count"),
                  "pose": o.get("pose"),
              }
              for o in real[:8]
          ], ensure_ascii=False))
          SCENE_READY_PY
              then
                echo "scene object readiness confirmed"
                exit 0
              fi
            fi
            sleep 2
          done

          echo "::error::scene did not expose a non-robot object before scenario execution"
          {
            echo "=== latest /api/state ==="
            cat "$state_file" || true
            echo
            echo "=== scene.log tail ==="
            tail -260 "examples/webots/rbnx-boot/logs/scene.log" || true
            echo
            echo "=== final caps ==="
            rbnx caps -v --server "$ROBONIX_CI_ATLAS" || true
            echo
            echo "=== rbnx boot tail ==="
            tail -240 "$RUNNER_TEMP/rbnx-boot.log" || true
          } | tee -a "$RUNNER_TEMP/sim-logs/scene-readiness.txt"
          exit 1

      - name: Verify scene perception backend
        run: |
          set -euo pipefail
          line="$(docker logs "$ROBONIX_SCENE_CONTAINER" 2>&1 | grep -m1 '\[scene\] perception: .*backend=' || true)"
          echo "${line:-no perception line in the scene log}"
          case "$line" in
            *"backend=${SCENE_PERCEPTION_BACKEND},"*) ;;
            *) echo "::error::scene is not running the ${SCENE_PERCEPTION_BACKEND} backend"; exit 1 ;;
          esac

      - name: Run scenario suite
        id: scenarios
        run: |
          set -o pipefail
          export ROBONIX_ATLAS="$ROBONIX_CI_ATLAS"
          python3 "$GITHUB_WORKSPACE/testing/run.py" --server "$ROBONIX_CI_ATLAS" \
            --first object_navigation \
            --summary-json "$GITHUB_WORKSPACE/testing/logs/summary.json" | tee "$RUNNER_TEMP/scenarios.out"
          # Surface the one-line RESULT for the reusable-workflow output / bot.
          echo "result=$(grep '^RESULT:' "$RUNNER_TEMP/scenarios.out" | tail -1)" >> "$GITHUB_OUTPUT"

      - name: Navigation stress
        # Explore, then send the robot to random reachable points through the
        # navigation contract. The simulator is the judge: any contact above
        # the floor fails the step (testing/nav_stress.py). The seed is fixed
        # so a failure replays. Navigation does not depend on the scene
        # backend, so one leg runs it.
        if: ${{ !cancelled() && matrix.key == 'dm' }}
        run: |
          set -euo pipefail
          mkdir -p "$RUNNER_TEMP/sim-logs"
          docker cp "$GITHUB_WORKSPACE/testing/nav_stress.py" "$ROBONIX_NAV2_CONTAINER:/tmp/nav_stress.py"
          status=0
          docker exec "$ROBONIX_NAV2_CONTAINER" bash -c '
            source /opt/ros/humble/setup.bash
            export $(tr "\0" "\n" < /proc/1/environ | grep -E "^PYTHONPATH=")
            export ZENOH_SESSION_CONFIG_URI=/tmp/robonix_zenoh_session.json5
            python3 -u /tmp/nav_stress.py --goals 10 --seed 1 --explore-s 120 --out /tmp/nav-stress.json
          ' || status=$?
          docker cp "$ROBONIX_NAV2_CONTAINER:/tmp/nav-stress.json" "$RUNNER_TEMP/sim-logs/nav-stress.json" || true
          exit "$status"

      - name: Drive a fixed mapping loop
        # Chassis motion is a gRPC contract and the executor only dispatches
        # MCP, so this drives cmd_vel directly. A scripted rectangle returns to
        # its start, which makes the saved map comparable between runs and
        # forces the SLAM graph to close a loop.
        if: ${{ !cancelled() }}
        continue-on-error: true
        run: |
          set -uo pipefail
          docker exec -i "$ROBONIX_SIM_CONTAINER" bash <<'EOS'
          source /opt/ros/humble/setup.bash
          export RMW_IMPLEMENTATION=${RMW_IMPLEMENTATION:-rmw_zenoh_cpp}
          python3 - <<'PY'
          import math, time
          import rclpy
          from geometry_msgs.msg import Twist
          rclpy.init()
          node = rclpy.create_node("ci_mapping_loop")
          pub = node.create_publisher(Twist, "/cmd_vel", 10)
          time.sleep(2.0)

          def drive(lin, ang, seconds):
              msg = Twist()
              msg.linear.x, msg.angular.z = lin, ang
              end = time.time() + seconds
              while time.time() < end:
                  pub.publish(msg)
                  rclpy.spin_once(node, timeout_sec=0.05)
              pub.publish(Twist())
              for _ in range(5):
                  rclpy.spin_once(node, timeout_sec=0.02)

          for corner in range(4):
              drive(0.25, 0.0, 6.0)          # 1.5 m leg
              drive(0.0, 0.5, math.pi / 1.0) # 90 deg turn
              print(f"corner {corner + 1}/4 done", flush=True)
          node.destroy_node(); rclpy.shutdown()
          PY
          EOS

      - name: Verify scene map persistence
        # A failed suite is exactly when the saved map matters most, so this
        # runs even then; it still fails the job on its own checks.
        if: ${{ !cancelled() }}
        run: |
          set -euo pipefail
          mkdir -p "$RUNNER_TEMP/sim-logs"
          map_id="ci_scene_map_${RUN_KEY}"
          python3 "$GITHUB_WORKSPACE/testing/verify_scene_map_persistence.py" \
            --scene-url "http://127.0.0.1:${SCENE_WEB_PORT}" \
            --map-id "$map_id" \
            --mapping-container "${ROBONIX_MAPPING_CONTAINER:-robonix_mapping}" \
            --maps-dir "$MAPPING_MAPS_DIR" \
            --sim-container "$ROBONIX_SIM_CONTAINER" \
            --export-preview "$RUNNER_TEMP/sim-logs/slam-map.png" \
            --delete-after \
            --timeout 420 | tee "$RUNNER_TEMP/sim-logs/scene-map-persistence.txt"

      - name: Validate boot/shutdown lifecycle
        # boot → shutdown → re-boot. Proves the runtime keeps
        # state under <manifest-dir>/rbnx-boot/ (not ~/.robonix/), that
        # Soma package children are stopped cleanly, and that the stack
        # can come back up after a clean teardown (the issue #128
        # regression). The second boot reuses both the per-run sim container
        # and the persistent mapping/scene state: a process restart must not
        # silently turn into a new-map session while the physical robot stays
        # at its post-scenario pose. CI leaves the second boot running so the
        # next step can exercise scenarios against the restarted stack; the
        # final cleanup step shuts it down.
        working-directory: examples/webots
        run: |
          set -euo pipefail
          milvus_lock="$(dirname "$AGENT_MILVUS_URI")/.$(basename "$AGENT_MILVUS_URI").lock"
          # Scenario memory uses fixed fixture keys and must be isolated between
          # passes. Mapping and scene data are deliberately preserved because
          # they are runtime state whose reboot recovery this test exercises.
          export ROBONIX_LIFECYCLE_RESET_PATHS="${AGENT_MEMORY_DIR}:${AGENT_MILVUS_URI}:${milvus_lock}"
          python3 "$GITHUB_WORKSPACE/testing/validate_lifecycle.py" \
            --rbnx rbnx \
            --manifest robonix_manifest.ci.generated.yaml \
            --server "$ROBONIX_CI_ATLAS" \
            --sim-container "$ROBONIX_SIM_CONTAINER" \
            --leave-running \
            --log-dir "$RUNNER_TEMP/lifecycle-logs" | tee "$RUNNER_TEMP/lifecycle.out"

      - name: Re-run scenario suite after lifecycle validator
        # After the validator re-boots the stack, run the scenarios one
        # more time so the report covers both a fresh boot and a
        # post-shutdown boot. This is what actually exercises the
        # "restart doesn't crash" fix — the original suite proved
        # boot-once, this proves boot-after-shutdown.
        working-directory: examples/webots
        run: |
          set -euo pipefail
          export ROBONIX_ATLAS="$ROBONIX_CI_ATLAS"
          # Wait for atlas to settle (the validator just re-booted
          # everything; reuse the same caps wait).
          for i in $(seq 1 60); do
            if rbnx caps -v --server "$ROBONIX_CI_ATLAS" >"$RUNNER_TEMP/caps.post-restart.txt" 2>&1 \
              && grep -q "robonix/service/navigation/navigate" "$RUNNER_TEMP/caps.post-restart.txt" \
              && grep -q "^● nav2 \[ACTIVE\]" "$RUNNER_TEMP/caps.post-restart.txt" \
              && grep -qE "^● explore \[(INACTIVE|ACTIVE)\]" "$RUNNER_TEMP/caps.post-restart.txt"; then
              echo "post-restart atlas ready"
              break
            fi
            if [ "$i" -eq 60 ]; then
              echo "::error::atlas not ready after lifecycle reboot"
              tail -120 "$RUNNER_TEMP/caps.post-restart.txt" || true
              exit 1
            fi
            sleep 5
          done
          # The restart suite is an independent pass over the same scenario set.
          # save_map is intentionally immutable, so remove the first suite's
          # test-only saved map before exercising save_map again. Do not touch
          # the live Webots/RTAB-Map session database (webots_lab).
          if [ -d "${MAPPING_MAPS_DIR:?}/ci_test_map" ]; then
            docker run --rm --network none \
              -v "$MAPPING_MAPS_DIR:/maps:rw" \
              busybox:latest sh -lc 'rm -rf /maps/ci_test_map' \
              || sudo -n rm -rf "$MAPPING_MAPS_DIR/ci_test_map" \
              || rm -rf "$MAPPING_MAPS_DIR/ci_test_map"
          fi
          # The fake VLM keeps per-scenario pending-step state so it can avoid
          # duplicate calls while one RTDL batch is still executing. The first
          # suite and the post-lifecycle suite are independent test runs, so
          # restart the deterministic planner before the second suite.
          if [ -n "${FAKE_VLM_PID:-}" ]; then
            kill "$FAKE_VLM_PID" 2>/dev/null || true
          fi
          for _ in $(seq 1 30); do
            if ! curl -fsS --noproxy '*' "http://127.0.0.1:${FAKE_VLM_PORT}/v1/models" >/dev/null 2>&1; then
              break
            fi
            sleep 0.2
          done
          echo "[fake-vlm] --- restart after lifecycle validator ---" >>"$RUNNER_TEMP/fake_vlm.log"
          python3 "$GITHUB_WORKSPACE/testing/fake_vlm/server.py" \
            --port "$FAKE_VLM_PORT" >>"$RUNNER_TEMP/fake_vlm.log" 2>&1 &
          export FAKE_VLM_PID="$!"
          echo "FAKE_VLM_PID=$FAKE_VLM_PID" >> "$GITHUB_ENV"
          for i in $(seq 1 20); do
            curl -fsS --noproxy '*' "http://127.0.0.1:${FAKE_VLM_PORT}/v1/models" >/dev/null 2>&1 && break
            if [ "$i" -eq 20 ]; then
              echo "::error::fake VLM did not restart after lifecycle validator"
              tail -120 "$RUNNER_TEMP/fake_vlm.log" || true
              exit 1
            fi
            sleep 0.5
          done
          # Atlas readiness only proves that Scene registered its contracts.
          # A restarted Scene still needs fresh camera frames before its object
          # registry contains a real object, so apply the same readiness gate
          # used before the first scenario pass.
          scene_url="http://127.0.0.1:${SCENE_WEB_PORT}/api/state"
          state_file="$RUNNER_TEMP/sim-logs/scene-state-post-restart.json"
          scene_ready=0
          echo "waiting for a real non-robot scene object after lifecycle restart: ${scene_url}"
          for i in $(seq 1 180); do
            if curl -fsS --max-time 2 "$scene_url" -o "$state_file"; then
              if python3 - "$state_file" <<'SCENE_READY_PY'
          import json, sys
          path = sys.argv[1]
          with open(path, "r", encoding="utf-8") as f:
              state = json.load(f)
          objects = state.get("objects") or []
          for obj in objects:
              cls = str(obj.get("cls") or obj.get("label") or "").strip().lower()
              oid = str(obj.get("id") or "")
              # A missing object is one restored from a saved map and not seen
              # since; list_objects leaves those out, so they cannot count.
              if cls and cls != "robot" and oid.startswith("scene.object.") and not obj.get("missing"):
                  print(json.dumps({
                      "id": obj.get("id"),
                      "cls": obj.get("cls") or obj.get("label"),
                      "confidence": obj.get("confidence"),
                      "observation_count": obj.get("observation_count"),
                      "pose": obj.get("pose"),
                  }, ensure_ascii=False))
                  sys.exit(0)
          sys.exit(1)
          SCENE_READY_PY
              then
                echo "post-restart scene object readiness confirmed"
                scene_ready=1
                break
              fi
            fi
            if [ "$i" -eq 30 ]; then
              # The mapping loop turns open-loop, so the robot can come back
              # facing a bare wall and a still camera then sees nothing. One
              # slow turn in place shows it the room.
              echo "no object after 60 s; turning the robot once in place"
              docker exec -i "$ROBONIX_SIM_CONTAINER" bash <<'EOS' || true
          source /opt/ros/humble/setup.bash
          export RMW_IMPLEMENTATION=${RMW_IMPLEMENTATION:-rmw_zenoh_cpp}
          python3 - <<'PY'
          import math, time
          import rclpy
          from geometry_msgs.msg import Twist
          rclpy.init()
          node = rclpy.create_node("ci_look_around")
          pub = node.create_publisher(Twist, "/cmd_vel", 10)
          time.sleep(2.0)
          msg = Twist()
          msg.angular.z = 0.4
          end = time.time() + 2 * math.pi / 0.4
          while time.time() < end:
              pub.publish(msg)
              rclpy.spin_once(node, timeout_sec=0.05)
          pub.publish(Twist())
          node.destroy_node(); rclpy.shutdown()
          PY
          EOS
            fi
            sleep 2
          done
          if [ "$scene_ready" -ne 1 ]; then
            echo "::error::scene did not expose a non-robot object after lifecycle restart"
            cat "$state_file" || true
            tail -260 "$GITHUB_WORKSPACE/examples/webots/rbnx-boot/logs/scene.log" || true
            exit 1
          fi
          # A Webots boot starts a fresh live SLAM session by design — see the
          # comment at the top of examples/webots/robonix_manifest.yaml — so the
          # rebooted RTAB-Map opens an empty database and refills it from the
          # sensor stream. Until it has, /map is published and entirely unknown.
          #
          # object_navigation is the only scenario that needs a global plan, and
          # --first puts it at the head of this pass. Without this gate it asked
          # the planner to route across a map that did not exist yet: nav2
          # answered "GridBased: failed to create plan", Pilot retried, and the
          # scenario spent its whole 300s that way.
          #
          # Waiting for a publisher would not help, because the publisher exists
          # from the moment the node starts, and neither would waiting for a
          # message, because an all-unknown grid is a valid message. The gate has
          # to be free space. The reasoning is the one written above for Scene:
          # Atlas readiness proves a provider registered its contracts, not that
          # it has produced anything usable.
          #
          # The probe's own --timeout bounds the loop it spins waiting for a
          # grid, not the middleware setup that happens before it. An
          # rmw_zenoh session whose router is not where it was retries
          # forever, so the probe can block having printed nothing at all, and
          # the step's only other bound is the job's 90 minute timeout. That
          # is exactly what happened once: RTAB-Map published for the whole 80
          # minutes while the readiness file stayed empty, and the run was
          # cancelled rather than failed. Bound the probe here too, and run it
          # unbuffered so a hang leaves the line it reached behind.
          map_readiness="$RUNNER_TEMP/sim-logs/map-readiness-post-restart.txt"
          echo "waiting for /map to carry known cells after lifecycle restart"
          probe=0
          timeout --signal=TERM --kill-after=15 300 \
            docker exec -i "$ROBONIX_SIM_CONTAINER" bash -lc '
            set -eo pipefail
            source /opt/ros/humble/setup.bash
            set -u
            export ROS_DOMAIN_ID='"${ROS_DOMAIN_ID}"'
            export RMW_IMPLEMENTATION="${RMW_IMPLEMENTATION:-rmw_zenoh_cpp}"
            python3 -u - --timeout 200 "$@"
          ' _ <"$GITHUB_WORKSPACE/testing/wait_for_map.py" \
            >"$map_readiness" 2>&1 || probe=$?
          if [ "$probe" -ne 0 ]; then
            if [ "$probe" -ge 124 ]; then
              echo "::error::the /map readiness probe never returned; it hung before its own timeout could fire"
            else
              echo "::error::/map had no usable occupancy grid after lifecycle restart"
            fi
            echo "--- probe output (empty means it never reached its first print) ---"
            cat "$map_readiness" || true
            tail -260 "$GITHUB_WORKSPACE/examples/webots/rbnx-boot/logs/mapping.log" || true
            exit 1
          fi
          cat "$map_readiness"

          python3 "$GITHUB_WORKSPACE/testing/run.py" --server "$ROBONIX_CI_ATLAS" \
            --first object_navigation \
            --summary-json "$GITHUB_WORKSPACE/testing/logs/summary.restart.json" \
            | tee "$RUNNER_TEMP/scenarios.restart.out"

      - name: Capture runtime logs
        if: always()
        run: |
          if [ "${ROBONIX_CI_ISOLATION_READY:-false}" != "true" ]; then
            echo "per-run isolation was not initialized; refusing to inspect default or unrelated runtime resources"
            exit 0
          fi
          # Capture the sim/webots container stdout and the in-container ROS
          # logs so a failure leaves a full trace alongside the scribe logs.
          mkdir -p "$RUNNER_TEMP/sim-logs"
          docker logs "$ROBONIX_SIM_CONTAINER" >"$RUNNER_TEMP/sim-logs/webots.stdout.log" 2>&1 || true
          for name in \
            "${ROBONIX_MAPPING_CONTAINER:-ci-${RUN_KEY}-mapping}" \
            "${ROBONIX_NAV2_CONTAINER:-ci-${RUN_KEY}-nav2}" \
            "${ROBONIX_EXPLORE_CONTAINER:-ci-${RUN_KEY}-explore}" \
            "${ROBONIX_SCENE_CONTAINER:-ci-${RUN_KEY}-scene}"; do
            [ -n "$name" ] || continue
            docker logs "$name" >"$RUNNER_TEMP/sim-logs/${name}.stdout.log" 2>&1 || true
            docker inspect "$name" >"$RUNNER_TEMP/sim-logs/${name}.inspect.json" 2>&1 || true
          done
          docker exec "$ROBONIX_SIM_CONTAINER" bash -c 'tar c -C /root/.ros log 2>/dev/null' \
            2>/dev/null | tar x -C "$RUNNER_TEMP/sim-logs" 2>/dev/null || true
          docker exec "$ROBONIX_SIM_CONTAINER" bash -lc '
            version="$(timeout 10s webots --version 2>&1 | awk "/^Cyberbotics Webots|^Webots[[:space:]]/ {print; exit}")"
            if [ -z "$version" ]; then
              version="$(dpkg-query -W webots 2>/dev/null | cut -f2 | sed "s/^/webots package /" | head -1)"
            fi
            printf "WEBOTS_VERSION=%s\n" "$version"
            printf "ROS_DISTRO=%s\n" "${ROS_DISTRO:-}"
          ' \
            >"$RUNNER_TEMP/sim-logs/webots.version.txt" 2>&1 || true
          docker inspect "$ROBONIX_SIM_CONTAINER" --format '{{.Config.Image}}' \
            >"$RUNNER_TEMP/sim-logs/sim-container-image.txt" 2>&1 || true
          # rbnx caps snapshot at end-of-run for post-mortem.
          rbnx caps --server "${ROBONIX_CI_ATLAS:-127.0.0.1:50051}" --json >"$RUNNER_TEMP/sim-logs/caps.final.json" 2>&1 || true

      - name: Stop run resources
        if: always()
        run: |
          if [ "${ROBONIX_CI_ISOLATION_READY:-false}" != "true" ]; then
            echo "per-run isolation was not initialized; refusing unscoped shutdown"
            docker ps -aq --filter "name=ci-${RUN_KEY}-" | while read -r cid; do
              [ -n "$cid" ] || continue
              docker rm -f "$cid" >/dev/null 2>&1 || true
            done
            docker volume ls -q --filter "name=ci-${RUN_KEY}_" | while read -r vid; do
              [ -n "$vid" ] || continue
              docker volume rm -f "$vid" >/dev/null 2>&1 || true
            done
            exit 0
          fi
          # never touch other users' containers/processes; match on the run-id
          # prefix. $RUN_ID is the GitHub-assigned numeric id (job-level env).
          rbnx shutdown --server "$ROBONIX_CI_ATLAS" 2>/dev/null || true
          [ -n "${RBNX_BOOT_PID:-}" ] && kill "${RBNX_BOOT_PID}" 2>/dev/null || true
          [ -n "${FAKE_VLM_PID:-}" ] && kill "${FAKE_VLM_PID}" 2>/dev/null || true
          for port in "${ATLAS_PORT:-}" "${EXECUTOR_PORT:-}" "${SOMA_PORT:-}" "${PILOT_PORT:-}" "${LIAISON_PORT:-}" "${FAKE_VLM_PORT:-}" "${ROBONIX_SIM_STREAM_PORT:-}" "${ROBONIX_SIM_VIEWER_PORT:-}" "${MAPPING_GRPC_PORT:-}" "${NAV2_DRIVER_PORT:-}" "${SCENE_WEB_PORT:-}"; do
            [ -z "$port" ] && continue
            pids=$(ss -ltnp 2>/dev/null | awk -v p=":$port" '$0 ~ p { while (match($0, /pid=[0-9]+/)) { print substr($0, RSTART+4, RLENGTH-4); $0=substr($0, RSTART+RLENGTH) } }' | sort -u)
            [ -n "$pids" ] && kill $pids 2>/dev/null || true
          done
          remove_x_socket() {
            local xnum="$1"
            local socket="/tmp/.X11-unix/X${xnum}"
            [ -n "$xnum" ] || return 0
            [ -S "$socket" ] || return 0
            rm -f "$socket" 2>/dev/null || true
            [ -S "$socket" ] || return 0
            sudo -n rm -f "$socket" 2>/dev/null || true
            [ -S "$socket" ] || return 0
            docker exec "${ROBONIX_SIM_CONTAINER:-ci-${RUN_KEY}-sim}" rm -f "$socket" >/dev/null 2>&1 || true
            [ -S "$socket" ] || return 0
            if docker image inspect busybox:latest >/dev/null 2>&1; then
              timeout 10s docker run --rm --network none -v /tmp/.X11-unix:/tmp/.X11-unix:rw busybox:latest rm -f "$socket" >/dev/null 2>&1 || true
            fi
          }

          docker ps -aq --filter "name=ci-${RUN_KEY}-" | while read -r cid; do
            [ -n "$cid" ] || continue
            docker rm -f "$cid" >/dev/null 2>&1 || true
          done
          if [ -n "${ROBONIX_SIM_XDISPLAY:-}" ]; then
            xnum="${ROBONIX_SIM_XDISPLAY#:}"
            x_pids=$(ps -eo pid=,args= | awk -v d=":${xnum}" '$0 ~ /(Xorg|Xvfb)/ && index($0, d) {print $1}' | sort -u)
            if [ -z "$x_pids" ]; then
              remove_x_socket "$xnum"
            fi
          fi
          rm -rf "/tmp/robonix-ci-${RUN_KEY}"
          (cd examples/webots/sim && docker compose -p "ci-${RUN_KEY}" down --remove-orphans -v) 2>/dev/null || true
          if [ -n "${ROBONIX_SIM_XDISPLAY:-}" ]; then
            xnum="${ROBONIX_SIM_XDISPLAY#:}"
            for _ in $(seq 1 30); do
              [ -n "$xnum" ] || break
              x_pids=$(ps -eo pid=,args= | awk -v d=":${xnum}" '$0 ~ /(Xorg|Xvfb)/ && index($0, d) {print $1}' | sort -u)
              [ -n "$x_pids" ] && kill $x_pids >/dev/null 2>&1 || true
              remove_x_socket "$xnum"
              [ -S "/tmp/.X11-unix/X${xnum}" ] || break
              sleep 1
            done
            if [ -n "$xnum" ] && [ -S "/tmp/.X11-unix/X${xnum}" ]; then
              echo "::warning::stale per-run X socket remains after cleanup: /tmp/.X11-unix/X${xnum}"
              ls -l "/tmp/.X11-unix/X${xnum}" || true
            fi
          fi
          docker volume ls -q --filter "name=ci-${RUN_KEY}_" | while read -r vid; do
            [ -n "$vid" ] || continue
            docker volume rm -f "$vid" >/dev/null 2>&1 || true
          done

      - name: Prepare self-contained report viewer assets
        if: ${{ always() && env.ROBONIX_CI_CHECKOUT_READY == 'true' }}
        run: |
          set -euo pipefail
          python3 testing/prepare_report_assets.py --asset-dir "$RUNNER_TEMP/report-assets"

      - name: Generate test report
        if: ${{ always() && env.ROBONIX_CI_CHECKOUT_READY == 'true' }}
        env:
          REPORT_PR_NUMBER: ${{ inputs.pr_number || github.event.pull_request.number || '' }}
          REPORT_PR_TITLE: ${{ inputs.pr_title || github.event.pull_request.title || '' }}
          REPORT_PR_AUTHOR: ${{ inputs.pr_author || github.event.pull_request.user.login || '' }}
          REPORT_PR_URL: ${{ inputs.pr_url || github.event.pull_request.html_url || '' }}
          REPORT_PR_HEAD_REF: ${{ inputs.pr_head_ref || github.event.pull_request.head.ref || '' }}
          REPORT_PR_BASE_REF: ${{ inputs.pr_base_ref || github.event.pull_request.base.ref || '' }}
          REPORT_PR_HEAD_REPO: ${{ inputs.pr_head_repo || github.event.pull_request.head.repo.full_name || '' }}
          REPORT_PR_HEAD_SHA: ${{ inputs.pr_head_sha || github.event.pull_request.head.sha || '' }}
          REPORT_TRIGGERED_BY: ${{ inputs.triggered_by || github.triggering_actor || github.actor }}
        run: |
          set -euo pipefail
          tested_commit="$(git rev-parse HEAD 2>/dev/null || true)"
          tested_subject="$(git log -1 --format=%s 2>/dev/null || true)"
          tested_author="$(git log -1 --format='%an <%ae>' 2>/dev/null || true)"
          tested_date="$(git log -1 --format=%cI 2>/dev/null || true)"
          origin_url="$(git config --get remote.origin.url 2>/dev/null || true)"
          commit_url=""
          if [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ] && [ -n "$tested_commit" ]; then
            commit_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${tested_commit}"
          fi
          pr_head_commit_url=""
          if [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${REPORT_PR_HEAD_REPO:-}" ] && [ -n "${REPORT_PR_HEAD_SHA:-}" ]; then
            pr_head_commit_url="${GITHUB_SERVER_URL}/${REPORT_PR_HEAD_REPO}/commit/${REPORT_PR_HEAD_SHA}"
          fi
          os_pretty_name="$(. /etc/os-release 2>/dev/null && printf '%s' "${PRETTY_NAME:-}" || true)"
          kernel="$(uname -srmo 2>/dev/null || true)"
          arch="$(uname -m 2>/dev/null || true)"
          cpu_model="$(awk -F: '/model name/ {gsub(/^[ \t]+/, "", $2); print $2; exit}' /proc/cpuinfo 2>/dev/null || true)"
          cpu_cores="$(nproc 2>/dev/null || true)"
          mem_total="$(free -h 2>/dev/null | awk '/^Mem:/ {print $2}' || true)"
          gpu_summary="$(nvidia-smi --query-gpu=index,name,memory.total,driver_version --format=csv,noheader 2>/dev/null | paste -sd ';' - || true)"
          nvidia_driver="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1 || true)"
          cuda_version="$(nvidia-smi 2>/dev/null | sed -n 's/.*CUDA Version: \([^ |]*\).*/\1/p' | head -1 || true)"
          docker_version="$(docker version --format '{{.Server.Version}}' 2>/dev/null || docker --version 2>/dev/null || true)"
          docker_compose_version="$(docker compose version --short 2>/dev/null || docker-compose --version 2>/dev/null || true)"
          webots_version="$(sed -n 's/^WEBOTS_VERSION=//p' "$RUNNER_TEMP/sim-logs/webots.version.txt" 2>/dev/null | head -1 || true)"
          ros_distro="$(sed -n 's/^ROS_DISTRO=//p' "$RUNNER_TEMP/sim-logs/webots.version.txt" 2>/dev/null | head -1 || true)"
          sim_image="$(cat "$RUNNER_TEMP/sim-logs/sim-container-image.txt" 2>/dev/null || true)"
          python_version="$(python3 --version 2>/dev/null || true)"
          rustc_version="$(rustc --version 2>/dev/null || true)"
          rbnx_version="$(rbnx --version 2>/dev/null || true)"
          cache_log_dir="$RUNNER_TEMP/cache-provider-logs"
          rm -rf "$cache_log_dir"
          mkdir -p "$cache_log_dir"
          if [ -d "$GITHUB_WORKSPACE/examples/webots/rbnx-boot/cache" ]; then
            while IFS= read -r -d '' f; do
              rel="${f#"$GITHUB_WORKSPACE/examples/webots/rbnx-boot/cache/"}"
              mkdir -p "$cache_log_dir/$(dirname "$rel")"
              cp "$f" "$cache_log_dir/$rel"
            done < <(find "$GITHUB_WORKSPACE/examples/webots/rbnx-boot/cache" -path '*/logs/*' -type f -print0)
          fi
          REPORT_METADATA_JSON="$RUNNER_TEMP/report-metadata.json" \
          TESTED_COMMIT="$tested_commit" \
          TESTED_COMMIT_SUBJECT="$tested_subject" \
          TESTED_COMMIT_AUTHOR="$tested_author" \
          TESTED_COMMIT_DATE="$tested_date" \
          COMMIT_URL="$commit_url" \
          PR_HEAD_COMMIT_URL="$pr_head_commit_url" \
          ORIGIN_URL="$origin_url" \
          ENV_HOSTNAME="$(hostname 2>/dev/null || true)" \
          ENV_OS_PRETTY_NAME="$os_pretty_name" \
          ENV_KERNEL="$kernel" \
          ENV_ARCH="$arch" \
          ENV_CPU_MODEL="$cpu_model" \
          ENV_CPU_CORES="$cpu_cores" \
          ENV_MEM_TOTAL="$mem_total" \
          ENV_GPU_SUMMARY="$gpu_summary" \
          ENV_NVIDIA_DRIVER="$nvidia_driver" \
          ENV_CUDA_VERSION="$cuda_version" \
          ENV_DOCKER_VERSION="$docker_version" \
          ENV_DOCKER_COMPOSE_VERSION="$docker_compose_version" \
          ENV_WEBOTS_VERSION="$webots_version" \
          ENV_ROS_DISTRO="$ros_distro" \
          ENV_SIM_IMAGE="$sim_image" \
          ENV_PYTHON_VERSION="$python_version" \
          ENV_RUSTC_VERSION="$rustc_version" \
          ENV_RBNX_VERSION="$rbnx_version" \
          python3 - <<'PY'
          import json
          import os
          from datetime import datetime, timezone

          def env(name: str) -> str:
              return os.environ.get(name, "")

          metadata = {
              "generated_on": datetime.now(timezone.utc).isoformat(timespec="seconds"),
              "workflow": env("GITHUB_WORKFLOW"),
              "run_id": env("GITHUB_RUN_ID"),
              "run_attempt": env("GITHUB_RUN_ATTEMPT"),
              "event": env("GITHUB_EVENT_NAME"),
              "repository": env("GITHUB_REPOSITORY"),
              "ref": env("GITHUB_REF"),
              "sha": env("GITHUB_SHA"),
              "actor": env("GITHUB_ACTOR"),
              "triggering_actor": env("GITHUB_TRIGGERING_ACTOR"),
              "runner_name": env("RUNNER_NAME"),
              "runner_os": env("RUNNER_OS"),
              "runner_arch": env("RUNNER_ARCH"),
              "run_url": f"{env('GITHUB_SERVER_URL')}/{env('GITHUB_REPOSITORY')}/actions/runs/{env('GITHUB_RUN_ID')}",
              "tested_commit": env("TESTED_COMMIT"),
              "tested_commit_subject": env("TESTED_COMMIT_SUBJECT"),
              "tested_commit_author": env("TESTED_COMMIT_AUTHOR"),
              "tested_commit_date": env("TESTED_COMMIT_DATE"),
              "commit_url": env("COMMIT_URL"),
              "origin_url": env("ORIGIN_URL"),
              "pr_number": env("REPORT_PR_NUMBER"),
              "pr_title": env("REPORT_PR_TITLE"),
              "pr_author": env("REPORT_PR_AUTHOR"),
              "pr_url": env("REPORT_PR_URL"),
              "pr_head_ref": env("REPORT_PR_HEAD_REF"),
              "pr_base_ref": env("REPORT_PR_BASE_REF"),
              "pr_head_repo": env("REPORT_PR_HEAD_REPO"),
              "pr_head_sha": env("REPORT_PR_HEAD_SHA"),
              "pr_head_commit_url": env("PR_HEAD_COMMIT_URL"),
              "triggered_by": env("REPORT_TRIGGERED_BY"),
              "env_hostname": env("ENV_HOSTNAME"),
              "env_os_pretty_name": env("ENV_OS_PRETTY_NAME"),
              "env_kernel": env("ENV_KERNEL"),
              "env_arch": env("ENV_ARCH"),
              "env_cpu_model": env("ENV_CPU_MODEL"),
              "env_cpu_cores": env("ENV_CPU_CORES"),
              "env_mem_total": env("ENV_MEM_TOTAL"),
              "env_gpu_summary": env("ENV_GPU_SUMMARY"),
              "env_nvidia_driver": env("ENV_NVIDIA_DRIVER"),
              "env_cuda_version": env("ENV_CUDA_VERSION"),
              "env_docker_version": env("ENV_DOCKER_VERSION"),
              "env_docker_compose_version": env("ENV_DOCKER_COMPOSE_VERSION"),
              "env_webots_version": env("ENV_WEBOTS_VERSION"),
              "env_ros_distro": env("ENV_ROS_DISTRO"),
              "env_ros_domain_id": env("ROS_DOMAIN_ID"),
              "env_sim_container": f"ci-{env('RUN_ID')}-sim",
              "env_sim_image": env("ENV_SIM_IMAGE"),
              "env_webots_stream_port": env("ROBONIX_SIM_STREAM_PORT"),
              "env_webots_viewer_port": env("ROBONIX_SIM_VIEWER_PORT"),
              "env_python_version": env("ENV_PYTHON_VERSION"),
              "env_rustc_version": env("ENV_RUSTC_VERSION"),
              "env_rbnx_version": env("ENV_RBNX_VERSION"),
          }
          with open(env("REPORT_METADATA_JSON"), "w", encoding="utf-8") as f:
              json.dump({k: v for k, v in metadata.items() if v}, f, indent=2, sort_keys=True)
              f.write("\n")
          PY
          # URL-less here: the inline image comes from the report job, which
          # knows whether the report site publishes for this event.
          map_preview_args=()
          [ -f "$RUNNER_TEMP/sim-logs/slam-map.png" ] && \
            map_preview_args=(--map-preview "$RUNNER_TEMP/sim-logs/slam-map.png")
          python3 "$GITHUB_WORKSPACE/testing/report.py" \
            --summary-json "$GITHUB_WORKSPACE/testing/logs/summary.json" \
            --out-dir "$GITHUB_WORKSPACE/testing/report" \
            "${map_preview_args[@]}" \
            --metadata-json "$RUNNER_TEMP/report-metadata.json" \
            --log-root "testing/logs=$GITHUB_WORKSPACE/testing/logs" \
            --log-root "boot_logs=$GITHUB_WORKSPACE/examples/webots/rbnx-boot/logs" \
            --log-root "lifecycle_logs=$RUNNER_TEMP/lifecycle-logs" \
            --log-root "cache_provider_logs=$cache_log_dir" \
            --log-root "provider_logs=$GITHUB_WORKSPACE/examples/webots/logs" \
            --log-root "scene_logs=$GITHUB_WORKSPACE/system/scene/logs" \
            --log-root "sim_logs=$RUNNER_TEMP/sim-logs" \
            --log-file "fake_vlm.log=$RUNNER_TEMP/fake_vlm.log" \
            --log-file "rbnx-boot.log=$RUNNER_TEMP/rbnx-boot.log" \
            --log-file "remote-provenance.txt=$RUNNER_TEMP/remote-provenance.txt" \
            --max-log-bytes 524288 \
            --max-total-log-bytes 12582912 \
            --metadata "tested_commit=$tested_commit" \
            --metadata "tested_commit_subject=$tested_subject" \
            --metadata "tested_commit_author=$tested_author" \
            --metadata "tested_commit_date=$tested_date" \
            --metadata "commit_url=$commit_url" \
            --metadata "origin_url=$origin_url" \
            --metadata "pr_number=${REPORT_PR_NUMBER:-}" \
            --metadata "pr_title=${REPORT_PR_TITLE:-}" \
            --metadata "pr_author=${REPORT_PR_AUTHOR:-}" \
            --metadata "pr_url=${REPORT_PR_URL:-}" \
            --metadata "pr_head_ref=${REPORT_PR_HEAD_REF:-}" \
            --metadata "pr_base_ref=${REPORT_PR_BASE_REF:-}" \
            --metadata "pr_head_repo=${REPORT_PR_HEAD_REPO:-}" \
            --metadata "pr_head_sha=${REPORT_PR_HEAD_SHA:-}" \
            --metadata "pr_head_commit_url=$pr_head_commit_url" \
            --metadata "triggered_by=${REPORT_TRIGGERED_BY:-}" \
            --metadata "env_hostname=$(hostname 2>/dev/null || true)" \
            --metadata "env_os_pretty_name=$os_pretty_name" \
            --metadata "env_kernel=$kernel" \
            --metadata "env_arch=$arch" \
            --metadata "env_cpu_model=$cpu_model" \
            --metadata "env_cpu_cores=$cpu_cores" \
            --metadata "env_mem_total=$mem_total" \
            --metadata "env_gpu_summary=$gpu_summary" \
            --metadata "env_nvidia_driver=$nvidia_driver" \
            --metadata "env_cuda_version=$cuda_version" \
            --metadata "env_docker_version=$docker_version" \
            --metadata "env_docker_compose_version=$docker_compose_version" \
            --metadata "env_webots_version=$webots_version" \
            --metadata "env_ros_distro=$ros_distro" \
            --metadata "env_ros_domain_id=${ROS_DOMAIN_ID:-}" \
            --metadata "env_sim_container=ci-${RUN_KEY}-sim" \
            --metadata "env_sim_image=$sim_image" \
            --metadata "env_webots_stream_port=${ROBONIX_SIM_STREAM_PORT:-}" \
            --metadata "env_webots_viewer_port=${ROBONIX_SIM_VIEWER_PORT:-}" \
            --metadata "env_python_version=$python_version" \
            --metadata "env_rustc_version=$rustc_version" \
            --metadata "env_rbnx_version=$rbnx_version" || true
          if [ -f "$GITHUB_WORKSPACE/testing/report/summary.md" ]; then
            cat "$GITHUB_WORKSPACE/testing/report/summary.md" >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Upload logs and report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: testing-logs-${{ github.run_id }}-${{ matrix.key }}
          path: |
            examples/webots/rbnx-boot/logs/
            examples/webots/logs/
            system/scene/logs/
            testing/logs/
            testing/report/
            ${{ runner.temp }}/report-metadata.json
            ${{ runner.temp }}/remote-provenance.txt
            ${{ runner.temp }}/fake_vlm.log
            ${{ runner.temp }}/rbnx-boot.log
            ${{ runner.temp }}/cache-provider-logs/
            ${{ runner.temp }}/sim-logs/
            ${{ runner.temp }}/lifecycle.out
            ${{ runner.temp }}/lifecycle-logs/
            ${{ runner.temp }}/scenarios.out
            ${{ runner.temp }}/scenarios.restart.out
            examples/webots/rbnx-boot/logs/lifecycle-*.log
            examples/webots/rbnx-boot/processes.json
          if-no-files-found: ignore

      - name: Clean runner-local run directory
        if: always()
        run: |
          set -euo pipefail
          target="${ROBONIX_CI_RUNTIME_DIR:-}"
          if [ -z "$target" ]; then
            echo "runner-local runtime directory was not initialized; nothing to clean"
            exit 0
          fi
          case "$target" in
            "$RUNNER_TEMP/robonix-ci-$RUN_KEY") ;;
            *) echo "::error::refusing to clean unexpected runtime directory: ${target:-<unset>}"; exit 1 ;;
          esac
          rm -rf -- "$target" 2>/dev/null || true
          if [ -e "$target" ]; then
            timeout 60s docker run --rm --network none \
              -v "$RUNNER_TEMP:$RUNNER_TEMP:rw" \
              busybox:latest sh -c 'rm -rf -- "$1"' sh "$target"
          fi
          if [ -e "$target" ]; then
            echo "::error::runner-local runtime directory remains after cleanup: $target"
            exit 1
          fi

  report:
    name: Webots report generation (${{ matrix.backend }})
    needs: gpu_testing
    if: always()
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        include:
          - backend: concept_graphs
            key: cg
          - backend: dualmap
            key: dm
    env:
      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
      ENABLE_LLM_ANALYSIS: ${{ inputs.enable_llm_analysis == true }}
      GITHUB_TOKEN: ${{ github.token }}
      REPORT_PR_NUMBER: ${{ inputs.pr_number || github.event.pull_request.number || '' }}
      REPORT_PR_TITLE: ${{ inputs.pr_title || github.event.pull_request.title || '' }}
      REPORT_PR_AUTHOR: ${{ inputs.pr_author || github.event.pull_request.user.login || '' }}
      REPORT_PR_URL: ${{ inputs.pr_url || github.event.pull_request.html_url || '' }}
      REPORT_PR_HEAD_REF: ${{ inputs.pr_head_ref || github.event.pull_request.head.ref || '' }}
      REPORT_PR_BASE_REF: ${{ inputs.pr_base_ref || github.event.pull_request.base.ref || '' }}
      REPORT_PR_HEAD_REPO: ${{ inputs.pr_head_repo || github.event.pull_request.head.repo.full_name || '' }}
      REPORT_PR_HEAD_SHA: ${{ inputs.pr_head_sha || github.event.pull_request.head.sha || '' }}
      REPORT_TEST_REF: ${{ inputs.ref || github.ref }}
      REPORT_TRIGGERED_BY: ${{ inputs.triggered_by || github.triggering_actor || github.actor }}
      # Must mirror PUBLISH_REPORT_SITE in the publish_report job: the map
      # image URL only resolves when the report site actually publishes.
      REPORT_SITE_PUBLISHED: >-
        ${{ inputs.authorized == true ||
            github.event_name == 'push' ||
            github.event_name == 'workflow_dispatch' ||
            github.event_name == 'workflow_call' }}
    steps:
      - name: Checkout trusted report generator
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.repository.default_branch }}

      - name: Download raw Webots artifacts
        uses: actions/download-artifact@v4
        continue-on-error: true
        with:
          name: testing-logs-${{ github.run_id }}-${{ matrix.key }}
          path: dl

      - name: Prepare self-contained report viewer assets
        run: |
          set -euo pipefail
          python3 testing/prepare_report_assets.py --asset-dir "$RUNNER_TEMP/report-assets"

      - name: Build Webots HTML report
        run: |
          set -euo pipefail
          mkdir -p report dl
          run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
          pr_head_commit_url=""
          if [ -n "${REPORT_PR_HEAD_REPO:-}" ] && [ -n "${REPORT_PR_HEAD_SHA:-}" ]; then
            pr_head_commit_url="${GITHUB_SERVER_URL}/${REPORT_PR_HEAD_REPO}/commit/${REPORT_PR_HEAD_SHA}"
          fi
          llm_args=()
          if [ "${ENABLE_LLM_ANALYSIS:-false}" = "true" ]; then
            llm_args+=(--llm)
          fi
          if [ "${REPORT_SITE_PUBLISHED:-false}" = "true" ]; then
            export ROBONIX_REPORT_MAP_URL="https://ci-reports.robonix.ai/reports/runs/${GITHUB_RUN_ID}-${{ matrix.key }}/slam-map.png"
          fi
          python3 testing/build_report_from_artifacts.py \
            --artifact-root dl \
            --out-dir report \
            --repo . \
            "${llm_args[@]}" \
            --metadata "report_id=${GITHUB_RUN_ID}-${{ matrix.key }}" \
            --metadata "scene_backend=${{ matrix.backend }}" \
            --metadata "run_url=${run_url}" \
            --metadata "pr_number=${REPORT_PR_NUMBER:-}" \
            --metadata "pr_title=${REPORT_PR_TITLE:-}" \
            --metadata "pr_author=${REPORT_PR_AUTHOR:-}" \
            --metadata "pr_url=${REPORT_PR_URL:-}" \
            --metadata "pr_head_ref=${REPORT_PR_HEAD_REF:-}" \
            --metadata "pr_base_ref=${REPORT_PR_BASE_REF:-}" \
            --metadata "pr_head_repo=${REPORT_PR_HEAD_REPO:-}" \
            --metadata "pr_head_sha=${REPORT_PR_HEAD_SHA:-}" \
            --metadata "pr_head_commit_url=${pr_head_commit_url}" \
            --metadata "test_ref=${REPORT_TEST_REF:-}" \
            --metadata "triggered_by=${REPORT_TRIGGERED_BY:-}"
          if [ -f report/summary.md ]; then
            cat report/summary.md >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Upload generated report
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: testing-report-${{ github.run_id }}-${{ matrix.key }}
          path: report/
          if-no-files-found: ignore

  publish_report:
    name: Publish Webots report site
    needs: report
    if: always()
    runs-on: ubuntu-22.04
    permissions:
      contents: write
      actions: read
    concurrency:
      group: webots-report-pages
      cancel-in-progress: false
    outputs:
      report_url: ${{ steps.publish_url.outputs.report_url }}
    env:
      REPORT_SITE_DIR: /tmp/robonix-report-site-${{ github.run_id }}
      REPORTS_REPOSITORY: syswonder/robonix-ci-reports
      REPORTS_BRANCH: main
      REPORT_BASE_URL: https://ci-reports.robonix.ai
      PUBLISH_REPORT_SITE: >-
        ${{ inputs.authorized == true ||
            github.event_name == 'push' ||
            github.event_name == 'workflow_dispatch' ||
            github.event_name == 'workflow_call' }}
    steps:
      - name: Mark Pages publish skipped for pull request
        if: env.PUBLISH_REPORT_SITE != 'true'
        run: |
          set -euo pipefail
          echo "Pages publishing is skipped for this pull_request event, but the required job succeeds."
          echo "Download the testing-report-${GITHUB_RUN_ID} artifact for the generated HTML report." >> "$GITHUB_STEP_SUMMARY"

      - name: Checkout repository for report history
        if: env.PUBLISH_REPORT_SITE == 'true'
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1

      - name: Download generated report
        if: env.PUBLISH_REPORT_SITE == 'true'
        uses: actions/download-artifact@v4
        continue-on-error: true
        with:
          pattern: testing-report-${{ github.run_id }}-*
          path: report-dl

      - name: Configure report repository deploy key
        if: env.PUBLISH_REPORT_SITE == 'true'
        env:
          ROBONIX_REPORTS_DEPLOY_KEY: ${{ secrets.ROBONIX_REPORTS_DEPLOY_KEY }}
        run: |
          set -euo pipefail
          if [ -z "${ROBONIX_REPORTS_DEPLOY_KEY:-}" ]; then
            echo "ROBONIX_REPORTS_DEPLOY_KEY is required to publish reports to ${REPORTS_REPOSITORY}" >&2
            exit 1
          fi
          mkdir -p ~/.ssh
          printf '%s\n' "$ROBONIX_REPORTS_DEPLOY_KEY" > ~/.ssh/robonix_reports
          chmod 600 ~/.ssh/robonix_reports
          cat > ~/.ssh/config <<'EOF'
          Host github.com
            HostName github.com
            User git
            IdentityFile ~/.ssh/robonix_reports
            IdentitiesOnly yes
          EOF
          chmod 600 ~/.ssh/config
          ssh-keyscan github.com >> ~/.ssh/known_hosts

      - name: Restore report history
        if: env.PUBLISH_REPORT_SITE == 'true'
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          rm -rf "$REPORT_SITE_DIR"
          if git ls-remote --exit-code "https://github.com/${REPORTS_REPOSITORY}.git" "refs/heads/${REPORTS_BRANCH}" >/dev/null 2>&1; then
            git clone --depth=1 --branch "${REPORTS_BRANCH}" \
              "https://github.com/${REPORTS_REPOSITORY}.git" "$REPORT_SITE_DIR"
          else
            mkdir -p "$REPORT_SITE_DIR"
            git -C "$REPORT_SITE_DIR" init -b "${REPORTS_BRANCH}"
            git -C "$REPORT_SITE_DIR" remote add origin "git@github.com:${REPORTS_REPOSITORY}.git"
          fi

      - name: Stage report site
        id: stage
        if: env.PUBLISH_REPORT_SITE == 'true'
        run: |
          set -euo pipefail
          mkdir -p "${REPORT_SITE_DIR}/reports" report-dl
          # One report per scene backend, each under its own run directory.
          report_id=""
          for src in report-dl/testing-report-*/; do
            [ -f "$src/index.html" ] || continue
            id="$(basename "$src")"; id="${id#testing-report-}"
            rm -rf "${REPORT_SITE_DIR}/reports/runs/${id}"
            mkdir -p "${REPORT_SITE_DIR}/reports/runs/${id}"
            cp -a "$src". "${REPORT_SITE_DIR}/reports/runs/${id}/"
            [ -n "$report_id" ] || report_id="$id"
          done
          if [ -z "$report_id" ]; then
            report_id="${GITHUB_RUN_ID}"
            mkdir -p "${REPORT_SITE_DIR}/reports/runs/${report_id}"
            printf '%s\n' \
              '<!doctype html>' \
              '<meta charset="utf-8">' \
              '<title>Robonix Webots CI Report</title>' \
              '<body><h1>Robonix Webots CI Report</h1><p>No generated report artifact was available for this run.</p></body>' \
              > "${REPORT_SITE_DIR}/reports/runs/${report_id}/index.html"
          fi
          expected_report_url="${REPORT_BASE_URL}/reports/runs/${report_id}/"
          python3 testing/report_index.py --site-dir "$REPORT_SITE_DIR" --retention-days 90
          printf '%s\n' \
            '<!doctype html>' \
            '<meta charset="utf-8">' \
            "<meta http-equiv=\"refresh\" content=\"0; url=reports/runs/${report_id}/\">" \
            '<title>Robonix Webots CI Reports</title>' \
            "<a href=\"reports/runs/${report_id}/\">Latest Robonix Webots CI report</a>" \
            > "${REPORT_SITE_DIR}/index.html"
          : > "${REPORT_SITE_DIR}/.nojekyll"
          echo "report_url=${expected_report_url}" >> "$GITHUB_OUTPUT"

      - name: Persist report history
        if: env.PUBLISH_REPORT_SITE == 'true'
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          git -C "$REPORT_SITE_DIR" config user.name "github-actions[bot]"
          git -C "$REPORT_SITE_DIR" config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          work_tree="$(mktemp -d)"
          rsync -a --exclude='.git' "$REPORT_SITE_DIR"/ "$work_tree"/
          rm -rf "$REPORT_SITE_DIR"
          mkdir -p "$REPORT_SITE_DIR"
          git -C "$REPORT_SITE_DIR" init -b "${REPORTS_BRANCH}"
          git -C "$REPORT_SITE_DIR" remote add origin "git@github.com:${REPORTS_REPOSITORY}.git"
          git -C "$REPORT_SITE_DIR" config user.name "github-actions[bot]"
          git -C "$REPORT_SITE_DIR" config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          rsync -a "$work_tree"/ "$REPORT_SITE_DIR"/
          git -C "$REPORT_SITE_DIR" add -A
          if git -C "$REPORT_SITE_DIR" diff --cached --quiet; then
            echo "report history unchanged"
            exit 0
          fi
          git -C "$REPORT_SITE_DIR" commit -m "ci: publish Webots report ${GITHUB_RUN_ID}"
          git -C "$REPORT_SITE_DIR" push --force origin "HEAD:${REPORTS_BRANCH}"

      - name: Set report URL
        id: publish_url
        if: env.PUBLISH_REPORT_SITE == 'true'
        run: |
          set -euo pipefail
          page_url="${REPORT_BASE_URL}"
          echo "report_url=${{ steps.stage.outputs.report_url }}" >> "$GITHUB_OUTPUT"

      - name: Probe report Pages URL
        if: env.PUBLISH_REPORT_SITE == 'true'
        env:
          REPORT_URL: ${{ steps.publish_url.outputs.report_url }}
        run: |
          set -euo pipefail
          for attempt in $(seq 1 30); do
            if curl -fsSI "$REPORT_URL" >/dev/null; then
              exit 0
            fi
            echo "report URL is not ready yet: ${REPORT_URL} (attempt ${attempt}/30)"
            sleep 10
          done
          echo "::warning::report URL is not reachable yet; GitHub Pages may still be building: ${REPORT_URL}"
          echo "Report URL is not reachable yet; GitHub Pages may still be building: ${REPORT_URL}" >> "$GITHUB_STEP_SUMMARY"

      - name: Record Pages deployment status
        if: env.PUBLISH_REPORT_SITE == 'true'
        env:
          GH_TOKEN: ${{ github.token }}
          REPORT_URL: ${{ steps.publish_url.outputs.report_url }}
        run: |
          set -euo pipefail
          page_url="${REPORT_BASE_URL}/"
          deployment_payload="$(mktemp)"
          status_payload="$(mktemp)"
          jq -n \
            --arg ref "${GITHUB_SHA}" \
            --arg env "robonix-ci-reports" \
            --arg desc "Robonix Webots report site" \
            '{
              ref: $ref,
              environment: $env,
              description: $desc,
              auto_merge: false,
              required_contexts: []
            }' > "$deployment_payload"
          if ! deployment_id="$(
            gh api \
              -X POST \
              "repos/${GITHUB_REPOSITORY}/deployments" \
              --input "$deployment_payload" \
              --jq ".id"
          )"; then
            echo "::warning::unable to create deployment status record for report site; skipping GitHub deployment metadata"
            echo "Skipped GitHub deployment metadata for the report site because the workflow token cannot create deployments." >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi
          jq -n \
            --arg state "success" \
            --arg target_url "$REPORT_URL" \
            --arg environment_url "$page_url" \
            --arg desc "Robonix Webots report is published." \
            '{
              state: $state,
              target_url: $target_url,
              environment_url: $environment_url,
              description: $desc
            }' > "$status_payload"
          if ! gh api \
            -X POST \
            "repos/${GITHUB_REPOSITORY}/deployments/${deployment_id}/statuses" \
            --input "$status_payload" >/dev/null; then
            echo "::warning::unable to update deployment status for report site; report was still published"
            echo "Skipped GitHub deployment status update for the report site because the workflow token cannot update deployments." >> "$GITHUB_STEP_SUMMARY"
          fi