From 5d996f5343d23ddd1ccb795519dcb340b0e71d08 Mon Sep 17 00:00:00 2001
From: Jian Wu <wujia@microsoft.com>
Date: Thu, 11 Jun 2026 16:25:09 +0800
Subject: [PATCH 1/9] Add E2E test pipeline for azure.ai.agents extension
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Copilot CLI-driven pipeline using cli-interactive-tester MCP tool.
Same architecture as local testing — Copilot reads scenario goals
and drives terminal via MCP protocol.

Implementation:
- Copilot CLI installed via npm install -g @github/copilot
- Auth via COPILOT_GITHUB_TOKEN (Fine-grained PAT, Copilot Requests perm)
- MCP config in ~/.copilot/mcp-config.json (auto-loaded by Copilot)
- Execution: copilot -p prompt --allow-tool=... --no-ask-user
- workflow_dispatch with tier selector (0 / 0+1 / 0+1+2)
- ubuntu-22.04 runner
- Checkout: trangevi/test-scenarios (until PR #8524 merges)
- Tier 2 has always-run teardown for Azure resource cleanup
- Results uploaded as artifacts

TODO:
- Confirm --allow-tool syntax for MCP-registered tools
- Configure COPILOT_PAT secret (Fine-grained PAT)
- Confirm cli-interactive-tester repo visibility
- Create prompt-ci-run.md in scenarios directory

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .github/workflows/e2e-ext-azure-ai-agents.yml | 168 ++++++++++++++++++
 1 file changed, 168 insertions(+)
 create mode 100644 .github/workflows/e2e-ext-azure-ai-agents.yml

diff --git a/.github/workflows/e2e-ext-azure-ai-agents.yml b/.github/workflows/e2e-ext-azure-ai-agents.yml
new file mode 100644
index 00000000000..82261369a43
--- /dev/null
+++ b/.github/workflows/e2e-ext-azure-ai-agents.yml
@@ -0,0 +1,168 @@
+name: "ext-azure-ai-agents-e2e"
+
+# E2E tests for the azure.ai.agents CLI extension.
+#
+# Architecture (identical to local testing):
+#   Copilot CLI (LLM brain, reads goals from scenario YAML)
+#     ↕ MCP protocol (stdio)
+#   cli-interactive-tester (MCP server, manages tmux sessions)
+#     ↕ tmux
+#   azd ai agent CLI (under test)
+#
+# Manual trigger only — switch to PR-trigger once pipeline is stable.
+# Scenarios live on branch trangevi/test-scenarios until PR #8524 merges.
+
+on:
+  workflow_dispatch:
+    inputs:
+      tier:
+        description: "Which tiers to run"
+        type: choice
+        options:
+          - "0"
+          - "0+1"
+          - "0+1+2"
+        default: "0"
+      confirm_tier2_cost:
+        description: "Confirm Tier 2 Azure costs (~$2-5)"
+        type: boolean
+        default: false
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  SCENARIOS_DIR: cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios
+  AZD_AGENTS_FIXTURES: cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures
+  # TODO: change to 'main' after PR #8524 merges
+  TARGET_BRANCH: trangevi/test-scenarios
+
+jobs:
+  e2e-test:
+    name: "E2E scenarios (Tier ${{ inputs.tier }})"
+    runs-on: ubuntu-22.04
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          # TODO: remove ref after PR #8524 merges to main
+          ref: ${{ env.TARGET_BRANCH }}
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: "cli/azd/go.mod"
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build azd + install extension
+        working-directory: cli/azd
+        run: |
+          go build -o ./azd .
+          export PATH="$PWD:$PATH"
+          azd extension install azure.ai.agents --source ./extensions/azure.ai.agents
+
+      - name: Install cli-interactive-tester
+        run: |
+          sudo apt-get install -y tmux
+          # TODO: confirm repo visibility. If private, use PAT:
+          #   git clone https://x-access-token:${{ secrets.GH_TOKEN }}@github.com/...
+          git clone https://github.com/coreai-microsoft/cli-interactive-tester.git /tmp/cli-interactive-tester
+          cd /tmp/cli-interactive-tester
+          pip install -e .
+          python -c "from auto_test_tool.mcp_server import main; print('MCP tool ready')"
+
+      - name: Install uv (for Tier 2 run-local scenario)
+        if: contains(inputs.tier, '2')
+        uses: astral-sh/setup-uv@v6
+
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: GitHub CLI Auth
+        run: echo "${{ secrets.GH_TOKEN }}" | gh auth login --with-token
+
+      - name: Create test profile
+        working-directory: ${{ env.SCENARIOS_DIR }}
+        run: |
+          cat > profile.local.yaml <<EOF
+          prefix: "ci-${{ github.run_number }}"
+          subscription: "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
+          region: "North Central US"
+          foundry_project_endpoint: "${{ secrets.FOUNDRY_PROJECT_ENDPOINT }}"
+          EOF
+
+      - name: Generate MCP config
+        run: |
+          cat > ${{ github.workspace }}/.copilot-mcp.json <<EOF
+          {
+            "mcpServers": {
+              "cli-interactive-tester": {
+                "type": "stdio",
+                "command": "python",
+                "args": ["-m", "auto_test_tool.mcp_server"],
+                "cwd": "/tmp/cli-interactive-tester"
+              }
+            }
+          }
+          EOF
+
+      - name: Run scenarios via Copilot + MCP tool
+        working-directory: ${{ env.SCENARIOS_DIR }}
+        env:
+          AZD_AGENTS_FIXTURES: ${{ github.workspace }}/${{ env.AZD_AGENTS_FIXTURES }}
+          UV_HTTP_TIMEOUT: "300"
+          GITHUB_TOKEN: ${{ secrets.COPILOT_TOKEN }}
+        run: |
+          export PATH="${{ github.workspace }}/cli/azd:$PATH"
+          #
+          # Copilot drives the MCP tool to execute each scenario:
+          #   load_scenario → run_pre_hooks → start_session
+          #   → observe terminal + send keystrokes to accomplish goals
+          #   → finish_session → run_post_hooks
+          #
+          # This is the SAME architecture as local testing — Copilot reads
+          # the scenario YAML goals and autonomously navigates the CLI.
+          #
+          # TODO: confirm how to invoke Copilot CLI headlessly in CI.
+          # Options being evaluated:
+          #   1. `copilot --mcp-config ... --prompt-file ...` (needs COPILOT_TOKEN)
+          #   2. `gh copilot run` with piped prompt
+          #   3. GitHub Copilot Extensions API (programmatic invocation)
+          #
+          copilot \
+            --mcp-config ${{ github.workspace }}/.copilot-mcp.json \
+            --prompt-file prompt-ci-run.md
+
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-results-tier-${{ inputs.tier }}
+          path: |
+            ${{ env.SCENARIOS_DIR }}/reports/
+            ${{ env.SCENARIOS_DIR }}/full-pipeline-run-results.md
+          retention-days: 30
+
+      - name: Teardown (Tier 2 always cleanup)
+        if: always() && contains(inputs.tier, '2')
+        working-directory: ${{ env.SCENARIOS_DIR }}
+        env:
+          AZD_AGENTS_FIXTURES: ${{ github.workspace }}/${{ env.AZD_AGENTS_FIXTURES }}
+        run: |
+          export PATH="${{ github.workspace }}/cli/azd:$PATH"
+          # Direct teardown (not Copilot) for reliability on failure
+          cd ~/working/azd-agents-shared/*/ 2>/dev/null && \
+            azd down --force --purge || true
+
+
+

From 1fe7bdf06b74d36593cb1fa23a9e49040b9c5764 Mon Sep 17 00:00:00 2001
From: Jian Wu <wujia@microsoft.com>
Date: Thu, 11 Jun 2026 18:34:31 +0800
Subject: [PATCH 2/9] fix: align workflow with official Copilot CLI docs

- Add setup-node + npm install -g @github/copilot
- Use COPILOT_GITHUB_TOKEN env var with COPILOT_PAT secret
- MCP config at ~/.copilot/mcp-config.json (auto-loaded)
- Use copilot -p with --allow-tool and --no-ask-user
- Add conditional on Azure Login (skip for Tier 0)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .github/workflows/e2e-ext-azure-ai-agents.yml | 62 ++++++++++---------
 1 file changed, 32 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/e2e-ext-azure-ai-agents.yml b/.github/workflows/e2e-ext-azure-ai-agents.yml
index 82261369a43..6be8e07d83f 100644
--- a/.github/workflows/e2e-ext-azure-ai-agents.yml
+++ b/.github/workflows/e2e-ext-azure-ai-agents.yml
@@ -3,12 +3,15 @@ name: "ext-azure-ai-agents-e2e"
 # E2E tests for the azure.ai.agents CLI extension.
 #
 # Architecture (identical to local testing):
-#   Copilot CLI (LLM brain, reads goals from scenario YAML)
-#     ↕ MCP protocol (stdio)
+#   Copilot CLI (LLM brain, installed via npm)
+#     ↕ MCP protocol (stdio, config in ~/.copilot/mcp-config.json)
 #   cli-interactive-tester (MCP server, manages tmux sessions)
 #     ↕ tmux
 #   azd ai agent CLI (under test)
 #
+# Copilot CLI runs in programmatic mode (-p), reads scenario goals from YAML,
+# and drives the terminal via MCP tool — same flow as local testing.
+#
 # Manual trigger only — switch to PR-trigger once pipeline is stable.
 # Scenarios live on branch trangevi/test-scenarios until PR #8524 merges.
 
@@ -38,8 +41,6 @@ permissions:
 env:
   SCENARIOS_DIR: cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios
   AZD_AGENTS_FIXTURES: cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures
-  # TODO: change to 'main' after PR #8524 merges
-  TARGET_BRANCH: trangevi/test-scenarios
 
 jobs:
   e2e-test:
@@ -49,8 +50,10 @@ jobs:
     steps:
       - uses: actions/checkout@v6
         with:
-          # TODO: remove ref after PR #8524 merges to main
-          ref: ${{ env.TARGET_BRANCH }}
+          # TODO: change to 'main' after PR #8524 merges
+          ref: trangevi/test-scenarios
+
+      - uses: actions/setup-node@v4
 
       - uses: actions/setup-go@v6
         with:
@@ -60,6 +63,9 @@ jobs:
         with:
           python-version: "3.12"
 
+      - name: Install Copilot CLI
+        run: npm install -g @github/copilot
+
       - name: Build azd + install extension
         working-directory: cli/azd
         run: |
@@ -70,18 +76,18 @@ jobs:
       - name: Install cli-interactive-tester
         run: |
           sudo apt-get install -y tmux
-          # TODO: confirm repo visibility. If private, use PAT:
-          #   git clone https://x-access-token:${{ secrets.GH_TOKEN }}@github.com/...
+          # TODO: confirm repo visibility. If private, use:
+          #   git clone https://x-access-token:${{ secrets.GH_TOKEN }}@github.com/coreai-microsoft/cli-interactive-tester.git
           git clone https://github.com/coreai-microsoft/cli-interactive-tester.git /tmp/cli-interactive-tester
           cd /tmp/cli-interactive-tester
           pip install -e .
-          python -c "from auto_test_tool.mcp_server import main; print('MCP tool ready')"
 
       - name: Install uv (for Tier 2 run-local scenario)
         if: contains(inputs.tier, '2')
         uses: astral-sh/setup-uv@v6
 
       - name: Azure Login
+        if: contains(inputs.tier, '1') || contains(inputs.tier, '2')
         uses: azure/login@v2
         with:
           client-id: ${{ secrets.AZURE_CLIENT_ID }}
@@ -89,6 +95,7 @@ jobs:
           subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
 
       - name: GitHub CLI Auth
+        if: contains(inputs.tier, '1') || contains(inputs.tier, '2')
         run: echo "${{ secrets.GH_TOKEN }}" | gh auth login --with-token
 
       - name: Create test profile
@@ -101,9 +108,10 @@ jobs:
           foundry_project_endpoint: "${{ secrets.FOUNDRY_PROJECT_ENDPOINT }}"
           EOF
 
-      - name: Generate MCP config
+      - name: Configure MCP for Copilot
         run: |
-          cat > ${{ github.workspace }}/.copilot-mcp.json <<EOF
+          mkdir -p ~/.copilot
+          cat > ~/.copilot/mcp-config.json <<'EOF'
           {
             "mcpServers": {
               "cli-interactive-tester": {
@@ -116,32 +124,24 @@ jobs:
           }
           EOF
 
-      - name: Run scenarios via Copilot + MCP tool
+      - name: Run E2E scenarios via Copilot + MCP tool
         working-directory: ${{ env.SCENARIOS_DIR }}
         env:
+          COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_PAT }}
           AZD_AGENTS_FIXTURES: ${{ github.workspace }}/${{ env.AZD_AGENTS_FIXTURES }}
           UV_HTTP_TIMEOUT: "300"
-          GITHUB_TOKEN: ${{ secrets.COPILOT_TOKEN }}
         run: |
           export PATH="${{ github.workspace }}/cli/azd:$PATH"
+          # Copilot CLI in programmatic mode:
+          #   -p = non-interactive, exit on completion
+          #   --no-ask-user = never prompt for user input
+          #   --allow-tool = pre-authorize MCP tool access
           #
-          # Copilot drives the MCP tool to execute each scenario:
-          #   load_scenario → run_pre_hooks → start_session
-          #   → observe terminal + send keystrokes to accomplish goals
-          #   → finish_session → run_post_hooks
-          #
-          # This is the SAME architecture as local testing — Copilot reads
-          # the scenario YAML goals and autonomously navigates the CLI.
-          #
-          # TODO: confirm how to invoke Copilot CLI headlessly in CI.
-          # Options being evaluated:
-          #   1. `copilot --mcp-config ... --prompt-file ...` (needs COPILOT_TOKEN)
-          #   2. `gh copilot run` with piped prompt
-          #   3. GitHub Copilot Extensions API (programmatic invocation)
-          #
-          copilot \
-            --mcp-config ${{ github.workspace }}/.copilot-mcp.json \
-            --prompt-file prompt-ci-run.md
+          # TODO: confirm --allow-tool syntax for MCP-registered tools.
+          # Possible forms: 'cli-interactive-tester(*)', 'cli-interactive-tester'
+          copilot -p "$(cat prompt-ci-run.md)" \
+            --allow-tool='cli-interactive-tester(*)' \
+            --no-ask-user
 
       - name: Upload results
         if: always()
@@ -166,3 +166,5 @@ jobs:
 
 
 
+
+

From 8b8b0e7f7b0d35542bbd6bcfe3a89157a9bddece Mon Sep 17 00:00:00 2001
From: Jian Wu <wujia@microsoft.com>
Date: Thu, 11 Jun 2026 18:38:53 +0800
Subject: [PATCH 3/9] fix: heredoc indentation, concurrency group, prompt file
 check

- Use printf instead of heredoc to avoid leading whitespace in
  profile.local.yaml and mcp-config.json
- Fix concurrency group to prevent parallel runs (was using run_id)
- Add existence check for prompt-ci-run.md before Copilot invocation

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .github/workflows/e2e-ext-azure-ai-agents.yml | 42 ++++++++++---------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/e2e-ext-azure-ai-agents.yml b/.github/workflows/e2e-ext-azure-ai-agents.yml
index 6be8e07d83f..6276fc918af 100644
--- a/.github/workflows/e2e-ext-azure-ai-agents.yml
+++ b/.github/workflows/e2e-ext-azure-ai-agents.yml
@@ -32,8 +32,8 @@ on:
         default: false
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.run_id }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
 
 permissions:
   contents: read
@@ -101,28 +101,26 @@ jobs:
       - name: Create test profile
         working-directory: ${{ env.SCENARIOS_DIR }}
         run: |
-          cat > profile.local.yaml <<EOF
-          prefix: "ci-${{ github.run_number }}"
-          subscription: "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
-          region: "North Central US"
-          foundry_project_endpoint: "${{ secrets.FOUNDRY_PROJECT_ENDPOINT }}"
-          EOF
+          printf '%s\n' \
+            'prefix: "ci-${{ github.run_number }}"' \
+            'subscription: "${{ secrets.AZURE_SUBSCRIPTION_ID }}"' \
+            'region: "North Central US"' \
+            'foundry_project_endpoint: "${{ secrets.FOUNDRY_PROJECT_ENDPOINT }}"' \
+            > profile.local.yaml
 
       - name: Configure MCP for Copilot
         run: |
           mkdir -p ~/.copilot
-          cat > ~/.copilot/mcp-config.json <<'EOF'
-          {
-            "mcpServers": {
-              "cli-interactive-tester": {
-                "type": "stdio",
-                "command": "python",
-                "args": ["-m", "auto_test_tool.mcp_server"],
-                "cwd": "/tmp/cli-interactive-tester"
-              }
-            }
-          }
-          EOF
+          printf '%s\n' '{' \
+            '  "mcpServers": {' \
+            '    "cli-interactive-tester": {' \
+            '      "type": "stdio",' \
+            '      "command": "python",' \
+            '      "args": ["-m", "auto_test_tool.mcp_server"],' \
+            '      "cwd": "/tmp/cli-interactive-tester"' \
+            '    }' \
+            '  }' \
+            '}' > ~/.copilot/mcp-config.json
 
       - name: Run E2E scenarios via Copilot + MCP tool
         working-directory: ${{ env.SCENARIOS_DIR }}
@@ -132,6 +130,10 @@ jobs:
           UV_HTTP_TIMEOUT: "300"
         run: |
           export PATH="${{ github.workspace }}/cli/azd:$PATH"
+          if [ ! -f prompt-ci-run.md ]; then
+            echo "::error::prompt-ci-run.md not found in $(pwd)"
+            exit 1
+          fi
           # Copilot CLI in programmatic mode:
           #   -p = non-interactive, exit on completion
           #   --no-ask-user = never prompt for user input

From 47beb1eb880f3e3f4519adb6dba8eea05e12bd2b Mon Sep 17 00:00:00 2001
From: Jian Wu <wujia@microsoft.com>
Date: Thu, 11 Jun 2026 18:43:06 +0800
Subject: [PATCH 4/9] feat: add prompt-ci-run.md and pass TIER env to Copilot

- Create CI-adapted prompt file for Copilot CLI execution
- Pass TIER input as env var so Copilot knows which phases to run
- Prompt includes profile loading, scenario ordering, and output format

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .github/workflows/e2e-ext-azure-ai-agents.yml |   1 +
 .../prompt-ci-run.md                          | 114 ++++++++++++++++++
 2 files changed, 115 insertions(+)
 create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/prompt-ci-run.md

diff --git a/.github/workflows/e2e-ext-azure-ai-agents.yml b/.github/workflows/e2e-ext-azure-ai-agents.yml
index 6276fc918af..0cd94b65209 100644
--- a/.github/workflows/e2e-ext-azure-ai-agents.yml
+++ b/.github/workflows/e2e-ext-azure-ai-agents.yml
@@ -126,6 +126,7 @@ jobs:
         working-directory: ${{ env.SCENARIOS_DIR }}
         env:
           COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_PAT }}
+          TIER: ${{ inputs.tier }}
           AZD_AGENTS_FIXTURES: ${{ github.workspace }}/${{ env.AZD_AGENTS_FIXTURES }}
           UV_HTTP_TIMEOUT: "300"
         run: |
diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/prompt-ci-run.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/prompt-ci-run.md
new file mode 100644
index 00000000000..22d433715e4
--- /dev/null
+++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/prompt-ci-run.md
@@ -0,0 +1,114 @@
+# CI Pipeline Prompt — E2E Test Scenarios
+
+Autonomously run azure.ai.agents CLI test scenarios using the cli-interactive-tester MCP tool. Do NOT ask questions — make all decisions yourself and run to completion.
+
+## Environment
+
+- Runner: Ubuntu 22.04 (GitHub Actions)
+- azd is on PATH (pre-built in earlier step)
+- Azure auth: active (federated identity, earlier step)
+- GitHub CLI auth: active (earlier step)
+- Scenarios directory: current working directory
+- All paths are POSIX (Linux runner)
+
+## Profile Setup
+
+Read and merge (local overrides shared):
+1. `./profile.yaml`
+2. `./profile.local.yaml`
+
+Derive `shared_agent_name = "{prefix}-{shared_agent_suffix}"`. Pass merged map as `session_vars` on every MCP call.
+
+## Tier Selection
+
+Check the environment variable `TIER` to determine which tiers to run:
+- `TIER=0` → run Phase 0 only
+- `TIER=0+1` → run Phase 0 + Phase 1
+- `TIER=0+1+2` → run all phases
+
+If `TIER` is not set, default to `0`.
+
+## Run Plan
+
+Record the START TIME at the beginning.
+
+### Phase 0 — Tier 0 (offline, no auth, fast)
+
+Run ALL `00-*.yaml` scenarios sequentially:
+1. `00-version.yaml`
+2. `00-help-root.yaml`
+3. `00-sample-list-text.yaml`
+4. `00-sample-list-json-filters.yaml`
+5. `00-doctor-empty-dir.yaml`
+6. `00-doctor-local-only.yaml`
+7. `00-doctor-partial-failure.yaml`
+8. `00-init-validate-mutually-exclusive.yaml`
+9. `00-init-validate-no-prompt-missing.yaml`
+10. `00-invoke-validate-protocol.yaml`
+11. `00-eval-context-required.yaml`
+12. `00-optimize-apply-requires-candidate.yaml`
+13. `00-delete-help.yaml`
+14. `00-endpoint-show-help.yaml`
+15. `00-code-download-help.yaml`
+16. `00-init-picker-navigation.yaml` — interactive picker UX, abort with Ctrl-C
+
+### Phase 1 — Tier 1 (auth required, scaffold only, NO azd provision)
+
+1. `10-init-template-python.yaml`
+2. `10-init-template-dotnet.yaml`
+3. `10-init-deploy-mode-code.yaml`
+4. `10-init-deploy-mode-container.yaml`
+5. `10-init-from-code.yaml`
+6. `10-init-from-manifest-url.yaml`
+7. `10-init-flags-agent-name-model.yaml`
+8. `10-init-validate-deploy-mode.yaml`
+
+### Phase 2 — Tier 2 (real Azure resources, strict order)
+
+1. `20-setup-deploy-shared-agent.yaml` — FIRST
+2. `21-show.yaml`
+3. `21-show-json.yaml`
+4. `22-invoke-remote.yaml`
+5. `22-invoke-input-file.yaml`
+6. `22-invoke-new-session.yaml`
+7. `23-invoke-protocol-invocations.yaml`
+8. `23-sessions-lifecycle.yaml`
+9. `24-files-lifecycle.yaml`
+10. `25-monitor-console.yaml`
+11. `25-monitor-system.yaml`
+12. `26-endpoint-update.yaml`
+13. `27-run-local-and-invoke-local.yaml` — needs two sessions (port allocation)
+14. `28-eval-lifecycle.yaml`
+15. `29-optimize-submit-and-cancel.yaml`
+16. `2A-doctor-provisioned-all-pass.yaml`
+17. `2B-endpoint-show.yaml`
+18. `2C-code-download.yaml`
+19. `2D-delete.yaml`
+20. `2Z-teardown-down.yaml` — LAST
+
+## Rules
+
+- Per scenario: `load_scenario` → `run_pre_hooks` (if any) → `start_session` → accomplish goals → `finish_session` → `run_post_hooks` (if any).
+- Use `run_name=<scenario-stem>` for each `start_session`.
+- Record start/end time for EACH scenario (wall clock).
+- If a scenario fails, record FAIL with reason and **continue to next** (do not abort).
+- Don't verify/retry after a select — treat select miss as hard failure.
+- Prefer `choice_text` over `choice_index`.
+- Clear pre-filled text fields before typing (select-all + delete).
+- For Tier 2 setup: select Container deploy mode, Python language, Basic Responses template.
+- For subscription selection: use the subscription from profile.
+- For region: select region from profile.
+- For model: select model from profile.
+
+## Output
+
+When all scenarios are done, write a markdown report to `./full-pipeline-run-results.md` with:
+
+1. **Summary table**: total scenarios, PASS/FAIL/SKIP counts per tier
+2. **Per-scenario detail**: name, tier, result, duration, notes
+3. **Timing**: total wall clock, time per tier, top 5 slowest
+4. **Issues**: any bugs, failures, or unexpected behaviors
+
+Record END TIME and total duration.
+
+Start now. Begin with Phase 0.

From cd001898232f5198b062b8876fddce2f16c03738 Mon Sep 17 00:00:00 2001
From: Jian Wu <wujia@microsoft.com>
Date: Thu, 11 Jun 2026 18:46:21 +0800
Subject: [PATCH 5/9] fix: remove cwd from MCP config, use bash -c cd instead

cwd may not be supported by Copilot CLI's MCP config schema.
Use bash wrapper to cd before launching the MCP server.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .github/workflows/e2e-ext-azure-ai-agents.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-ext-azure-ai-agents.yml b/.github/workflows/e2e-ext-azure-ai-agents.yml
index 0cd94b65209..b9737eab262 100644
--- a/.github/workflows/e2e-ext-azure-ai-agents.yml
+++ b/.github/workflows/e2e-ext-azure-ai-agents.yml
@@ -115,9 +115,8 @@ jobs:
             '  "mcpServers": {' \
             '    "cli-interactive-tester": {' \
             '      "type": "stdio",' \
-            '      "command": "python",' \
-            '      "args": ["-m", "auto_test_tool.mcp_server"],' \
-            '      "cwd": "/tmp/cli-interactive-tester"' \
+            '      "command": "bash",' \
+            '      "args": ["-c", "cd /tmp/cli-interactive-tester && python -m auto_test_tool.mcp_server"]' \
             '    }' \
             '  }' \
             '}' > ~/.copilot/mcp-config.json

From 15ec2396794455ca0484aa8f77ac019abda70246 Mon Sep 17 00:00:00 2001
From: Jian Wu <wujia@microsoft.com>
Date: Fri, 12 Jun 2026 11:01:34 +0800
Subject: [PATCH 6/9] fix: add id-token:write permission, clean trailing lines

Required for azure/login OIDC federated credentials (Tier 1/2).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .github/workflows/e2e-ext-azure-ai-agents.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/e2e-ext-azure-ai-agents.yml b/.github/workflows/e2e-ext-azure-ai-agents.yml
index b9737eab262..da4f01d0432 100644
--- a/.github/workflows/e2e-ext-azure-ai-agents.yml
+++ b/.github/workflows/e2e-ext-azure-ai-agents.yml
@@ -37,6 +37,7 @@ concurrency:
 
 permissions:
   contents: read
+  id-token: write  # Required for azure/login OIDC federated credentials
 
 env:
   SCENARIOS_DIR: cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios
@@ -166,7 +167,3 @@ jobs:
           cd ~/working/azd-agents-shared/*/ 2>/dev/null && \
             azd down --force --purge || true
 
-
-
-
-

From 424d3098ed191ae1e7f768b15733c2e486019efe Mon Sep 17 00:00:00 2001
From: Jian Wu <wujia@microsoft.com>
Date: Fri, 12 Jun 2026 11:21:22 +0800
Subject: [PATCH 7/9] =?UTF-8?q?fix:=20address=20Glen's=20review=20?=
 =?UTF-8?q?=E2=80=94=20Tier=200=20only,=20comment=20out=20Tier=201/2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Remove id-token:write (not needed for Tier 0)
- Comment out Tier 1/2 steps with TODO explaining ADO vs GHA decision
- Fix #3: teardown glob guard with warning (commented out)
- Fix #4: secrets via env: not inline shell (commented out)
- Fix #5: pin node-version: 20
- Fix #7: use GITHUB_PATH instead of per-step export

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .github/workflows/e2e-ext-azure-ai-agents.yml | 88 +++++++++++--------
 1 file changed, 51 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/e2e-ext-azure-ai-agents.yml b/.github/workflows/e2e-ext-azure-ai-agents.yml
index da4f01d0432..78d9cc3eff8 100644
--- a/.github/workflows/e2e-ext-azure-ai-agents.yml
+++ b/.github/workflows/e2e-ext-azure-ai-agents.yml
@@ -37,7 +37,7 @@ concurrency:
 
 permissions:
   contents: read
-  id-token: write  # Required for azure/login OIDC federated credentials
+  # NOTE: If Tier 1/2 re-enabled in GitHub Actions, add: id-token: write (required for azure/login OIDC)
 
 env:
   SCENARIOS_DIR: cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios
@@ -55,6 +55,8 @@ jobs:
           ref: trangevi/test-scenarios
 
       - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
 
       - uses: actions/setup-go@v6
         with:
@@ -73,6 +75,7 @@ jobs:
           go build -o ./azd .
           export PATH="$PWD:$PATH"
           azd extension install azure.ai.agents --source ./extensions/azure.ai.agents
+          echo "${{ github.workspace }}/cli/azd" >> "$GITHUB_PATH"
 
       - name: Install cli-interactive-tester
         run: |
@@ -83,31 +86,41 @@ jobs:
           cd /tmp/cli-interactive-tester
           pip install -e .
 
-      - name: Install uv (for Tier 2 run-local scenario)
-        if: contains(inputs.tier, '2')
-        uses: astral-sh/setup-uv@v6
-
-      - name: Azure Login
-        if: contains(inputs.tier, '1') || contains(inputs.tier, '2')
-        uses: azure/login@v2
-        with:
-          client-id: ${{ secrets.AZURE_CLIENT_ID }}
-          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
-          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-
-      - name: GitHub CLI Auth
-        if: contains(inputs.tier, '1') || contains(inputs.tier, '2')
-        run: echo "${{ secrets.GH_TOKEN }}" | gh auth login --with-token
-
-      - name: Create test profile
-        working-directory: ${{ env.SCENARIOS_DIR }}
-        run: |
-          printf '%s\n' \
-            'prefix: "ci-${{ github.run_number }}"' \
-            'subscription: "${{ secrets.AZURE_SUBSCRIPTION_ID }}"' \
-            'region: "North Central US"' \
-            'foundry_project_endpoint: "${{ secrets.FOUNDRY_PROJECT_ENDPOINT }}"' \
-            > profile.local.yaml
+      # ─────────────────────────────────────────────────────────────────
+      # TODO: Tier 1/2 steps below are commented out pending discussion:
+      #   - Option A: Move Tier 1/2 to ADO pipeline (per cli/azd/AGENTS.md convention)
+      #   - Option B: Keep in GitHub Actions (requires id-token: write, OIDC secrets)
+      # Re-enable once decision is made.
+      # ─────────────────────────────────────────────────────────────────
+
+      # - name: Install uv (for Tier 2 run-local scenario)
+      #   if: contains(inputs.tier, '2')
+      #   uses: astral-sh/setup-uv@v6
+
+      # - name: Azure Login
+      #   if: contains(inputs.tier, '1') || contains(inputs.tier, '2')
+      #   uses: azure/login@v2
+      #   with:
+      #     client-id: ${{ secrets.AZURE_CLIENT_ID }}
+      #     tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+      #     subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      # - name: GitHub CLI Auth
+      #   if: contains(inputs.tier, '1') || contains(inputs.tier, '2')
+      #   run: echo "${{ secrets.GH_TOKEN }}" | gh auth login --with-token
+
+      # - name: Create test profile
+      #   working-directory: ${{ env.SCENARIOS_DIR }}
+      #   env:
+      #     AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      #     FOUNDRY_PROJECT_ENDPOINT: ${{ secrets.FOUNDRY_PROJECT_ENDPOINT }}
+      #   run: |
+      #     printf '%s\n' \
+      #       "prefix: \"ci-${{ github.run_number }}\"" \
+      #       "subscription: \"${AZURE_SUBSCRIPTION_ID}\"" \
+      #       "region: \"North Central US\"" \
+      #       "foundry_project_endpoint: \"${FOUNDRY_PROJECT_ENDPOINT}\"" \
+      #       > profile.local.yaml
 
       - name: Configure MCP for Copilot
         run: |
@@ -130,7 +143,6 @@ jobs:
           AZD_AGENTS_FIXTURES: ${{ github.workspace }}/${{ env.AZD_AGENTS_FIXTURES }}
           UV_HTTP_TIMEOUT: "300"
         run: |
-          export PATH="${{ github.workspace }}/cli/azd:$PATH"
           if [ ! -f prompt-ci-run.md ]; then
             echo "::error::prompt-ci-run.md not found in $(pwd)"
             exit 1
@@ -156,14 +168,16 @@ jobs:
             ${{ env.SCENARIOS_DIR }}/full-pipeline-run-results.md
           retention-days: 30
 
-      - name: Teardown (Tier 2 always cleanup)
-        if: always() && contains(inputs.tier, '2')
-        working-directory: ${{ env.SCENARIOS_DIR }}
-        env:
-          AZD_AGENTS_FIXTURES: ${{ github.workspace }}/${{ env.AZD_AGENTS_FIXTURES }}
-        run: |
-          export PATH="${{ github.workspace }}/cli/azd:$PATH"
-          # Direct teardown (not Copilot) for reliability on failure
-          cd ~/working/azd-agents-shared/*/ 2>/dev/null && \
-            azd down --force --purge || true
+      # - name: Teardown (Tier 2 always cleanup)
+      #   if: always() && contains(inputs.tier, '2')
+      #   working-directory: ${{ env.SCENARIOS_DIR }}
+      #   env:
+      #     AZD_AGENTS_FIXTURES: ${{ github.workspace }}/${{ env.AZD_AGENTS_FIXTURES }}
+      #   run: |
+      #     WORK_DIR=$(ls -d ~/working/azd-agents-shared/*/ 2>/dev/null | head -1)
+      #     if [ -z "$WORK_DIR" ]; then
+      #       echo "::warning::No working directory found — teardown skipped, check for leaked resources"
+      #       exit 0
+      #     fi
+      #     cd "$WORK_DIR" && azd down --force --purge
 

From d08b5c92aeb5ea6966cfa449f2c15c4472bf0064 Mon Sep 17 00:00:00 2001
From: Jian Wu <wujia@microsoft.com>
Date: Fri, 12 Jun 2026 11:33:37 +0800
Subject: [PATCH 8/9] refactor: re-enable all tiers, apply review fixes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

All tiers (0/1/2) are now active — if team decides GHA is not
appropriate for authenticated workloads, PR can be abandoned.

Fixes from Glen's review:
- id-token: write restored (required for azure/login OIDC)
- Teardown: glob guard with ::warning:: instead of silent || true
- Secrets: passed via env: instead of inline shell interpolation
- setup-node: pin node-version 20
- PATH: use GITHUB_PATH once instead of per-step export
- Create test profile: add if: condition (skip for Tier 0)
- Trailing whitespace cleaned

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .github/workflows/e2e-ext-azure-ai-agents.yml | 89 +++++++++----------
 1 file changed, 40 insertions(+), 49 deletions(-)

diff --git a/.github/workflows/e2e-ext-azure-ai-agents.yml b/.github/workflows/e2e-ext-azure-ai-agents.yml
index 78d9cc3eff8..cb34c39a8d9 100644
--- a/.github/workflows/e2e-ext-azure-ai-agents.yml
+++ b/.github/workflows/e2e-ext-azure-ai-agents.yml
@@ -37,7 +37,7 @@ concurrency:
 
 permissions:
   contents: read
-  # NOTE: If Tier 1/2 re-enabled in GitHub Actions, add: id-token: write (required for azure/login OIDC)
+  id-token: write  # Required for azure/login OIDC federated credentials (Tier 1/2)
 
 env:
   SCENARIOS_DIR: cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios
@@ -86,41 +86,35 @@ jobs:
           cd /tmp/cli-interactive-tester
           pip install -e .
 
-      # ─────────────────────────────────────────────────────────────────
-      # TODO: Tier 1/2 steps below are commented out pending discussion:
-      #   - Option A: Move Tier 1/2 to ADO pipeline (per cli/azd/AGENTS.md convention)
-      #   - Option B: Keep in GitHub Actions (requires id-token: write, OIDC secrets)
-      # Re-enable once decision is made.
-      # ─────────────────────────────────────────────────────────────────
-
-      # - name: Install uv (for Tier 2 run-local scenario)
-      #   if: contains(inputs.tier, '2')
-      #   uses: astral-sh/setup-uv@v6
-
-      # - name: Azure Login
-      #   if: contains(inputs.tier, '1') || contains(inputs.tier, '2')
-      #   uses: azure/login@v2
-      #   with:
-      #     client-id: ${{ secrets.AZURE_CLIENT_ID }}
-      #     tenant-id: ${{ secrets.AZURE_TENANT_ID }}
-      #     subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-
-      # - name: GitHub CLI Auth
-      #   if: contains(inputs.tier, '1') || contains(inputs.tier, '2')
-      #   run: echo "${{ secrets.GH_TOKEN }}" | gh auth login --with-token
-
-      # - name: Create test profile
-      #   working-directory: ${{ env.SCENARIOS_DIR }}
-      #   env:
-      #     AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-      #     FOUNDRY_PROJECT_ENDPOINT: ${{ secrets.FOUNDRY_PROJECT_ENDPOINT }}
-      #   run: |
-      #     printf '%s\n' \
-      #       "prefix: \"ci-${{ github.run_number }}\"" \
-      #       "subscription: \"${AZURE_SUBSCRIPTION_ID}\"" \
-      #       "region: \"North Central US\"" \
-      #       "foundry_project_endpoint: \"${FOUNDRY_PROJECT_ENDPOINT}\"" \
-      #       > profile.local.yaml
+      - name: Install uv (for Tier 2 run-local scenario)
+        if: contains(inputs.tier, '2')
+        uses: astral-sh/setup-uv@v6
+
+      - name: Azure Login
+        if: contains(inputs.tier, '1') || contains(inputs.tier, '2')
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: GitHub CLI Auth
+        if: contains(inputs.tier, '1') || contains(inputs.tier, '2')
+        run: echo "${{ secrets.GH_TOKEN }}" | gh auth login --with-token
+
+      - name: Create test profile
+        if: contains(inputs.tier, '1') || contains(inputs.tier, '2')
+        working-directory: ${{ env.SCENARIOS_DIR }}
+        env:
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          FOUNDRY_PROJECT_ENDPOINT: ${{ secrets.FOUNDRY_PROJECT_ENDPOINT }}
+        run: |
+          printf '%s\n' \
+            "prefix: \"ci-${{ github.run_number }}\"" \
+            "subscription: \"${AZURE_SUBSCRIPTION_ID}\"" \
+            "region: \"North Central US\"" \
+            "foundry_project_endpoint: \"${FOUNDRY_PROJECT_ENDPOINT}\"" \
+            > profile.local.yaml
 
       - name: Configure MCP for Copilot
         run: |
@@ -168,16 +162,13 @@ jobs:
             ${{ env.SCENARIOS_DIR }}/full-pipeline-run-results.md
           retention-days: 30
 
-      # - name: Teardown (Tier 2 always cleanup)
-      #   if: always() && contains(inputs.tier, '2')
-      #   working-directory: ${{ env.SCENARIOS_DIR }}
-      #   env:
-      #     AZD_AGENTS_FIXTURES: ${{ github.workspace }}/${{ env.AZD_AGENTS_FIXTURES }}
-      #   run: |
-      #     WORK_DIR=$(ls -d ~/working/azd-agents-shared/*/ 2>/dev/null | head -1)
-      #     if [ -z "$WORK_DIR" ]; then
-      #       echo "::warning::No working directory found — teardown skipped, check for leaked resources"
-      #       exit 0
-      #     fi
-      #     cd "$WORK_DIR" && azd down --force --purge
-
+      - name: Teardown (Tier 2 always cleanup)
+        if: always() && contains(inputs.tier, '2')
+        working-directory: ${{ env.SCENARIOS_DIR }}
+        run: |
+          WORK_DIR=$(ls -d ~/working/azd-agents-shared/*/ 2>/dev/null | head -1)
+          if [ -z "$WORK_DIR" ]; then
+            echo "::warning::No working directory found — teardown skipped, check for leaked resources"
+            exit 0
+          fi
+          cd "$WORK_DIR" && azd down --force --purge

From be27a0575b4a8467306e6db136f9743e8b5b039a Mon Sep 17 00:00:00 2001
From: Jian Wu <wujia@microsoft.com>
Date: Fri, 12 Jun 2026 15:02:27 +0800
Subject: [PATCH 9/9] fix: add Tier 2 cost confirmation guard, fix prompt env
 description

- Fail fast if Tier 2 selected without confirm_tier2_cost=true
- Clarify that Azure/GitHub auth is only active for Tier 1/2

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .github/workflows/e2e-ext-azure-ai-agents.yml               | 6 ++++++
 .../tests/cli-interactive-tester-scenarios/prompt-ci-run.md | 4 ++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-ext-azure-ai-agents.yml b/.github/workflows/e2e-ext-azure-ai-agents.yml
index cb34c39a8d9..682c519d4e3 100644
--- a/.github/workflows/e2e-ext-azure-ai-agents.yml
+++ b/.github/workflows/e2e-ext-azure-ai-agents.yml
@@ -90,6 +90,12 @@ jobs:
         if: contains(inputs.tier, '2')
         uses: astral-sh/setup-uv@v6
 
+      - name: Validate Tier 2 cost confirmation
+        if: contains(inputs.tier, '2') && !inputs.confirm_tier2_cost
+        run: |
+          echo "::error::Tier 2 creates Azure resources (~\$2-5). Set confirm_tier2_cost=true to proceed."
+          exit 1
+
       - name: Azure Login
         if: contains(inputs.tier, '1') || contains(inputs.tier, '2')
         uses: azure/login@v2
diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/prompt-ci-run.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/prompt-ci-run.md
index 22d433715e4..d5fe42104ad 100644
--- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/prompt-ci-run.md
+++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/prompt-ci-run.md
@@ -6,8 +6,8 @@ Autonomously run azure.ai.agents CLI test scenarios using the cli-interactive-te
 
 - Runner: Ubuntu 22.04 (GitHub Actions)
 - azd is on PATH (pre-built in earlier step)
-- Azure auth: active (federated identity, earlier step)
-- GitHub CLI auth: active (earlier step)
+- Azure auth: active if TIER includes 1 or 2 (federated identity via earlier step)
+- GitHub CLI auth: active if TIER includes 1 or 2 (earlier step)
 - Scenarios directory: current working directory
 - All paths are POSIX (Linux runner)