fix: resolve recovered production smoke incidents

.github/workflows/smoke.yml

@@ -25,6 +25,24 @@ jobs:
           SMOKE_API_KEY: ${{ secrets.SMOKE_API_KEY }}
           SMOKE_PLAN: "true"
         run: node scripts/production-smoke.mjs
+      - name: Resolve recovered smoke incidents
+        if: success()
+        env: { GH_TOKEN: "${{ github.token }}" }
+        run: |
+          gh issue list \
+            --repo "$GITHUB_REPOSITORY" \
+            --label incident \
+            --state open \
+            --search '"Production smoke test failure" in:title' \
+            --json number \
+            --jq '.[].number' |
+          while read -r issue_number; do
+            test -n "$issue_number" || continue
+            gh issue comment "$issue_number" \
+              --repo "$GITHUB_REPOSITORY" \
+              --body "Resolved automatically after production checks recovered in [run $GITHUB_RUN_ID]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)."
+            gh issue close "$issue_number" --repo "$GITHUB_REPOSITORY" --reason completed
+          done
       - name: Open public incident on failure
         if: failure()
         env: { GH_TOKEN: "${{ github.token }}" }

scripts/test/smoke-incident-workflow.test.mjs

@@ -0,0 +1,12 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+
+const workflow = await readFile(new URL("../../.github/workflows/smoke.yml", import.meta.url), "utf8");
+
+test("production smoke incidents follow the probe state", () => {
+  assert.match(workflow, /Open public incident on failure[\s\S]*if: failure\(\)/);
+  assert.match(workflow, /Resolve recovered smoke incidents[\s\S]*if: success\(\)/);
+  assert.match(workflow, /gh issue comment[\s\S]*Resolved automatically/);
+  assert.match(workflow, /gh issue close[\s\S]*--reason completed/);
+});