feat(ops): GitOps pipeline, incident management, cost optimization, backup verification

.github/workflows/backup-verification.yml

@@ -0,0 +1,54 @@
+name: Backup Verification
+
+# #1066: Weekly off-cluster restore test (the in-cluster CronJob runs daily).
+# Running it from a second environment proves backups are restorable even if
+# the production cluster is gone.
+
+on:
+  schedule:
+    - cron: "0 6 * * 1"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    environment: backup-verification
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.BACKUP_VERIFY_ROLE_ARN }}
+          aws-region: ${{ vars.BACKUP_REGION }}
+
+      - name: Install tools
+        run: sudo apt-get update && sudo apt-get install -y zstd jq
+
+      - name: Restore and verify latest backup
+        env:
+          BACKUP_BUCKET: ${{ secrets.BACKUP_BUCKET }}
+          PUSHGATEWAY_URL: ${{ secrets.PUSHGATEWAY_URL }}
+          ALERTMANAGER_URL: ${{ secrets.ALERTMANAGER_URL }}
+        run: scripts/ops/backup-verify.sh
+
+      - name: Upload verification report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: backup-verification-${{ github.run_id }}
+          path: backup-verification.json
+          retention-days: 90
+
+      - name: Open issue on failure
+        if: failure()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh issue create --title "Backup verification failed ($(date -u +%F))" \
+            --label incident,backups \
+            --body "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. See docs/backup-strategy.md §5."

.github/workflows/cost-report.yml

@@ -0,0 +1,58 @@
+name: Cost Report
+
+# #1067: Weekly cost report generated from the savings ledger written by
+# scripts/ops/cost-optimize.sh, published as a workflow summary + artifact.
+
+on:
+  schedule:
+    - cron: "0 7 * * 1"
+  workflow_dispatch:
+    inputs:
+      days:
+        description: Reporting window in days
+        default: "30"
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  report:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.COST_REPORT_ROLE_ARN }}
+          aws-region: ${{ vars.BACKUP_REGION }}
+
+      - name: Fetch savings ledger
+        run: aws s3 cp "${{ secrets.COST_LEDGER_URI }}" cost-ledger.jsonl
+
+      - name: Build report
+        env:
+          COST_LEDGER: cost-ledger.jsonl
+          REPORT_DAYS: ${{ inputs.days || '30' }}
+          REPORT_OUT: cost-report.md
+        run: scripts/ops/cost-optimize.sh report
+
+      - name: Append cloud spend (AWS Cost Explorer)
+        run: |
+          start=$(date -u -d "-${{ inputs.days || '30' }} days" +%F); end=$(date -u +%F)
+          aws ce get-cost-and-usage --time-period "Start=$start,End=$end" \
+            --granularity MONTHLY --metrics UnblendedCost \
+            --filter '{"Tags":{"Key":"project","Values":["atomicip"]}}' \
+            --group-by Type=DIMENSION,Key=SERVICE \
+            | jq -r '"\n## Cloud spend by service\n\n| Service | USD |\n|---|---:|",
+                     (.ResultsByTime[].Groups[] | "| \(.Keys[0]) | \(.Metrics.UnblendedCost.Amount | tonumber * 100 | round / 100) |")' \
+            >> cost-report.md
+
+      - name: Publish summary
+        run: cat cost-report.md >> "$GITHUB_STEP_SUMMARY"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cost-report-${{ github.run_id }}
+          path: cost-report.md
+          retention-days: 365

.github/workflows/gitops-promote.yml

@@ -0,0 +1,104 @@
+name: GitOps Promote
+
+# #1069: Pull-request based deployments.
+#  - push to main          -> build image, commit new tag to the staging overlay
+#                             (Argo CD auto-syncs staging)
+#  - workflow_dispatch     -> open a PR bumping the production overlay to a
+#                             given tag; merging that PR deploys production.
+# Rollback is `git revert` of the promotion commit (scripts/gitops-rollback.sh).
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "api-server/**"
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: Image tag (git SHA) already running in staging to promote to production
+        required: true
+        type: string
+
+permissions:
+  contents: write
+  packages: write
+  pull-requests: write
+
+concurrency:
+  group: gitops-promote
+  cancel-in-progress: false
+
+env:
+  IMAGE: ghcr.io/atomicip/api-server
+
+jobs:
+  build-and-stage:
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: docker/build-push-action@v6
+        with:
+          context: api-server
+          push: true
+          tags: ${{ env.IMAGE }}:${{ github.sha }}
+
+      - name: Install kustomize
+        run: |
+          curl -sSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.4.3/kustomize_v5.4.3_linux_amd64.tar.gz" \
+            | tar -xz -C /usr/local/bin kustomize
+
+      - name: Bump staging overlay
+        shell: bash
+        run: |
+          set -euo pipefail
+          cd deploy/k8s/overlays/staging
+          kustomize edit set image "${IMAGE}=${IMAGE}:${GITHUB_SHA}"
+          cd -
+          git config user.name "atomicip-gitops-bot"
+          git config user.email "gitops-bot@users.noreply.github.com"
+          git add deploy/k8s/overlays/staging/kustomization.yaml
+          git commit -m "deploy(staging): api-server ${GITHUB_SHA::12}"
+          git push origin HEAD:main
+
+  promote-production:
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Verify image exists
+        run: docker manifest inspect "${IMAGE}:${{ inputs.tag }}" > /dev/null
+
+      - name: Install kustomize
+        run: |
+          curl -sSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.4.3/kustomize_v5.4.3_linux_amd64.tar.gz" \
+            | tar -xz -C /usr/local/bin kustomize
+
+      - name: Bump production overlay
+        working-directory: deploy/k8s/overlays/production
+        run: kustomize edit set image "${IMAGE}=${IMAGE}:${{ inputs.tag }}"
+
+      - name: Open promotion pull request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          branch: gitops/promote-production-${{ inputs.tag }}
+          commit-message: "deploy(production): api-server ${{ inputs.tag }}"
+          title: "deploy(production): api-server ${{ inputs.tag }}"
+          labels: deployment, production
+          body: |
+            Promotes `${{ env.IMAGE }}:${{ inputs.tag }}` from staging to production.
+
+            Merging this PR deploys it: Argo CD syncs `deploy/k8s/overlays/production`
+            automatically. To roll back, revert the merge commit
+            (`scripts/gitops-rollback.sh production`).
+
+            - [ ] Tag verified healthy in staging
+            - [ ] No open SEV-1/SEV-2 incidents

.github/workflows/gitops-validate.yml

@@ -0,0 +1,65 @@
+name: GitOps Manifest Validation
+
+# #1069: Every pull request touching deployment manifests is rendered and
+# schema-validated, and the rendered diff against main is posted for review.
+# Merging the PR *is* the deployment: Argo CD syncs main to the cluster.
+
+on:
+  pull_request:
+    paths:
+      - "deploy/**"
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        overlay: [staging, production]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install kustomize and kubeconform
+        shell: bash
+        run: |
+          set -euo pipefail
+          curl -sSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.4.3/kustomize_v5.4.3_linux_amd64.tar.gz" \
+            | tar -xz -C /usr/local/bin kustomize
+          curl -sSL "https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz" \
+            | tar -xz -C /usr/local/bin kubeconform
+
+      - name: Render overlay
+        run: kustomize build --load-restrictor LoadRestrictionsNone "deploy/k8s/overlays/${{ matrix.overlay }}" > rendered.yaml
+
+      - name: Validate schemas
+        run: kubeconform -strict -summary -ignore-missing-schemas rendered.yaml
+
+      - name: Diff against main
+        id: diff
+        shell: bash
+        run: |
+          set -euo pipefail
+          git worktree add /tmp/base origin/${{ github.base_ref }}
+          if [ -d "/tmp/base/deploy/k8s/overlays/${{ matrix.overlay }}" ]; then
+            kustomize build --load-restrictor LoadRestrictionsNone "/tmp/base/deploy/k8s/overlays/${{ matrix.overlay }}" > base.yaml
+          else
+            : > base.yaml
+          fi
+          diff -u base.yaml rendered.yaml > manifest.diff || true
+          {
+            echo "### Rendered diff: \`${{ matrix.overlay }}\`"
+            echo '```diff'
+            head -c 60000 manifest.diff
+            echo '```'
+          } > comment.md
+
+      - name: Comment diff on PR
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: gitops-diff-${{ matrix.overlay }}
+          path: comment.md

CHANGELOG.md

@@ -71,6 +71,12 @@ Each entry below references an issue number from our GitHub repository. When rev
 - **#781** — Arbitrator committee mechanism with M-of-N signatures and time-locked ruling enforcement
 - **#906** — Treasury address validation: guard against hardcoded placeholder addresses
 
+### Operations & Reliability
+- **#1066** — Automated backup verification: daily/weekly restore tests, integrity checks, RTO measurement, failure alerts (`scripts/ops/backup-verify.sh`, `docs/backup-strategy.md`)
+- **#1067** — Cost optimization automation: api-server HPA, orphaned data cleanup, compression/tiering, savings ledger and reports (`scripts/ops/cost-optimize.sh`, `docs/cost-optimization.md`)
+- **#1068** — Incident management: PagerDuty/Opsgenie integration, incidents from alerts, acknowledgment, postmortems (`api-server/src/incidents.rs`, `docs/incident-response.md`)
+- **#1069** — GitOps deployment pipeline: Argo CD + Kustomize, PR-based promotion, rollback via `git revert` (`deploy/`, `docs/gitops.md`)
+
 ## Contributing
 
 When adding new features or fixes, update this file with:

api-server/src/commitment_monitoring.rs

@@ -371,6 +371,8 @@ pub fn spawn_background_evaluator() {
             COMMITMENT_MONITOR.evaluate(now);
             alerting::ALERT_MANAGER.tick(now);
             for n in alerting::ALERT_MANAGER.drain_notifications() {
+                // #1068: critical notifications open (or re-trigger) an incident.
+                crate::incidents::INCIDENTS.open_from_notification(&n, now);
                 tracing::warn!(
                     alert = %n.alert_name,
                     severity = ?n.severity,