ci(backend): experiment: backend test shards on ubuntu-26.04

.github/workflows/backend.yml

@@ -203,7 +203,7 @@ jobs:
       needs.files-changed.outputs.backend == 'true' && needs.calculate-shards.outputs.shard-count != '0'
     needs: [files-changed, select-tests, calculate-shards]
     name: backend test
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
     timeout-minutes: 60
     permissions:
       contents: read
@@ -224,11 +224,16 @@ jobs:
       TEST_GROUP_STRATEGY: roundrobin
       PYTHONHASHSEED: '0'
       XDIST_PER_WORKER_SNUBA: '1'
-      XDIST_WORKERS: ${{ github.event_name == 'pull_request' && '2' || '3' }}
+      XDIST_WORKERS: '4'
+      # Set to '1' to sample pytest and its xdist workers with py-spy (adds ~0.4 cores of overhead).
+      PROFILE_BACKEND_TESTS: '0'
 
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
+      - name: Start resource monitor
+        run: python3 .github/workflows/scripts/monitor-resources.py --start
+
       # Overlaps the per-worker ClickHouse bootstrap with devservices up.
       - name: Start per-worker Snuba bootstrap in the background
         if: env.XDIST_PER_WORKER_SNUBA == '1'
@@ -258,7 +263,8 @@ jobs:
             -v /tmp/pg-sock:/var/run/postgresql \
             -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=sentry \
             "$PG_IMAGE" postgres \
-              -c wal_level=logical -c max_replication_slots=1 -c max_wal_senders=1
+              -c wal_level=logical -c max_replication_slots=1 -c max_wal_senders=1 \
+              -c fsync=off -c synchronous_commit=off -c full_page_writes=off
           # docker run -d returns 0 even if postgres dies right after — bound the
           # wait so a crashing postgres surfaces fast instead of hitting the job
           # timeout silently.
@@ -284,6 +290,27 @@ jobs:
         if: env.XDIST_PER_WORKER_SNUBA == '1'
         run: python3 .github/workflows/scripts/bootstrap-snuba.py --wait
 
+      # sentry-shared-kafka's healthcheck runs `kafka-topics --list` every 5s, which starts
+      # a JVM each time: about 0.25 of the runner's 4 cores during tests. Everything that
+      # waits on Kafka's health is done by now, so answer that exact command with a TCP
+      # probe instead (kafka-topics-healthcheck.sh).
+      - name: Make the Kafka healthcheck cheap
+        continue-on-error: true
+        run: |
+          real=$(docker exec kafka-kafka-1 sh -c 'command -v kafka-topics')
+          docker exec -u 0 kafka-kafka-1 mv "$real" "$real.real"
+          docker cp .github/workflows/scripts/kafka-topics-healthcheck.sh "kafka-kafka-1:$real"
+          time docker exec kafka-kafka-1 kafka-topics --bootstrap-server 127.0.0.1:9092 --list
+
+      - name: Install py-spy
+        if: env.PROFILE_BACKEND_TESTS == '1'
+        run: |
+          uv tool install py-spy==0.4.1
+          echo "PY_SPY=$(uv tool dir --bin)/py-spy" >> "$GITHUB_ENV"
+          echo "PY_SPY_OUTPUT=$PWD/.artifacts/py-spy.txt" >> "$GITHUB_ENV"
+          # py-spy attaches to the running pytest controller, which isn't its child.
+          sudo sysctl -w kernel.yama.ptrace_scope=0
+
       - name: Download selected tests artifact
         if: needs.select-tests.outputs.has-selected-tests == 'true'
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -297,7 +324,11 @@ jobs:
         run: |
           if [ -n "${XDIST_WORKERS}" ]; then
             export PYTEST_ADDOPTS="$PYTEST_ADDOPTS -n ${XDIST_WORKERS} --dist=loadfile"
-            timeout -k 30 1200 make test-python-ci || {
+            profile=()
+            if [ -n "${PY_SPY:-}" ]; then
+              profile=(.github/workflows/scripts/profile-backend-tests.sh)
+            fi
+            "${profile[@]}" timeout -k 30 1200 make test-python-ci || {
               rc=$?
               if [ "$rc" -eq 124 ]; then
                 echo "::error::Test run timed out after 20 minutes (possible xdist hang)"
@@ -308,6 +339,11 @@ jobs:
             make test-python-ci
           fi
 
+      - name: Stop resource monitor
+        if: always()
+        continue-on-error: true
+        run: python3 .github/workflows/scripts/monitor-resources.py --stop
+
       # pytest-selected.py balances full-run shards with these. See
       # .github/workflows/scripts/backend-test-durations.py to refresh them.
       - name: Extract test durations
@@ -327,6 +363,18 @@ jobs:
           path: .artifacts/test-durations.json
           retention-days: 30
 
+      - name: Upload resource usage
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: backend-resource-usage-${{ github.run_id }}-${{ matrix.instance }}
+          path: |
+            .artifacts/resource-usage.jsonl
+            .artifacts/resource-usage-summary.json
+            .artifacts/py-spy.txt
+          retention-days: 30
+
       - name: Report failures
         if: ${{ !cancelled() && github.event_name == 'pull_request' }}
         continue-on-error: true

.github/workflows/scripts/kafka-topics-healthcheck.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Stands in for kafka-topics inside the CI Kafka container (see backend.yml).
+# The healthcheck's exact command gets a TCP probe instead of a JVM start;
+# anything else runs the real kafka-topics.
+if [ "$*" = "--bootstrap-server 127.0.0.1:9092 --list" ] &&
+  (exec 3<>/dev/tcp/127.0.0.1/9092) 2>/dev/null; then
+  exit 0
+fi
+exec "$0.real" "$@"