From 89a8b0d2f06775810c3584663e1430fca1e0d552 Mon Sep 17 00:00:00 2001 From: ruscher Date: Sun, 11 Oct 2026 22:04:52 -0300 Subject: [PATCH] GitLab Status: recognize and retry builder runner failures The testing-2026-10-11 request reached the builder 1 s after the run (pipeline 37669), but its first job died with runner_system_failure (Kubernetes 502 Bad Gateway) before any build job existed. The tracker only looked for build jobs, so it waited 10 min and reported that no job started. - track-build.sh: a failed request pipeline for the package right after the request, or a build job that failed for a runner reason, ends with exit status 3 and the GitLab job link. - gitlab-status.yml: on status 3, re-run the "build" job of that Build Package run (at most twice, only from the tracker of attempt 1) and follow the new request. The organization webhook fires for re-runs too (pipeline 37694, 2 s after the manual re-run). --- .github/scripts/track-build.sh | 45 ++++++++++++++++++++++++----- .github/workflows/gitlab-status.yml | 30 +++++++++++++++++-- README.md | 2 +- docs/ci-packaging.md | 16 ++++++++-- 4 files changed, 80 insertions(+), 13 deletions(-) diff --git a/.github/scripts/track-build.sh b/.github/scripts/track-build.sh index b432768..edc558f 100755 --- a/.github/scripts/track-build.sh +++ b/.github/scripts/track-build.sh @@ -12,7 +12,9 @@ # # Exits non-zero when no matching job starts, when it does not succeed, or when # a testing/stable package does not reach https://repo.biglinux.com.br: the -# server drops some uploads without any error in the job. +# server drops some uploads without any error in the job. Exit status 3 means +# the builder's own infrastructure failed (GitLab runner), so nothing was built +# and requesting the build again is safe. set -euo pipefail : "${PACKAGE:?}" "${ORG:?}" "${BRANCH:?}" "${SINCE:?}" @@ -30,6 +32,15 @@ get() { curl -fsS --retry 3 --retry-delay 5 --retry-all-errors --max-time 30 "$@" } +# GitLab failure reasons that come from the runners, not from the package +infra_failure='^(runner_system_failure|scheduler_failure|api_failure|data_integrity_failure)$' + +infra_error() { + echo "::error::The BigLinux builder's infrastructure failed ($1): $2. Nothing was built; request the build again." + printf '### %s: builder infrastructure failure\n%s: [%s](%s)\n' "$PACKAGE" "$1" "$2" "$2" >>"$summary" + exit 3 +} + # One of the header lines the build job prints ("org: biglinux", ...) header() { sed -nE "s/^[^ ]+ [0-9]+[OE]\+? ?$1: (.*)$/\1/p" <<<"$2" | head -n1 @@ -53,10 +64,25 @@ echo "Looking for the $PACKAGE build of $url (branch $BRANCH) requested at $SINC # has one job named after the package. job='' child='' rejected=" " deadline=$((SECONDS + START_TIMEOUT)) while [ -z "$job" ]; do - while read -r pid created; do - [ "$(date -u -d "$created" +%s)" -ge $((since - 30)) ] || continue + while read -r pid created status; do + t=$(date -u -d "$created" +%s) + [ "$t" -ge $((since - 30)) ] || continue child=$(get "$api/pipelines/$pid/bridges" | jq -r '.[0].downstream_pipeline.id // empty') - [ -n "$child" ] || continue + if [ -z "$child" ]; then + # The request failed before the build job existed. Only the + # generating job's log names the package, so also require the + # pipeline to start right after the request (the builder needs 1-3 s). + if [[ "$status" =~ ^(failed|canceled)$ ]] && [ "$t" -le $((since + 15)) ]; then + read -r gid reason < <(get "$api/pipelines/$pid/jobs" | + jq -r '.[] | select(.name == "generate-PackageBuild") | "\(.id) \(.failure_reason // "none")"') + if [ -n "${gid:-}" ] && get "$web/-/jobs/$gid/raw" | grep -qE "[0-9]+[OE]\+? ?package_name=$PACKAGE$"; then + [[ "$reason" =~ $infra_failure ]] && infra_error "$reason" "$web/-/jobs/$gid" + echo "::error::The builder pipeline $web/-/pipelines/$pid failed before building ($reason)" + exit 1 + fi + fi + continue + fi id=$(get "$api/pipelines/$child/jobs" | jq -r --arg n "$PACKAGE" '.[] | select(.name == $n) | .id') if [ -z "$id" ] || [[ "$rejected" == *" $id "* ]]; then continue; fi log=$(get "$web/-/jobs/$id/raw" || true) @@ -69,7 +95,7 @@ while [ -z "$job" ]; do fi rejected+="$id " done < <(get "$api/pipelines?source=trigger&updated_after=$after&order_by=id&sort=asc&per_page=100" | - jq -r '.[] | "\(.id) \(.created_at)"') + jq -r '.[] | "\(.id) \(.created_at) \(.status)"') if [ -z "$job" ]; then if [ "$SECONDS" -ge "$deadline" ]; then @@ -85,11 +111,16 @@ done # 2. Wait for it to finish deadline=$((SECONDS + BUILD_TIMEOUT)) last= while :; do - status=$(get "$api/pipelines/$child/jobs" | jq -r --argjson id "$job" '.[] | select(.id == $id) | .status') + read -r status reason < <(get "$api/pipelines/$child/jobs" | + jq -r --argjson id "$job" '.[] | select(.id == $id) | "\(.status) \(.failure_reason // "none")"') [ "$status" = "$last" ] || echo "Job status: $status" last=$status case "$status" in - success | failed | canceled | skipped) break ;; + failed) + if [[ "$reason" =~ $infra_failure ]]; then infra_error "$reason" "$web/-/jobs/$job"; fi + break + ;; + success | canceled | skipped) break ;; esac if [ "$SECONDS" -ge "$deadline" ]; then echo "::error::Builder job $web/-/jobs/$job still '$status' after ${BUILD_TIMEOUT}s" diff --git a/.github/workflows/gitlab-status.yml b/.github/workflows/gitlab-status.yml index a09dc72..4e4cd76 100644 --- a/.github/workflows/gitlab-status.yml +++ b/.github/workflows/gitlab-status.yml @@ -2,7 +2,9 @@ name: GitLab Status # Follows the job the BigLinux builder starts after a successful # "Build Package" run, and fails when it does not start or does not succeed. -# Read only: it uses the public GitLab API and no secret. The name avoids the +# It uses the public GitLab API and no secret. When the builder's own runners +# fail before building, it re-runs the "build" job of that run (at most twice), +# which makes the organization request the build again. The name avoids the # words "Build" and "Package", so this run never requests a build itself. on: @@ -20,6 +22,7 @@ on: permissions: contents: read + actions: write # re-run the build job after a builder infrastructure failure jobs: track: @@ -41,4 +44,27 @@ jobs: ORG: biglinux BRANCH: ${{ github.event.workflow_run.head_branch || inputs.branch }} SINCE: ${{ github.event.workflow_run.updated_at || inputs.since }} - run: .github/scripts/track-build.sh + # Only the tracker of a run's first attempt re-runs it + RUN_ID: ${{ github.event.workflow_run.run_attempt == 1 && github.event.workflow_run.id || '' }} + GH_TOKEN: ${{ github.token }} + run: | + for retry in 0 1 2; do + status=0 + .github/scripts/track-build.sh || status=$? + if [ "$status" -ne 3 ] || [ -z "$RUN_ID" ] || [ "$retry" -eq 2 ]; then + exit "$status" + fi + build=$(gh run view "$RUN_ID" -R "$GITHUB_REPOSITORY" --json jobs --jq '.jobs[] | select(.name == "build") | .databaseId') + echo "::warning::Builder infrastructure failure; re-running the build job of run $RUN_ID" + attempt=$(gh run view "$RUN_ID" -R "$GITHUB_REPOSITORY" --json attempt --jq .attempt) + gh run rerun "$RUN_ID" -R "$GITHUB_REPOSITORY" --job "$build" + # Wait for the new attempt to finish + until gh run view "$RUN_ID" -R "$GITHUB_REPOSITORY" --json attempt,status \ + --jq ".attempt > $attempt and .status == \"completed\"" | grep -qx true; do + sleep 15 + done + read -r conclusion SINCE < <(gh run view "$RUN_ID" -R "$GITHUB_REPOSITORY" \ + --json conclusion,updatedAt --jq '"\(.conclusion) \(.updatedAt)"') + [ "$conclusion" = success ] || { echo "::error::The re-run ended '$conclusion'"; exit 1; } + export SINCE + done diff --git a/README.md b/README.md index b0f73e2..2ad39ad 100644 --- a/README.md +++ b/README.md @@ -263,7 +263,7 @@ The suite checks the page configuration, translations, version consistency and t |---|---|---| | Tests | pull requests, manual | tests, lint, translations, package build/install/remove | | Build Package | pushes to `main`, `testing-*`, `stable-*`, manual | the tests above, then the `build` job | -| GitLab Status | after Build Package (biglinux only), manual | follows the builder job; fails if it fails or if testing/stable never publish the file | +| GitLab Status | after Build Package (biglinux only), manual | follows the builder job, retries runner failures; fails if the build fails or if testing/stable never publish the file | The BigLinux builder ([packagebuilder](https://gitlab.bigib.org/builder/packagebuilder)) starts by itself when a **Build Package** run in `biglinux/biglinux-welcome` succeeds; no secret is needed. `main` is built for the unstable repository, a `testing-*` branch for testing and a `stable-*` branch for stable. Failed tests mean no build. Details, evidence and the release checklist: [docs/ci-packaging.md](docs/ci-packaging.md). diff --git a/docs/ci-packaging.md b/docs/ci-packaging.md index 6fe01d3..93392b9 100644 --- a/docs/ci-packaging.md +++ b/docs/ci-packaging.md @@ -19,9 +19,18 @@ push to main / testing-* / stable-* (biglinux/biglinux-welcome) └─ GitLab Status workflow (workflow_run) follows that job through the public GitLab API and fails if it does not start within 10 min or does not succeed; for testing/stable it then waits up to 1 h for - repo.biglinux.com.br//x86_64/ and fails if it never appears + repo.biglinux.com.br//x86_64/ and fails if it never appears. + If the builder's runners fail (GitLab failure reason runner_system_failure, + scheduler_failure, api_failure, data_integrity_failure), nothing was + built: it re-runs the "build" job of that run, at most twice, and follows + the new request ``` +Runner failures are not rare. On 2026-10-11 the request for +`testing-2026-10-11` reached the builder 1 s after the run (pipeline 37669), +but its first job died with `runner_system_failure` (Kubernetes +`502 Bad Gateway`), as did two other pipelines that day. + Destination repository, decided by the builder from `git_branch` (`package-build` in packagebuilder): @@ -158,8 +167,9 @@ None is needed for builds. Recommended, by a `biglinux` admin: 4. To publish to testing: create branch `testing-YYYY-MM-DD` from `main` (`git push origin main:refs/heads/testing-$(date +%F)`). Same for stable with `stable-YYYY-MM-DD_HH-MM`. Check **GitLab Status** again. -5. If the builder job failed, open the job link, fix, and rerun - **Build Package** on that branch (Actions → Run workflow). +5. If the builder job failed, open the job link. Runner failures are retried + automatically; for anything else, fix it and re-run the `build` job of the + **Build Package** run for that branch (or Actions → Run workflow). Each branch gets one build per successful run; rerunning the workflow requests another build of the same branch.