From ae648bae07e39c6ac540978af59ab6fbd6947d3a Mon Sep 17 00:00:00 2001 From: Daniel Falster Date: Wed, 19 Aug 2026 18:15:30 +1000 Subject: [PATCH 1/3] Stop apt-get update hanging the Linux jobs `azure.archive.ubuntu.com` stopped accepting connections from some runners today, and `apt-get update` has no per-connection timeout: it retried across ~30 index files while holding a runner. One job sat on that line for 26 minutes, and three re-runs did the same, while the g++ job on a byte-identical step passed -- the fault is per-VM network reachability, so nothing in the config distinguishes them. `apt-get install` already falls back to archive.ubuntu.com. So the update gets a short timeout and stops being joined to the install with `&&`, which is what made a hung update block the install entirely. --- .github/workflows/cpp-tests.yml | 25 ++++++++++++++++++++++++- .github/workflows/docs.yml | 25 ++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cpp-tests.yml b/.github/workflows/cpp-tests.yml index 7ce47d7..3d11cd7 100644 --- a/.github/workflows/cpp-tests.yml +++ b/.github/workflows/cpp-tests.yml @@ -58,9 +58,32 @@ jobs: path: odelia fetch-depth: 1 + # ⚠️ `apt-get update` CAN HANG FOR HOURS HERE, and the two guards below are + # both load-bearing. Measured on 2026-08-19: azure.archive.ubuntu.com -- + # the mirror the runner image prefers -- stopped accepting connections from + # some runners, and `apt-get update` sat retrying across ~30 index files + # with no per-connection timeout. One job held a runner for 26 minutes on + # this line and was still going; three re-runs did the same while the g++ + # job on an identical step passed, because the fault is per-VM network + # reachability and nothing in the config distinguishes them. + # + # `apt-get install` does NOT have the problem: apt's mirror list falls back + # to archive.ubuntu.com, which the log shows it doing -- `Ign:` on the Azure + # host, then a successful fetch over https. So the update is what needs a + # short timeout, and it must not be joined to the install with `&&`: a + # failed update still leaves the image's package lists usable, while a + # hung one otherwise means the install never runs at all. + # + # `timeout-minutes` is the backstop for whatever this mirror does next. - name: Install Boost headers (Linux) if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y libboost-dev + timeout-minutes: 5 + run: | + set -eu + apt_opts="-o Acquire::http::Timeout=15 -o Acquire::https::Timeout=15 -o Acquire::Retries=2" + sudo apt-get $apt_opts update \ + || echo "apt-get update failed; continuing, the install falls back to another mirror" + sudo apt-get $apt_opts install -y libboost-dev - name: Install Boost headers (macOS) if: runner.os == 'macOS' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 62d8f71..bd587e7 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -32,8 +32,31 @@ jobs: steps: - uses: actions/checkout@v4 + # ⚠️ `apt-get update` CAN HANG FOR HOURS HERE, and the two guards below are + # both load-bearing. Measured on 2026-08-19: azure.archive.ubuntu.com -- + # the mirror the runner image prefers -- stopped accepting connections from + # some runners, and `apt-get update` sat retrying across ~30 index files + # with no per-connection timeout. One job held a runner for 26 minutes on + # this line and was still going; three re-runs did the same while the g++ + # job on an identical step passed, because the fault is per-VM network + # reachability and nothing in the config distinguishes them. + # + # `apt-get install` does NOT have the problem: apt's mirror list falls back + # to archive.ubuntu.com, which the log shows it doing -- `Ign:` on the Azure + # host, then a successful fetch over https. So the update is what needs a + # short timeout, and it must not be joined to the install with `&&`: a + # failed update still leaves the image's package lists usable, while a + # hung one otherwise means the install never runs at all. + # + # `timeout-minutes` is the backstop for whatever this mirror does next. - name: Install Doxygen and graphviz - run: sudo apt-get update && sudo apt-get install -y doxygen graphviz + timeout-minutes: 5 + run: | + set -eu + apt_opts="-o Acquire::http::Timeout=15 -o Acquire::https::Timeout=15 -o Acquire::Retries=2" + sudo apt-get $apt_opts update \ + || echo "apt-get update failed; continuing, the install falls back to another mirror" + sudo apt-get $apt_opts install -y doxygen graphviz # The renderer reads the headers through tools/doxygen_filter.awk, which # rewrites comments so Doxygen can see them. A bug there could silently From 040a95fb1a9b380e4d65f8e41627ae7f4fd8c1c8 Mon Sep 17 00:00:00 2001 From: Daniel Falster Date: Wed, 19 Aug 2026 18:23:45 +1000 Subject: [PATCH 2/3] Strip the unreachable mirror, not just its symptom --- .github/workflows/cpp-tests.yml | 42 ++++++++++++++++++++------------- .github/workflows/docs.yml | 42 ++++++++++++++++++++------------- 2 files changed, 50 insertions(+), 34 deletions(-) diff --git a/.github/workflows/cpp-tests.yml b/.github/workflows/cpp-tests.yml index 3d11cd7..837bd6d 100644 --- a/.github/workflows/cpp-tests.yml +++ b/.github/workflows/cpp-tests.yml @@ -58,31 +58,39 @@ jobs: path: odelia fetch-depth: 1 - # ⚠️ `apt-get update` CAN HANG FOR HOURS HERE, and the two guards below are - # both load-bearing. Measured on 2026-08-19: azure.archive.ubuntu.com -- - # the mirror the runner image prefers -- stopped accepting connections from - # some runners, and `apt-get update` sat retrying across ~30 index files - # with no per-connection timeout. One job held a runner for 26 minutes on - # this line and was still going; three re-runs did the same while the g++ - # job on an identical step passed, because the fault is per-VM network - # reachability and nothing in the config distinguishes them. + # ⚠️ `apt-get update` CAN HANG FOR HOURS HERE, and the mirror line below is + # what stops it. Measured on 2026-08-19: azure.archive.ubuntu.com -- the + # mirror the runner image prefers -- stopped accepting connections from some + # runners. One job held a runner for 26 minutes on this line and was still + # going; three re-runs did the same, while the g++ job on a byte-identical + # step passed. The step references nothing from the matrix, so the fault is + # per-VM network reachability rather than anything in this file. # - # `apt-get install` does NOT have the problem: apt's mirror list falls back - # to archive.ubuntu.com, which the log shows it doing -- `Ign:` on the Azure - # host, then a successful fetch over https. So the update is what needs a - # short timeout, and it must not be joined to the install with `&&`: a - # failed update still leaves the image's package lists usable, while a - # hung one otherwise means the install never runs at all. + # The runner lists that host in /etc/apt/apt-mirrors.txt with + # archive.ubuntu.com behind it, so DELETING THE LINE is the fix: apt then + # goes straight to a mirror that answers. # - # `timeout-minutes` is the backstop for whatever this mirror does next. + # ⚠️ Per-connection timeouts alone were tried first and are NOT sufficient, + # which is worth knowing before someone simplifies this. With + # `Acquire::http::Timeout=15` and `Retries=2` the dead host is still + # consulted for every one of ~30 index targets, so `update` blew a 5-minute + # budget anyway -- a red job instead of a hung one, but still red. They stay + # as a backstop for a mirror that is slow rather than absent, not as the + # mechanism. + # + # `timeout-minutes` bounds whatever this mirror does next. - name: Install Boost headers (Linux) if: runner.os == 'Linux' timeout-minutes: 5 run: | set -eu + list=/etc/apt/apt-mirrors.txt + if [ -f "$list" ] && + grep -E "^[^#]*https?://" "$list" | grep -qv azure.archive.ubuntu.com; then + sudo sed -i "/azure.archive.ubuntu.com/d" "$list" + fi apt_opts="-o Acquire::http::Timeout=15 -o Acquire::https::Timeout=15 -o Acquire::Retries=2" - sudo apt-get $apt_opts update \ - || echo "apt-get update failed; continuing, the install falls back to another mirror" + sudo apt-get $apt_opts update sudo apt-get $apt_opts install -y libboost-dev - name: Install Boost headers (macOS) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bd587e7..5e22936 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -32,30 +32,38 @@ jobs: steps: - uses: actions/checkout@v4 - # ⚠️ `apt-get update` CAN HANG FOR HOURS HERE, and the two guards below are - # both load-bearing. Measured on 2026-08-19: azure.archive.ubuntu.com -- - # the mirror the runner image prefers -- stopped accepting connections from - # some runners, and `apt-get update` sat retrying across ~30 index files - # with no per-connection timeout. One job held a runner for 26 minutes on - # this line and was still going; three re-runs did the same while the g++ - # job on an identical step passed, because the fault is per-VM network - # reachability and nothing in the config distinguishes them. + # ⚠️ `apt-get update` CAN HANG FOR HOURS HERE, and the mirror line below is + # what stops it. Measured on 2026-08-19: azure.archive.ubuntu.com -- the + # mirror the runner image prefers -- stopped accepting connections from some + # runners. One job held a runner for 26 minutes on this line and was still + # going; three re-runs did the same, while the g++ job on a byte-identical + # step passed. The step references nothing from the matrix, so the fault is + # per-VM network reachability rather than anything in this file. # - # `apt-get install` does NOT have the problem: apt's mirror list falls back - # to archive.ubuntu.com, which the log shows it doing -- `Ign:` on the Azure - # host, then a successful fetch over https. So the update is what needs a - # short timeout, and it must not be joined to the install with `&&`: a - # failed update still leaves the image's package lists usable, while a - # hung one otherwise means the install never runs at all. + # The runner lists that host in /etc/apt/apt-mirrors.txt with + # archive.ubuntu.com behind it, so DELETING THE LINE is the fix: apt then + # goes straight to a mirror that answers. # - # `timeout-minutes` is the backstop for whatever this mirror does next. + # ⚠️ Per-connection timeouts alone were tried first and are NOT sufficient, + # which is worth knowing before someone simplifies this. With + # `Acquire::http::Timeout=15` and `Retries=2` the dead host is still + # consulted for every one of ~30 index targets, so `update` blew a 5-minute + # budget anyway -- a red job instead of a hung one, but still red. They stay + # as a backstop for a mirror that is slow rather than absent, not as the + # mechanism. + # + # `timeout-minutes` bounds whatever this mirror does next. - name: Install Doxygen and graphviz timeout-minutes: 5 run: | set -eu + list=/etc/apt/apt-mirrors.txt + if [ -f "$list" ] && + grep -E "^[^#]*https?://" "$list" | grep -qv azure.archive.ubuntu.com; then + sudo sed -i "/azure.archive.ubuntu.com/d" "$list" + fi apt_opts="-o Acquire::http::Timeout=15 -o Acquire::https::Timeout=15 -o Acquire::Retries=2" - sudo apt-get $apt_opts update \ - || echo "apt-get update failed; continuing, the install falls back to another mirror" + sudo apt-get $apt_opts update sudo apt-get $apt_opts install -y doxygen graphviz # The renderer reads the headers through tools/doxygen_filter.awk, which From 9c3b46188b37f0d8bd6446d4a6ad3f5b00c71d20 Mon Sep 17 00:00:00 2001 From: Daniel Falster Date: Wed, 19 Aug 2026 19:02:04 +1000 Subject: [PATCH 3/3] Apply the mirror fix before every apt caller --- .github/actions/working-apt-mirror/action.yml | 52 +++++++++++++++++++ .github/workflows/R-CMD-check.yml | 5 ++ .github/workflows/cpp-tests.yml | 30 ++--------- .github/workflows/docs.yml | 30 ++--------- 4 files changed, 65 insertions(+), 52 deletions(-) create mode 100644 .github/actions/working-apt-mirror/action.yml diff --git a/.github/actions/working-apt-mirror/action.yml b/.github/actions/working-apt-mirror/action.yml new file mode 100644 index 0000000..5e66e26 --- /dev/null +++ b/.github/actions/working-apt-mirror/action.yml @@ -0,0 +1,52 @@ +name: Prefer an apt mirror that answers +description: > + Drop azure.archive.ubuntu.com from the runner's mirror list, when a fallback + remains. A no-op off Linux. + +# ⚠️ WHY THIS EXISTS. On 2026-08-19 azure.archive.ubuntu.com -- the mirror the +# ubuntu runner image prefers -- stopped accepting connections from some runners. +# `apt-get update` consults it for every index target, so a job held a runner for +# 26 minutes on one apt line and three re-runs did the same, while a byte-identical +# step in a sibling job passed. The fault is per-VM network reachability: nothing +# in a workflow file distinguishes the jobs that hang from the ones that do not. +# +# The runner lists that host in /etc/apt/apt-mirrors.txt with archive.ubuntu.com +# behind it, so DELETING THE LINE is the fix -- apt then goes straight to a mirror +# that answers. Measured after: 11-19s for steps that had been hanging. +# +# ⚠️ PER-CONNECTION TIMEOUTS ALONE ARE NOT SUFFICIENT, which is worth knowing +# before someone simplifies this away. With `Acquire::http::Timeout=15` and +# `Retries=2` the dead host is still consulted for each of ~30 index targets, so +# `apt-get update` blew a 5-minute budget anyway: a red job instead of a hung one, +# but still red. +# +# ⚠️ IT MUST RUN BEFORE ANY ACTION THAT INSTALLS PACKAGES, not only before our own +# apt steps. `r-lib/actions/setup-r` installs R's system dependencies with apt +# inside the action, where a step-level fix cannot reach it -- that job sat on +# setup-r for 37 minutes while this was happening. +# +# This is a composite action rather than three copied blocks so the explanation +# above lives in one place. A duplicated comment is a claim that rots. + +runs: + using: composite + steps: + - shell: bash + run: | + set -eu + if [ "$RUNNER_OS" != "Linux" ]; then + echo "not Linux; nothing to do" + exit 0 + fi + list=/etc/apt/apt-mirrors.txt + if [ ! -f "$list" ]; then + echo "$list absent; the image does not use a mirror list" + exit 0 + fi + if grep -E "^[^#]*https?://" "$list" | grep -qv azure.archive.ubuntu.com; then + sudo sed -i "/azure.archive.ubuntu.com/d" "$list" + echo "dropped azure.archive.ubuntu.com; remaining mirrors:" + else + echo "azure.archive.ubuntu.com is the only mirror listed; left alone:" + fi + cat "$list" diff --git a/.github/workflows/R-CMD-check.yml b/.github/workflows/R-CMD-check.yml index c077f6b..9e7a60f 100644 --- a/.github/workflows/R-CMD-check.yml +++ b/.github/workflows/R-CMD-check.yml @@ -39,6 +39,11 @@ jobs: steps: - uses: actions/checkout@v4 + # ⚠️ BEFORE setup-r, not after: it installs R's system dependencies with + # apt inside the action, so a step-level fix cannot reach them. This job + # sat on setup-r for 37 minutes during the 2026-08-19 mirror outage. + - uses: ./.github/actions/working-apt-mirror + - uses: r-lib/actions/setup-r@v2 with: r-version: ${{ matrix.config.r }} diff --git a/.github/workflows/cpp-tests.yml b/.github/workflows/cpp-tests.yml index 837bd6d..b01b89b 100644 --- a/.github/workflows/cpp-tests.yml +++ b/.github/workflows/cpp-tests.yml @@ -58,37 +58,15 @@ jobs: path: odelia fetch-depth: 1 - # ⚠️ `apt-get update` CAN HANG FOR HOURS HERE, and the mirror line below is - # what stops it. Measured on 2026-08-19: azure.archive.ubuntu.com -- the - # mirror the runner image prefers -- stopped accepting connections from some - # runners. One job held a runner for 26 minutes on this line and was still - # going; three re-runs did the same, while the g++ job on a byte-identical - # step passed. The step references nothing from the matrix, so the fault is - # per-VM network reachability rather than anything in this file. - # - # The runner lists that host in /etc/apt/apt-mirrors.txt with - # archive.ubuntu.com behind it, so DELETING THE LINE is the fix: apt then - # goes straight to a mirror that answers. - # - # ⚠️ Per-connection timeouts alone were tried first and are NOT sufficient, - # which is worth knowing before someone simplifies this. With - # `Acquire::http::Timeout=15` and `Retries=2` the dead host is still - # consulted for every one of ~30 index targets, so `update` blew a 5-minute - # budget anyway -- a red job instead of a hung one, but still red. They stay - # as a backstop for a mirror that is slow rather than absent, not as the - # mechanism. - # - # `timeout-minutes` bounds whatever this mirror does next. + # The mirror fix is a local composite action; its `action.yml` records why + # this is needed and what was measured. It must precede every apt call. + - uses: ./.github/actions/working-apt-mirror + - name: Install Boost headers (Linux) if: runner.os == 'Linux' timeout-minutes: 5 run: | set -eu - list=/etc/apt/apt-mirrors.txt - if [ -f "$list" ] && - grep -E "^[^#]*https?://" "$list" | grep -qv azure.archive.ubuntu.com; then - sudo sed -i "/azure.archive.ubuntu.com/d" "$list" - fi apt_opts="-o Acquire::http::Timeout=15 -o Acquire::https::Timeout=15 -o Acquire::Retries=2" sudo apt-get $apt_opts update sudo apt-get $apt_opts install -y libboost-dev diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5e22936..eeac8b8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -32,36 +32,14 @@ jobs: steps: - uses: actions/checkout@v4 - # ⚠️ `apt-get update` CAN HANG FOR HOURS HERE, and the mirror line below is - # what stops it. Measured on 2026-08-19: azure.archive.ubuntu.com -- the - # mirror the runner image prefers -- stopped accepting connections from some - # runners. One job held a runner for 26 minutes on this line and was still - # going; three re-runs did the same, while the g++ job on a byte-identical - # step passed. The step references nothing from the matrix, so the fault is - # per-VM network reachability rather than anything in this file. - # - # The runner lists that host in /etc/apt/apt-mirrors.txt with - # archive.ubuntu.com behind it, so DELETING THE LINE is the fix: apt then - # goes straight to a mirror that answers. - # - # ⚠️ Per-connection timeouts alone were tried first and are NOT sufficient, - # which is worth knowing before someone simplifies this. With - # `Acquire::http::Timeout=15` and `Retries=2` the dead host is still - # consulted for every one of ~30 index targets, so `update` blew a 5-minute - # budget anyway -- a red job instead of a hung one, but still red. They stay - # as a backstop for a mirror that is slow rather than absent, not as the - # mechanism. - # - # `timeout-minutes` bounds whatever this mirror does next. + # The mirror fix is a local composite action; its `action.yml` records why + # this is needed and what was measured. It must precede every apt call. + - uses: ./.github/actions/working-apt-mirror + - name: Install Doxygen and graphviz timeout-minutes: 5 run: | set -eu - list=/etc/apt/apt-mirrors.txt - if [ -f "$list" ] && - grep -E "^[^#]*https?://" "$list" | grep -qv azure.archive.ubuntu.com; then - sudo sed -i "/azure.archive.ubuntu.com/d" "$list" - fi apt_opts="-o Acquire::http::Timeout=15 -o Acquire::https::Timeout=15 -o Acquire::Retries=2" sudo apt-get $apt_opts update sudo apt-get $apt_opts install -y doxygen graphviz