From 2ae81a27178ff042d28a8e8f5ed2f8d2204a5a30 Mon Sep 17 00:00:00 2001 From: Brent Vatne Date: Fri, 23 Oct 2026 17:44:17 -0700 Subject: [PATCH] Workflows: the launch decoder's time limit kills a hung apt-get timeout without -k sends one SIGTERM and then waits for the command however long it takes, and through sudo the signal reached an apt-get that did not exit on it: in brentvatne/euxy review run 40042718931 the first attempt took 7 minutes under a 2-minute limit. timeout now runs under sudo, so it signals apt-get itself, and -k 10 kills it if it is still running. --- templates/explore-baseline.yml | 8 +++++--- templates/explore-compare.yml | 8 +++++--- templates/explore-review.yml | 8 +++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/templates/explore-baseline.yml b/templates/explore-baseline.yml index 9aac4e5..a448289 100644 --- a/templates/explore-baseline.yml +++ b/templates/explore-baseline.yml @@ -87,14 +87,16 @@ jobs: shell: bash # Launch recordings (capture.launch) are decoded with ffmpeg; GitHub's Ubuntu runners do not ship it. A package # mirror can stall without failing, so each apt-get call has network timeouts and a time limit, and a stalled - # attempt is tried once more: the step fails in minutes rather than holding the job to its timeout. + # attempt is tried once more: the step fails in minutes rather than holding the job to its timeout. timeout runs + # under sudo so its signal reaches apt-get itself, and -k kills an apt-get that does not exit on it: without -k, + # timeout waits for the command however long it takes (brentvatne/euxy review run 40042718931 waited 7 minutes). timeout-minutes: 10 run: | set -euo pipefail if ! command -v ffmpeg > /dev/null; then - apt=(sudo apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120) + apt=(apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120) for attempt in 1 2; do - if timeout 120 "${apt[@]}" update && timeout 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi + if sudo timeout -k 10 120 "${apt[@]}" update && sudo timeout -k 10 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi if [ "$attempt" = 2 ]; then echo '::error title=LAUNCH_DECODER_INSTALL_FAILED::apt-get could not install ffmpeg; rerun the job.'; exit 1; fi echo "::warning title=LAUNCH_DECODER_INSTALL_RETRY::apt-get did not install ffmpeg on attempt $attempt; trying again." done diff --git a/templates/explore-compare.yml b/templates/explore-compare.yml index 9a87e5b..0219160 100644 --- a/templates/explore-compare.yml +++ b/templates/explore-compare.yml @@ -124,14 +124,16 @@ jobs: shell: bash # Launch recordings (capture.launch) are decoded with ffmpeg; GitHub's Ubuntu runners do not ship it. A package # mirror can stall without failing, so each apt-get call has network timeouts and a time limit, and a stalled - # attempt is tried once more: the step fails in minutes rather than holding the job to its timeout. + # attempt is tried once more: the step fails in minutes rather than holding the job to its timeout. timeout runs + # under sudo so its signal reaches apt-get itself, and -k kills an apt-get that does not exit on it: without -k, + # timeout waits for the command however long it takes (brentvatne/euxy review run 40042718931 waited 7 minutes). timeout-minutes: 10 run: | set -euo pipefail if ! command -v ffmpeg > /dev/null; then - apt=(sudo apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120) + apt=(apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120) for attempt in 1 2; do - if timeout 120 "${apt[@]}" update && timeout 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi + if sudo timeout -k 10 120 "${apt[@]}" update && sudo timeout -k 10 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi if [ "$attempt" = 2 ]; then echo '::error title=LAUNCH_DECODER_INSTALL_FAILED::apt-get could not install ffmpeg; rerun the job.'; exit 1; fi echo "::warning title=LAUNCH_DECODER_INSTALL_RETRY::apt-get did not install ffmpeg on attempt $attempt; trying again." done diff --git a/templates/explore-review.yml b/templates/explore-review.yml index db693ca..f81cc22 100644 --- a/templates/explore-review.yml +++ b/templates/explore-review.yml @@ -113,14 +113,16 @@ jobs: shell: bash # Launch recordings (capture.launch) are decoded with ffmpeg; GitHub's Ubuntu runners do not ship it. A package # mirror can stall without failing, so each apt-get call has network timeouts and a time limit, and a stalled - # attempt is tried once more: the step fails in minutes rather than holding the job to its timeout. + # attempt is tried once more: the step fails in minutes rather than holding the job to its timeout. timeout runs + # under sudo so its signal reaches apt-get itself, and -k kills an apt-get that does not exit on it: without -k, + # timeout waits for the command however long it takes (brentvatne/euxy review run 40042718931 waited 7 minutes). timeout-minutes: 10 run: | set -euo pipefail if ! command -v ffmpeg > /dev/null; then - apt=(sudo apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120) + apt=(apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120) for attempt in 1 2; do - if timeout 120 "${apt[@]}" update && timeout 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi + if sudo timeout -k 10 120 "${apt[@]}" update && sudo timeout -k 10 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi if [ "$attempt" = 2 ]; then echo '::error title=LAUNCH_DECODER_INSTALL_FAILED::apt-get could not install ffmpeg; rerun the job.'; exit 1; fi echo "::warning title=LAUNCH_DECODER_INSTALL_RETRY::apt-get did not install ffmpeg on attempt $attempt; trying again." done