Workflows: the launch decoder's time limit kills a hung apt-get

templates/explore-baseline.yml

@@ -87,14 +87,16 @@ jobs:
         shell: bash
         # Launch recordings (capture.launch) are decoded with ffmpeg; GitHub's Ubuntu runners do not ship it. A package
         # mirror can stall without failing, so each apt-get call has network timeouts and a time limit, and a stalled
-        # attempt is tried once more: the step fails in minutes rather than holding the job to its timeout.
+        # attempt is tried once more: the step fails in minutes rather than holding the job to its timeout. timeout runs
+        # under sudo so its signal reaches apt-get itself, and -k kills an apt-get that does not exit on it: without -k,
+        # timeout waits for the command however long it takes (brentvatne/euxy review run 40042718931 waited 7 minutes).
         timeout-minutes: 10
         run: |
           set -euo pipefail
           if ! command -v ffmpeg > /dev/null; then
-            apt=(sudo apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120)
+            apt=(apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120)
             for attempt in 1 2; do
-              if timeout 120 "${apt[@]}" update && timeout 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi
+              if sudo timeout -k 10 120 "${apt[@]}" update && sudo timeout -k 10 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi
               if [ "$attempt" = 2 ]; then echo '::error title=LAUNCH_DECODER_INSTALL_FAILED::apt-get could not install ffmpeg; rerun the job.'; exit 1; fi
               echo "::warning title=LAUNCH_DECODER_INSTALL_RETRY::apt-get did not install ffmpeg on attempt $attempt; trying again."
             done

templates/explore-compare.yml

@@ -124,14 +124,16 @@ jobs:
         shell: bash
         # Launch recordings (capture.launch) are decoded with ffmpeg; GitHub's Ubuntu runners do not ship it. A package
         # mirror can stall without failing, so each apt-get call has network timeouts and a time limit, and a stalled
-        # attempt is tried once more: the step fails in minutes rather than holding the job to its timeout.
+        # attempt is tried once more: the step fails in minutes rather than holding the job to its timeout. timeout runs
+        # under sudo so its signal reaches apt-get itself, and -k kills an apt-get that does not exit on it: without -k,
+        # timeout waits for the command however long it takes (brentvatne/euxy review run 40042718931 waited 7 minutes).
         timeout-minutes: 10
         run: |
           set -euo pipefail
           if ! command -v ffmpeg > /dev/null; then
-            apt=(sudo apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120)
+            apt=(apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120)
             for attempt in 1 2; do
-              if timeout 120 "${apt[@]}" update && timeout 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi
+              if sudo timeout -k 10 120 "${apt[@]}" update && sudo timeout -k 10 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi
               if [ "$attempt" = 2 ]; then echo '::error title=LAUNCH_DECODER_INSTALL_FAILED::apt-get could not install ffmpeg; rerun the job.'; exit 1; fi
               echo "::warning title=LAUNCH_DECODER_INSTALL_RETRY::apt-get did not install ffmpeg on attempt $attempt; trying again."
             done

templates/explore-review.yml

@@ -113,14 +113,16 @@ jobs:
         shell: bash
         # Launch recordings (capture.launch) are decoded with ffmpeg; GitHub's Ubuntu runners do not ship it. A package
         # mirror can stall without failing, so each apt-get call has network timeouts and a time limit, and a stalled
-        # attempt is tried once more: the step fails in minutes rather than holding the job to its timeout.
+        # attempt is tried once more: the step fails in minutes rather than holding the job to its timeout. timeout runs
+        # under sudo so its signal reaches apt-get itself, and -k kills an apt-get that does not exit on it: without -k,
+        # timeout waits for the command however long it takes (brentvatne/euxy review run 40042718931 waited 7 minutes).
         timeout-minutes: 10
         run: |
           set -euo pipefail
           if ! command -v ffmpeg > /dev/null; then
-            apt=(sudo apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120)
+            apt=(apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120)
             for attempt in 1 2; do
-              if timeout 120 "${apt[@]}" update && timeout 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi
+              if sudo timeout -k 10 120 "${apt[@]}" update && sudo timeout -k 10 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi
               if [ "$attempt" = 2 ]; then echo '::error title=LAUNCH_DECODER_INSTALL_FAILED::apt-get could not install ffmpeg; rerun the job.'; exit 1; fi
               echo "::warning title=LAUNCH_DECODER_INSTALL_RETRY::apt-get did not install ffmpeg on attempt $attempt; trying again."
             done