From 8f5928c2253fd1288819771bd16d2a325d016518 Mon Sep 17 00:00:00 2001 From: Hayden Roche Date: Tue, 11 Aug 2026 22:24:27 +0000 Subject: [PATCH] Fix flaky/hanging note-c card.binary HIL tests The nightly HIL job kept failing, and recently hung for hours. Fixes: - Use the current notestation-client (v3.5.1) via the install_notestation_client action, replacing the old v2.2.0 client. The old one briefly flickered the Swan's USB test port while the board rebooted after flashing, so the tests couldn't open it and none ran. - Wait for the board's prompt and send the "start" command a few times. The board can miss a single "start" sent while its USB is still settling (~2s), which left the run hung with no output. - Add a concurrency group so a new push cancels the previous run instead of two runs fighting over the single test station. - Cap the job at 30 minutes so a stall fails fast. - Drop the "Install notecard CLI" and socat steps; the current client no longer needs them (a plain cat streams the Notecard trace). Verified on barcelona-notestation-1. Authored by Claude Code (AI agent). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/notecard-binary-tests.yml | 93 ++++++++------------- test/hitl/card.binary/after_upload.py | 76 +++++++++++++++-- 2 files changed, 106 insertions(+), 63 deletions(-) diff --git a/.github/workflows/notecard-binary-tests.yml b/.github/workflows/notecard-binary-tests.yml index 47f8f1be..abd1b262 100644 --- a/.github/workflows/notecard-binary-tests.yml +++ b/.github/workflows/notecard-binary-tests.yml @@ -19,6 +19,14 @@ permissions: contents: read id-token: write +# Don't let a run collide with itself: a new push to the same ref (e.g. a PR +# update) cancels the in-flight run instead of racing it for the single +# barcelona-notestation-1 reservation. Different refs (other PRs, the nightly +# schedule) get separate groups and are unaffected. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: md5srv-test: uses: ./.github/workflows/md5srv-tests.yml @@ -30,6 +38,10 @@ jobs: github.event.repository.fork != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) runs-on: ubuntu-latest + # A healthy run reserves, flashes, and runs the suite in well under this. + # The cap ensures a stuck serial monitor fails fast instead of burning the + # default 6h job timeout. + timeout-minutes: 30 defaults: run: shell: bash @@ -69,7 +81,7 @@ jobs: aws-region: us-east-1 mask-aws-account-id: 'false' - # Needed for asyncio.TaskGroup, which the card_client code uses. + # Needed to run PlatformIO (installed and invoked below). - name: Set up Python 3.11 uses: actions/setup-python@v4 with: @@ -98,18 +110,6 @@ jobs: sudo apt update sudo apt install gdb-multiarch ngrok jq - # We need socat 1.7.4.4 because prior to that there are some issues with - # setting the baud rate properly. socat is used by the card_client code - # for various tunnels (e.g. Swan USB port, Swan OpenOCD server, etc.). - - name: Install socat 1.7.4.4 - run: | - wget http://www.dest-unreach.org/socat/download/socat-1.7.4.4.tar.gz - tar xvf socat-1.7.4.4.tar.gz - cd socat-1.7.4.4 - ./configure - make -j - sudo make install - - name: Install PlatformIO dependencies run: | python -m pip install --upgrade pip @@ -193,49 +193,28 @@ jobs: exit 1 fi - # The notestation CLI uses the notecard CLI. - - name: Install notecard CLI - run: | - git clone --depth=1 https://github.com/blues/note-cli.git - cd note-cli/notecard - go build . - sudo cp notecard /usr/bin/ - - - name: Download and install notestation CLI v2.2.0 - env: - GH_TOKEN: ${{ secrets.NOTESTATION_RELEASE_DOWNLOAD_TOKEN }} - run: | - api_url="https://api.github.com/repos/blues/notestation/releases/tags/v2.2.0" - - # Query the release asset metadata. - asset_info=$(curl -s -H "Authorization: token $GH_TOKEN" "$api_url") - - # Extract the release asset URL and name. - asset_url=$(echo "$asset_info" | jq -r '.assets[] | select(.name | endswith(".whl")) | .url') - asset_name=$(echo "$asset_info" | jq -r '.assets[] | select(.name | endswith(".whl")) | .name') - - if [ -z "$asset_url" ] || [ -z "$asset_name" ]; then - echo "ERROR: No wheel found in release assets." - exit 1 - fi - - # Download the wheel using the original filename. - curl -L -H "Authorization: token $GH_TOKEN" \ - -H "Accept: application/octet-stream" \ - "$asset_url" -o "$asset_name" - - # Create a virtual environment and install the notestation CLI in it. - python -m venv venv - source venv/bin/activate - wheel_file=$(ls *.whl) - pip install "$wheel_file" + # Use the Go notestation-client (notestation-v3.2.1+). Its SerialTunnel + # publishes the host_mcu_usb symlink only after the server confirms the + # USB device is live, and removes it on drop -- so the symlink is a + # reliable device-liveness signal, which is what after_upload.py checks + # with os.path.exists() before running the tests. The old v2.2.0 Python + # client built the tunnel with `socat PTY,link=host_mcu_usb TCP:...`, + # which created the symlink as soon as socat started -- before the Swan's + # USB was up -- so it flickered while the Swan re-enumerated after + # flashing: after_upload.py caught a blink and let the run proceed, then + # socat removed the symlink before PlatformIO opened the port, giving + # "could not open port .../host_mcu_usb" and 0 tests. See + # blues/notestation f6fc011 (first released in notestation-v3.2.1). + - name: Install notestation-client + uses: blues/notestation-actions/install_notestation_client@v1.0.2 + with: + version: notestation-v3.5.1 + token: ${{ secrets.NOTESTATION_RELEASE_DOWNLOAD_TOKEN }} - name: Build and upload test firmware and run tests run: | - source venv/bin/activate - # Reserve the Notestation in the background - nohup notestation client reserve --notestation barcelona-notestation-1 &> card_client.log & + nohup notestation-client reserve --notestation barcelona-notestation-1 &> card_client.log & PID=$! timeout=600 # 10 minutes in seconds @@ -265,10 +244,10 @@ jobs: echo "Notestation reserved." - # If firmware update is requested, flash it using notestation client flash + # If firmware update is requested, flash it using notestation-client flash if [ "${{ env.FIRMWARE_S3_URI }}" == "nightly" ]; then echo "Flashing nightly firmware..." - notestation client flash --notecard-nightly + notestation-client flash --notecard-nightly --notestation barcelona-notestation-1 echo "Firmware flash complete." fi @@ -299,8 +278,10 @@ jobs: # Enable Notecard USB trace output. printf '{"req":"card.trace","mode":"usb","trace":"req,comm,sync,mem"}\n' > "$RESERVATION_DIR/notecard_usb" - # Stream Notecard USB output to trace.log in the background. - socat -u GOPEN:"$RESERVATION_DIR/notecard_usb",rawer - >> "$GITHUB_WORKSPACE/trace.log" & + # Stream Notecard USB output to trace.log in the background. The + # notestation-client already exposes notecard_usb as a raw PTY, so a + # plain read is enough -- no socat needed. + cat "$RESERVATION_DIR/notecard_usb" >> "$GITHUB_WORKSPACE/trace.log" & TRACE_PID=$! echo "Notecard trace capture started (PID: $TRACE_PID)." diff --git a/test/hitl/card.binary/after_upload.py b/test/hitl/card.binary/after_upload.py index 95bee943..e3beedc4 100644 --- a/test/hitl/card.binary/after_upload.py +++ b/test/hitl/card.binary/after_upload.py @@ -2,6 +2,7 @@ # runs the after_upload method registered in this file. import os +import select import time Import("env") @@ -9,13 +10,13 @@ # There's an issue with PlatformIO where, after uploading, it expects to see the # test port (i.e. the Swan's USB, which outputs test logging) too quickly. After -# upload, it may take a second for the device to appear, and so we don't want to +# upload, it may take a moment for the device to appear, and so we don't want to # proceed past the upload phase until it's appeared. def _wait_for_test_port(env): port = env.GetProjectOption("test_port") print("Waiting for test port: " + port) - timeout_secs = 10 + timeout_secs = 30 start = time.time() while not os.path.exists(port): if (time.time() - start) > timeout_secs: @@ -25,12 +26,73 @@ def _wait_for_test_port(env): return port -# The test firmware doesn't kick off until it receives the string "start\n" or -# "start\r\n" on the Swan's USB port. +# The test firmware (test/test_main.cpp) does not run the tests until it receives +# "start\n" on the Swan's USB. It only starts listening once it has booted far +# enough to reach its read loop, at which point it prints: +# +# Waiting to receive "start"... +# +# The freshly-reset Swan and the test port can become ready at nearly the same +# instant, so sending "start" the moment the port appears races the firmware's +# boot: the byte can be dropped before the read loop is running, leaving the +# firmware waiting forever with no output (a silent hang of the test step). +# +# To avoid that race, wait until we have actually seen the firmware's prompt, +# then send "start" -- at that point the firmware is guaranteed to be in its +# read loop. If the prompt isn't seen within the timeout (e.g. it was emitted +# before we opened the port and wasn't buffered), fall back to sending anyway, +# which is no worse than the previous unconditional behaviour. +# +# Use plain os-level file I/O rather than pyserial: opening via pyserial toggles +# the DTR/RTS modem-control lines, which errors on this virtual (TCP-backed) +# serial device -- the same reason the tests run with --no-reset. def _start_tests(port): - print("Sending 'start\\n' to Swan to kick off tests...") - with open(port, "wb", buffering=0) as f: - f.write(b"start\n") + prompt = b'Waiting to receive "start"' + prompt_timeout = 20 # seconds to wait for the firmware's prompt + + fd = os.open(port, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK) + try: + # 1) Wait until the firmware prints its prompt, i.e. it has booted into + # its read loop and is listening. + buf = b"" + deadline = time.time() + prompt_timeout + saw_prompt = False + while time.time() < deadline: + r, _, _ = select.select([fd], [], [], 0.5) + if not r: + continue + try: + data = os.read(fd, 4096) + except (BlockingIOError, OSError): + continue + if data: + buf = (buf + data)[-4096:] # the prompt is short; keep the tail + if prompt in buf: + saw_prompt = True + break + if saw_prompt: + print('Saw prompt; sending "start" to kick off tests...') + else: + print( + f"Did not see the firmware prompt within {prompt_timeout}s; " + 'sending "start" anyway.' + ) + + # 2) Send "start" as a short burst rather than once. The very first send + # can be dropped while the just-reset Swan's USB re-enumeration + # settles (a ~2s window observed on hardware); a repeat a couple + # seconds later lands. Extra "start"s after the firmware has begun + # are harmless -- it has already left its read loop. Kept short (a few + # seconds) so it finishes before the firmware's post-start delay + # elapses and the test-monitor step takes over. + for _ in range(8): + try: + os.write(fd, b"start\n") + except OSError: + pass + time.sleep(0.4) + finally: + os.close(fd) def after_upload(source, target, env):