From dd670821c6ff43549c3e6dd3a1ca1fe698e491a3 Mon Sep 17 00:00:00 2001 From: Kailai-Wang <7630809+Kailai-Wang@users.noreply.github.com> Date: Wed, 1 Jul 2026 20:06:43 +0000 Subject: [PATCH 1/2] chore(ci): capture zombienet node logs on ts-test failure When the heima/paseo ts-test network fails to produce blocks, CI showed only "timed out after 30 minutes" with no logs, because: - launch-network.sh ran zombienet with `-l silent` and `> /dev/null`, dropping the orchestrator output. Now `-l info` with output written to $ZOMBIENET_DIR/zombienet.log (under the archived dir); per-node *.log files are unchanged. - The "Archive logs if test fails" step uploaded all of /tmp/parachain_dev/, but the downloaded `polkadot` relay binary has no owner read bit, so the zip failed with EACCES and uploaded nothing. Now archives only **/*.log and **/*.json (readable text). Test-infra observability only; no runtime/node change. --- .github/workflows/ci.yml | 7 ++++++- .github/workflows/create-release-draft.yml | 8 +++++++- parachain/scripts/launch-network.sh | 7 ++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 876940668c..96be0e980a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -463,7 +463,12 @@ jobs: if: failure() with: name: ${{ matrix.chain }}-ts-tests-artifact - path: /tmp/parachain_dev/ + # Only log/spec files, NOT the downloaded relay binaries: the `polkadot` binary + # has no owner read bit, so archiving the whole dir fails the zip with EACCES and + # loses all logs. Globs below match readable text only. + path: | + /tmp/parachain_dev/**/*.log + /tmp/parachain_dev/**/*.json if-no-files-found: ignore retention-days: 3 diff --git a/.github/workflows/create-release-draft.yml b/.github/workflows/create-release-draft.yml index 0180856cfe..fa915bb944 100644 --- a/.github/workflows/create-release-draft.yml +++ b/.github/workflows/create-release-draft.yml @@ -249,8 +249,14 @@ jobs: if: ${{ failure() }} with: name: ${{ matrix.chain }}-ts-tests-artifacts - path: /tmp/parachain_dev/ + # Only the log/spec files, NOT the downloaded relay binaries: the `polkadot` + # binary is written without an owner read bit, so archiving the whole dir fails + # the zip with EACCES and we lose all logs. Globs below are readable text only. + path: | + /tmp/parachain_dev/**/*.log + /tmp/parachain_dev/**/*.json retention-days: 3 + if-no-files-found: warn - name: Dockerhub login uses: docker/login-action@v4 diff --git a/parachain/scripts/launch-network.sh b/parachain/scripts/launch-network.sh index d27a1ba1a4..ee284211c2 100755 --- a/parachain/scripts/launch-network.sh +++ b/parachain/scripts/launch-network.sh @@ -80,7 +80,12 @@ fi print_divider echo "launching zombienet network (in background), dir = $ZOMBIENET_DIR ..." -nohup $ZOMBIENET_BIN -d $ZOMBIENET_DIR -l silent spawn config.toml > /dev/null 2>&1 & +# Log at `info` (not `silent`) and keep the orchestrator output in a file under the +# zombienet dir so it is captured by CI's "Archive logs if test fails" step. Per-node +# logs already land in $ZOMBIENET_DIR/*.log; this adds zombienet's own startup/scheduling +# output, which is what reveals stalls (e.g. a collator that never produces block #1). +mkdir -p "$HEIMA_DIR/$ZOMBIENET_DIR" +nohup $ZOMBIENET_BIN -d $ZOMBIENET_DIR -l info spawn config.toml > "$HEIMA_DIR/$ZOMBIENET_DIR/zombienet.log" 2>&1 & cd "$ROOTDIR/parachain/ts-tests" From c507d3d428d3f427772583c1f4e1e6b3853032d1 Mon Sep 17 00:00:00 2001 From: Kailai-Wang <7630809+Kailai-Wang@users.noreply.github.com> Date: Wed, 1 Jul 2026 21:18:41 +0000 Subject: [PATCH 2/2] fix(ci): use `-l text` and don't pre-create the zombienet dir Two mistakes in the previous commit's launch-network.sh change: - `-l info` is not a valid zombienet log level (only table|text|silent), so zombienet errored out immediately and no network started. Use `-l text`. - `mkdir -p $ZOMBIENET_DIR` pre-created the spawn dir, which broke zombienet's spawn. Removed; the orchestrator log is written to a sibling path `$HEIMA_DIR/zombienet-$ZOMBIENET_DIR.log` instead of inside the dir. Verified locally: zombienet spawns cleanly and the sibling log captures the orchestrator (text) output. --- parachain/scripts/launch-network.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parachain/scripts/launch-network.sh b/parachain/scripts/launch-network.sh index ee284211c2..dbee610feb 100755 --- a/parachain/scripts/launch-network.sh +++ b/parachain/scripts/launch-network.sh @@ -80,12 +80,12 @@ fi print_divider echo "launching zombienet network (in background), dir = $ZOMBIENET_DIR ..." -# Log at `info` (not `silent`) and keep the orchestrator output in a file under the -# zombienet dir so it is captured by CI's "Archive logs if test fails" step. Per-node -# logs already land in $ZOMBIENET_DIR/*.log; this adds zombienet's own startup/scheduling -# output, which is what reveals stalls (e.g. a collator that never produces block #1). -mkdir -p "$HEIMA_DIR/$ZOMBIENET_DIR" -nohup $ZOMBIENET_BIN -d $ZOMBIENET_DIR -l info spawn config.toml > "$HEIMA_DIR/$ZOMBIENET_DIR/zombienet.log" 2>&1 & +# Log at `text` (not `silent`) and keep the orchestrator output in a file so it is captured +# by CI's "Archive logs if test fails" step. Write it as a sibling of the zombienet dir +# (NOT inside it — zombienet requires `-d` to be a fresh/absent directory and pre-creating +# it breaks spawn). Per-node logs still land in $ZOMBIENET_DIR/*.log. +# (`-l` accepts only table|text|silent; `text` is the plain-text verbose mode.) +nohup $ZOMBIENET_BIN -d $ZOMBIENET_DIR -l text spawn config.toml > "$HEIMA_DIR/zombienet-$ZOMBIENET_DIR.log" 2>&1 & cd "$ROOTDIR/parachain/ts-tests"