From b9aee27fbdee3ad4af393cbf41557be98eb7788c Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:38:01 +0000 Subject: [PATCH 1/2] Move run logs out of the repository Untrack 236 run-log files (run-logs/, *.log, *.jsonl, *.out, *.err) under openspec/changes/*. Copies live in infra-drafts/evidence//; each old directory keeps an EVIDENCE.md pointer. Drop log re-include lines from .gitignore and add trailing log rules. Add tests/test_no_tracked_logs.py; test_fn_learning_signal reads the try27 log from the out-of-repo evidence dir. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 235 +- .../run-logs/EVIDENCE.md | 24 + .../task-8.7-cleanup-verification.txt | 26 - .../run-logs/task-8.7-command.sh | 25 - .../task-8.7-modal-island-metrics.txt | 140 - .../run-logs/task-8.7-syncer-tape.txt | 22 - .../task-9.2-budget60-post-change.json | 83 - .../task-9.2-budget60-post-change.stderr | 0 .../task-9.2-budget60-pre-change.json | 81 - .../task-9.2-budget60-pre-change.stderr | 0 .../run-logs/task-9.2-command.txt | 2 - .../run-logs/task-9.2-post-change.json | 55 - .../run-logs/task-9.2-post-change.stderr | 0 .../run-logs/task-9.2-pre-change.json | 54 - .../run-logs/task-9.2-pre-change.stderr | 0 .../run-logs/verify_cleanup.sh | 33 - .../yeto-decoupled-m2/island-0/events.jsonl | 26 - .../yeto-decoupled-m2/island-0/events.jsonl | 68 - .../evidence/2026-09-29-gpu/parse-args-p.log | 41 - .../yeto-federated-m2/island-0/events.jsonl | 43 - .../ports/head-yeto-tape.jsonl | 16 - .../evidence/EVIDENCE.md | 15 + .../2026-09-29-algo1b-cpu-img/run3.log | 64 - .../attempt1/out/setup.log | 10 - .../attempt1/out/teardown.log | 1 - .../out/baseline/island-0/events.jsonl | 29 - .../out/clip_higher/island-0/events.jsonl | 29 - .../out/drgrpo/island-0/events.jsonl | 29 - .../out/dual_clip/island-0/events.jsonl | 29 - .../out/entropy/island-0/events.jsonl | 29 - .../out/kl_k3/island-0/events.jsonl | 29 - .../out/over_sampling/island-0/events.jsonl | 29 - .../overlong_penalty/island-0/events.jsonl | 32 - .../2026-09-29-algo1b-g1/out/setup.log | 10 - .../2026-09-29-algo1b-g1/out/teardown.log | 1 - .../out/token/island-0/events.jsonl | 29 - .../algo1b-g1b-a-5-l0-modal.jsonl | 33 - .../attempt1/out/setup.log | 1 - .../attempt1/out/teardown.log | 1 - .../out/baseline/island-0/events.jsonl | 33 - .../out/no_std/island-0/events.jsonl | 33 - .../2026-09-29-algo1b-g1c/out/setup.log | 8 - .../2026-09-29-algo1b-g1c/out/teardown.log | 1 - .../out/token/island-0/events.jsonl | 33 - .../out/os_off/island-0/events.jsonl | 51 - .../out/over_sampling/island-0/events.jsonl | 51 - .../2026-09-29-algo1b-g1d/out/setup.log | 8 - .../2026-09-29-algo1b-g1d/out/teardown.log | 1 - .../out/clip_hi/island-0/events.jsonl | 33 - .../out/clip_sym/island-0/events.jsonl | 33 - .../2026-09-29-algo1b-g1e/out/setup.log | 8 - .../2026-09-29-algo1b-g1e/out/teardown.log | 1 - .../out/clip_hi/island-0/events.jsonl | 33 - .../out/clip_sym/island-0/events.jsonl | 33 - .../2026-09-29-algo1b-g1g/out/setup.log | 8 - .../2026-09-29-algo1b-g1g/out/teardown.log | 1 - .../out/os_off/island-0/events.jsonl | 60 - .../out/over_sampling/island-0/events.jsonl | 60 - .../2026-09-29-algo1b-g1h/out/setup.log | 8 - .../2026-09-29-algo1b-g1h/out/teardown.log | 1 - .../out/of_off/island-0/events.jsonl | 33 - .../out/of_on/island-0/events.jsonl | 34 - .../2026-09-29-algo1b-g1i/out/setup.log | 8 - .../2026-09-29-algo1b-g1i/out/teardown.log | 1 - .../attempt1/out/setup.log | 8 - .../attempt1/out/teardown.log | 1 - .../out/clip_hi/island-0/events.jsonl | 33 - .../out/clip_sym/island-0/events.jsonl | 33 - .../2026-09-29-algo1b-g1j/out/setup.log | 8 - .../2026-09-29-algo1b-g1j/out/teardown.log | 1 - .../rl-algo-grpo-knobs/evidence/EVIDENCE.md | 58 + .../upstream_flags_test.log | 2 - .../evidence/EVIDENCE.md | 11 + .../evidence/2026-09-29-g1/five_one.log | 14 - .../runs/icepop/island-0/events.jsonl | 29 - .../2026-09-29-g1/runs/observe.driver.log | 109 - .../runs/opsm-trainer/island-0/events.jsonl | 29 - .../runs/tis/island-0/events.jsonl | 29 - .../runs/g2-observe/island-0/events.jsonl | 165 - .../runs/g2-observe/metrics.jsonl | 20 - .../runs/mis-mask/island-0/events.jsonl | 29 - .../runs/mis/island-0/events.jsonl | 29 - .../2026-09-29-g1b/runs/observe.driver.log | 3 - .../runs/observe/island-0/events.jsonl | 29 - .../runs/icepop/island-0/events.jsonl | 29 - .../runs/opsm-trainer/island-0/events.jsonl | 30 - .../runs/opsm-trainer/launcher-events.jsonl | 30 - .../runs/tis/island-0/events.jsonl | 29 - .../2026-09-29-g3/run/yeto-tape.jsonl | 3 - .../2026-09-29-g3b/run/yeto-tape.jsonl | 3 - .../evidence/EVIDENCE.md | 27 + .../rl-algo-seq-and-adv/evidence/EVIDENCE.md | 17 + .../g1/attempt2/gspo_s2/island-0/events.jsonl | 9 - .../g1/attempt4/gdpo/island-0/events.jsonl | 36 - .../g1/attempt4/mapo/island-0/events.jsonl | 36 - .../g1/attempt4/maxrl/island-0/events.jsonl | 36 - .../evidence/g3/run/yeto-tape.jsonl | 3 - .../evidence/r1-gspo/first/run/watchdog.log | 11 - .../evidence/r1-gspo/run/watchdog.log | 11 - .../attempt1/run.log | 43 - .../evidence/EVIDENCE.md | 11 + .../evidence/EVIDENCE.md | 13 + .../codex-smoke-12-launcher-events.jsonl | 24 - .../codex-smoke-12-local-rounds.jsonl | 2 - .../codex-smoke-12-rl-island-0.jsonl | 24 - .../2026-09-29-decoupled2/verify_peft.log | 242 - .../yeto-decoupled-m2/island-0/events.jsonl | 132 - .../export.log | 2 - .../pulled/rl-island-0.jsonl | 43 - .../yeto-federated-m2/island-0/events.jsonl | 23 - .../yeto-federated-m2/island-0/events.jsonl | 43 - .../yeto-federated-m2/island-0/events.jsonl | 11 - .../yeto-federated-m2/island-0/events.jsonl | 19 - .../yeto-federated-m2/island-0/events.jsonl | 23 - .../yeto-federated-m2/island-0/events.jsonl | 23 - .../yeto-federated-m2/island-0/events.jsonl | 23 - .../yeto-federated-m2/island-0/events.jsonl | 23 - .../yeto-federated-m2/island-0/events.jsonl | 23 - .../yeto-federated-m2/island-0/events.jsonl | 43 - .../yeto-federated-m2/island-0/events.jsonl | 43 - .../yeto-federated-m2/island-0/events.jsonl | 43 - .../yeto-federated-m2/island-0/events.jsonl | 43 - .../yeto-federated-m2/island-0/events.jsonl | 43 - .../yeto-federated-m2/island-0/events.jsonl | 11 - .../yeto-federated-m2/island-0/events.jsonl | 19 - .../yeto-decoupled-m2/island-0/events.jsonl | 52 - .../yeto-decoupled-m2/island-0/events.jsonl | 52 - .../yeto-decoupled-m2/island-0/events.jsonl | 52 - .../yeto-decoupled-m2/island-0/events.jsonl | 52 - .../yeto-decoupled-m2/island-0/events.jsonl | 52 - .../yeto-decoupled-m2/island-0/events.jsonl | 132 - .../yeto-decoupled-m2/island-0/events.jsonl | 132 - .../yeto-decoupled-m2/island-0/events.jsonl | 132 - .../yeto-decoupled-m2/island-0/events.jsonl | 132 - .../yeto-decoupled-m2/island-0/events.jsonl | 132 - .../yeto-decoupled-m2/island-0/events.jsonl | 14 - .../yeto-decoupled-m2/island-0/events.jsonl | 28 - .../drive-legacy.log | 15 - .../drive-ports.log | 15 - .../kill44-legacy/island-0/events.jsonl | 34 - .../kill44-legacy/miles-first.log | 401 - .../kill44-legacy/miles-restart.log | 401 - .../kill44-legacy/run.log | 7 - .../kill44-legacy/syncer.log | 31 - .../kill44-ports/island-0/events.jsonl | 65 - .../kill44-ports/miles-first.log | 401 - .../kill44-ports/miles-restart.log | 401 - .../kill44-ports/run.log | 7 - .../kill44-ports/syncer.log | 31 - .../island-0/events.jsonl | 65 - .../legacy-0/benchmark.log | 103 - .../legacy-0/report/results.jsonl | 1 - .../legacy-0/work/data/m2/combined.jsonl | 24 - .../legacy-0/work/data/m2/eval.jsonl | 8 - .../legacy-0/work/data/m2/island-0.jsonl | 12 - .../legacy-0/work/data/m2/island-1.jsonl | 12 - .../yeto-federated-m2/eval-samples.jsonl | 8 - .../work/seed-17/yeto-federated-m2/eval.log | 6 - .../yeto-federated-m2/island-0/events.jsonl | 23 - .../yeto-federated-m2/island-0/miles.log | 401 - .../yeto-federated-m2/island-1/events.jsonl | 23 - .../yeto-federated-m2/island-1/miles.log | 401 - .../seed-17/yeto-federated-m2/syncer.jsonl | 3 - .../work/seed-17/yeto-federated-m2/syncer.log | 28 - .../yeto-federated-m2/island-0/events.jsonl | 23 - .../yeto-federated-m2/island-0/events.jsonl | 23 - .../yeto-federated-m2/island-0/events.jsonl | 23 - .../yeto-federated-m2/island-0/events.jsonl | 23 - .../yeto-federated-m2/island-0/events.jsonl | 23 - .../yeto-decoupled-m2/island-0/events.jsonl | 132 - .../island-0/events.jsonl | 65 - .../island-0/events.jsonl | 14 - .../island-0/events.jsonl | 29 - .../2026-09-29-rerun-smoke-normal/probe.jsonl | 13 - .../island-0/events.jsonl | 11 - .../island-0/events.jsonl | 9 - .../yeto-federated-m2/island-0/events.jsonl | 43 - .../island-0/events.jsonl | 14 - .../island-0/events.jsonl | 29 - .../island-0/events.jsonl | 11 - .../island-0/events.jsonl | 9 - .../benchmark.log | 103 - .../report/results.jsonl | 1 - .../work/data/m2/combined.jsonl | 24 - .../work/data/m2/eval.jsonl | 8 - .../work/data/m2/island-0.jsonl | 12 - .../work/data/m2/island-1.jsonl | 12 - .../yeto-federated-m2/eval-samples.jsonl | 8 - .../work/seed-17/yeto-federated-m2/eval.log | 1 - .../yeto-federated-m2/island-0/events.jsonl | 43 - .../yeto-federated-m2/island-0/miles.log | 401 - .../yeto-federated-m2/island-1/events.jsonl | 43 - .../yeto-federated-m2/island-1/miles.log | 401 - .../seed-17/yeto-federated-m2/syncer.jsonl | 3 - .../work/seed-17/yeto-federated-m2/syncer.log | 29 - .../yeto-federated-m2/island-0/events.jsonl | 5 - .../yeto-federated-m2/island-0/events.jsonl | 11 - .../yeto-federated-m2/island-0/events.jsonl | 11 - .../yeto-federated-m2/island-0/events.jsonl | 19 - .../yeto-federated-m2/island-0/events.jsonl | 19 - .../island-0/events.jsonl | 29 - .../rl-engine-ports/evidence/EVIDENCE.md | 106 + .../2026-09-30-img-5c1b49e/check_flags.log | 9 - .../rl-infra-spec/evidence/EVIDENCE.md | 54 + .../fna/pulled/rl-island-0.jsonl | 128 - .../fna/pulled/run.log | 9075 ----------------- .../fnprep/fnprep-remote.log | 16 - .../evidence/image/attempt2/imports_cpu.log | 8 - .../evidence/image/attempt2/ns_probe.log | 22 - .../infra-a/2.2-2.3/round2-e0a/tape.jsonl | 42 - .../state/elastic-state/ledger/journal.jsonl | 30 - .../elastic-state/reconfig/journal.jsonl | 1 - .../state/elastic-state/ledger/journal.jsonl | 15 - .../elastic-state/reconfig/journal.jsonl | 5 - .../state/elastic-state/ledger/journal.jsonl | 15 - .../elastic-state/reconfig/journal.jsonl | 5 - .../state/elastic-state/ledger/journal.jsonl | 15 - .../elastic-state/reconfig/journal.jsonl | 4 - .../state/elastic-state/ledger/journal.jsonl | 13 - .../elastic-state/reconfig/journal.jsonl | 0 .../state/elastic-state/ledger/journal.jsonl | 13 - .../elastic-state/reconfig/journal.jsonl | 0 .../state/elastic-state/ledger/journal.jsonl | 15 - .../elastic-state/reconfig/journal.jsonl | 6 - .../state/elastic-state/ledger/journal.jsonl | 5 - .../elastic-state/reconfig/journal.jsonl | 0 .../state/elastic-state/ledger/journal.jsonl | 28 - .../elastic-state/reconfig/journal.jsonl | 6 - .../base/elastic-state/ledger/journal.jsonl | 60 - .../base/elastic-state/reconfig/journal.jsonl | 1 - .../a4/e1b/elastic-state/ledger/journal.jsonl | 60 - .../e1b/elastic-state/reconfig/journal.jsonl | 15 - .../a4/x2/elastic-state/ledger/journal.jsonl | 60 - .../x2/elastic-state/reconfig/journal.jsonl | 27 - .../e1b/elastic-state/ledger/journal.jsonl | 14 - .../e1b/elastic-state/reconfig/journal.jsonl | 14 - .../elastic-state/ledger/journal.jsonl | 20 - .../elastic-state/reconfig/journal.jsonl | 16 - .../infra-v2-b1/a4s3/watchdog/watchdog.out | 2 - .../fe1/elastic-state/ledger/journal.jsonl | 23 - .../fe1/elastic-state/reconfig/journal.jsonl | 11 - .../fe1r/elastic-state/ledger/journal.jsonl | 8 - .../fe1r/elastic-state/reconfig/journal.jsonl | 14 - .../fe1r5/elastic-state/ledger/journal.jsonl | 23 - .../elastic-state/reconfig/journal.jsonl | 26 - .../evidence/infra-v2-b1/nsmoke/launcher.log | 0 .../2026-10-02-m3a27/check-old.log | 15 - tests/test_fn_learning_signal.py | 5 +- tests/test_no_tracked_logs.py | 49 + 249 files changed, 402 insertions(+), 19153 deletions(-) create mode 100644 openspec/changes/add-nebius-verda-modal-clouds/run-logs/EVIDENCE.md delete mode 100644 openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-cleanup-verification.txt delete mode 100755 openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-command.sh delete mode 100644 openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-modal-island-metrics.txt delete mode 100644 openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-syncer-tape.txt delete mode 100644 openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-post-change.json delete mode 100644 openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-post-change.stderr delete mode 100644 openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-pre-change.json delete mode 100644 openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-pre-change.stderr delete mode 100644 openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-command.txt delete mode 100644 openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-post-change.json delete mode 100644 openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-post-change.stderr delete mode 100644 openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-pre-change.json delete mode 100644 openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-pre-change.stderr delete mode 100755 openspec/changes/add-nebius-verda-modal-clouds/run-logs/verify_cleanup.sh delete mode 100644 openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/dec-legacy/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/dec-ports/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/parse-args-p.log delete mode 100644 openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/strict2-ports/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/fix-decoupled-lr-schedule/evidence/2026-10-07-gpu-head/ports/head-yeto-tape.jsonl create mode 100644 openspec/changes/fix-decoupled-lr-schedule/evidence/EVIDENCE.md delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-cpu-img/run3.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/attempt1/out/setup.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/attempt1/out/teardown.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/baseline/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/clip_higher/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/drgrpo/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/dual_clip/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/entropy/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/kl_k3/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/over_sampling/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/overlong_penalty/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/setup.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/teardown.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/token/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1b/attempt5-out-a/algo1b-g1b-a-5-l0-modal.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/attempt1/out/setup.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/attempt1/out/teardown.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/out/baseline/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/out/no_std/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/out/setup.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/out/teardown.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/out/token/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1d/out/os_off/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1d/out/over_sampling/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1d/out/setup.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1d/out/teardown.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1e/out/clip_hi/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1e/out/clip_sym/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1e/out/setup.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1e/out/teardown.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1g/out/clip_hi/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1g/out/clip_sym/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1g/out/setup.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1g/out/teardown.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1h/out/os_off/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1h/out/over_sampling/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1h/out/setup.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1h/out/teardown.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1i/out/of_off/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1i/out/of_on/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1i/out/setup.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1i/out/teardown.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1j/attempt1/out/setup.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1j/attempt1/out/teardown.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1j/out/clip_hi/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1j/out/clip_sym/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1j/out/setup.log delete mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1j/out/teardown.log create mode 100644 openspec/changes/rl-algo-grpo-knobs/evidence/EVIDENCE.md delete mode 100644 openspec/changes/rl-algo-loss-variants/evidence/2026-09-30-parse-5c1b49e/upstream_flags_test.log create mode 100644 openspec/changes/rl-algo-loss-variants/evidence/EVIDENCE.md delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/five_one.log delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/icepop/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/observe.driver.log delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/opsm-trainer/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/tis/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/g2-observe/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/g2-observe/metrics.jsonl delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/mis-mask/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/mis/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/observe.driver.log delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/observe/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1c/runs/icepop/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1c/runs/opsm-trainer/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1c/runs/opsm-trainer/launcher-events.jsonl delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1c/runs/tis/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g3/run/yeto-tape.jsonl delete mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g3b/run/yeto-tape.jsonl create mode 100644 openspec/changes/rl-algo-mismatch-correction/evidence/EVIDENCE.md create mode 100644 openspec/changes/rl-algo-seq-and-adv/evidence/EVIDENCE.md delete mode 100644 openspec/changes/rl-algo-seq-and-adv/evidence/g1/attempt2/gspo_s2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-seq-and-adv/evidence/g1/attempt4/gdpo/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-seq-and-adv/evidence/g1/attempt4/mapo/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-seq-and-adv/evidence/g1/attempt4/maxrl/island-0/events.jsonl delete mode 100644 openspec/changes/rl-algo-seq-and-adv/evidence/g3/run/yeto-tape.jsonl delete mode 100644 openspec/changes/rl-algo-seq-and-adv/evidence/r1-gspo/first/run/watchdog.log delete mode 100644 openspec/changes/rl-algo-seq-and-adv/evidence/r1-gspo/run/watchdog.log delete mode 100644 openspec/changes/rl-algorithm-capabilities/evidence/2026-09-29-megatron-parse/attempt1/run.log create mode 100644 openspec/changes/rl-algorithm-capabilities/evidence/EVIDENCE.md create mode 100644 openspec/changes/rl-codex-harness-rollout/evidence/EVIDENCE.md delete mode 100644 openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-launcher-events.jsonl delete mode 100644 openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-local-rounds.jsonl delete mode 100644 openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-rl-island-0.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-decoupled2/verify_peft.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-decoupled2/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-default-params-modal/export.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-default-params-modal/pulled/rl-island-0.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62-v2/legacy-s17/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62-v2/ports-s17/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62-v2/tf-legacy/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62-v2/tf-ports/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/legacy-s17/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/legacy-s18/work/seed-18/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/legacy-s19/work/seed-19/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/legacy-s20/work/seed-20/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/legacy-s21/work/seed-21/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/ports-s17/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/ports-s18/work/seed-18/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/ports-s19/work/seed-19/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/ports-s20/work/seed-20/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/ports-s21/work/seed-21/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/tf-legacy/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/tf-ports/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/legacy-s17/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/legacy-s18/work/seed-18/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/legacy-s19/work/seed-19/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/legacy-s20/work/seed-20/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/legacy-s21/work/seed-21/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/ports-s17/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/ports-s18/work/seed-18/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/ports-s19/work/seed-19/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/ports-s20/work/seed-20/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/ports-s21/work/seed-21/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/tf-legacy/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/tf-ports/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/drive-legacy.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/drive-ports.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/miles-first.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/miles-restart.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/run.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/syncer.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/miles-first.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/miles-restart.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/run.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/syncer.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-strict1/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/benchmark.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/report/results.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/data/m2/combined.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/data/m2/eval.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/data/m2/island-0.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/data/m2/island-1.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/eval-samples.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/eval.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-0/miles.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-1/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-1/miles.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/syncer.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/syncer.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-1/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-2/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-0/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-1/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-2/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-decoupled/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-kill44/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-badtoken/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-normal/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-normal/probe.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-pubfail/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-zerograd/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-strict2/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-badtoken/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-normal/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-pubfail/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-zerograd/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/benchmark.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/report/results.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/data/m2/combined.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/data/m2/eval.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/data/m2/island-0.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/data/m2/island-1.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/eval-samples.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/eval.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-0/miles.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-1/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-1/miles.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/syncer.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/syncer.log delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-a-failed-race/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-a/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-b/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfp-a/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfp-b/work/seed-17/yeto-federated-m2/island-0/events.jsonl delete mode 100644 openspec/changes/rl-engine-ports/evidence/2026-09-29-verda-smoke/island-0/events.jsonl create mode 100644 openspec/changes/rl-engine-ports/evidence/EVIDENCE.md delete mode 100644 openspec/changes/rl-infra-spec/evidence/2026-09-30-img-5c1b49e/check_flags.log create mode 100644 openspec/changes/rl-infra-spec/evidence/EVIDENCE.md delete mode 100644 openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fna/pulled/rl-island-0.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fna/pulled/run.log delete mode 100644 openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fnprep/fnprep-remote.log delete mode 100644 openspec/changes/rl-infra-spec/evidence/image/attempt2/imports_cpu.log delete mode 100644 openspec/changes/rl-infra-spec/evidence/image/attempt2/ns_probe.log delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-a/2.2-2.3/round2-e0a/tape.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-b1/state/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-b1/state/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f1/state/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f1/state/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f2/state/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f2/state/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f3/state/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f3/state/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5-e2r/state/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5-e2r/state/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5/state/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5/state/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f6a/state/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f6a/state/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rb/state/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rb/state/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rbold/state/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rbold/state/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/base/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/base/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/e1b/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/e1b/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/x2/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/x2/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/e1b/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/e1b/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/watchdog.out delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r/fe1r/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r/fe1r/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r4/fe1r5/elastic-state/ledger/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r4/fe1r5/elastic-state/reconfig/journal.jsonl delete mode 100644 openspec/changes/rl-infra-spec/evidence/infra-v2-b1/nsmoke/launcher.log delete mode 100644 openspec/changes/rl-infra-spec/evidence/ports-image/2026-10-02-m3a27/check-old.log create mode 100644 tests/test_no_tracked_logs.py diff --git a/.gitignore b/.gitignore index 12aa221a4..dcb050ac0 100644 --- a/.gitignore +++ b/.gitignore @@ -50,253 +50,44 @@ openspec/changes/*/evidence/**/*.pyc !openspec/changes/*/evidence/**/*.md !openspec/changes/*/evidence/**/ARCHIVE-MANIFEST.tsv # 放行：仍在仓库中的现有 evidence 文件（被测试/代码读取，或被文档引用的小文件） -!/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/dec-legacy/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/dec-ports/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/parse-args-p.log -!/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/strict2-ports/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-10-07-gpu-head/ports/head-yeto-tape.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-cpu-img/run3.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/attempt1/out/setup.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/attempt1/out/teardown.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/baseline/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/clip_higher/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/drgrpo/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/dual_clip/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/entropy/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/kl_k3/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/over_sampling/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/overlong_penalty/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/setup.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/teardown.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/out/token/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1b/attempt5-out-a/algo1b-g1b-a-5-l0-modal.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/attempt1/out/setup.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/attempt1/out/teardown.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/out/baseline/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/out/no_std/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/out/setup.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/out/teardown.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1c/out/token/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1d/out/os_off/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1d/out/over_sampling/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1d/out/setup.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1d/out/teardown.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1e/out/clip_hi/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1e/out/clip_sym/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1e/out/setup.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1e/out/teardown.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1g/out/clip_hi/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1g/out/clip_sym/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1g/out/setup.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1g/out/teardown.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1h/out/os_off/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1h/out/over_sampling/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1h/out/setup.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1h/out/teardown.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1i/out/of_off/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1i/out/of_on/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1i/out/setup.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1i/out/teardown.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1j/attempt1/out/setup.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1j/attempt1/out/teardown.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1j/out/clip_hi/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1j/out/clip_sym/island-0/events.jsonl -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1j/out/setup.log -!/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1j/out/teardown.log -!/openspec/changes/rl-algo-loss-variants/evidence/2026-09-30-parse-5c1b49e/upstream_flags_test.log -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/five_one.log !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/gpu_name.txt !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/icepop/check.json -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/icepop/island-0/events.jsonl !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/image-manifest.json -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/observe.driver.log !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/observe/check.json !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/opsm-trainer/check.json -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/opsm-trainer/island-0/events.jsonl !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/tis/check.json -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/tis/island-0/events.jsonl -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/g2-observe/island-0/events.jsonl -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/g2-observe/metrics.jsonl !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/g2-observe/report.md !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/gpu_name.txt !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/image-manifest.json !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/mis-mask/check.json -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/mis-mask/island-0/events.jsonl !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/mis/check.json -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/mis/island-0/events.jsonl !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/mis/spec.json -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/observe.driver.log !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/observe/check.json -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/observe/island-0/events.jsonl !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1c/runs/icepop/check.json -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1c/runs/icepop/island-0/events.jsonl !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1c/runs/opsm-trainer/check.json -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1c/runs/opsm-trainer/island-0/events.jsonl -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1c/runs/opsm-trainer/launcher-events.jsonl !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1c/runs/tis-attempt4-fleetcontroller/app_list_check.txt !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1c/runs/tis/check.json -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1c/runs/tis/island-0/events.jsonl -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g3/run/yeto-tape.jsonl -!/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g3b/run/yeto-tape.jsonl !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-trigger/runs/icepop/check.json !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-trigger/runs/mis-mask/check.json !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-trigger/runs/opsm-trainer/check.json !/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-trigger/runs/tis/check.json -!/openspec/changes/rl-algo-seq-and-adv/evidence/g1/attempt2/gspo_s2/island-0/events.jsonl -!/openspec/changes/rl-algo-seq-and-adv/evidence/g1/attempt4/gdpo/island-0/events.jsonl -!/openspec/changes/rl-algo-seq-and-adv/evidence/g1/attempt4/mapo/island-0/events.jsonl -!/openspec/changes/rl-algo-seq-and-adv/evidence/g1/attempt4/maxrl/island-0/events.jsonl -!/openspec/changes/rl-algo-seq-and-adv/evidence/g3/run/yeto-tape.jsonl -!/openspec/changes/rl-algo-seq-and-adv/evidence/r1-gspo/first/run/watchdog.log -!/openspec/changes/rl-algo-seq-and-adv/evidence/r1-gspo/run/watchdog.log -!/openspec/changes/rl-algorithm-capabilities/evidence/2026-09-29-megatron-parse/attempt1/run.log -!/openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-launcher-events.jsonl -!/openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-local-rounds.jsonl -!/openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-rl-island-0.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-decoupled2/verify_peft.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-decoupled2/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-default-params-modal/export.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-default-params-modal/pulled/rl-island-0.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62-v2/legacy-s17/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62-v2/ports-s17/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62-v2/tf-legacy/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62-v2/tf-ports/work/seed-17/yeto-federated-m2/island-0/events.jsonl !/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62-v3/replay/legacy-s17/work/seed-17/yeto-federated-m2/rollouts/island-0/0.pt -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/legacy-s17/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/legacy-s18/work/seed-18/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/legacy-s19/work/seed-19/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/legacy-s20/work/seed-20/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/legacy-s21/work/seed-21/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/ports-s17/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/ports-s18/work/seed-18/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/ports-s19/work/seed-19/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/ports-s20/work/seed-20/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/ports-s21/work/seed-21/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/tf-legacy/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq62/tf-ports/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/legacy-s17/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/legacy-s18/work/seed-18/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/legacy-s19/work/seed-19/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/legacy-s20/work/seed-20/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/legacy-s21/work/seed-21/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/ports-s17/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/ports-s18/work/seed-18/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/ports-s19/work/seed-19/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/ports-s20/work/seed-20/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/ports-s21/work/seed-21/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/tf-legacy/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-eq63/tf-ports/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/drive-legacy.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/drive-ports.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/miles-first.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/miles-restart.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/run.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/syncer.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/miles-first.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/miles-restart.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/run.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/syncer.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-strict1/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/benchmark.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/report/results.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/data/m2/combined.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/data/m2/eval.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/data/m2/island-0.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/data/m2/island-1.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/eval-samples.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/eval.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-0/miles.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-1/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-1/miles.log !/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/rollouts/dashboard_columns/rollout_0.parquet !/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/rollouts/dashboard_columns/rollout_1.parquet !/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/rollouts/dashboard_columns/rollout_2.parquet -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/syncer.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/syncer.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-1/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-2/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-0/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-1/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-2/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-decoupled/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-kill44/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-badtoken/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-normal/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-normal/probe.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-pubfail/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-zerograd/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-strict2/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-badtoken/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-normal/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-pubfail/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-zerograd/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/benchmark.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/report/results.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/data/m2/combined.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/data/m2/eval.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/data/m2/island-0.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/data/m2/island-1.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/eval-samples.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/eval.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-0/miles.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-1/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-1/miles.log !/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/rollouts/dashboard_columns/rollout_0.parquet !/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/rollouts/dashboard_columns/rollout_1.parquet !/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/rollouts/dashboard_columns/rollout_2.parquet -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/syncer.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/syncer.log -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-a-failed-race/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-a/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-b/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfp-a/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfp-b/work/seed-17/yeto-federated-m2/island-0/events.jsonl -!/openspec/changes/rl-engine-ports/evidence/2026-09-29-verda-smoke/island-0/events.jsonl -!/openspec/changes/rl-infra-spec/evidence/2026-09-30-img-5c1b49e/check_flags.log -!/openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fna/pulled/rl-island-0.jsonl -!/openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fna/pulled/run.log -!/openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fnprep/fnprep-remote.log -!/openspec/changes/rl-infra-spec/evidence/image/attempt2/imports_cpu.log -!/openspec/changes/rl-infra-spec/evidence/image/attempt2/ns_probe.log -!/openspec/changes/rl-infra-spec/evidence/infra-a/2.2-2.3/round2-e0a/tape.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-b1/state/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-b1/state/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f1/state/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f1/state/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f2/state/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f2/state/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f3/state/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f3/state/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5-e2r/state/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5-e2r/state/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5/state/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5/state/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f6a/state/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f6a/state/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rb/state/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rb/state/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rbold/state/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rbold/state/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/base/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/base/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/e1b/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/e1b/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/x2/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/x2/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/e1b/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/e1b/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/watchdog.out -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r/fe1r/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r/fe1r/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r4/fe1r5/elastic-state/ledger/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r4/fe1r5/elastic-state/reconfig/journal.jsonl -!/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/nsmoke/launcher.log -!/openspec/changes/rl-infra-spec/evidence/ports-image/2026-10-02-m3a27/check-old.log + +# Run logs never enter the repository (rl-code-quality-cleanup). Keep evidence +# logs outside the repo (/home/michael/work/infra-drafts/evidence//) and +# leave an EVIDENCE.md pointer. These rules come last so no earlier `!` re-includes logs. +**/run-logs/** +!**/run-logs/EVIDENCE.md +*.log +*.stderr +*.stdout +*metrics*.json +openspec/**/*.jsonl +openspec/**/*.out +openspec/**/*.err diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/EVIDENCE.md b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/EVIDENCE.md new file mode 100644 index 000000000..74139f8d6 --- /dev/null +++ b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/EVIDENCE.md @@ -0,0 +1,24 @@ +# Evidence moved out of the repository + +Run logs that used to live under this directory were removed from git by change `rl-code-quality-cleanup` (2026-10-12). +Copies are kept outside the repository at: + + /home/michael/work/infra-drafts/evidence/add-nebius-verda-modal-clouds/run-logs/ + +The relative path under that directory is the same as the old path under this directory. +Moved files: + +- task-8.7-cleanup-verification.txt +- task-8.7-command.sh +- task-8.7-modal-island-metrics.txt +- task-8.7-syncer-tape.txt +- task-9.2-budget60-post-change.json +- task-9.2-budget60-post-change.stderr +- task-9.2-budget60-pre-change.json +- task-9.2-budget60-pre-change.stderr +- task-9.2-command.txt +- task-9.2-post-change.json +- task-9.2-post-change.stderr +- task-9.2-pre-change.json +- task-9.2-pre-change.stderr +- verify_cleanup.sh diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-cleanup-verification.txt b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-cleanup-verification.txt deleted file mode 100644 index 68a5b89fb..000000000 --- a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-cleanup-verification.txt +++ /dev/null @@ -1,26 +0,0 @@ -=== verified at 2026-09-23T14:35:41Z === - -=== sky status === -Enabled Infra: aws, nebius, verda - -Clusters -NAME INFRA RESOURCES STATUS AUTOSTOP LAUNCHED -yeto-rl84f-head Nebius (eu-north1) 1x(cpus=8, mem=32, cpu-d3_8vcpu-32..., ...) UP - 26 mins ago - -Managed jobs -No in-progress managed jobs. (See: sky jobs -h) - -Services -No live services. (See: sky serve -h) - -=== modal app list === -yeto-yeto-mix87b | stopped | tasks 0 | stopped_at 2026-09-23 14:34:19+00:00 -yeto-yeto-rl84f | stopped | tasks 0 | stopped_at 2026-09-23 14:32:28+00:00 -yeto-yeto-mix87 | stopped | tasks 0 | stopped_at 2026-09-23 14:22:02+00:00 - -=== nebius instances (eu-north1 project) === -instances: 2 - - yeto-rl84f-head-2aebdd34-8456-head RUNNING - - cyberrl-verl-smoke-20260923 STOPPED - -=== aws us-east-1 running/pending instances === diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-command.sh b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-command.sh deleted file mode 100755 index f168844c1..000000000 --- a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-command.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Task 8.7 mixed-fleet acceptance run, 2026-09-23 (attempt 2, prefix yeto-mix87b). -# -# The spec's literal fleet (aws:1xl4@us-east-1 + modal:1xh100) cannot run on this -# AWS account: the "Running On-Demand G and VT instances" vCPU quota is 0 in -# us-east-1 (spot G quota is 0 too), so no g4/g5/g6 L4 instance can ever start. -# Attempt 1 (yeto-mix87) failed with VcpuLimitExceeded in every zone. -# -# Substitution, keeping all three configured clouds in one launch: -# - head + syncer on AWS us-east-1 (CPU only; standard On-Demand quota is 256) -# - learner island 0: Nebius 1xH100 @ eu-north1 -# - learner island 1: Modal 1xH100 (unpinned region) -# Both learner islands still sync to one syncer across the public internet. -set -euo pipefail -cd /home/michael/yeto -exec .venv/bin/python -m yeto.cli launch \ - --gpu nebius:1xh100@eu-north1,modal:1xh100 \ - --syncer-region us-east-1 \ - --model qwen3-0.6b \ - --data HuggingFaceH4/no_robots \ - --assistant-mask-mode legacy \ - --max-rows 2000 --seq-len 1024 \ - --total-steps 16 --fragments 8 --quorum 2 \ - --on-demand \ - --cluster-prefix yeto-mix87b diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-modal-island-metrics.txt b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-modal-island-metrics.txt deleted file mode 100644 index 6887ed269..000000000 --- a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-modal-island-metrics.txt +++ /dev/null @@ -1,140 +0,0 @@ -# yeto-mix87b per-step learner metrics (SFT: loss/token, target_tokens) -# Modal island (learner 1) only -- the Nebius island's training logs were not -# streamed into the head job log and were lost with the cluster. See note in -# live-run-failures.md item 24. - -learner1.r0 INFO local_step=10 global_step=0 loss/token=2.5589 target_tokens=28605 (0.65 s/step) -learner1.r0 INFO local_step=20 global_step=0 loss/token=2.4548 target_tokens=54884 (0.17 s/step) -learner1.r0 INFO local_step=30 global_step=0 loss/token=2.4194 target_tokens=83799 (0.17 s/step) -learner1.r0 INFO local_step=40 global_step=0 loss/token=2.7540 target_tokens=111704 (0.17 s/step) -learner1.r0 INFO local_step=50 global_step=0 loss/token=2.2123 target_tokens=139473 (0.17 s/step) -learner1.r0 INFO local_step=60 global_step=0 loss/token=2.5241 target_tokens=168090 (0.17 s/step) -learner1.r0 INFO local_step=70 global_step=0 loss/token=2.5021 target_tokens=196518 (0.17 s/step) -learner1.r0 INFO local_step=80 global_step=0 loss/token=2.1775 target_tokens=224464 (0.17 s/step) -learner1.r0 INFO local_step=90 global_step=0 loss/token=2.6823 target_tokens=252506 (0.17 s/step) -learner1.r0 INFO local_step=100 global_step=0 loss/token=2.3980 target_tokens=281288 (0.17 s/step) -learner1.r0 INFO local_step=110 global_step=0 loss/token=2.5555 target_tokens=309817 (0.17 s/step) -learner1.r0 INFO local_step=120 global_step=0 loss/token=2.5641 target_tokens=338655 (0.17 s/step) -learner1.r0 INFO local_step=130 global_step=0 loss/token=2.4695 target_tokens=367936 (0.17 s/step) -learner1.r0 INFO local_step=140 global_step=0 loss/token=2.2848 target_tokens=397675 (0.17 s/step) -learner1.r0 INFO local_step=150 global_step=0 loss/token=2.2980 target_tokens=425848 (0.17 s/step) -learner1.r0 INFO local_step=160 global_step=0 loss/token=2.6155 target_tokens=454588 (0.17 s/step) -learner1.r0 INFO local_step=170 global_step=0 loss/token=2.2572 target_tokens=483342 (0.17 s/step) -learner1.r0 INFO local_step=180 global_step=0 loss/token=2.3194 target_tokens=511816 (0.17 s/step) -learner1.r0 INFO local_step=190 global_step=0 loss/token=2.0833 target_tokens=540161 (0.17 s/step) -learner1.r0 INFO local_step=200 global_step=0 loss/token=2.1662 target_tokens=568912 (0.17 s/step) -learner1.r0 INFO local_step=210 global_step=0 loss/token=2.1449 target_tokens=597921 (0.17 s/step) -learner1.r0 INFO local_step=220 global_step=0 loss/token=2.2336 target_tokens=626743 (0.17 s/step) -learner1.r0 INFO local_step=230 global_step=0 loss/token=2.1309 target_tokens=654735 (0.17 s/step) -learner1.r0 INFO local_step=240 global_step=0 loss/token=2.3703 target_tokens=682682 (0.17 s/step) -learner1.r0 INFO local_step=250 global_step=0 loss/token=2.0174 target_tokens=710253 (0.17 s/step) -learner1.r0 INFO local_step=260 global_step=0 loss/token=2.5727 target_tokens=739349 (0.17 s/step) -learner1.r0 INFO local_step=270 global_step=0 loss/token=2.3085 target_tokens=766907 (0.17 s/step) -learner1.r0 INFO local_step=280 global_step=0 loss/token=2.2394 target_tokens=793670 (0.17 s/step) -learner1.r0 INFO local_step=290 global_step=0 loss/token=1.8265 target_tokens=820688 (0.17 s/step) -learner1.r0 INFO local_step=300 global_step=0 loss/token=2.1192 target_tokens=848778 (0.19 s/step) -learner1.r0 INFO local_step=310 global_step=0 loss/token=1.6574 target_tokens=876844 (0.17 s/step) -learner1.r0 INFO local_step=320 global_step=0 loss/token=2.0434 target_tokens=902760 (0.17 s/step) -learner1.r0 INFO local_step=330 global_step=0 loss/token=2.0685 target_tokens=931769 (0.17 s/step) -learner1.r0 INFO local_step=340 global_step=0 loss/token=1.8710 target_tokens=960335 (0.17 s/step) -learner1.r0 INFO local_step=350 global_step=0 loss/token=2.1341 target_tokens=988979 (0.17 s/step) -learner1.r0 INFO local_step=360 global_step=0 loss/token=1.9661 target_tokens=1019216 (0.17 s/step) -learner1.r0 INFO local_step=370 global_step=0 loss/token=2.3635 target_tokens=1047639 (0.17 s/step) -learner1.r0 INFO local_step=380 global_step=0 loss/token=2.1636 target_tokens=1075714 (0.17 s/step) -learner1.r0 INFO local_step=390 global_step=0 loss/token=2.2054 target_tokens=1105166 (0.17 s/step) -learner1.r0 INFO local_step=400 global_step=0 loss/token=2.1477 target_tokens=1133538 (0.17 s/step) -learner1.r0 INFO local_step=410 global_step=0 loss/token=2.2388 target_tokens=1163107 (0.17 s/step) -learner1.r0 INFO local_step=420 global_step=0 loss/token=1.6487 target_tokens=1193046 (0.17 s/step) -learner1.r0 INFO local_step=430 global_step=0 loss/token=1.7137 target_tokens=1223179 (0.17 s/step) -learner1.r0 INFO local_step=440 global_step=0 loss/token=1.6047 target_tokens=1249662 (0.17 s/step) -learner1.r0 INFO local_step=450 global_step=0 loss/token=1.7247 target_tokens=1279533 (0.17 s/step) -learner1.r0 INFO local_step=460 global_step=0 loss/token=1.4407 target_tokens=1308158 (0.17 s/step) -learner1.r0 INFO local_step=470 global_step=0 loss/token=1.5673 target_tokens=1336218 (0.17 s/step) -learner1.r0 INFO local_step=480 global_step=0 loss/token=1.8043 target_tokens=1362731 (0.17 s/step) -learner1.r0 INFO local_step=490 global_step=0 loss/token=1.7806 target_tokens=1390361 (0.17 s/step) -learner1.r0 INFO local_step=500 global_step=0 loss/token=1.9597 target_tokens=1418929 (0.17 s/step) -learner1.r0 INFO local_step=510 global_step=0 loss/token=1.4792 target_tokens=1448350 (0.17 s/step) -learner1.r0 INFO local_step=520 global_step=0 loss/token=1.7596 target_tokens=1476311 (0.17 s/step) -learner1.r0 INFO local_step=530 global_step=0 loss/token=1.8396 target_tokens=1505306 (0.17 s/step) -learner1.r0 INFO local_step=540 global_step=0 loss/token=1.8236 target_tokens=1532014 (0.17 s/step) -learner1.r0 INFO local_step=550 global_step=0 loss/token=1.5466 target_tokens=1561939 (0.17 s/step) -learner1.r0 INFO local_step=560 global_step=0 loss/token=1.5180 target_tokens=1589109 (0.17 s/step) -learner1.r0 INFO local_step=570 global_step=0 loss/token=1.1422 target_tokens=1617288 (0.17 s/step) -learner1.r0 INFO local_step=580 global_step=0 loss/token=1.2781 target_tokens=1646160 (0.17 s/step) -learner1.r0 INFO local_step=590 global_step=0 loss/token=1.5443 target_tokens=1675125 (0.17 s/step) -learner1.r0 INFO local_step=600 global_step=0 loss/token=1.4058 target_tokens=1702978 (0.17 s/step) -learner1.r0 INFO local_step=610 global_step=0 loss/token=1.0965 target_tokens=1732068 (0.17 s/step) -learner1.r0 INFO local_step=620 global_step=0 loss/token=1.6037 target_tokens=1760712 (0.17 s/step) -learner1.r0 INFO local_step=630 global_step=0 loss/token=1.4631 target_tokens=1788772 (0.17 s/step) -learner1.r0 INFO local_step=640 global_step=0 loss/token=1.2847 target_tokens=1817329 (0.17 s/step) -learner1.r0 INFO local_step=650 global_step=0 loss/token=1.3313 target_tokens=1845718 (0.17 s/step) -learner1.r0 INFO local_step=660 global_step=0 loss/token=1.3097 target_tokens=1872325 (0.17 s/step) -learner1.r0 INFO local_step=670 global_step=0 loss/token=1.6081 target_tokens=1900119 (0.17 s/step) -learner1.r0 INFO local_step=680 global_step=0 loss/token=1.2875 target_tokens=1931461 (0.17 s/step) -learner1.r0 INFO local_step=690 global_step=0 loss/token=1.3801 target_tokens=1959652 (0.17 s/step) -learner1.r0 INFO local_step=700 global_step=0 loss/token=0.9371 target_tokens=1987795 (0.17 s/step) -learner1.r0 INFO local_step=710 global_step=0 loss/token=0.8789 target_tokens=2017415 (0.17 s/step) -learner1.r0 INFO local_step=720 global_step=0 loss/token=1.1211 target_tokens=2046389 (0.17 s/step) -learner1.r0 INFO local_step=730 global_step=0 loss/token=1.1016 target_tokens=2073567 (0.17 s/step) -learner1.r0 INFO local_step=740 global_step=0 loss/token=0.9761 target_tokens=2104389 (0.17 s/step) -learner1.r0 INFO local_step=750 global_step=0 loss/token=1.0867 target_tokens=2132560 (0.17 s/step) -learner1.r0 INFO local_step=760 global_step=0 loss/token=1.2100 target_tokens=2160269 (0.17 s/step) -learner1.r0 INFO local_step=770 global_step=0 loss/token=1.0714 target_tokens=2187712 (0.17 s/step) -learner1.r0 INFO local_step=780 global_step=0 loss/token=1.1799 target_tokens=2215121 (0.17 s/step) -learner1.r0 INFO local_step=790 global_step=0 loss/token=1.0908 target_tokens=2243190 (0.17 s/step) -learner1.r0 INFO local_step=800 global_step=0 loss/token=1.1151 target_tokens=2270963 (0.17 s/step) -learner1.r0 INFO local_step=810 global_step=0 loss/token=1.0538 target_tokens=2299956 (0.17 s/step) -learner1.r0 INFO local_step=820 global_step=0 loss/token=0.9287 target_tokens=2328427 (0.17 s/step) -learner1.r0 INFO local_step=830 global_step=0 loss/token=0.9958 target_tokens=2356465 (0.17 s/step) -learner1.r0 INFO local_step=840 global_step=0 loss/token=0.6022 target_tokens=2385834 (0.17 s/step) -learner1.r0 INFO local_step=850 global_step=0 loss/token=0.5885 target_tokens=2414474 (0.17 s/step) -learner1.r0 INFO local_step=860 global_step=0 loss/token=0.9293 target_tokens=2444659 (0.17 s/step) -learner1.r0 INFO local_step=870 global_step=0 loss/token=0.9642 target_tokens=2474556 (0.17 s/step) -learner1.r0 INFO local_step=880 global_step=0 loss/token=0.7807 target_tokens=2502211 (0.17 s/step) -learner1.r0 INFO local_step=890 global_step=0 loss/token=0.8472 target_tokens=2529253 (0.17 s/step) -learner1.r0 INFO local_step=900 global_step=0 loss/token=0.7424 target_tokens=2556195 (0.17 s/step) -learner1.r0 INFO local_step=910 global_step=0 loss/token=0.8783 target_tokens=2583787 (0.17 s/step) -learner1.r0 INFO local_step=920 global_step=0 loss/token=0.7522 target_tokens=2612714 (0.17 s/step) -learner1.r0 INFO local_step=930 global_step=0 loss/token=0.8417 target_tokens=2640863 (0.17 s/step) -learner1.r0 INFO local_step=940 global_step=0 loss/token=1.0607 target_tokens=2669911 (0.17 s/step) -learner1.r0 INFO local_step=950 global_step=0 loss/token=0.8686 target_tokens=2698032 (0.17 s/step) -learner1.r0 INFO local_step=960 global_step=0 loss/token=1.0213 target_tokens=2726453 (0.17 s/step) -learner1.r0 INFO local_step=970 global_step=0 loss/token=0.8592 target_tokens=2754833 (0.17 s/step) -learner1.r0 INFO local_step=980 global_step=0 loss/token=0.6148 target_tokens=2783234 (0.17 s/step) -learner1.r0 INFO local_step=990 global_step=0 loss/token=0.5817 target_tokens=2810996 (0.17 s/step) -learner1.r0 INFO local_step=1000 global_step=0 loss/token=0.6390 target_tokens=2838536 (0.18 s/step) -learner1.r0 INFO local_step=1010 global_step=0 loss/token=0.5169 target_tokens=2866302 (0.17 s/step) -learner1.r0 INFO local_step=1020 global_step=0 loss/token=0.7541 target_tokens=2896729 (0.17 s/step) -learner1.r0 INFO local_step=1030 global_step=0 loss/token=0.6113 target_tokens=2922881 (0.17 s/step) -learner1.r0 INFO local_step=1040 global_step=0 loss/token=0.6085 target_tokens=2950075 (0.17 s/step) -learner1.r0 INFO local_step=1050 global_step=0 loss/token=0.6285 target_tokens=2978846 (0.17 s/step) -learner1.r0 INFO local_step=1060 global_step=0 loss/token=0.5775 target_tokens=3008111 (0.17 s/step) -learner1.r0 INFO local_step=1070 global_step=0 loss/token=0.9757 target_tokens=3037085 (0.17 s/step) -learner1.r0 INFO local_step=1080 global_step=0 loss/token=0.6330 target_tokens=3065190 (0.17 s/step) -learner1.r0 INFO local_step=1090 global_step=0 loss/token=0.6943 target_tokens=3094612 (0.17 s/step) -learner1.r0 INFO local_step=1100 global_step=0 loss/token=0.6273 target_tokens=3122526 (0.17 s/step) -learner1.r0 INFO local_step=1110 global_step=0 loss/token=0.3897 target_tokens=3151605 (0.17 s/step) -learner1.r0 INFO local_step=1120 global_step=0 loss/token=0.5497 target_tokens=3179078 (0.17 s/step) -learner1.r0 INFO local_step=1130 global_step=0 loss/token=0.5109 target_tokens=3205951 (0.17 s/step) -learner1.r0 INFO local_step=1140 global_step=0 loss/token=0.6352 target_tokens=3236428 (0.17 s/step) -learner1.r0 INFO local_step=1150 global_step=0 loss/token=0.5308 target_tokens=3263677 (0.17 s/step) -learner1.r0 INFO local_step=1160 global_step=0 loss/token=0.4957 target_tokens=3293111 (0.17 s/step) -learner1.r0 INFO local_step=1170 global_step=0 loss/token=0.5031 target_tokens=3320707 (0.17 s/step) -learner1.r0 INFO local_step=1180 global_step=0 loss/token=0.4423 target_tokens=3348522 (0.17 s/step) -learner1.r0 INFO local_step=1190 global_step=0 loss/token=0.6046 target_tokens=3377617 (0.17 s/step) -learner1.r0 INFO local_step=1200 global_step=0 loss/token=0.4268 target_tokens=3407371 (0.17 s/step) -learner1.r0 INFO local_step=1210 global_step=0 loss/token=0.5226 target_tokens=3436283 (0.17 s/step) -learner1.r0 INFO local_step=1220 global_step=0 loss/token=0.5036 target_tokens=3465778 (0.17 s/step) -learner1.r0 INFO local_step=1230 global_step=0 loss/token=0.5581 target_tokens=3492219 (0.17 s/step) -learner1.r0 INFO local_step=1240 global_step=0 loss/token=0.5917 target_tokens=3520836 (0.17 s/step) -learner1.r0 INFO local_step=1250 global_step=0 loss/token=0.4442 target_tokens=3549763 (0.17 s/step) -learner1.r0 INFO local_step=1260 global_step=0 loss/token=0.3529 target_tokens=3580606 (0.17 s/step) -learner1.r0 INFO local_step=1270 global_step=0 loss/token=0.4759 target_tokens=3608057 (0.17 s/step) -learner1.r0 INFO local_step=1280 global_step=0 loss/token=0.4425 target_tokens=3635681 (0.17 s/step) -learner1.r0 INFO local_step=1290 global_step=0 loss/token=1.6040 target_tokens=3661972 (0.18 s/step) -learner1.r0 INFO local_step=1300 global_step=0 loss/token=1.6507 target_tokens=3690502 (0.17 s/step) -learner1.r0 INFO local_step=1310 global_step=0 loss/token=1.4601 target_tokens=3718239 (0.17 s/step) -learner1.r0 INFO local_step=1320 global_step=5 loss/token=1.7331 target_tokens=3746093 (0.17 s/step) -learner1.r0 INFO local_step=1330 global_step=8 loss/token=1.7922 target_tokens=3776305 (0.17 s/step) -learner1.r0 INFO local_step=1340 global_step=11 loss/token=1.6058 target_tokens=3802718 (0.17 s/step) -learner1.r0 INFO local_step=1350 global_step=14 loss/token=1.5552 target_tokens=3832825 (0.17 s/step) diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-syncer-tape.txt b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-syncer-tape.txt deleted file mode 100644 index 37d5cb8ee..000000000 --- a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-8.7-syncer-tape.txt +++ /dev/null @@ -1,22 +0,0 @@ -2026-09-23T14:28:48.889943Z INFO yeto_syncer::server: learner connected (layout: 8 fragments) learner_id=1 generation=17716224334033667103 num_streams=4 -2026-09-23T14:32:28.621692Z INFO yeto_syncer::server: learner connected (layout: 8 fragments) learner_id=0 generation=12705452410956145537 num_streams=4 -2026-09-23T14:32:34.969022Z INFO yeto_syncer::server: outer step step=1 fragment=0 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="4.6840" ms=4616 -2026-09-23T14:32:34.986272Z INFO yeto_syncer::server: outer step step=2 fragment=1 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="4.5054" ms=4633 -2026-09-23T14:32:35.335961Z INFO yeto_syncer::server: outer step step=3 fragment=2 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="8.5878" ms=366 -2026-09-23T14:32:35.378098Z INFO yeto_syncer::server: outer step step=4 fragment=3 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="8.4471" ms=391 -2026-09-23T14:32:35.706346Z INFO yeto_syncer::server: outer step step=5 fragment=4 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="8.2491" ms=369 -2026-09-23T14:32:36.246907Z INFO yeto_syncer::server: outer step step=6 fragment=5 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="7.6038" ms=358 -2026-09-23T14:32:36.879867Z INFO yeto_syncer::server: outer step step=7 fragment=6 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="7.8732" ms=448 -2026-09-23T14:32:37.387502Z INFO yeto_syncer::server: outer step step=8 fragment=7 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="7.4388" ms=412 -2026-09-23T14:32:37.914532Z INFO yeto_syncer::server: outer step step=9 fragment=0 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="3.3872" ms=355 -2026-09-23T14:32:38.414774Z INFO yeto_syncer::server: outer step step=10 fragment=1 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="3.3140" ms=313 -2026-09-23T14:32:39.095259Z INFO yeto_syncer::server: outer step step=11 fragment=2 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="3.2641" ms=452 -2026-09-23T14:32:39.603707Z INFO yeto_syncer::server: outer step step=12 fragment=3 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="3.3407" ms=418 -2026-09-23T14:32:40.126514Z INFO yeto_syncer::server: outer step step=13 fragment=4 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="3.3352" ms=399 -2026-09-23T14:32:40.632759Z INFO yeto_syncer::server: outer step step=14 fragment=5 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="3.2449" ms=362 -2026-09-23T14:32:41.157168Z INFO yeto_syncer::server: outer step step=15 fragment=6 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="3.4433" ms=344 -2026-09-23T14:32:41.655405Z INFO yeto_syncer::server: outer step step=16 fragment=7 responders=[Member { learner_id: 0, generation: 12705452410956145537 }, Member { learner_id: 1, generation: 17716224334033667103 }] gnorm="3.3660" ms=299 -2026-09-23T14:32:42.440471Z INFO yeto_syncer::server: learner finalized authoritative parameters learner_id=1 generation=17716224334033667103 step=16 remaining=1 -2026-09-23T14:32:42.760755Z INFO yeto_syncer::server: learner finalized authoritative parameters learner_id=0 generation=12705452410956145537 step=16 remaining=0 -2026-09-23T14:32:42.760790Z INFO yeto_syncer::server: all learners acknowledged final cut learners=2 -2026-09-23T14:32:42.760793Z INFO yeto_syncer::server: training complete after 16 outer steps diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-post-change.json b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-post-change.json deleted file mode 100644 index f836e8110..000000000 --- a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-post-change.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "model": "qwen3-8b", - "budget_per_hour": 60.0, - "weights_gb_bf16": 17.0, - "shard": "ddp", - "islands": [ - { - "key": "aws:8xh100@us-east-2", - "count": 2, - "cloud": "aws", - "region": "us-east-2", - "instance_type": "p5.48xlarge", - "nodes": 1, - "score": 9, - "score_assumed": false, - "est_price_per_hour": 20.257, - "price_source": "catalog", - "eff_tflops": 2630.74 - } - ], - "total_eff_tflops": 5261.48, - "est_cost_per_hour": 40.914, - "budget_cost_with_margin": 46.991099999999996, - "price_margin": 0.15, - "head_cost_per_hour": 0.4, - "binding": [ - "budget", - "quota:us-east-2:L-7212CCBC" - ], - "island_shape": null, - "rejections": [ - "aws:1xh100@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:1xh100@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:1xl40s@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:1xl40s@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:1xt4@us-east-1: T4 predates bf16 training", - "aws:1xt4@us-east-2: T4 predates bf16 training", - "aws:2x1xa10g@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:2x1xa10g@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:2x1xl4@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:2x1xl4@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xa10g@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:4xa10g@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xl40s@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:4xl40s@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xl4@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:4xl4@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xt4@us-east-1: T4 predates bf16 training", - "aws:4xt4@us-east-2: T4 predates bf16 training", - "aws:8xa100@us-east-1: placement score 3 \u2264 7", - "aws:8xa100@us-east-2: placement score 2 \u2264 7", - "aws:8xa10g@us-east-1: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-1)", - "aws:8xa10g@us-east-2: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-2)", - "aws:8xb200@us-east-1: placement score 1 \u2264 7", - "aws:8xb200@us-east-2: placement score 3 \u2264 7", - "aws:8xh100@us-east-1: placement score 3 \u2264 7", - "aws:8xh200@us-east-1: placement score 1 \u2264 7", - "aws:8xh200@us-east-2: placement score 1 \u2264 7", - "aws:8xl40s@us-east-1: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-1)", - "aws:8xl40s@us-east-2: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-2)", - "aws:8xl4@us-east-1: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-1)", - "aws:8xl4@us-east-2: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-2)", - "aws:8xt4@us-east-1: T4 predates bf16 training", - "aws:8xt4@us-east-2: T4 predates bf16 training" - ], - "warnings": [], - "launch_argv": [ - "yeto", - "launch", - "--gpu", - "aws:8xh100@us-east-2,aws:8xh100@us-east-2", - "--model", - "qwen3-8b", - "--tuning", - "lora", - "--shard", - "ddp", - "--disk-size", - "512", - "--data", - "" - ] -} diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-post-change.stderr b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-post-change.stderr deleted file mode 100644 index e69de29bb..000000000 diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-pre-change.json b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-pre-change.json deleted file mode 100644 index cb1ea4d04..000000000 --- a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-pre-change.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "model": "qwen3-8b", - "budget_per_hour": 60.0, - "weights_gb_bf16": 17.0, - "shard": "ddp", - "islands": [ - { - "key": "aws:8xh100@us-east-2", - "count": 2, - "cloud": "aws", - "region": "us-east-2", - "instance_type": "p5.48xlarge", - "nodes": 1, - "score": 9, - "score_assumed": false, - "est_price_per_hour": 20.257, - "eff_tflops": 2630.74 - } - ], - "total_eff_tflops": 5261.48, - "est_cost_per_hour": 40.914, - "budget_cost_with_margin": 46.991099999999996, - "price_margin": 0.15, - "head_cost_per_hour": 0.4, - "binding": [ - "budget", - "quota:us-east-2:L-7212CCBC" - ], - "rejections": [ - "aws:1xh100@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:1xh100@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:1xl40s@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:1xl40s@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:1xt4@us-east-1: T4 predates bf16 training", - "aws:1xt4@us-east-2: T4 predates bf16 training", - "aws:2x1xa10g@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:2x1xa10g@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:2x1xl4@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:2x1xl4@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xa10g@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:4xa10g@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xl40s@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:4xl40s@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xl4@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:4xl4@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xt4@us-east-1: T4 predates bf16 training", - "aws:4xt4@us-east-2: T4 predates bf16 training", - "aws:8xa100@us-east-1: placement score 3 \u2264 7", - "aws:8xa100@us-east-2: placement score 2 \u2264 7", - "aws:8xa10g@us-east-1: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-1)", - "aws:8xa10g@us-east-2: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-2)", - "aws:8xb200@us-east-1: placement score 1 \u2264 7", - "aws:8xb200@us-east-2: placement score 3 \u2264 7", - "aws:8xh100@us-east-1: placement score 3 \u2264 7", - "aws:8xh200@us-east-1: placement score 1 \u2264 7", - "aws:8xh200@us-east-2: placement score 1 \u2264 7", - "aws:8xl40s@us-east-1: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-1)", - "aws:8xl40s@us-east-2: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-2)", - "aws:8xl4@us-east-1: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-1)", - "aws:8xl4@us-east-2: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-2)", - "aws:8xt4@us-east-1: T4 predates bf16 training", - "aws:8xt4@us-east-2: T4 predates bf16 training" - ], - "warnings": [], - "launch_argv": [ - "yeto", - "launch", - "--gpu", - "aws:8xh100@us-east-2,aws:8xh100@us-east-2", - "--model", - "qwen3-8b", - "--tuning", - "lora", - "--shard", - "ddp", - "--disk-size", - "512", - "--data", - "" - ] -} diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-pre-change.stderr b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-budget60-pre-change.stderr deleted file mode 100644 index e69de29bb..000000000 diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-command.txt b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-command.txt deleted file mode 100644 index f24268d5e..000000000 --- a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-command.txt +++ /dev/null @@ -1,2 +0,0 @@ -yeto shape --model qwen3-8b --budget 20 --clouds aws --regions us-east-1,us-east-2 --json -yeto shape --model qwen3-8b --budget 60 --clouds aws --regions us-east-1,us-east-2 --json diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-post-change.json b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-post-change.json deleted file mode 100644 index 0794cfae5..000000000 --- a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-post-change.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "model": "qwen3-8b", - "budget_per_hour": 20.0, - "weights_gb_bf16": 17.0, - "shard": "fsdp", - "islands": [], - "total_eff_tflops": 0.0, - "est_cost_per_hour": 0.4, - "budget_cost_with_margin": 0.0, - "price_margin": 0.15, - "head_cost_per_hour": 0.4, - "binding": [ - "budget" - ], - "island_shape": null, - "rejections": [ - "aws:1xh100@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:1xh100@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:1xl40s@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:1xl40s@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:1xt4@us-east-1: T4 predates bf16 training", - "aws:1xt4@us-east-2: T4 predates bf16 training", - "aws:2x1xa10g@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:2x1xa10g@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:2x1xl4@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:2x1xl4@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xa10g@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:4xa10g@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xl40s@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:4xl40s@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xl4@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:4xl4@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xt4@us-east-1: T4 predates bf16 training", - "aws:4xt4@us-east-2: T4 predates bf16 training", - "aws:8xa100-80gb@us-east-1: one island ($21.96/hr) exceeds the budget", - "aws:8xa100@us-east-1: placement score 3 \u2264 7", - "aws:8xa100@us-east-2: placement score 2 \u2264 7", - "aws:8xa10g@us-east-1: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-1)", - "aws:8xa10g@us-east-2: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-2)", - "aws:8xb200@us-east-1: one island ($42.35/hr) exceeds the budget", - "aws:8xb200@us-east-2: one island ($42.34/hr) exceeds the budget", - "aws:8xh100@us-east-1: one island ($20.31/hr) exceeds the budget", - "aws:8xh100@us-east-2: one island ($20.26/hr) exceeds the budget", - "aws:8xh200@us-east-1: one island ($27.07/hr) exceeds the budget", - "aws:8xh200@us-east-2: one island ($24.56/hr) exceeds the budget", - "aws:8xl40s@us-east-1: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-1)", - "aws:8xl40s@us-east-2: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-2)", - "aws:8xl4@us-east-1: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-1)", - "aws:8xl4@us-east-2: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-2)", - "aws:8xt4@us-east-1: T4 predates bf16 training", - "aws:8xt4@us-east-2: T4 predates bf16 training" - ], - "warnings": [], - "launch_argv": null -} diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-post-change.stderr b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-post-change.stderr deleted file mode 100644 index e69de29bb..000000000 diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-pre-change.json b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-pre-change.json deleted file mode 100644 index 8dea74748..000000000 --- a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-pre-change.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "model": "qwen3-8b", - "budget_per_hour": 20.0, - "weights_gb_bf16": 17.0, - "shard": "fsdp", - "islands": [], - "total_eff_tflops": 0.0, - "est_cost_per_hour": 0.4, - "budget_cost_with_margin": 0.0, - "price_margin": 0.15, - "head_cost_per_hour": 0.4, - "binding": [ - "budget" - ], - "rejections": [ - "aws:1xh100@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:1xh100@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:1xl40s@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:1xl40s@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:1xt4@us-east-1: T4 predates bf16 training", - "aws:1xt4@us-east-2: T4 predates bf16 training", - "aws:2x1xa10g@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:2x1xa10g@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:2x1xl4@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:2x1xl4@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xa10g@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:4xa10g@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xl40s@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:4xl40s@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xl4@us-east-1: dominated by a fatter-node island of the same GPU", - "aws:4xl4@us-east-2: dominated by a fatter-node island of the same GPU", - "aws:4xt4@us-east-1: T4 predates bf16 training", - "aws:4xt4@us-east-2: T4 predates bf16 training", - "aws:8xa100-80gb@us-east-1: one island ($21.96/hr) exceeds the budget", - "aws:8xa100@us-east-1: placement score 3 \u2264 7", - "aws:8xa100@us-east-2: placement score 2 \u2264 7", - "aws:8xa10g@us-east-1: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-1)", - "aws:8xa10g@us-east-2: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-2)", - "aws:8xb200@us-east-1: one island ($42.35/hr) exceeds the budget", - "aws:8xb200@us-east-2: one island ($42.34/hr) exceeds the budget", - "aws:8xh100@us-east-1: one island ($20.31/hr) exceeds the budget", - "aws:8xh100@us-east-2: one island ($20.26/hr) exceeds the budget", - "aws:8xh200@us-east-1: one island ($27.07/hr) exceeds the budget", - "aws:8xh200@us-east-2: one island ($24.56/hr) exceeds the budget", - "aws:8xl40s@us-east-1: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-1)", - "aws:8xl40s@us-east-2: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-2)", - "aws:8xl4@us-east-1: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-1)", - "aws:8xl4@us-east-2: no spot quota room (0/0 vCPUs in use, L-3819A6DF@us-east-2)", - "aws:8xt4@us-east-1: T4 predates bf16 training", - "aws:8xt4@us-east-2: T4 predates bf16 training" - ], - "warnings": [], - "launch_argv": null -} diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-pre-change.stderr b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/task-9.2-pre-change.stderr deleted file mode 100644 index e69de29bb..000000000 diff --git a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/verify_cleanup.sh b/openspec/changes/add-nebius-verda-modal-clouds/run-logs/verify_cleanup.sh deleted file mode 100755 index 566082d1f..000000000 --- a/openspec/changes/add-nebius-verda-modal-clouds/run-logs/verify_cleanup.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# Post-run cleanup verification across the three clouds used by task 8.7. -cd /home/michael/yeto -echo "=== verified at $(date -u +%FT%TZ) ===" -echo -echo "=== sky status ===" -timeout 300 .venv/bin/sky status 2>&1 | grep -v FutureWarning - -echo -echo "=== modal app list ===" -timeout 180 .venv/bin/modal app list --json 2>/dev/null | .venv/bin/python -c ' -import json, sys -for a in json.load(sys.stdin): - print(a.get("description"), "|", a.get("state"), "| tasks", a.get("tasks"), "| stopped_at", a.get("stopped_at")) -' - -echo -echo "=== nebius instances (eu-north1 project) ===" -PID=$(.venv/bin/python -c "import yaml, os; print(yaml.safe_load(open(os.path.expanduser('~/.sky/config.yaml')))['nebius']['region_configs']['eu-north1']['project_id'])") -~/.nebius/bin/nebius compute instance list --parent-id "$PID" --format json 2>/dev/null | .venv/bin/python -c ' -import json, sys -items = json.load(sys.stdin).get("items", []) -print("instances:", len(items)) -for i in items: - print(" -", i["metadata"]["name"], i.get("status", {}).get("state")) -' - -echo -echo "=== aws us-east-1 running/pending instances ===" -AWS_PAGER= .venv/bin/python -m awscli ec2 describe-instances --region us-east-1 \ - --filters Name=instance-state-name,Values=running,pending \ - --query 'Reservations[].Instances[].[InstanceId,InstanceType,Tags[?Key==`Name`].Value|[0]]' \ - --output text 2>&1 | head -10 diff --git a/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/dec-legacy/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl b/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/dec-legacy/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl deleted file mode 100644 index ec8ab07cf..000000000 --- a/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/dec-legacy/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl +++ /dev/null @@ -1,26 +0,0 @@ -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":1,"partial_fragment_apply":false,"policy_version":0,"reset_parameter_count":392,"rl/optimizer_reset_count":1,"sync/apply_seconds":18.084768674999964,"sync/global_policy_hash":"c1573d237d410918ad6f36e0e0692faa04e5f8d1e6ec95e3b3411d46aa917797","time_unix":1790690554.6668603} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[0,0,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"c1573d237d410918ad6f36e0e0692faa04e5f8d1e6ec95e3b3411d46aa917797","rl/policy_token":"yeto:0:c1573d237d410918ad6f36e0e0692faa04e5f8d1e6ec95e3b3411d46aa917797","rl/rollout_id":0,"rl/sync_layout_fingerprint":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","time_unix":1790690560.605594} -{"action_tokens":5422,"active_groups":4,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":0,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":15.404935460500013,"group_p95_seconds":18.903663187650043,"group_p99_seconds":19.083854348730046,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":0.0007075946778059006,"nonzero_advantage_count":24,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5422,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0007075946778059006,"rl/ess_ratio":1.0,"rl/fragment_versions":[0,0,0,0],"rl/group_p50_seconds":15.404935460500013,"rl/group_p95_seconds":18.903663187650043,"rl/group_p99_seconds":19.083854348730046,"rl/local_delta_norm":0.0,"rl/mixed_version_group_count":0,"rl/policy_hash":"c1573d237d410918ad6f36e0e0692faa04e5f8d1e6ec95e3b3411d46aa917797","rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_id":0,"rl/rollout_seconds":19.59027813,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":19.59027813,"sync/applied_fragments":[],"sync/fragment_payload_bytes_received":0,"sync/fragment_payload_bytes_sent":0,"sync/submitted_fragments":[],"time_unix":1790690619.9528277,"tool_wait_seconds":0.0,"train_seconds":21.506647063999935,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[0,0,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"8a3e63efd74b795f031a024cf1036dd949fa812a0a1ae172e426143e90a330e2","rl/policy_token":"yeto:1:8a3e63efd74b795f031a024cf1036dd949fa812a0a1ae172e426143e90a330e2","rl/rollout_id":1,"rl/sync_layout_fingerprint":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","time_unix":1790690620.1126633} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":0,"sync/finalization":false,"sync/hook_seconds":1.180844144000048,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790690620.1137807} -{"action_tokens":9326,"active_groups":4,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":1,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.02695125185334604,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":9.025549336999916,"group_p95_seconds":13.961909474499969,"group_p99_seconds":14.46206960769998,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":0.0008219872252084315,"nonzero_advantage_count":16,"pass_rate":null,"pg_loss":null,"reward_mean":0.15625,"reward_std":0.3630921887069453,"rl/action_tokens":9326,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0008219872252084315,"rl/ess_ratio":1.0,"rl/fragment_versions":[0,0,0,0],"rl/group_p50_seconds":9.025549336999916,"rl/group_p95_seconds":13.961909474499969,"rl/group_p99_seconds":14.46206960769998,"rl/local_delta_norm":0.02695125185334604,"rl/mixed_version_group_count":0,"rl/policy_hash":"8a3e63efd74b795f031a024cf1036dd949fa812a0a1ae172e426143e90a330e2","rl/reward_mean":0.15625,"rl/reward_std":0.3630921887069453,"rl/rollout_id":1,"rl/rollout_seconds":14.5985530129999,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.5,"rollout_seconds":14.5985530129999,"sync/applied_fragments":[],"sync/fragment_payload_bytes_received":0,"sync/fragment_payload_bytes_sent":20185088,"sync/submitted_fragments":[0,1],"time_unix":1790690655.9697196,"tool_wait_seconds":0.0,"train_seconds":12.08055850300002,"train_step":null,"zero_variance_group_ratio":0.5} -{"base_version":0,"c_steps":2,"c_tokens":14748,"delta_l2_norm":0.017602773383259773,"event":"rl_fragment_push","fragment_id":0,"global_step":1,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":127.943347572,"realized_h":2,"round_attempt":1,"time_unix":1790690655.9703672} -{"base_version":0,"c_steps":2,"c_tokens":14748,"delta_l2_norm":0.02040863409638405,"event":"rl_fragment_push","fragment_id":1,"global_step":2,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":127.95671075500002,"realized_h":2,"round_attempt":1,"time_unix":1790690655.9706812} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[0,0,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"71df6e0ff680cc1878ce56ee6c4891d77db740a28396cc900b6beb774d43bc6e","rl/policy_token":"yeto:2:71df6e0ff680cc1878ce56ee6c4891d77db740a28396cc900b6beb774d43bc6e","rl/rollout_id":2,"rl/sync_layout_fingerprint":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","time_unix":1790690656.300662} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":1,"sync/finalization":false,"sync/hook_seconds":1.280282642999964,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790690656.3022327} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":1,"partial_fragment_apply":true,"policy_version":3,"reset_parameter_count":0,"rl/optimizer_reset_count":1,"sync/apply_seconds":3.8359225699999797,"sync/global_policy_hash":"d10894ff091f5bd2bfb756d46d725639c87bf10c7d0003472782edfcc99eab98","time_unix":1790690694.1397955} -{"action_tokens":8051,"active_groups":4,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":2,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.03520575348295659,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":8.921028535500056,"group_p95_seconds":12.94758489230001,"group_p99_seconds":13.355434716860005,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":0.0007414693245664239,"nonzero_advantage_count":24,"pass_rate":null,"pg_loss":null,"reward_mean":0.28125,"reward_std":0.4496092053105675,"rl/action_tokens":8051,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0007414693245664239,"rl/ess_ratio":1.0,"rl/fragment_versions":[0,0,0,0],"rl/group_p50_seconds":8.921028535500056,"rl/group_p95_seconds":12.94758489230001,"rl/group_p99_seconds":13.355434716860005,"rl/local_delta_norm":0.03520575348295659,"rl/mixed_version_group_count":0,"rl/policy_hash":"71df6e0ff680cc1878ce56ee6c4891d77db740a28396cc900b6beb774d43bc6e","rl/reward_mean":0.28125,"rl/reward_std":0.4496092053105675,"rl/rollout_id":2,"rl/rollout_seconds":13.468478194999989,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":13.468478194999989,"sync/applied_fragments":[0,1],"sync/fragment_payload_bytes_received":20185088,"sync/fragment_payload_bytes_sent":20185088,"sync/submitted_fragments":[2,3],"time_unix":1790690697.8068292,"tool_wait_seconds":0.0,"train_seconds":11.65659791400003,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_fragment_bcast","fragment_id":0,"island_id":0,"payload_bytes":10092544,"queue_seconds":32.79853531200001,"time_unix":1790690697.807764,"version":1} -{"event":"rl_fragment_bcast","fragment_id":1,"island_id":0,"payload_bytes":10092544,"queue_seconds":32.47436389000006,"time_unix":1790690697.808514,"version":2} -{"base_version":0,"c_steps":3,"c_tokens":22799,"delta_l2_norm":0.023216163739562035,"event":"rl_fragment_push","fragment_id":2,"global_step":3,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":40.309663158000035,"realized_h":3,"round_attempt":1,"time_unix":1790690697.808783} -{"base_version":0,"c_steps":3,"c_tokens":22799,"delta_l2_norm":0.02646610699594021,"event":"rl_fragment_push","fragment_id":3,"global_step":4,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":40.121894238000095,"realized_h":3,"round_attempt":1,"time_unix":1790690697.8092644} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[1,2,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"d10894ff091f5bd2bfb756d46d725639c87bf10c7d0003472782edfcc99eab98","rl/policy_token":"yeto:3:d10894ff091f5bd2bfb756d46d725639c87bf10c7d0003472782edfcc99eab98","rl/rollout_id":3,"rl/sync_layout_fingerprint":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","time_unix":1790690700.5386055} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":2,"sync/finalization":false,"sync/hook_seconds":10.991187824000008,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790690700.5422935} -{"action_tokens":7373,"active_groups":4,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":3,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.03191237385068339,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":7.397883975000013,"group_p95_seconds":11.834091895099991,"group_p99_seconds":12.295264301419985,"island_id":0,"local_round_id":4,"loss":null,"lr":null,"mean_kl":0.0007870138506405056,"nonzero_advantage_count":8,"pass_rate":null,"pg_loss":null,"reward_mean":0.84375,"reward_std":0.3630921887069453,"rl/action_tokens":7373,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0007870138506405056,"rl/ess_ratio":1.0,"rl/fragment_versions":[1,2,0,0],"rl/group_p50_seconds":7.397883975000013,"rl/group_p95_seconds":11.834091895099991,"rl/group_p99_seconds":12.295264301419985,"rl/local_delta_norm":0.03191237385068339,"rl/mixed_version_group_count":0,"rl/policy_hash":"d10894ff091f5bd2bfb756d46d725639c87bf10c7d0003472782edfcc99eab98","rl/reward_mean":0.84375,"rl/reward_std":0.3630921887069453,"rl/rollout_id":3,"rl/rollout_seconds":12.424299983999958,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.75,"rollout_seconds":12.424299983999958,"sync/applied_fragments":[],"sync/fragment_payload_bytes_received":111017984,"sync/fragment_payload_bytes_sent":40370176,"sync/submitted_fragments":[0,1,2,3],"time_unix":1790690743.0868666,"tool_wait_seconds":0.0,"train_seconds":11.796269435999989,"train_step":null,"zero_variance_group_ratio":0.75} -{"base_version":1,"c_steps":4,"c_tokens":30172,"delta_l2_norm":0.0066511789336800575,"event":"rl_fragment_push","fragment_id":0,"global_step":5,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":0.10014572299996871,"realized_h":4,"round_attempt":1,"time_unix":1790690743.0899668} -{"base_version":2,"c_steps":4,"c_tokens":30172,"delta_l2_norm":0.0070272572338581085,"event":"rl_fragment_push","fragment_id":1,"global_step":6,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":0.0372448959999474,"realized_h":4,"round_attempt":1,"time_unix":1790690743.0916188} -{"base_version":3,"c_steps":4,"c_tokens":30172,"delta_l2_norm":0.02025582268834114,"event":"rl_fragment_push","fragment_id":2,"global_step":7,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":0.13971307700001034,"realized_h":4,"round_attempt":1,"time_unix":1790690743.0922148} -{"base_version":4,"c_steps":4,"c_tokens":30172,"delta_l2_norm":0.02268216758966446,"event":"rl_fragment_push","fragment_id":3,"global_step":8,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":0.16919961299993247,"realized_h":4,"round_attempt":1,"time_unix":1790690743.0924733} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":1,"partial_fragment_apply":true,"policy_version":4,"reset_parameter_count":0,"rl/optimizer_reset_count":1,"sync/apply_seconds":10.36009783499992,"sync/global_policy_hash":"feebb2caaacdb5e5f5f8beaf35b1187e518b0b33a87eb398f1050523d1845a07","time_unix":1790690753.4543216} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[5,6,7,8],"rl/mixed_version_group_count":0,"rl/policy_hash":"feebb2caaacdb5e5f5f8beaf35b1187e518b0b33a87eb398f1050523d1845a07","rl/policy_token":"yeto:4:feebb2caaacdb5e5f5f8beaf35b1187e518b0b33a87eb398f1050523d1845a07","rl/rollout_id":4,"rl/sync_layout_fingerprint":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","time_unix":1790690756.1853726} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":3,"sync/finalization":true,"sync/hook_seconds":23.36645816700002,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790690756.1884437} diff --git a/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/dec-ports/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl b/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/dec-ports/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl deleted file mode 100644 index f07cc9836..000000000 --- a/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/dec-ports/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl +++ /dev/null @@ -1,68 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790691216.2950532} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:f885696f526aedf45989e741080eeb88f3c28cff43bcd9f5f233f64e4ec62f9f","time_unix":1790691401.924346} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":0,"time_unix":1790691424.6145396} -{"event":"rl_policy_apply","island_id":0,"local_step":0,"optimizer":"reset","partial_fragment_apply":false,"policy_version":0,"sync/apply_seconds":15.89786987299999,"sync/global_policy_hash":"c1573d237d410918ad6f36e0e0692faa04e5f8d1e6ec95e3b3411d46aa917797","time_unix":1790691440.512947} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[0,0,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"c1573d237d410918ad6f36e0e0692faa04e5f8d1e6ec95e3b3411d46aa917797","rl/policy_token":"yeto:0:c1573d237d410918ad6f36e0e0692faa04e5f8d1e6ec95e3b3411d46aa917797","rl/rollout_id":0,"rl/sync_layout_fingerprint":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","time_unix":1790691446.1698055} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790691446.5686111} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:c1573d237d410918ad6f36e0e0692faa04e5f8d1e6ec95e3b3411d46aa917797","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"71e6d29923b28cca7e797d1fc3b0a119883e6cd8ce99fbd0ee3f4fd9c4b32c9e","time_unix":1790691490.4486902} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790691490.4489162} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790691490.4542375} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790691498.7818847} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790691499.5434742} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790691522.77219} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_bcast","rollout_id":0,"time_unix":1790691523.566635} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_pull","rollout_id":0,"time_unix":1790691524.1227174} -{"action_tokens":5422,"active_groups":4,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6637349128723145,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5422,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/fragment_versions":[0,0,0,0],"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.0,"rl/mixed_version_group_count":0,"rl/policy_hash":"c1573d237d410918ad6f36e0e0692faa04e5f8d1e6ec95e3b3411d46aa917797","rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_id":0,"rl/rollout_seconds":8.331014641000138,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":8.331014641000138,"sync/applied_fragments":[],"sync/fragment_payload_bytes_received":0,"sync/fragment_payload_bytes_sent":0,"sync/submitted_fragments":[],"time_unix":1790691524.291559,"tool_wait_seconds":0.0,"train_seconds":23.228243108000015,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[0,0,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"d297c9886bb36a58756b9496f51ed5776d986af4ac7f9172c64bf0a078d1ebce","rl/policy_token":"yeto:1:d297c9886bb36a58756b9496f51ed5776d986af4ac7f9172c64bf0a078d1ebce","rl/rollout_id":1,"rl/sync_layout_fingerprint":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","time_unix":1790691524.4311254} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":0,"sync/finalization":false,"sync/hook_seconds":1.7857967010002085,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790691524.558347} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790691524.6926703} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:d297c9886bb36a58756b9496f51ed5776d986af4ac7f9172c64bf0a078d1ebce","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"b51ed63cc9a9e93223e4198d8c80cc803ee5e7403bb5520065d49ea018a53d70","time_unix":1790691526.1724107} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790691526.1726656} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790691527.145265} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1790691532.1937935} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790691533.019629} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790691549.5292993} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_bcast","rollout_id":1,"time_unix":1790691550.3455734} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_pull","rollout_id":1,"time_unix":1790691550.7561855} -{"action_tokens":9659,"active_groups":4,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.026652837419060486,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.40616747736930847,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.15625,"reward_std":0.3630921887069453,"rl/action_tokens":9659,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/fragment_versions":[0,0,0,0],"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.026652837419060486,"rl/mixed_version_group_count":0,"rl/policy_hash":"d297c9886bb36a58756b9496f51ed5776d986af4ac7f9172c64bf0a078d1ebce","rl/reward_mean":0.15625,"rl/reward_std":0.3630921887069453,"rl/rollout_id":1,"rl/rollout_seconds":6.0197490380001,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":6.0197490380001,"sync/applied_fragments":[],"sync/fragment_payload_bytes_received":0,"sync/fragment_payload_bytes_sent":20185088,"sync/submitted_fragments":[0,1],"time_unix":1790691550.9840822,"tool_wait_seconds":0.0,"train_seconds":16.50906582100015,"train_step":null,"zero_variance_group_ratio":0.25} -{"base_version":0,"c_steps":2,"c_tokens":15081,"delta_l2_norm":0.017399510368704796,"event":"rl_fragment_push","fragment_id":0,"global_step":1,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":136.0343994599998,"realized_h":2,"round_attempt":1,"time_unix":1790691550.9847698} -{"base_version":0,"c_steps":2,"c_tokens":15081,"delta_l2_norm":0.020189868286252022,"event":"rl_fragment_push","fragment_id":1,"global_step":2,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":136.0524832110002,"realized_h":2,"round_attempt":1,"time_unix":1790691550.985139} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[0,0,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"56790b0c9a0f04f7b985a7a6d463170a42fc2b2724df179adbe6c86e55bb6d4f","rl/policy_token":"yeto:2:56790b0c9a0f04f7b985a7a6d463170a42fc2b2724df179adbe6c86e55bb6d4f","rl/rollout_id":2,"rl/sync_layout_fingerprint":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","time_unix":1790691551.1539993} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":1,"sync/finalization":false,"sync/hook_seconds":1.7521276840000155,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790691551.2817843} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790691551.441403} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:56790b0c9a0f04f7b985a7a6d463170a42fc2b2724df179adbe6c86e55bb6d4f","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"2da166ddfc51c114d5eaf9b3fe4bd85037806047e14a300dc71620bc7cfe2677","time_unix":1790691552.8110156} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790691552.8116198} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790691553.9967413} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790691558.4235177} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790691559.246034} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790691576.4882689} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_bcast","rollout_id":2,"time_unix":1790691577.5537386} -{"event":"rl_driver_phase","island_id":0,"optimizer":"preserve","phase":"apply","policy_version":3,"time_unix":1790691578.2703366} -{"event":"rl_policy_apply","island_id":0,"local_step":3,"optimizer":"preserve","partial_fragment_apply":true,"policy_version":3,"sync/apply_seconds":3.2871879859999353,"sync/global_policy_hash":"02a90c867edea54afcadf8931a3400a1c2b8defe92a909f4f12a32942eac0d8c","time_unix":1790691581.5586355} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_pull","rollout_id":2,"time_unix":1790691581.5623226} -{"action_tokens":8056,"active_groups":4,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.03479443910978905,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.42974650859832764,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.375,"reward_std":0.4841229182759271,"rl/action_tokens":8056,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/fragment_versions":[0,0,0,0],"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.03479443910978905,"rl/mixed_version_group_count":0,"rl/policy_hash":"56790b0c9a0f04f7b985a7a6d463170a42fc2b2724df179adbe6c86e55bb6d4f","rl/reward_mean":0.375,"rl/reward_std":0.4841229182759271,"rl/rollout_id":2,"rl/rollout_seconds":5.610740189000126,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.0,"rollout_seconds":5.610740189000126,"sync/applied_fragments":[0,1],"sync/fragment_payload_bytes_received":20185088,"sync/fragment_payload_bytes_sent":20185088,"sync/submitted_fragments":[2,3],"time_unix":1790691582.412332,"tool_wait_seconds":0.0,"train_seconds":17.241836482000053,"train_step":null,"zero_variance_group_ratio":0.0} -{"event":"rl_fragment_bcast","fragment_id":0,"island_id":0,"payload_bytes":10092544,"queue_seconds":22.201345791999984,"time_unix":1790691582.4137232,"version":1} -{"event":"rl_fragment_bcast","fragment_id":1,"island_id":0,"payload_bytes":10092544,"queue_seconds":21.835317481999937,"time_unix":1790691582.4143953,"version":2} -{"base_version":0,"c_steps":3,"c_tokens":23137,"delta_l2_norm":0.022993501275777817,"event":"rl_fragment_push","fragment_id":2,"global_step":3,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":26.67331394200005,"realized_h":3,"round_attempt":1,"time_unix":1790691582.4155812} -{"base_version":0,"c_steps":3,"c_tokens":23137,"delta_l2_norm":0.0261142086237669,"event":"rl_fragment_push","fragment_id":3,"global_step":4,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":26.343376001000024,"realized_h":3,"round_attempt":1,"time_unix":1790691582.4160066} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[1,2,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"02a90c867edea54afcadf8931a3400a1c2b8defe92a909f4f12a32942eac0d8c","rl/policy_token":"yeto:3:02a90c867edea54afcadf8931a3400a1c2b8defe92a909f4f12a32942eac0d8c","rl/rollout_id":3,"rl/sync_layout_fingerprint":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","time_unix":1790691582.9317567} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":2,"sync/finalization":false,"sync/hook_seconds":9.00478030399995,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790691585.4940171} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790691588.3822236} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:02a90c867edea54afcadf8931a3400a1c2b8defe92a909f4f12a32942eac0d8c","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"adcbd7cc1e8b11c321e6a2b6fcd0b8baf028eee6083da2de139fa364c8752517","time_unix":1790691592.7634528} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":3,"time_unix":1790691592.7640316} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":3,"rollout_id":3,"time_unix":1790691593.8979502} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":3,"time_unix":1790691598.597808} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":3,"time_unix":1790691599.4111912} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":3,"time_unix":1790691615.5916934} -{"action_tokens":7257,"active_groups":4,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":3,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.03210449527284614,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.17361636459827423,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":4,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":7257,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/fragment_versions":[1,2,0,0],"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.03210449527284614,"rl/mixed_version_group_count":0,"rl/policy_hash":"02a90c867edea54afcadf8931a3400a1c2b8defe92a909f4f12a32942eac0d8c","rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_id":3,"rl/rollout_seconds":5.832810805999998,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.75,"rollout_seconds":5.832810805999998,"sync/applied_fragments":[],"sync/fragment_payload_bytes_received":111017984,"sync/fragment_payload_bytes_sent":40370176,"sync/submitted_fragments":[0,1,2,3],"time_unix":1790691626.1956136,"tool_wait_seconds":0.0,"train_seconds":16.18017540699998,"train_step":null,"zero_variance_group_ratio":0.75} -{"base_version":1,"c_steps":4,"c_tokens":30394,"delta_l2_norm":0.006707613822072744,"event":"rl_fragment_push","fragment_id":0,"global_step":5,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":0.1881164940000417,"realized_h":4,"round_attempt":1,"time_unix":1790691626.196025} -{"base_version":2,"c_steps":4,"c_tokens":30394,"delta_l2_norm":0.00705877086147666,"event":"rl_fragment_push","fragment_id":1,"global_step":6,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":0.017443710999941686,"realized_h":4,"round_attempt":1,"time_unix":1790691626.1962347} -{"base_version":3,"c_steps":4,"c_tokens":30394,"delta_l2_norm":0.020380431786179543,"event":"rl_fragment_push","fragment_id":2,"global_step":7,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":0.13414412899987838,"realized_h":4,"round_attempt":1,"time_unix":1790691626.196398} -{"base_version":4,"c_steps":4,"c_tokens":30394,"delta_l2_norm":0.02281486988067627,"event":"rl_fragment_push","fragment_id":3,"global_step":8,"island_id":0,"payload_bytes":10092544,"pull_to_push_seconds":0.05837701600012224,"realized_h":4,"round_attempt":1,"time_unix":1790691626.1965115} -{"event":"rl_driver_phase","island_id":0,"optimizer":"preserve","phase":"apply","policy_version":4,"time_unix":1790691626.196654} -{"event":"rl_policy_apply","island_id":0,"local_step":4,"optimizer":"preserve","partial_fragment_apply":true,"policy_version":4,"sync/apply_seconds":9.346937944000047,"sync/global_policy_hash":"150ea162fe5782579c65f4bfa787617c3bc9aec68c5b0574b0f654a01b0c658e","time_unix":1790691635.5438645} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[5,6,7,8],"rl/mixed_version_group_count":0,"rl/policy_hash":"150ea162fe5782579c65f4bfa787617c3bc9aec68c5b0574b0f654a01b0c658e","rl/policy_token":"yeto:4:150ea162fe5782579c65f4bfa787617c3bc9aec68c5b0574b0f654a01b0c658e","rl/rollout_id":4,"rl/sync_layout_fingerprint":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","time_unix":1790691638.668711} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":3,"sync/finalization":true,"sync/hook_seconds":26.668770441000106,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790691642.262406} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":4,"time_unix":1790691645.167019} -{"event":"rl_publication","island_id":0,"policy_version":4,"rl/policy_token":"yeto:4:150ea162fe5782579c65f4bfa787617c3bc9aec68c5b0574b0f654a01b0c658e","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"459c560e33f743a8ff185c3991fe9fc6de9b44deaa225430cc887da8d96c47b3","time_unix":1790691654.7666576} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":4,"time_unix":1790691654.767229} diff --git a/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/parse-args-p.log b/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/parse-args-p.log deleted file mode 100644 index 9a0dc4d89..000000000 --- a/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/parse-args-p.log +++ /dev/null @@ -1,41 +0,0 @@ -imports ok -.........................s............................... [100%] -=============================== warnings summary =============================== -tests/test_rl_miles_adapter_config.py: 14 warnings - warnings.warn( - -tests/test_rl_miles_adapter_config.py::test_default_cell_count_without_miles -tests/test_rl_miles_adapter_config.py::test_default_cell_count_without_miles -tests/test_rl_miles_adapter_config.py::test_default_cell_count_without_miles - warnings.warn( - -tests/test_rl_miles_adapter_config.py::test_default_cell_count_without_miles - /opt/sglang/lib/python3.12/site-packages/transformer_engine/__init__.py:103: RuntimeWarning: Detected a Jax installation but could not find the shared object file for the Transformer Engine Jax extension library. If this is not intentional, please reinstall Transformer Engine with `pip install transformer_engine[jax]` or build from source with `NVTE_FRAMEWORK=jax`. - warnings.warn( - -tests/test_rl_miles_adapter_config.py::test_default_cell_count_without_miles - from transformer_engine.pytorch.float8_tensor import Float8Tensor - -tests/test_rl_miles_adapter_config.py::test_default_cell_count_without_miles - from transformer_engine.pytorch.fp8 import FP8GlobalStateManager, fp8_autocast - -tests/test_rl_miles_adapter_config.py::test_default_cell_count_without_miles - warnings.warn( - -tests/test_rl_miles_adapter_config.py: 10 warnings - action = action_class(**kwargs) - -tests/test_rl_miles_adapter_config.py::test_upstream_parse_args_accepts_translation[True] -tests/test_rl_miles_adapter_config.py::test_upstream_parse_args_accepts_translation[False] - /root/Megatron-LM/megatron/core/_rank_utils.py:39: UserWarning: Could not determine rank from torch.distributed, RANK, or SLURM_PROCID. Defaulting to rank 0. - warnings.warn( - --- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html -=========================== short test summary info ============================ -SKIPPED [1] tests/test_rl_miles_adapter_config.py:277: upstream cell counter importable; covered by parse_args check -56 passed, 1 skipped, 33 warnings in 43.80s -pytest_rc=0 -/opt/sglang/lib/python3.12/site-packages/transformer_engine/__init__.py:103: RuntimeWarning: Detected a Jax installation but could not find the shared object file for the Transformer Engine Jax extension library. If this is not intentional, please reinstall Transformer Engine with `pip install transformer_engine[jax]` or build from source with `NVTE_FRAMEWORK=jax`. - warnings.warn( -WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). diff --git a/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/strict2-ports/work/seed-17/yeto-federated-m2/island-0/events.jsonl b/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/strict2-ports/work/seed-17/yeto-federated-m2/island-0/events.jsonl deleted file mode 100644 index 60b52c3bf..000000000 --- a/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-09-29-gpu/strict2-ports/work/seed-17/yeto-federated-m2/island-0/events.jsonl +++ /dev/null @@ -1,43 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790690273.6630745} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:9c2740fdae43eddd935e0ae8587e2fc241aae69d5d97bad266e237ec6a6f6f70","time_unix":1790690465.3119874} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":0,"time_unix":1790690471.4297576} -{"event":"rl_policy_apply","island_id":0,"local_step":0,"optimizer":"reset","partial_fragment_apply":false,"policy_version":0,"sync/apply_seconds":149.84870418500003,"sync/global_policy_hash":"c1573d237d410918ad6f36e0e0692faa04e5f8d1e6ec95e3b3411d46aa917797","time_unix":1790690621.279177} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790690621.3850346} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:c1573d237d410918ad6f36e0e0692faa04e5f8d1e6ec95e3b3411d46aa917797","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"71e6d29923b28cca7e797d1fc3b0a119883e6cd8ce99fbd0ee3f4fd9c4b32c9e","time_unix":1790690680.9097319} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790690680.910066} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790690680.915623} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790690694.708541} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790690695.4701283} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790690736.6782143} -{"event":"rl_driver_phase","island_id":0,"phase":"export_push","rollout_id":0,"time_unix":1790690736.6784518} -{"action_tokens":5422,"active_groups":4,"applied_lr":0.0001,"applied_lrs":[0.0001],"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.23342058062553406,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6637349128723145,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5422,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":0,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.23342058062553406,"rl/mixed_version_group_count":0,"rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_policy_version":0,"rl/rollout_seconds":13.797616832000017,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":13.797616832000017,"sync/bytes_sent":40370224,"time_unix":1790690755.2389255,"tool_wait_seconds":0.0,"train_seconds":41.207739242,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"wait_global","policy_version":1,"time_unix":1790690755.3328753} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":1,"time_unix":1790690756.2615552} -{"event":"rl_policy_apply","island_id":0,"local_step":1,"optimizer":"reset","partial_fragment_apply":false,"policy_version":1,"sync/apply_seconds":75.13976189300001,"sync/global_policy_hash":"1488ce011bcfdce1d384b3e5cf613c91b94f5f18b9c9c3abca21dfc8d8b86be2","time_unix":1790690831.4021177} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790690831.6007807} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:1488ce011bcfdce1d384b3e5cf613c91b94f5f18b9c9c3abca21dfc8d8b86be2","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"8d176e7b08d34a78d880356da0c7b3d257c642bb2a4c3aa361e34c5a901ac260","time_unix":1790690833.1036549} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790690833.103963} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790690834.1546295} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1790690839.1858804} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790690840.0721755} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790690856.6757078} -{"event":"rl_driver_phase","island_id":0,"phase":"export_push","rollout_id":1,"time_unix":1790690856.6759164} -{"action_tokens":10617,"active_groups":4,"applied_lr":6.666666666666668e-05,"applied_lrs":[6.666666666666668e-05],"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.19769075512886047,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.26213788986206055,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.09375,"reward_std":0.2914805954090255,"rl/action_tokens":10617,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":1,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.19769075512886047,"rl/mixed_version_group_count":0,"rl/reward_mean":0.09375,"rl/reward_std":0.2914805954090255,"rl/rollout_policy_version":1,"rl/rollout_seconds":6.08059619800008,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.5,"rollout_seconds":6.08059619800008,"sync/bytes_sent":40370224,"time_unix":1790690858.9226196,"tool_wait_seconds":0.0,"train_seconds":16.60317477000001,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"wait_global","policy_version":2,"time_unix":1790690858.9784846} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":2,"time_unix":1790690862.671974} -{"event":"rl_policy_apply","island_id":0,"local_step":2,"optimizer":"reset","partial_fragment_apply":false,"policy_version":2,"sync/apply_seconds":47.13765966899996,"sync/global_policy_hash":"a57d3a4ea719a9fa7192617df1854d7dc9e524f0108bb60d3c5df40ef46dcf76","time_unix":1790690909.8105536} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790690912.9361017} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:a57d3a4ea719a9fa7192617df1854d7dc9e524f0108bb60d3c5df40ef46dcf76","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"3b4ee0a133d6e915e4b3cca0fefa5bd6de33082d195f60f2d15dabe17b4330ff","time_unix":1790690917.82747} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790690917.8282166} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790690918.8263662} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790690923.7120345} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790690924.6039498} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790690940.7210743} -{"event":"rl_driver_phase","island_id":0,"phase":"export_push","rollout_id":2,"time_unix":1790690940.7212274} -{"action_tokens":8509,"active_groups":4,"applied_lr":3.333333333333334e-05,"applied_lrs":[3.333333333333334e-05],"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.10056578367948532,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.32474952936172485,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.28125,"reward_std":0.4496092053105675,"rl/action_tokens":8509,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":2,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.10056578367948532,"rl/mixed_version_group_count":0,"rl/reward_mean":0.28125,"rl/reward_std":0.4496092053105675,"rl/rollout_policy_version":2,"rl/rollout_seconds":5.8821821980000095,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":5.8821821980000095,"sync/bytes_sent":40370224,"time_unix":1790690958.4509134,"tool_wait_seconds":0.0,"train_seconds":16.11675203699997,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"wait_global","policy_version":3,"time_unix":1790690958.530335} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":3,"time_unix":1790690959.5114903} -{"event":"rl_policy_apply","island_id":0,"local_step":3,"optimizer":"reset","partial_fragment_apply":false,"policy_version":3,"sync/apply_seconds":44.44098671500001,"sync/global_policy_hash":"175cd72ae984c4f38b076ccf18eb57169d05759a29a745368b5641504d649ee0","time_unix":1790691003.9538503} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790691004.1605477} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:175cd72ae984c4f38b076ccf18eb57169d05759a29a745368b5641504d649ee0","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"8ae3d8e4653c2443b42432d1db0d9b0dcf57f49fb3483aa788ebab7131a0e02e","time_unix":1790691010.8758833} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":3,"time_unix":1790691010.8774402} diff --git a/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-10-07-gpu-head/ports/head-yeto-tape.jsonl b/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-10-07-gpu-head/ports/head-yeto-tape.jsonl deleted file mode 100644 index bc1bb6efb..000000000 --- a/openspec/changes/fix-decoupled-lr-schedule/evidence/2026-10-07-gpu-head/ports/head-yeto-tape.jsonl +++ /dev/null @@ -1,16 +0,0 @@ -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":0,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.142,"sync/global_delta_norm":0.016904009297019223,"step":1,"fragment":0,"launch_base_version":0,"attempt":1,"gnorm":0.016904009297019223,"ms":176568,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":176424,"grace_ms":0,"sync_ms":142,"responders":[{"id":0,"generation":6757810205334505669,"base_version":0,"staleness":0,"c_steps":2,"c_tokens":9325,"weight":43477812.5,"contribution":0.5037674384517989},{"id":1,"generation":13073886985506235143,"base_version":0,"staleness":0,"c_steps":2,"c_tokens":9255,"weight":42827512.5,"contribution":0.4962325615482011}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":0,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.133,"sync/global_delta_norm":0.019526150103390594,"step":2,"fragment":1,"launch_base_version":0,"attempt":1,"gnorm":0.019526150103390594,"ms":180811,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":180676,"grace_ms":0,"sync_ms":133,"responders":[{"id":0,"generation":6757810205334505669,"base_version":0,"staleness":0,"c_steps":2,"c_tokens":9325,"weight":43477812.5,"contribution":0.5037674384517989},{"id":1,"generation":13073886985506235143,"base_version":0,"staleness":0,"c_steps":2,"c_tokens":9255,"weight":42827512.5,"contribution":0.4962325615482011}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":0,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.149,"sync/global_delta_norm":0.022542686964332877,"step":3,"fragment":2,"launch_base_version":0,"attempt":1,"gnorm":0.022542686964332877,"ms":19651,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":19499,"grace_ms":0,"sync_ms":149,"responders":[{"id":0,"generation":6757810205334505669,"base_version":0,"staleness":0,"c_steps":3,"c_tokens":17693,"weight":104347416.33333333,"contribution":0.4881338094770202},{"id":1,"generation":13073886985506235143,"base_version":0,"staleness":0,"c_steps":3,"c_tokens":18118,"weight":109420641.33333333,"contribution":0.5118661905229798}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":0,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.275,"sync/global_delta_norm":0.025284166540088658,"step":4,"fragment":3,"launch_base_version":0,"attempt":1,"gnorm":0.025284166540088658,"ms":15680,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":14357,"grace_ms":0,"sync_ms":275,"responders":[{"id":0,"generation":6757810205334505669,"base_version":0,"staleness":0,"c_steps":3,"c_tokens":17693,"weight":104347416.33333333,"contribution":0.4881338094770202},{"id":1,"generation":13073886985506235143,"base_version":0,"staleness":0,"c_steps":3,"c_tokens":18118,"weight":109420641.33333333,"contribution":0.5118661905229798}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":1,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.14,"sync/global_delta_norm":0.013213822411525975,"step":5,"fragment":0,"launch_base_version":1,"attempt":1,"gnorm":0.013213822411525975,"ms":50425,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":50282,"grace_ms":0,"sync_ms":140,"responders":[{"id":0,"generation":6757810205334505669,"base_version":1,"staleness":0,"c_steps":2,"c_tokens":13039,"weight":85007760.5,"contribution":0.48415372821775515},{"id":1,"generation":13073886985506235143,"base_version":1,"staleness":0,"c_steps":2,"c_tokens":13459,"weight":90572340.5,"contribution":0.5158462717822448}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":2,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.148,"sync/global_delta_norm":0.013095061785329867,"step":6,"fragment":1,"launch_base_version":2,"attempt":1,"gnorm":0.013095061785329867,"ms":42796,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":42645,"grace_ms":0,"sync_ms":148,"responders":[{"id":0,"generation":6757810205334505669,"base_version":2,"staleness":0,"c_steps":2,"c_tokens":13039,"weight":85007760.5,"contribution":0.48415372821775515},{"id":1,"generation":13073886985506235143,"base_version":2,"staleness":0,"c_steps":2,"c_tokens":13459,"weight":90572340.5,"contribution":0.5158462717822448}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":3,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.142,"sync/global_delta_norm":0.011910036760202681,"step":7,"fragment":2,"launch_base_version":3,"attempt":1,"gnorm":0.011910036760202681,"ms":17971,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":17826,"grace_ms":0,"sync_ms":142,"responders":[{"id":0,"generation":6757810205334505669,"base_version":3,"staleness":0,"c_steps":2,"c_tokens":14255,"weight":101602512.5,"contribution":0.4921703077871491},{"id":1,"generation":13073886985506235143,"base_version":3,"staleness":0,"c_steps":2,"c_tokens":14480,"weight":104835200,"contribution":0.5078296922128509}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":4,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.144,"sync/global_delta_norm":0.014588542998993776,"step":8,"fragment":3,"launch_base_version":4,"attempt":1,"gnorm":0.014588542998993776,"ms":23777,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":23630,"grace_ms":0,"sync_ms":144,"responders":[{"id":0,"generation":6757810205334505669,"base_version":4,"staleness":0,"c_steps":3,"c_tokens":19794,"weight":130600812,"contribution":0.5547189271962354},{"id":1,"generation":13073886985506235143,"base_version":4,"staleness":0,"c_steps":2,"c_tokens":14480,"weight":104835200,"contribution":0.4452810728037646}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":5,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.146,"sync/global_delta_norm":0.011552469953103276,"step":9,"fragment":0,"launch_base_version":5,"attempt":1,"gnorm":0.011552469953103276,"ms":43386,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":43238,"grace_ms":0,"sync_ms":146,"responders":[{"id":0,"generation":6757810205334505669,"base_version":5,"staleness":0,"c_steps":2,"c_tokens":11930,"weight":71162450,"contribution":0.48363797302600714},{"id":1,"generation":13073886985506235143,"base_version":5,"staleness":0,"c_steps":2,"c_tokens":12327,"weight":75977464.5,"contribution":0.5163620269739928}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":6,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.149,"sync/global_delta_norm":0.011021540856531682,"step":10,"fragment":1,"launch_base_version":6,"attempt":1,"gnorm":0.011021540856531682,"ms":34426,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":34273,"grace_ms":0,"sync_ms":149,"responders":[{"id":0,"generation":6757810205334505669,"base_version":6,"staleness":0,"c_steps":2,"c_tokens":11930,"weight":71162450,"contribution":0.48363797302600714},{"id":1,"generation":13073886985506235143,"base_version":6,"staleness":0,"c_steps":2,"c_tokens":12327,"weight":75977464.5,"contribution":0.5163620269739928}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":7,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.152,"sync/global_delta_norm":0.010939862641465976,"step":11,"fragment":2,"launch_base_version":7,"attempt":1,"gnorm":0.010939862641465976,"ms":18110,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":17956,"grace_ms":0,"sync_ms":152,"responders":[{"id":0,"generation":6757810205334505669,"base_version":7,"staleness":0,"c_steps":2,"c_tokens":11276,"weight":63574088,"contribution":0.4876951146459552},{"id":1,"generation":13073886985506235143,"base_version":7,"staleness":0,"c_steps":2,"c_tokens":11557,"weight":66782124.5,"contribution":0.5123048853540448}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":8,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.15,"sync/global_delta_norm":0.009526002874153615,"step":12,"fragment":3,"launch_base_version":8,"attempt":1,"gnorm":0.009526002874153615,"ms":28102,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":27949,"grace_ms":0,"sync_ms":150,"responders":[{"id":0,"generation":6757810205334505669,"base_version":8,"staleness":0,"c_steps":2,"c_tokens":10190,"weight":51918050,"contribution":0.500392695796501},{"id":1,"generation":13073886985506235143,"base_version":8,"staleness":0,"c_steps":2,"c_tokens":10182,"weight":51836562,"contribution":0.4996073042034989}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":9,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.153,"sync/global_delta_norm":0.00890202325721634,"step":13,"fragment":0,"launch_base_version":9,"attempt":1,"gnorm":0.00890202325721634,"ms":46305,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":46149,"grace_ms":0,"sync_ms":153,"responders":[{"id":0,"generation":6757810205334505669,"base_version":9,"staleness":0,"c_steps":2,"c_tokens":11223,"weight":62977864.5,"contribution":0.5045199345064776},{"id":1,"generation":13073886985506235143,"base_version":9,"staleness":0,"c_steps":2,"c_tokens":11122,"weight":61849442,"contribution":0.49548006549352247}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":10,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.15,"sync/global_delta_norm":0.008868444855429935,"step":14,"fragment":1,"launch_base_version":10,"attempt":1,"gnorm":0.008868444855429935,"ms":32247,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":32095,"grace_ms":0,"sync_ms":150,"responders":[{"id":0,"generation":6757810205334505669,"base_version":10,"staleness":0,"c_steps":2,"c_tokens":11223,"weight":62977864.5,"contribution":0.5045199345064776},{"id":1,"generation":13073886985506235143,"base_version":10,"staleness":0,"c_steps":2,"c_tokens":11122,"weight":61849442,"contribution":0.49548006549352247}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":11,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.141,"sync/global_delta_norm":0.00925157809174843,"step":15,"fragment":2,"launch_base_version":11,"attempt":1,"gnorm":0.00925157809174843,"ms":18640,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":18497,"grace_ms":0,"sync_ms":141,"responders":[{"id":0,"generation":6757810205334505669,"base_version":11,"staleness":0,"c_steps":2,"c_tokens":11603,"weight":67314804.5,"contribution":0.4920490663119819},{"id":1,"generation":13073886985506235143,"base_version":11,"staleness":0,"c_steps":2,"c_tokens":11789,"weight":69490260.5,"contribution":0.507950933688018}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"8aec5cf16e725739adb4d8fd825e59af89dd7783fc9ac2a413869ba823f03bc6","sync/base_version":12,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.126,"sync/global_delta_norm":0.008987127566754842,"step":16,"fragment":3,"launch_base_version":12,"attempt":1,"gnorm":0.008987127566754842,"ms":25430,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"responded":[0,1],"responded_members":[{"id":0,"generation":6757810205334505669},{"id":1,"generation":13073886985506235143}],"missed_grace":[],"missed_members":[],"quorum_ms":25302,"grace_ms":0,"sync_ms":126,"responders":[{"id":0,"generation":6757810205334505669,"base_version":12,"staleness":0,"c_steps":2,"c_tokens":12129,"weight":73556320.5,"contribution":0.4926348213758834},{"id":1,"generation":13073886985506235143,"base_version":12,"staleness":0,"c_steps":2,"c_tokens":12309,"weight":75755740.5,"contribution":0.5073651786241167}]} diff --git a/openspec/changes/fix-decoupled-lr-schedule/evidence/EVIDENCE.md b/openspec/changes/fix-decoupled-lr-schedule/evidence/EVIDENCE.md new file mode 100644 index 000000000..0e98b000d --- /dev/null +++ b/openspec/changes/fix-decoupled-lr-schedule/evidence/EVIDENCE.md @@ -0,0 +1,15 @@ +# Evidence moved out of the repository + +Run logs that used to live under this directory were removed from git by change `rl-code-quality-cleanup` (2026-10-12). +Copies are kept outside the repository at: + + /home/michael/work/infra-drafts/evidence/fix-decoupled-lr-schedule/evidence/ + +The relative path under that directory is the same as the old path under this directory. +Moved files: + +- 2026-09-29-gpu/dec-legacy/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-gpu/dec-ports/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-gpu/parse-args-p.log +- 2026-09-29-gpu/strict2-ports/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-10-07-gpu-head/ports/head-yeto-tape.jsonl diff --git a/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-cpu-img/run3.log b/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-cpu-img/run3.log deleted file mode 100644 index 5a2010af3..000000000 --- a/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-cpu-img/run3.log +++ /dev/null @@ -1,64 +0,0 @@ -✓ Initialized. View run at -https://modal.com/apps/yetalabs/main/ap-IFjK3spr0lcOfFUfQIopEL -✓ Created objects. -├── 🔨 Created mount -│ /home/michael/work/algo-1b/openspec/changes/rl-algo-grpo-knobs/evidence/2026 -│ -09-29-algo1b-cpu-img/modal_cpu_img.py -├── 🔨 Created mount /home/michael/work/algo-1b -├── 🔨 Created mount /home/michael/work/miles-next -└── 🔨 Created function run. - -========== -== CUDA == -========== - -CUDA Version 13.0.3 - -Container image Copyright (c) 2016-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -This container image and its contents are governed by the NVIDIA Deep Learning Container License. -By pulling and using the container, you accept the terms and conditions of this license: -https://developer.nvidia.com/ngc/nvidia-deep-learning-container-license - -A copy of this license is made available in this container at /NGC-DL-CONTAINER-LICENSE for your convenience. - -image miles None -name -Tesla T4 ---- full parse_args (miles-next 0394715 parser, image megatron) -......................... [100%] -=============================== warnings summary =============================== -tests/test_rl_grpo_knobs_upstream.py: 60 warnings - /usr/lib/python3.12/argparse.py:1489: DeprecationWarning: 'type' is deprecated as of Python 3.12 and will be removed in Python 3.14. - action = action_class(**kwargs) - -tests/test_rl_grpo_knobs_upstream.py: 12 warnings - /root/Megatron-LM/megatron/core/_rank_utils.py:39: UserWarning: Could not determine rank from torch.distributed, RANK, or SLURM_PROCID. Defaulting to rank 0. - warnings.warn( - --- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html -25 passed, 23 deselected, 72 warnings in 74.70s (0:01:14) - /root/Megatron-LM/megatron/core/_rank_utils.py:39: UserWarning: Could not determine rank from torch.distributed, RANK, or SLURM_PROCID. Defaulting to rank 0. - warnings.warn( - --- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html -2 passed, 36 deselected, 12 warnings in 36.77s - ---- stderr --- -+ cd /tmp -+ python -c 'import pytest' -+ python -c 'import miles, sys; print('\''image miles'\'', miles.__file__)' -+ nvidia-smi --query-gpu=name --format=csv -+ echo '--- full parse_args (miles-next 0394715 parser, image megatron)' -+ cd /yeto -+ PYTHONPATH=/yeto:/miles-next -+ python -m pytest -q -p no:cacheprovider -rfEs tests/test_rl_grpo_knobs_upstream.py -k 'full_parse_args or miles_parser or kl_loss_triggers' -+ tail -80 -+ PYTHONPATH=/yeto:/miles-next -+ python -m pytest -q -p no:cacheprovider -rfEs tests/test_rl_miles_adapter_config.py -k upstream_parse -+ tail -5 - -Stopping app - local entrypoint completed. -✓ App completed. View run at -https://modal.com/apps/yetalabs/main/ap-IFjK3spr0lcOfFUfQIopEL -exit=0 diff --git a/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/attempt1/out/setup.log b/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/attempt1/out/setup.log deleted file mode 100644 index ea5739de3..000000000 --- a/openspec/changes/rl-algo-grpo-knobs/evidence/2026-09-29-algo1b-g1/attempt1/out/setup.log +++ /dev/null @@ -1,10 +0,0 @@ -error: could not find `Cargo.toml` in `/work/yeto` or any parent directory -rows 7473 -0394715083c91182b5eb0c526eeee4196ac694b9 -9f29303bef1eea38eb613e5f454a52db1326422d -GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-103288e1-7a9f-7f8d-a85d-5912a0bfe5e9) -SETUP_OK -ls: cannot access 'target/release': No such file or directory -Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. - Generating train split: 0%| | 0/7473 [00:00 - model_path = Path(snapshot_download(repo_id=MODEL, revision=REV)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 91, in _inner_fn - return fn(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 266, in _inner - return fn(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/_snapshot_download.py", line 532, in snapshot_download - hf_thread_map( - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/utils/tqdm.py", line 441, in hf_thread_map - results[future_to_index[future]] = future.result() - ^^^^^^^^^^^^^^^ - File "/usr/lib/python3.12/concurrent/futures/_base.py", line 449, in result - return self.__get_result() - ^^^^^^^^^^^^^^^^^^^ - File "/usr/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result - raise self._exception - File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run - result = self.fn(*self.args, **self.kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/_snapshot_download.py", line 512, in _inner_hf_hub_download - hf_hub_download( # type: ignore - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 91, in _inner_fn - return fn(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 266, in _inner - return fn(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1043, in hf_hub_download - return _hf_hub_download_to_cache_dir( - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1113, in _hf_hub_download_to_cache_dir - (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_call_error) = _get_metadata_or_catch_error( - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1758, in _get_metadata_or_catch_error - metadata = get_hf_file_metadata( - ^^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 91, in _inner_fn - return fn(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1665, in get_hf_file_metadata - response = _httpx_follow_hub_redirects_with_backoff( - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 754, in _httpx_follow_hub_redirects_with_backoff - response = http_backoff( - ^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 627, in http_backoff - return next( - ^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 535, in _http_backoff_base - response = client.request(method=method, url=url, **kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/httpx/_client.py", line 825, in request - return self.send(request, auth=auth, follow_redirects=follow_redirects) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/httpx/_client.py", line 914, in send - response = self._send_handling_auth( - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/httpx/_client.py", line 942, in _send_handling_auth - response = self._send_handling_redirects( - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/httpx/_client.py", line 979, in _send_handling_redirects - response = self._send_single_request(request) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/httpx/_client.py", line 1014, in _send_single_request - response = transport.handle_request(request) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/httpx/_transports/default.py", line 249, in handle_request - with map_httpcore_exceptions(): - File "/usr/lib/python3.12/contextlib.py", line 158, in __exit__ - self.gen.throw(value) - File "/opt/sglang/lib/python3.12/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions - raise mapped_exc(message) from exc -httpx.RemoteProtocolError: Server disconnected without sending a response. diff --git a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/opsm-trainer/island-0/events.jsonl b/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/opsm-trainer/island-0/events.jsonl deleted file mode 100644 index 0ec075414..000000000 --- a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/opsm-trainer/island-0/events.jsonl +++ /dev/null @@ -1,29 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_absorbed_flags":{},"rl/algorithm_spec":"{\"advantage\":{\"estimator\":\"grpo\",\"reward_binary\":false,\"reward_postprocess\":null,\"rewards_normalization\":true,\"std_normalization\":true,\"whiten\":false},\"correction\":{\"function\":null,\"method\":\"opsm\",\"mismatch_metrics\":false,\"opsm_delta\":0.0001,\"tis_clip\":null,\"tis_clip_low\":null,\"use_rollout_logprobs\":false},\"entropy_coef\":0.0,\"execution\":{\"group_required\":true,\"max_policy_staleness\":0,\"needs_critic\":false,\"needs_rollout_logprobs\":false},\"kl\":{\"coef\":null,\"estimator\":null,\"placement\":\"none\",\"unbiased\":false},\"loss\":{\"aggregation\":\"default\",\"custom_loss\":null,\"eps_clip\":null,\"eps_clip_c\":null,\"eps_clip_high\":null,\"reducer\":null,\"variant\":\"policy_loss\"},\"plugins\":[],\"sampling\":{\"filter\":null,\"max_replacements\":null,\"over_sampling_batch_size\":null,\"overlong_filter\":false},\"schema\":\"yeto-rl-algorithm-spec-v2\"}","rl/algorithm_spec_sha256":"98dafc415d8a642e585aa9fdbefcf15d3297e4d268c547e4e8ee8e22934e2406","rl/unverified_mechanisms":["opsm","opsm_trainer"],"rl_engine":"ports","time_unix":1790701172.8410373} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"98dafc415d8a642e585aa9fdbefcf15d3297e4d268c547e4e8ee8e22934e2406","runtime_fingerprint":"sha256:64a751644593be8d8962b45b64a8ad745d758e133ca03e6f720faf4949c8cf10","time_unix":1790701291.1694884} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790701293.9039826} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790701381.259456} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790701381.2596607} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790701381.264356} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790701393.1369412} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790701393.8595002} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790701412.3141077} -{"action_tokens":5227,"active_groups":4,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6349620223045349,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rollout_seconds":11.877193374999933,"time_unix":1790701413.4473574,"tool_wait_seconds":0.0,"train_seconds":18.45422855799984,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790701413.513677} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:3641f63acd5660ac99a9bc4431414962b2276d384ec9ed4e3e7b4b2b0ac23d08","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"262cea640e3afca293442d588aaf34b54d975cef49d3723e0be2015e0a5cf33a","time_unix":1790701414.7383802} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790701414.7387137} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790701415.9094412} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1790701423.0442548} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790701423.7953825} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790701437.1995869} -{"action_tokens":5883,"active_groups":4,"applied_lr":6.666666666666668e-06,"applied_lrs":[6.666666666666668e-06],"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.2790665924549103,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.34375,"reward_std":0.47495887979908324,"rollout_seconds":8.305545674000086,"time_unix":1790701437.6926043,"tool_wait_seconds":0.0,"train_seconds":13.40388232500004,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790701437.759944} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:b054c89a5fa5204639c1baeb726bd83e39f252a24615e06de5c637a45fb8270a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"ca72856cba9381d95cceca169eabcfcc2b525499dcb1d1319971f2a763e9bc6f","time_unix":1790701438.9460857} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790701438.9466803} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790701440.2819474} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790701447.3798635} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790701448.1841605} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790701462.5351229} -{"action_tokens":10025,"active_groups":4,"applied_lr":3.333333333333334e-06,"applied_lrs":[3.333333333333334e-06],"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.17665135860443115,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.0625,"reward_std":0.24206145913796354,"rollout_seconds":8.433149026000137,"time_unix":1790701463.0024981,"tool_wait_seconds":0.0,"train_seconds":14.350626038999962,"train_step":null,"zero_variance_group_ratio":0.75} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790701463.0679064} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:d15240f60def7379a2f8cfe41e91fe4af2a7f758c73756ad8d5e9bd14e00175b","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"fec589f5c53061d39f7b7ce85f019ea2619b5094d27cd9a0773bb8406dd423e3","time_unix":1790701464.3062694} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":3,"time_unix":1790701464.3069181} diff --git a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/tis/island-0/events.jsonl b/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/tis/island-0/events.jsonl deleted file mode 100644 index def543483..000000000 --- a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1/runs/tis/island-0/events.jsonl +++ /dev/null @@ -1,29 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_absorbed_flags":{},"rl/algorithm_spec":"{\"advantage\":{\"estimator\":\"grpo\",\"reward_binary\":false,\"reward_postprocess\":null,\"rewards_normalization\":true,\"std_normalization\":true,\"whiten\":false},\"correction\":{\"function\":null,\"method\":\"tis\",\"mismatch_metrics\":false,\"opsm_delta\":null,\"tis_clip\":2.0,\"tis_clip_low\":0.0,\"use_rollout_logprobs\":false},\"entropy_coef\":0.0,\"execution\":{\"group_required\":true,\"max_policy_staleness\":0,\"needs_critic\":false,\"needs_rollout_logprobs\":false},\"kl\":{\"coef\":null,\"estimator\":null,\"placement\":\"none\",\"unbiased\":false},\"loss\":{\"aggregation\":\"default\",\"custom_loss\":null,\"eps_clip\":null,\"eps_clip_c\":null,\"eps_clip_high\":null,\"reducer\":null,\"variant\":\"policy_loss\"},\"plugins\":[],\"sampling\":{\"filter\":null,\"max_replacements\":null,\"over_sampling_batch_size\":null,\"overlong_filter\":false},\"schema\":\"yeto-rl-algorithm-spec-v2\"}","rl/algorithm_spec_sha256":"5a8a5a9ff5abb317fc8159ab56c07293dc3bc99e084542880b6d32cc5a02b441","rl/unverified_mechanisms":["tis"],"rl_engine":"ports","time_unix":1790700391.1052573} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"5a8a5a9ff5abb317fc8159ab56c07293dc3bc99e084542880b6d32cc5a02b441","runtime_fingerprint":"sha256:599c380b8137b62b399e36cd2d9b24fd1942b9cdf31ec120d6d44b510468fe5a","time_unix":1790700515.7886345} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790700519.0470111} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790700652.0304732} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790700652.0306783} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790700652.0361128} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790700667.2308261} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790700667.9848874} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790700698.659185} -{"action_tokens":5227,"active_groups":4,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.5960708260536194,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rollout_seconds":15.200056637000046,"time_unix":1790700699.2008004,"tool_wait_seconds":0.0,"train_seconds":30.67391332599999,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790700699.2683} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:1e15a17498130945673b99c5a3ffd60e937a95037657608f1553613ee1487bd7","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"21703c69fe9a7e85f1995676241c4ed6e76560f2c63388c5e5e1165c97e195f1","time_unix":1790700700.679065} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790700700.6793218} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790700701.9519563} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1790700709.1195254} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790700710.0081122} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790700723.3489735} -{"action_tokens":5595,"active_groups":4,"applied_lr":6.666666666666668e-06,"applied_lrs":[6.666666666666668e-06],"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.42718449234962463,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.46875,"reward_std":0.4990224819584785,"rollout_seconds":8.440112733000035,"time_unix":1790700723.886492,"tool_wait_seconds":0.0,"train_seconds":13.340431730999967,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790700723.955734} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:1bd19d8527e3c8d20fd8b519363fd10c08129dff8497556bdfc00e60026c75e0","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"f5ff342fadd92f21aa6c8f99ee6b3c8c47adb3ed067c5acf79faef050556fb04","time_unix":1790700725.0397933} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790700725.040472} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790700726.6164942} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790700735.5920954} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790700736.5123477} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790700750.9726384} -{"action_tokens":9952,"active_groups":4,"applied_lr":3.333333333333334e-06,"applied_lrs":[3.333333333333334e-06],"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.3378796875476837,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.125,"reward_std":0.33071891388307384,"rollout_seconds":10.551610433999997,"time_unix":1790700751.4599864,"tool_wait_seconds":0.0,"train_seconds":14.459914202999983,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790700751.52617} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:0e99b6d99d7ba0a14f3b1329a10091d741f09ae64936d2f68629e95a9a7249ef","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"548091ff2ec74682425d30beaf116709f59bbbc4ae2cafb719aa03b02fc0cac4","time_unix":1790700752.8178685} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":3,"time_unix":1790700752.8185794} diff --git a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/g2-observe/island-0/events.jsonl b/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/g2-observe/island-0/events.jsonl deleted file mode 100644 index 9eae24a1f..000000000 --- a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/g2-observe/island-0/events.jsonl +++ /dev/null @@ -1,165 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_absorbed_flags":{},"rl/algorithm_spec":"{\"advantage\":{\"estimator\":\"grpo\",\"reward_binary\":false,\"reward_postprocess\":null,\"rewards_normalization\":true,\"std_normalization\":true,\"whiten\":false},\"correction\":{\"function\":{\"path\":\"yeto.rl.algos.mismatch_observe.observe_mismatch\",\"sha256\":\"9d5209db978e940d9b246d6e08dcb56c23e114594da08bb8ac1c88c79b8d6255\"},\"method\":\"custom\",\"mismatch_metrics\":true,\"opsm_delta\":null,\"tis_clip\":null,\"tis_clip_low\":null,\"use_rollout_logprobs\":false},\"entropy_coef\":0.0,\"execution\":{\"group_required\":true,\"max_policy_staleness\":0,\"needs_critic\":false,\"needs_rollout_logprobs\":false},\"kl\":{\"coef\":null,\"estimator\":null,\"placement\":\"none\",\"unbiased\":false},\"loss\":{\"aggregation\":\"default\",\"custom_loss\":null,\"eps_clip\":null,\"eps_clip_c\":null,\"eps_clip_high\":null,\"reducer\":null,\"variant\":\"policy_loss\"},\"plugins\":[],\"sampling\":{\"filter\":null,\"max_replacements\":null,\"over_sampling_batch_size\":null,\"overlong_filter\":false},\"schema\":\"yeto-rl-algorithm-spec-v2\"}","rl/algorithm_spec_sha256":"2124e8b85a289043723be2f6e4be063e71accb95cf10e9bdcde83288a32c1213","rl/contains_unverified_mechanisms":true,"rl/outer_sync":false,"rl/unverified_mechanisms":["corrections:custom","corrections:mismatch_observe","features:mismatch_metrics"],"rl_engine":"ports","time_unix":1790702805.0905912} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"2124e8b85a289043723be2f6e4be063e71accb95cf10e9bdcde83288a32c1213","runtime_fingerprint":"sha256:8b98364adafde788bb85addca1ae8e349190376fe31d53b641d4e061243883fd","time_unix":1790702902.5544834} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790702905.1476965} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790702982.359026} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790702982.3592062} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790702982.3637457} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790702993.24626} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790702993.9946895} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790703010.1293383} -{"action_tokens":5227,"active_groups":4,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6349465847015381,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rollout_seconds":10.886985986999889,"time_unix":1790703010.6145139,"tool_wait_seconds":0.0,"train_seconds":16.134182034999867,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790703010.6765313} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"9b28cafc782f0959717e92aef89a30a92c3384a708cb818791ccb23937a34b9c","time_unix":1790703011.5874326} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790703011.5878406} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790703012.8402884} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1790703019.094822} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790703019.9384134} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790703031.4250662} -{"action_tokens":6022,"active_groups":4,"applied_lr":9.5e-06,"applied_lrs":[9.5e-06],"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.4057203531265259,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.375,"reward_std":0.4841229182759271,"rollout_seconds":7.507027211999912,"time_unix":1790703031.8859997,"tool_wait_seconds":0.0,"train_seconds":11.486257514000044,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790703031.9444852} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:81cab679658128d10b87d55da440e97112dfc4327aea2b93f5ab32f56c846726","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"b8f9d2f88497da1ef4e819558b39ede7b93959ee8dfb5e478c96c65660a45176","time_unix":1790703032.9596958} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790703032.960409} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790703033.9250896} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790703040.4728572} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790703041.4594188} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790703053.8318233} -{"action_tokens":9529,"active_groups":4,"applied_lr":9e-06,"applied_lrs":[9e-06],"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.26765966415405273,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.09375,"reward_std":0.2914805954090255,"rollout_seconds":7.512412605000009,"time_unix":1790703054.2552793,"tool_wait_seconds":0.0,"train_seconds":12.372067951999952,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790703054.3159492} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:c9729f17d927d1d248f91a11b7446a8ac4f8ee5b6b7e201b39fa680cd88168a1","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"5fb9266243c269188a7f942e12c255ea939d4ecee6b39314104cf07c99778bb0","time_unix":1790703055.2816927} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":3,"time_unix":1790703055.2822766} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":3,"rollout_id":3,"time_unix":1790703056.439771} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":3,"time_unix":1790703060.7594619} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":3,"time_unix":1790703061.5033398} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":3,"time_unix":1790703073.2978518} -{"action_tokens":7789,"active_groups":4,"applied_lr":8.5e-06,"applied_lrs":[8.5e-06],"base_policy_version":3,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.20429565012454987,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":4,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.5625,"reward_std":0.49607837082461076,"rollout_seconds":5.477143634000186,"time_unix":1790703073.7208726,"tool_wait_seconds":0.0,"train_seconds":11.794188641999881,"train_step":null,"zero_variance_group_ratio":0.75} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":4,"time_unix":1790703073.7781477} -{"event":"rl_publication","island_id":0,"policy_version":4,"rl/policy_token":"yeto:4:b316018d8707879e2712053ee5a467aa7bbb37a1a8765ad7af0d315c5eb13fcb","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"ceb8f66638f6288cb7d48aac470b14592c1c37e0a3522e1c6e9bc2c47e903ce1","time_unix":1790703074.6527233} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":4,"time_unix":1790703074.653461} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":4,"rollout_id":4,"time_unix":1790703075.8338587} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":4,"time_unix":1790703078.187907} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":4,"time_unix":1790703078.997548} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":4,"time_unix":1790703091.2039661} -{"action_tokens":8225,"active_groups":4,"applied_lr":8.000000000000001e-06,"applied_lrs":[8.000000000000001e-06],"base_policy_version":4,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.4026455581188202,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":5,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.40625,"reward_std":0.4911323014219285,"rollout_seconds":3.5344506619999265,"time_unix":1790703091.6241026,"tool_wait_seconds":0.0,"train_seconds":12.206011695000143,"train_step":null,"zero_variance_group_ratio":0.0} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":5,"time_unix":1790703091.6818209} -{"event":"rl_publication","island_id":0,"policy_version":5,"rl/policy_token":"yeto:5:9a038fae8ac8b18392baed4fb91a619c2c128e6a5a8665084678ea0c30d532c6","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"bb707b76028cff61a49447dbf1cb067243b5fa10f0f30099dd27a0949dbfe8bf","time_unix":1790703092.5867097} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":5,"time_unix":1790703092.5872767} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":5,"rollout_id":5,"time_unix":1790703093.6287987} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":5,"time_unix":1790703098.0958662} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":5,"time_unix":1790703098.8604426} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":5,"time_unix":1790703110.9344018} -{"action_tokens":8690,"active_groups":4,"applied_lr":7.500000000000001e-06,"applied_lrs":[7.500000000000001e-06],"base_policy_version":5,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.2790771722793579,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":6,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.65625,"reward_std":0.47495887979908324,"rollout_seconds":5.508542970999997,"time_unix":1790703111.3515964,"tool_wait_seconds":0.0,"train_seconds":12.073055337000142,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":6,"time_unix":1790703111.4079492} -{"event":"rl_publication","island_id":0,"policy_version":6,"rl/policy_token":"yeto:6:b7d366537edb8220acb2955837257c7dfdffc1e7ecbdbb6ba4c0aeb2992244e2","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"66485e7cbb6d9e7239ad444759abb52a92c534d19a72cc24728680e56d29cc40","time_unix":1790703112.2785568} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":6,"time_unix":1790703112.2793038} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":6,"rollout_id":6,"time_unix":1790703113.3042126} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":6,"time_unix":1790703117.7601407} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":6,"time_unix":1790703118.520049} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":6,"time_unix":1790703130.5453842} -{"action_tokens":7051,"active_groups":4,"applied_lr":7e-06,"applied_lrs":[7e-06],"base_policy_version":6,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.4568344056606293,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":7,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.78125,"reward_std":0.41339864235384227,"rollout_seconds":5.480837015999896,"time_unix":1790703131.0368967,"tool_wait_seconds":0.0,"train_seconds":12.024955355999964,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":7,"time_unix":1790703131.1075315} -{"event":"rl_publication","island_id":0,"policy_version":7,"rl/policy_token":"yeto:7:4feb48fc0e206f0839d84b0fe872fe8103fbb1b708e7d99b5fb808e7b4d82ccc","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"fe8c2769d308c12a464d496403cb6d2143152c97e7baeff0e7126f7c26022b66","time_unix":1790703132.6555438} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":7,"time_unix":1790703132.6562545} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":7,"rollout_id":7,"time_unix":1790703133.6997895} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":7,"time_unix":1790703136.9071057} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":7,"time_unix":1790703137.5982442} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":7,"time_unix":1790703149.6128013} -{"action_tokens":8149,"active_groups":4,"applied_lr":6.5000000000000004e-06,"applied_lrs":[6.5000000000000004e-06],"base_policy_version":7,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.3184625506401062,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":8,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.5,"reward_std":0.5,"rollout_seconds":4.250849568999911,"time_unix":1790703150.0208876,"tool_wait_seconds":0.0,"train_seconds":12.014198719000206,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":8,"time_unix":1790703150.0778987} -{"event":"rl_publication","island_id":0,"policy_version":8,"rl/policy_token":"yeto:8:75e7edacd78c77f5ad98408a80216204cc00b8e269a4aa42dc9bcc21ef94ead8","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"3750cf4f026f70eadcf784834049df7ac789c38370e111ab662a840605b13550","time_unix":1790703151.0905192} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":8,"time_unix":1790703151.091104} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":8,"rollout_id":8,"time_unix":1790703152.1838303} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":8,"time_unix":1790703154.2205164} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":8,"time_unix":1790703154.9011326} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":8,"time_unix":1790703166.2090595} -{"action_tokens":5316,"active_groups":4,"applied_lr":6e-06,"applied_lrs":[6e-06],"base_policy_version":8,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.20146283507347107,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":9,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.96875,"reward_std":0.17399263633843817,"rollout_seconds":3.1293554869998843,"time_unix":1790703166.6209283,"tool_wait_seconds":0.0,"train_seconds":11.307602794000104,"train_step":null,"zero_variance_group_ratio":0.75} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":9,"time_unix":1790703166.681184} -{"event":"rl_publication","island_id":0,"policy_version":9,"rl/policy_token":"yeto:9:86fce00b6228d2e601d4ff19b21604c547f6e86e5aba9f2dab0669575edc2cce","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"9c47f6b036163650e9720c69673ca8acfcf1d4314b3c3726ebc63c042f3822ad","time_unix":1790703167.6288707} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":9,"time_unix":1790703167.6296096} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":9,"rollout_id":9,"time_unix":1790703168.5974033} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":9,"time_unix":1790703170.9874256} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":9,"time_unix":1790703171.6584003} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":9,"time_unix":1790703183.1311052} -{"action_tokens":5647,"active_groups":4,"applied_lr":5.500000000000001e-06,"applied_lrs":[5.500000000000001e-06],"base_policy_version":9,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6483407616615295,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":10,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rollout_seconds":3.357758198000056,"time_unix":1790703183.5618665,"tool_wait_seconds":0.0,"train_seconds":11.472376641999972,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":10,"time_unix":1790703183.6204727} -{"event":"rl_publication","island_id":0,"policy_version":10,"rl/policy_token":"yeto:10:9f57fa94c10b6a62d0db4abbd51046f42fd41c257f0c90e155c243ce7da78389","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"e74aef8c556b975aef1708c06ce356f3afc381a7e4681232d1fcd6e772823e42","time_unix":1790703184.5359974} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":10,"time_unix":1790703184.5367699} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":10,"rollout_id":10,"time_unix":1790703185.6664994} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":10,"time_unix":1790703188.1323292} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":10,"time_unix":1790703188.8511362} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":10,"time_unix":1790703200.501703} -{"action_tokens":6772,"active_groups":4,"applied_lr":5e-06,"applied_lrs":[5e-06],"base_policy_version":10,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.31910285353660583,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":11,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.625,"reward_std":0.4841229182759271,"rollout_seconds":3.595623567999837,"time_unix":1790703200.9321797,"tool_wait_seconds":0.0,"train_seconds":11.650190221999992,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":11,"time_unix":1790703200.9908113} -{"event":"rl_publication","island_id":0,"policy_version":11,"rl/policy_token":"yeto:11:828dda838286e3f203fe66002255c648ba9a43c6ebb16d6cbba7a947780b7238","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"635b9e2a64fe733f667a9da6636c423100aa9ae5401218c127b3c4939abd0e29","time_unix":1790703201.926563} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":11,"time_unix":1790703201.9271095} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":11,"rollout_id":11,"time_unix":1790703203.2537932} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":11,"time_unix":1790703205.282439} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":11,"time_unix":1790703205.9579926} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":11,"time_unix":1790703217.9685254} -{"action_tokens":6854,"active_groups":4,"applied_lr":4.5e-06,"applied_lrs":[4.5e-06],"base_policy_version":11,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.4051735997200012,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":12,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.84375,"reward_std":0.3630921887069453,"rollout_seconds":3.3552865869999096,"time_unix":1790703218.3793395,"tool_wait_seconds":0.0,"train_seconds":12.010113505999925,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":12,"time_unix":1790703218.440363} -{"event":"rl_publication","island_id":0,"policy_version":12,"rl/policy_token":"yeto:12:dd6221e1a40542fb0e511c0f116ad97e28afce8d3ad54826a2ddd590138bc2b0","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"53b169a339fe7f4f4b55d985163eb4fd289b84d78d84d5304c2ce0a53468ec8d","time_unix":1790703219.349595} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":12,"time_unix":1790703219.3502493} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":12,"rollout_id":12,"time_unix":1790703220.3678036} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":12,"time_unix":1790703222.5337203} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":12,"time_unix":1790703223.3375437} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":12,"time_unix":1790703235.022066} -{"action_tokens":6815,"active_groups":4,"applied_lr":4.000000000000001e-06,"applied_lrs":[4.000000000000001e-06],"base_policy_version":12,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.1964692771434784,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":13,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.71875,"reward_std":0.4496092053105675,"rollout_seconds":3.183468318999985,"time_unix":1790703235.4426696,"tool_wait_seconds":0.0,"train_seconds":11.684111240999982,"train_step":null,"zero_variance_group_ratio":0.75} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":13,"time_unix":1790703235.5039485} -{"event":"rl_publication","island_id":0,"policy_version":13,"rl/policy_token":"yeto:13:ca4c3ae28cb2b0f1137ac6dcdffb835fef52f6974f45a57c5788a0d72cdb0f31","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"dbab3905b1ff5e2ed8e4d2315833b6f443a383c00120402e17a9717f6242c9da","time_unix":1790703236.3922856} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":13,"time_unix":1790703236.3931475} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":13,"rollout_id":13,"time_unix":1790703237.3542557} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":13,"time_unix":1790703239.3826435} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":13,"time_unix":1790703240.1210597} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":13,"time_unix":1790703251.9546638} -{"action_tokens":6507,"active_groups":4,"applied_lr":3.5e-06,"applied_lrs":[3.5e-06],"base_policy_version":13,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.30722108483314514,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":14,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.625,"reward_std":0.4841229182759271,"rollout_seconds":2.9896103640001,"time_unix":1790703252.3943512,"tool_wait_seconds":0.0,"train_seconds":11.833131769999909,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":14,"time_unix":1790703252.452408} -{"event":"rl_publication","island_id":0,"policy_version":14,"rl/policy_token":"yeto:14:b7b44baa910ee19280ae94a04fb2db26188c10fd05d0df0c42fe347c8ea4df27","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"2b4f9269b6b9e0dac317b75ffc2293afbfa48c16a708bf677752060d2c589eb2","time_unix":1790703253.3657243} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":14,"time_unix":1790703253.3669605} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":14,"rollout_id":14,"time_unix":1790703254.406477} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":14,"time_unix":1790703258.8655722} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":14,"time_unix":1790703259.5399678} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":14,"time_unix":1790703271.4399607} -{"action_tokens":7489,"active_groups":4,"applied_lr":3.0000000000000005e-06,"applied_lrs":[3.0000000000000005e-06],"base_policy_version":14,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.25498318672180176,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":15,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.71875,"reward_std":0.4496092053105675,"rollout_seconds":5.498607566999908,"time_unix":1790703271.8495567,"tool_wait_seconds":0.0,"train_seconds":11.899468566999985,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":15,"time_unix":1790703271.9060063} -{"event":"rl_publication","island_id":0,"policy_version":15,"rl/policy_token":"yeto:15:27f06e1b6cdff4547f77fa38fe6da914c9f89ffad61d101e71b5f15842c4d95e","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"fab2f86d4e6bfa2c065b0f9ae2a05617921229930037931c1128d1463bfdeaf2","time_unix":1790703272.9192991} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":15,"time_unix":1790703272.9200225} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":15,"rollout_id":15,"time_unix":1790703273.9938703} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":15,"time_unix":1790703276.495002} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":15,"time_unix":1790703277.1640055} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":15,"time_unix":1790703289.1476223} -{"action_tokens":8064,"active_groups":4,"applied_lr":2.5e-06,"applied_lrs":[2.5e-06],"base_policy_version":15,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.290456086397171,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":16,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.375,"reward_std":0.4841229182759271,"rollout_seconds":3.574984652000012,"time_unix":1790703289.5587397,"tool_wait_seconds":0.0,"train_seconds":11.98323716799996,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":16,"time_unix":1790703289.620723} -{"event":"rl_publication","island_id":0,"policy_version":16,"rl/policy_token":"yeto:16:cc673a5ae49b711afb065a5fd6d0784a9d9d61afbcac6271fce3243fdadef11c","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"86fb021397ed2f613749d12b348f80be2afbad1d117acddf61632c0f0d4afd0b","time_unix":1790703290.5246813} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":16,"time_unix":1790703290.525352} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":16,"rollout_id":16,"time_unix":1790703291.579483} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":16,"time_unix":1790703294.1044297} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":16,"time_unix":1790703294.8480248} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":16,"time_unix":1790703306.792501} -{"action_tokens":8191,"active_groups":4,"applied_lr":2e-06,"applied_lrs":[2e-06],"base_policy_version":16,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.4378105103969574,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":17,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.46875,"reward_std":0.4990224819584785,"rollout_seconds":3.579074191000018,"time_unix":1790703307.2065215,"tool_wait_seconds":0.0,"train_seconds":11.944113718999915,"train_step":null,"zero_variance_group_ratio":0.0} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":17,"time_unix":1790703307.2653565} -{"event":"rl_publication","island_id":0,"policy_version":17,"rl/policy_token":"yeto:17:b7a15884d08f5df681934ca2bea511ce1d69d1505fbab39cf5669ac48e59b368","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"e1bc2d3f19d2dac51e8bc474deb0a6ec3966e0054b61606ab8d32e75caca4bfb","time_unix":1790703308.1691535} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":17,"time_unix":1790703308.1699464} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":17,"rollout_id":17,"time_unix":1790703309.2214491} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":17,"time_unix":1790703311.6847606} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":17,"time_unix":1790703312.4348326} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":17,"time_unix":1790703324.5318966} -{"action_tokens":6017,"active_groups":4,"applied_lr":1.5000000000000002e-06,"applied_lrs":[1.5000000000000002e-06],"base_policy_version":17,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.3114717900753021,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":18,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.90625,"reward_std":0.2914805954090255,"rollout_seconds":3.5148283169999104,"time_unix":1790703324.9848368,"tool_wait_seconds":0.0,"train_seconds":12.096549792999895,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":18,"time_unix":1790703325.047379} -{"event":"rl_publication","island_id":0,"policy_version":18,"rl/policy_token":"yeto:18:77e5362e5a9d5a08a6e4448f29eda4945e20a34ee9a98f6746f04706c47ae05b","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"94cb5fac9f89ef2c7961ee646fd7f474b8aa6b0ddd3b87b24fbaa75d8d2d999c","time_unix":1790703326.0207589} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":18,"time_unix":1790703326.0214694} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":18,"rollout_id":18,"time_unix":1790703327.1570776} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":18,"time_unix":1790703329.4406312} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":18,"time_unix":1790703330.286916} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":18,"time_unix":1790703342.100688} -{"action_tokens":6729,"active_groups":4,"applied_lr":1e-06,"applied_lrs":[1e-06],"base_policy_version":18,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.414722204208374,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":19,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.625,"reward_std":0.4841229182759271,"rollout_seconds":3.419183376999854,"time_unix":1790703342.5226483,"tool_wait_seconds":0.0,"train_seconds":11.813399493000134,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":19,"time_unix":1790703342.580803} -{"event":"rl_publication","island_id":0,"policy_version":19,"rl/policy_token":"yeto:19:547d905e2382ef3fac617c3297d1512840ba9dd8c5d19b5486e278f47773b06a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"27914c458b841fa2eb4b8f7aaa1a622b9864783c711f65a3ac78859ecf500a29","time_unix":1790703343.448865} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":19,"time_unix":1790703343.4495976} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":19,"rollout_id":19,"time_unix":1790703344.5737329} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":19,"time_unix":1790703349.171589} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":19,"time_unix":1790703349.8682597} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":19,"time_unix":1790703361.2877824} -{"action_tokens":6092,"active_groups":4,"applied_lr":5.000000000000005e-07,"applied_lrs":[5.000000000000005e-07],"base_policy_version":19,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.2948116362094879,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":20,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rollout_seconds":5.721970743000156,"time_unix":1790703361.6961086,"tool_wait_seconds":0.0,"train_seconds":11.419177900000022,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":20,"time_unix":1790703361.7529263} -{"event":"rl_publication","island_id":0,"policy_version":20,"rl/policy_token":"yeto:20:8f803df374464f5d6e79023f085d0a765d46064fde6d79861eef4ec008e3607a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"945c70cfe502523cdea0483f2983d12c2f6ecf83d5d5ef54984df54c433b4d50","time_unix":1790703362.6172771} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":20,"time_unix":1790703362.6179433} diff --git a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/g2-observe/metrics.jsonl b/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/g2-observe/metrics.jsonl deleted file mode 100644 index dbf5bcce2..000000000 --- a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/g2-observe/metrics.jsonl +++ /dev/null @@ -1,20 +0,0 @@ -{"step": 0, "train_rollout_kl": 0.0005710568511858582, "train_rollout_logprob_abs_diff": 0.012201795354485512, "tis_abs": 0.012203935533761978, "tis_abs_p50": 0.00018197298049926758, "tis_abs_p90": 0.043325260281562805, "tis_abs_p99": 0.13560795783996582, "tis": 0.9999749064445496, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.6349465847015381, "pg_loss": 1.210719347000122e-08} -{"step": 1, "train_rollout_kl": 0.0006510338862426579, "train_rollout_logprob_abs_diff": 0.01430277619510889, "tis_abs": 0.014332459308207035, "tis_abs_p50": 0.0005605844780802727, "tis_abs_p90": 0.049723051488399506, "tis_abs_p99": 0.13852258026599884, "tis": 1.0003305673599243, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.4057203531265259, "pg_loss": -1.862645149230957e-08} -{"step": 2, "train_rollout_kl": 0.0007765759946778417, "train_rollout_logprob_abs_diff": 0.016046907752752304, "tis_abs": 0.0160166397690773, "tis_abs_p50": 0.0008178334683179855, "tis_abs_p90": 0.05709638074040413, "tis_abs_p99": 0.15308727324008942, "tis": 1.0002163648605347, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.26765966415405273, "pg_loss": 1.862645149230957e-09} -{"step": 3, "train_rollout_kl": 0.0004784369084518403, "train_rollout_logprob_abs_diff": 0.010751387104392052, "tis_abs": 0.010726195760071278, "tis_abs_p50": 0.00011226348578929901, "tis_abs_p90": 0.03578929603099823, "tis_abs_p99": 0.125135138630867, "tis": 0.999543309211731, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.20429565012454987, "pg_loss": 0.0} -{"step": 4, "train_rollout_kl": 0.0006709932931698859, "train_rollout_logprob_abs_diff": 0.013619203120470047, "tis_abs": 0.013596654869616032, "tis_abs_p50": 0.0003055538982152939, "tis_abs_p90": 0.04572919011116028, "tis_abs_p99": 0.14935272932052612, "tis": 0.9997783899307251, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.4026455581188202, "pg_loss": 1.4901161193847656e-08} -{"step": 5, "train_rollout_kl": 0.0005538229597732425, "train_rollout_logprob_abs_diff": 0.012460410594940186, "tis_abs": 0.012447223998606205, "tis_abs_p50": 0.0004703737795352936, "tis_abs_p90": 0.04175793379545212, "tis_abs_p99": 0.13496477901935577, "tis": 0.9996656179428101, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.2790771722793579, "pg_loss": -2.7939677238464355e-09} -{"step": 6, "train_rollout_kl": 0.0006854034727439284, "train_rollout_logprob_abs_diff": 0.01445272471755743, "tis_abs": 0.014464765787124634, "tis_abs_p50": 0.0004054578021168709, "tis_abs_p90": 0.04947845637798309, "tis_abs_p99": 0.15034446120262146, "tis": 1.000496506690979, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.4568344056606293, "pg_loss": -7.450580596923828e-09} -{"step": 7, "train_rollout_kl": 0.0006364723667502403, "train_rollout_logprob_abs_diff": 0.013671725988388062, "tis_abs": 0.013664714992046356, "tis_abs_p50": 0.00042282044887542725, "tis_abs_p90": 0.047329146414995193, "tis_abs_p99": 0.14568601548671722, "tis": 1.0004161596298218, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.3184625506401062, "pg_loss": -1.862645149230957e-09} -{"step": 8, "train_rollout_kl": 0.000490161357447505, "train_rollout_logprob_abs_diff": 0.011313282884657383, "tis_abs": 0.011292749084532261, "tis_abs_p50": 0.00015522632747888565, "tis_abs_p90": 0.03782808408141136, "tis_abs_p99": 0.1275923252105713, "tis": 0.9998920559883118, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.20146283507347107, "pg_loss": -1.862645149230957e-09} -{"step": 9, "train_rollout_kl": 0.000597719568759203, "train_rollout_logprob_abs_diff": 0.013261339627206326, "tis_abs": 0.01322176307439804, "tis_abs_p50": 0.0007331008091568947, "tis_abs_p90": 0.04307105019688606, "tis_abs_p99": 0.13593189418315887, "tis": 0.9995669722557068, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.6483407616615295, "pg_loss": -1.4901161193847656e-08} -{"step": 10, "train_rollout_kl": 0.0005486300797201693, "train_rollout_logprob_abs_diff": 0.012660158798098564, "tis_abs": 0.01267577987164259, "tis_abs_p50": 0.0003475463017821312, "tis_abs_p90": 0.04278182238340378, "tis_abs_p99": 0.1383649706840515, "tis": 1.0004006624221802, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.31910285353660583, "pg_loss": -1.30385160446167e-08} -{"step": 11, "train_rollout_kl": 0.00044946433627046645, "train_rollout_logprob_abs_diff": 0.009670611470937729, "tis_abs": 0.009674709290266037, "tis_abs_p50": 4.432071000337601e-05, "tis_abs_p90": 0.0299384705722332, "tis_abs_p99": 0.12577709555625916, "tis": 1.000178575515747, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.4051735997200012, "pg_loss": 9.313225746154785e-09} -{"step": 12, "train_rollout_kl": 0.000505218980833888, "train_rollout_logprob_abs_diff": 0.010860621929168701, "tis_abs": 0.010885526426136494, "tis_abs_p50": 0.0002384614199399948, "tis_abs_p90": 0.033088989555835724, "tis_abs_p99": 0.13239625096321106, "tis": 1.0003488063812256, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.1964692771434784, "pg_loss": 3.725290298461914e-09} -{"step": 13, "train_rollout_kl": 0.0005429807933978736, "train_rollout_logprob_abs_diff": 0.011821040883660316, "tis_abs": 0.01174576673656702, "tis_abs_p50": 0.0003156391903758049, "tis_abs_p90": 0.037584997713565826, "tis_abs_p99": 0.13145102560520172, "tis": 0.9994498491287231, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.30722108483314514, "pg_loss": 1.1175870895385742e-08} -{"step": 14, "train_rollout_kl": 0.0004687801992986351, "train_rollout_logprob_abs_diff": 0.011300696060061455, "tis_abs": 0.011295455507934093, "tis_abs_p50": 0.00013717170804738998, "tis_abs_p90": 0.037828221917152405, "tis_abs_p99": 0.1271980255842209, "tis": 0.9998524188995361, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.25498318672180176, "pg_loss": 1.1175870895385742e-08} -{"step": 15, "train_rollout_kl": 0.0007064578239805996, "train_rollout_logprob_abs_diff": 0.015531898476183414, "tis_abs": 0.015483738854527473, "tis_abs_p50": 0.0007773423567414284, "tis_abs_p90": 0.05363350361585617, "tis_abs_p99": 0.1432761400938034, "tis": 0.9995055198669434, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.290456086397171, "pg_loss": 7.450580596923828e-09} -{"step": 16, "train_rollout_kl": 0.00068295793607831, "train_rollout_logprob_abs_diff": 0.014719825237989426, "tis_abs": 0.014723001047968864, "tis_abs_p50": 0.00043346360325813293, "tis_abs_p90": 0.051930077373981476, "tis_abs_p99": 0.14943595230579376, "tis": 1.0003637075424194, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.4378105103969574, "pg_loss": -1.30385160446167e-08} -{"step": 17, "train_rollout_kl": 0.0004534937033895403, "train_rollout_logprob_abs_diff": 0.009905938990414143, "tis_abs": 0.009857618249952793, "tis_abs_p50": 7.857102900743484e-05, "tis_abs_p90": 0.030025186017155647, "tis_abs_p99": 0.12247707694768906, "tis": 1.0000369548797607, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.3114717900753021, "pg_loss": 0.0} -{"step": 18, "train_rollout_kl": 0.0005796962068416178, "train_rollout_logprob_abs_diff": 0.012262513861060143, "tis_abs": 0.012239771895110607, "tis_abs_p50": 0.0001731961965560913, "tis_abs_p90": 0.04045763984322548, "tis_abs_p99": 0.1360016018152237, "tis": 1.0002073049545288, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.414722204208374, "pg_loss": 1.862645149230957e-09} -{"step": 19, "train_rollout_kl": 0.0005742553039453924, "train_rollout_logprob_abs_diff": 0.012810970656573772, "tis_abs": 0.012796175666153431, "tis_abs_p50": 0.0003550676628947258, "tis_abs_p90": 0.042201198637485504, "tis_abs_p99": 0.13719645142555237, "tis": 0.999658465385437, "ois": 1.0, "ess_ratio": 1.0, "mismatch_outside_0p5_5": 0.0, "grad_norm": 0.2948116362094879, "pg_loss": 1.1175870895385742e-08} diff --git a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/mis-mask/island-0/events.jsonl b/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/mis-mask/island-0/events.jsonl deleted file mode 100644 index cf524502a..000000000 --- a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/mis-mask/island-0/events.jsonl +++ /dev/null @@ -1,29 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_absorbed_flags":{},"rl/algorithm_spec":"{\"advantage\":{\"estimator\":\"grpo\",\"reward_binary\":false,\"reward_postprocess\":null,\"rewards_normalization\":true,\"std_normalization\":true,\"whiten\":false},\"correction\":{\"function\":{\"path\":\"yeto.rl.algos.vendor.miles_mis.compute_mis_weights_with_cp\",\"sha256\":\"f75f86c302edb7563ae8026b3bf4dda992217d9eed23b5c7ad0ae93936fd9096\"},\"method\":\"custom\",\"mis_level\":\"token\",\"mis_lower_bound\":0.5,\"mis_mode\":\"mask\",\"mis_upper_bound\":2.0,\"mismatch_metrics\":false,\"opsm_delta\":null,\"tis_clip\":null,\"tis_clip_low\":null,\"use_rollout_logprobs\":false},\"entropy_coef\":0.0,\"execution\":{\"group_required\":true,\"max_policy_staleness\":0,\"needs_critic\":false,\"needs_rollout_logprobs\":false},\"kl\":{\"coef\":null,\"estimator\":null,\"placement\":\"none\",\"unbiased\":false},\"loss\":{\"aggregation\":\"default\",\"custom_loss\":null,\"eps_clip\":null,\"eps_clip_c\":null,\"eps_clip_high\":null,\"reducer\":null,\"variant\":\"policy_loss\"},\"plugins\":[],\"sampling\":{\"filter\":null,\"max_replacements\":null,\"over_sampling_batch_size\":null,\"overlong_filter\":false},\"schema\":\"yeto-rl-algorithm-spec-v2\"}","rl/algorithm_spec_sha256":"51f8877f93b4584b8cd110c0e9f0cf4a4c7ed9078853e70fd2c7199fead79010","rl/contains_unverified_mechanisms":true,"rl/outer_sync":false,"rl/unverified_mechanisms":["corrections:custom","corrections:mis_mask"],"rl_engine":"ports","time_unix":1790702505.965834} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"51f8877f93b4584b8cd110c0e9f0cf4a4c7ed9078853e70fd2c7199fead79010","runtime_fingerprint":"sha256:229bb216b6a4b914760134615f9e42bc68e818e0982680832b7088fe025b1ad2","time_unix":1790702603.1222625} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790702605.6743526} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790702683.4132886} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790702683.4135015} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790702683.418157} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790702694.1852262} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790702694.8604286} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790702711.1016634} -{"action_tokens":5227,"active_groups":4,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.5959617495536804,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rollout_seconds":10.771616102000053,"time_unix":1790702711.574223,"tool_wait_seconds":0.0,"train_seconds":16.240816472000006,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790702711.6317847} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:0b01bd3202797592027869eb5b63947ba8de8eef07211940f2bba66ae324de96","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"d2627549d4b109be25760f85dbd0fb309712c033ddf572015aba4c18b022681c","time_unix":1790702712.5435603} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790702712.5439563} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790702713.5970764} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1790702719.753983} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790702720.5828378} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790702732.0909436} -{"action_tokens":5906,"active_groups":4,"applied_lr":6.666666666666668e-06,"applied_lrs":[6.666666666666668e-06],"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.4092763662338257,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.46875,"reward_std":0.4990224819584785,"rollout_seconds":7.210051248000013,"time_unix":1790702732.5398045,"tool_wait_seconds":0.0,"train_seconds":11.507686206000017,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790702732.5979548} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:13adaac4f0236b9e720c5855519ca9c1ff1f096aacbeb89ff3f8aa86195f3c2a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"71a45ce496e8726688fcf075a1baee3aa9469fe0d3b9e7a7acf32cf630f258d1","time_unix":1790702733.578345} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790702733.5790615} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790702734.7530642} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790702741.5015712} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790702742.2367818} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790702754.6840446} -{"action_tokens":9537,"active_groups":4,"applied_lr":3.333333333333334e-06,"applied_lrs":[3.333333333333334e-06],"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.26750707626342773,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.125,"reward_std":0.33071891388307384,"rollout_seconds":7.922510181000007,"time_unix":1790702755.101845,"tool_wait_seconds":0.0,"train_seconds":12.446701777000044,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790702755.1597853} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:42898b5ad4e5f020949cc3625908423b5396d9fe8df75b771dbf51bf4e3dff6c","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"1c1e069f240c732e5f53ba93fe6d9c2ed379d3b50c8f40372ac577a0c08aa649","time_unix":1790702756.1659021} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":3,"time_unix":1790702756.166624} diff --git a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/mis/island-0/events.jsonl b/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/mis/island-0/events.jsonl deleted file mode 100644 index f74e1fe18..000000000 --- a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/mis/island-0/events.jsonl +++ /dev/null @@ -1,29 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_absorbed_flags":{},"rl/algorithm_spec":"{\"advantage\":{\"estimator\":\"grpo\",\"reward_binary\":false,\"reward_postprocess\":null,\"rewards_normalization\":true,\"std_normalization\":true,\"whiten\":false},\"correction\":{\"function\":{\"path\":\"yeto.rl.algos.vendor.miles_mis.compute_mis_weights_with_cp\",\"sha256\":\"f75f86c302edb7563ae8026b3bf4dda992217d9eed23b5c7ad0ae93936fd9096\"},\"method\":\"custom\",\"mis_level\":\"token\",\"mis_mode\":\"truncate\",\"mis_upper_bound\":2.0,\"mismatch_metrics\":false,\"opsm_delta\":null,\"tis_clip\":null,\"tis_clip_low\":null,\"use_rollout_logprobs\":false},\"entropy_coef\":0.0,\"execution\":{\"group_required\":true,\"max_policy_staleness\":0,\"needs_critic\":false,\"needs_rollout_logprobs\":false},\"kl\":{\"coef\":null,\"estimator\":null,\"placement\":\"none\",\"unbiased\":false},\"loss\":{\"aggregation\":\"default\",\"custom_loss\":null,\"eps_clip\":null,\"eps_clip_c\":null,\"eps_clip_high\":null,\"reducer\":null,\"variant\":\"policy_loss\"},\"plugins\":[],\"sampling\":{\"filter\":null,\"max_replacements\":null,\"over_sampling_batch_size\":null,\"overlong_filter\":false},\"schema\":\"yeto-rl-algorithm-spec-v2\"}","rl/algorithm_spec_sha256":"d3ae37a8782b1017ee39fbb6e6dac96c85fedf5c0a3853e657beedb348b79681","rl/contains_unverified_mechanisms":true,"rl/outer_sync":false,"rl/unverified_mechanisms":["corrections:custom","corrections:mis"],"rl_engine":"ports","time_unix":1790702206.306172} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"d3ae37a8782b1017ee39fbb6e6dac96c85fedf5c0a3853e657beedb348b79681","runtime_fingerprint":"sha256:18844d751f69119c4d2b72f3bf719656ea0ceef49b2121fae4914e93acf28c9f","time_unix":1790702303.9076278} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790702306.522381} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790702383.4529958} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790702383.4531646} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790702383.458539} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790702394.82603} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790702395.4886956} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790702411.5708156} -{"action_tokens":5227,"active_groups":4,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.5959617495536804,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rollout_seconds":11.372752659999946,"time_unix":1790702412.03488,"tool_wait_seconds":0.0,"train_seconds":16.081627828000023,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790702412.0988204} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:0b01bd3202797592027869eb5b63947ba8de8eef07211940f2bba66ae324de96","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"d2627549d4b109be25760f85dbd0fb309712c033ddf572015aba4c18b022681c","time_unix":1790702413.0869203} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790702413.0873182} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790702414.2199647} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1790702420.5051649} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790702421.4609935} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790702433.1651816} -{"action_tokens":5906,"active_groups":4,"applied_lr":6.666666666666668e-06,"applied_lrs":[6.666666666666668e-06],"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.4092763662338257,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.46875,"reward_std":0.4990224819584785,"rollout_seconds":7.417816452000011,"time_unix":1790702433.6272366,"tool_wait_seconds":0.0,"train_seconds":11.703713393000044,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790702433.6856697} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:13adaac4f0236b9e720c5855519ca9c1ff1f096aacbeb89ff3f8aa86195f3c2a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"71a45ce496e8726688fcf075a1baee3aa9469fe0d3b9e7a7acf32cf630f258d1","time_unix":1790702434.591919} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790702434.5926821} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790702435.7788491} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790702442.2656183} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790702443.036989} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790702455.447101} -{"action_tokens":9537,"active_groups":4,"applied_lr":3.333333333333334e-06,"applied_lrs":[3.333333333333334e-06],"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.26750707626342773,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.125,"reward_std":0.33071891388307384,"rollout_seconds":7.673014130999945,"time_unix":1790702455.870409,"tool_wait_seconds":0.0,"train_seconds":12.409690325999918,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790702455.9275303} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:42898b5ad4e5f020949cc3625908423b5396d9fe8df75b771dbf51bf4e3dff6c","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"1c1e069f240c732e5f53ba93fe6d9c2ed379d3b50c8f40372ac577a0c08aa649","time_unix":1790702456.9429007} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":3,"time_unix":1790702456.9435904} diff --git a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/observe.driver.log b/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/observe.driver.log deleted file mode 100644 index 91c7637e3..000000000 --- a/openspec/changes/rl-algo-mismatch-correction/evidence/2026-09-29-g1b/runs/observe.driver.log +++ /dev/null @@ -1,3 +0,0 @@ - Fetching 10 files: 0%| | 0/10 [00:00", line 1, in - File "/root/Megatron-LM/megatron/training/__init__.py", line 5, in - from .global_vars import get_args - File "/root/Megatron-LM/megatron/training/global_vars.py", line 12, in - from megatron.core import Timers - File "/root/Megatron-LM/megatron/core/__init__.py", line 5, in - import megatron.core.tensor_parallel - File "/root/Megatron-LM/megatron/core/tensor_parallel/__init__.py", line 2, in - from .cross_entropy import vocab_parallel_cross_entropy - File "/root/Megatron-LM/megatron/core/tensor_parallel/cross_entropy.py", line 7, in - from megatron.core.parallel_state import get_tensor_model_parallel_group - File "/root/Megatron-LM/megatron/core/parallel_state.py", line 17, in - from .utils import GlobalMemoryBuffer, is_torch_min_version - File "/root/Megatron-LM/megatron/core/utils.py", line 59, in - import transformer_engine # pylint: disable=unused-import - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/transformer_engine/__init__.py", line 14, in - import transformer_engine.common - File "/opt/sglang/lib/python3.12/site-packages/transformer_engine/common/__init__.py", line 382, in - _TE_LIB_CTYPES = _load_core_library() - ^^^^^^^^^^^^^^^^^^^^ - File "/opt/sglang/lib/python3.12/site-packages/transformer_engine/common/__init__.py", line 360, in _load_core_library - return ctypes.CDLL(_get_shared_object_file("core"), mode=ctypes.RTLD_GLOBAL | os.RTLD_LAZY) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/lib/python3.12/ctypes/__init__.py", line 379, in __init__ - self._handle = _dlopen(self._name, mode) - ^^^^^^^^^^^^^^^^^^^^^^^^^ -OSError: libcuda.so.1: cannot open shared object file: No such file or directory - -EXIT 1 -TERMINATED sb-19VMbFwFBWDkDbQKMCyrsA -Traceback (most recent call last): - File "/home/michael/work/algo-cap/openspec/changes/rl-algorithm-capabilities/evidence/2026-09-29-megatron-parse/modal_run.py", line 37, in - open(sys.argv[5], "wb").write(sb.filesystem.read_bytes("/work/results.json")) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/tmp/modal-venv/lib/python3.12/site-packages/modal/sandbox_fs.py", line 311, in read_bytes - raise_read_file_error(returncode, stderr, remote_path) - File "/tmp/modal-venv/lib/python3.12/site-packages/modal/_utils/sandbox_fs_utils.py", line 163, in raise_read_file_error - raise SandboxFilesystemNotFoundError(f"{payload.message}: {remote_path}") -modal.exception.SandboxFilesystemNotFoundError: path does not exist: /work/results.json diff --git a/openspec/changes/rl-algorithm-capabilities/evidence/EVIDENCE.md b/openspec/changes/rl-algorithm-capabilities/evidence/EVIDENCE.md new file mode 100644 index 000000000..f28c617a0 --- /dev/null +++ b/openspec/changes/rl-algorithm-capabilities/evidence/EVIDENCE.md @@ -0,0 +1,11 @@ +# Evidence moved out of the repository + +Run logs that used to live under this directory were removed from git by change `rl-code-quality-cleanup` (2026-10-12). +Copies are kept outside the repository at: + + /home/michael/work/infra-drafts/evidence/rl-algorithm-capabilities/evidence/ + +The relative path under that directory is the same as the old path under this directory. +Moved files: + +- 2026-09-29-megatron-parse/attempt1/run.log diff --git a/openspec/changes/rl-codex-harness-rollout/evidence/EVIDENCE.md b/openspec/changes/rl-codex-harness-rollout/evidence/EVIDENCE.md new file mode 100644 index 000000000..dd4672d22 --- /dev/null +++ b/openspec/changes/rl-codex-harness-rollout/evidence/EVIDENCE.md @@ -0,0 +1,13 @@ +# Evidence moved out of the repository + +Run logs that used to live under this directory were removed from git by change `rl-code-quality-cleanup` (2026-10-12). +Copies are kept outside the repository at: + + /home/michael/work/infra-drafts/evidence/rl-codex-harness-rollout/evidence/ + +The relative path under that directory is the same as the old path under this directory. +Moved files: + +- gpu-20261003/codex-smoke-12-launcher-events.jsonl +- gpu-20261003/codex-smoke-12-local-rounds.jsonl +- gpu-20261003/codex-smoke-12-rl-island-0.jsonl diff --git a/openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-launcher-events.jsonl b/openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-launcher-events.jsonl deleted file mode 100644 index 8b7ba96ac..000000000 --- a/openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-launcher-events.jsonl +++ /dev/null @@ -1,24 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"e3a11ab38cbb7fd911b23fdd62a4eb6dfbb1c841","placement":"colocated","rl/algorithm_absorbed_flags":{},"rl/algorithm_spec":"{\"advantage\":{\"estimator\":\"grpo\",\"reward_binary\":false,\"reward_postprocess\":null,\"rewards_normalization\":true,\"std_normalization\":true,\"whiten\":false},\"correction\":{\"function\":null,\"method\":\"tis\",\"mismatch_metrics\":false,\"opsm_delta\":null,\"tis_clip\":2.0,\"tis_clip_low\":0.0,\"use_rollout_logprobs\":false},\"entropy_coef\":0.0,\"execution\":{\"group_required\":true,\"max_policy_staleness\":0,\"needs_critic\":false,\"needs_rollout_logprobs\":false},\"kl\":{\"coef\":null,\"estimator\":null,\"placement\":\"none\",\"unbiased\":false},\"loss\":{\"aggregation\":\"default\",\"custom_loss\":null,\"eps_clip\":null,\"eps_clip_c\":null,\"eps_clip_high\":null,\"reducer\":null,\"variant\":\"policy_loss\"},\"plugins\":[],\"sampling\":{\"filter\":null,\"max_replacements\":null,\"over_sampling_batch_size\":null,\"overlong_filter\":false},\"schema\":\"yeto-rl-algorithm-spec-v2\"}","rl/algorithm_spec_sha256":"5a8a5a9ff5abb317fc8159ab56c07293dc3bc99e084542880b6d32cc5a02b441","rl/outer_sync":false,"rl_engine":"ports","time_unix":1791056211.5022855} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"5a8a5a9ff5abb317fc8159ab56c07293dc3bc99e084542880b6d32cc5a02b441","runtime_fingerprint":"sha256:91c2e58b16d48d984c8ba5dafac6c210f4703d76985775b282103e50060d29c8","time_unix":1791056661.0445838} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1791056663.200235} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:f532d77408ad236f20f1de8b9b1f92b2fd8acfcb0f1207d08b9dc707cd064919","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":4718592,"sync/publication_payload_hash":"26532268a80cc0f63d879b1a21bf8e2314d25dec18979ee8230d8a3292f31de6","time_unix":1791056767.4482992} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1791056767.44865} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1791056767.454223} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1791056944.6112895} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1791056945.339233} -{"aborted_in_flight_groups":null,"applied_lrs":[1e-05],"clip_fraction":0.0,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"label/execution_mode":"colocated-serial","label/profile_hash":"sha256:adff0d99b326e777137839198e12646a45225b803d088dfa7605f4f575301ed3","label/weight_transport":"cuda-ipc","masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0006213188171387,"tis_abs":0.017334792762994766,"tis_clipfrac":0.0,"train_rollout_kl":0.0006736259286602339},"nonzero_advantages":null,"rollout_id":0,"submitted_groups":null,"time_unix":1791057142.724791,"tool_wait_seconds":261.8106750370007,"trained_groups":6,"trained_sample_ids_sha256":"707628e31f6d45772058a87142753d1928ab0b7b3f5af09d5855dd44c445efbe","trained_samples":24} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1791057142.7261853} -{"action_tokens":33877,"active_groups":6,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":0,"cancelled_groups":3,"clip_fraction":0.0,"completed_groups":6,"completed_trajectories":24,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":3,"dynamic_filter_generated_groups":9,"dynamic_filter_replacement_attempts":3,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.05284938961267471,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":177.16254627700005,"time_unix":1791057142.7863326,"tool_wait_seconds":261.8106750370007,"train_seconds":197.3848163230001,"train_step":null,"zero_variance_group_ratio":1.0} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1791057142.7946963} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:8e0534f3dc552a92f98d561ba34e10c1c4b257990e08a4264c3dda4d0c1040f5","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":4718592,"sync/publication_payload_hash":"b1e10095098f4c73dca4e000ac87f7bafbb4e82116c81d324186f0161bc1386e","time_unix":1791057143.2502723} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1791057143.2506843} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1791057146.930764} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1791057224.2019403} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1791057225.5482666} -{"aborted_in_flight_groups":0,"applied_lrs":[5e-06],"clip_fraction":0.0,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"label/execution_mode":"colocated-serial","label/profile_hash":"sha256:adff0d99b326e777137839198e12646a45225b803d088dfa7605f4f575301ed3","label/weight_transport":"cuda-ipc","masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0005265076955159,"tis_abs":0.018604476004838943,"tis_clipfrac":0.0,"train_rollout_kl":0.0007514981552958488},"nonzero_advantages":null,"rollout_id":1,"submitted_groups":0,"time_unix":1791057292.9470909,"tool_wait_seconds":124.1331004190015,"trained_groups":6,"trained_sample_ids_sha256":"8e834f43960dba58578bec39225d36a09d4364f0ed2425da968ee27727e3d4ec","trained_samples":24} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1791057292.9475033} -{"action_tokens":30673,"active_groups":6,"applied_lr":5e-06,"applied_lrs":[5e-06],"base_policy_version":1,"cancelled_groups":1,"clip_fraction":0.0,"completed_groups":6,"completed_trajectories":24,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":1,"dynamic_filter_generated_groups":7,"dynamic_filter_replacement_attempts":1,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.09851840883493423,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":80.95116212499988,"time_unix":1791057293.0197885,"tool_wait_seconds":124.1331004190015,"train_seconds":67.39810137899985,"train_step":null,"zero_variance_group_ratio":1.0} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1791057293.0278099} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:cdd2b3d46b7f49f907b5ffadcca39e5dcc2d0eeb53ce3234bf07e575a1985ea4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":4718592,"sync/publication_payload_hash":"94dbf0df16079717d0536ab8960a987f3c093d02a3f61e68fb22a28af2cb2c59","time_unix":1791057293.4093134} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":2,"time_unix":1791057293.4098613} -{"event":"rl_learner_finalized","island_id":0,"time_unix":1791057298.5279248} diff --git a/openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-local-rounds.jsonl b/openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-local-rounds.jsonl deleted file mode 100644 index 9c4d23c83..000000000 --- a/openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-local-rounds.jsonl +++ /dev/null @@ -1,2 +0,0 @@ -{"action_tokens":33877,"active_groups":6,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":0,"cancelled_groups":3,"clip_fraction":0.0,"completed_groups":6,"completed_trajectories":24,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":3,"dynamic_filter_generated_groups":9,"dynamic_filter_replacement_attempts":3,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.05284938961267471,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":177.16254627700005,"time_unix":1791057142.7863326,"tool_wait_seconds":261.8106750370007,"train_seconds":197.3848163230001,"train_step":null,"zero_variance_group_ratio":1.0} -{"action_tokens":30673,"active_groups":6,"applied_lr":5e-06,"applied_lrs":[5e-06],"base_policy_version":1,"cancelled_groups":1,"clip_fraction":0.0,"completed_groups":6,"completed_trajectories":24,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":1,"dynamic_filter_generated_groups":7,"dynamic_filter_replacement_attempts":1,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.09851840883493423,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":80.95116212499988,"time_unix":1791057293.0197885,"tool_wait_seconds":124.1331004190015,"train_seconds":67.39810137899985,"train_step":null,"zero_variance_group_ratio":1.0} diff --git a/openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-rl-island-0.jsonl b/openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-rl-island-0.jsonl deleted file mode 100644 index 8b7ba96ac..000000000 --- a/openspec/changes/rl-codex-harness-rollout/evidence/gpu-20261003/codex-smoke-12-rl-island-0.jsonl +++ /dev/null @@ -1,24 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"e3a11ab38cbb7fd911b23fdd62a4eb6dfbb1c841","placement":"colocated","rl/algorithm_absorbed_flags":{},"rl/algorithm_spec":"{\"advantage\":{\"estimator\":\"grpo\",\"reward_binary\":false,\"reward_postprocess\":null,\"rewards_normalization\":true,\"std_normalization\":true,\"whiten\":false},\"correction\":{\"function\":null,\"method\":\"tis\",\"mismatch_metrics\":false,\"opsm_delta\":null,\"tis_clip\":2.0,\"tis_clip_low\":0.0,\"use_rollout_logprobs\":false},\"entropy_coef\":0.0,\"execution\":{\"group_required\":true,\"max_policy_staleness\":0,\"needs_critic\":false,\"needs_rollout_logprobs\":false},\"kl\":{\"coef\":null,\"estimator\":null,\"placement\":\"none\",\"unbiased\":false},\"loss\":{\"aggregation\":\"default\",\"custom_loss\":null,\"eps_clip\":null,\"eps_clip_c\":null,\"eps_clip_high\":null,\"reducer\":null,\"variant\":\"policy_loss\"},\"plugins\":[],\"sampling\":{\"filter\":null,\"max_replacements\":null,\"over_sampling_batch_size\":null,\"overlong_filter\":false},\"schema\":\"yeto-rl-algorithm-spec-v2\"}","rl/algorithm_spec_sha256":"5a8a5a9ff5abb317fc8159ab56c07293dc3bc99e084542880b6d32cc5a02b441","rl/outer_sync":false,"rl_engine":"ports","time_unix":1791056211.5022855} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"5a8a5a9ff5abb317fc8159ab56c07293dc3bc99e084542880b6d32cc5a02b441","runtime_fingerprint":"sha256:91c2e58b16d48d984c8ba5dafac6c210f4703d76985775b282103e50060d29c8","time_unix":1791056661.0445838} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1791056663.200235} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:f532d77408ad236f20f1de8b9b1f92b2fd8acfcb0f1207d08b9dc707cd064919","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":4718592,"sync/publication_payload_hash":"26532268a80cc0f63d879b1a21bf8e2314d25dec18979ee8230d8a3292f31de6","time_unix":1791056767.4482992} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1791056767.44865} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1791056767.454223} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1791056944.6112895} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1791056945.339233} -{"aborted_in_flight_groups":null,"applied_lrs":[1e-05],"clip_fraction":0.0,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"label/execution_mode":"colocated-serial","label/profile_hash":"sha256:adff0d99b326e777137839198e12646a45225b803d088dfa7605f4f575301ed3","label/weight_transport":"cuda-ipc","masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0006213188171387,"tis_abs":0.017334792762994766,"tis_clipfrac":0.0,"train_rollout_kl":0.0006736259286602339},"nonzero_advantages":null,"rollout_id":0,"submitted_groups":null,"time_unix":1791057142.724791,"tool_wait_seconds":261.8106750370007,"trained_groups":6,"trained_sample_ids_sha256":"707628e31f6d45772058a87142753d1928ab0b7b3f5af09d5855dd44c445efbe","trained_samples":24} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1791057142.7261853} -{"action_tokens":33877,"active_groups":6,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":0,"cancelled_groups":3,"clip_fraction":0.0,"completed_groups":6,"completed_trajectories":24,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":3,"dynamic_filter_generated_groups":9,"dynamic_filter_replacement_attempts":3,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.05284938961267471,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":177.16254627700005,"time_unix":1791057142.7863326,"tool_wait_seconds":261.8106750370007,"train_seconds":197.3848163230001,"train_step":null,"zero_variance_group_ratio":1.0} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1791057142.7946963} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:8e0534f3dc552a92f98d561ba34e10c1c4b257990e08a4264c3dda4d0c1040f5","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":4718592,"sync/publication_payload_hash":"b1e10095098f4c73dca4e000ac87f7bafbb4e82116c81d324186f0161bc1386e","time_unix":1791057143.2502723} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1791057143.2506843} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1791057146.930764} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1791057224.2019403} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1791057225.5482666} -{"aborted_in_flight_groups":0,"applied_lrs":[5e-06],"clip_fraction":0.0,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"label/execution_mode":"colocated-serial","label/profile_hash":"sha256:adff0d99b326e777137839198e12646a45225b803d088dfa7605f4f575301ed3","label/weight_transport":"cuda-ipc","masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0005265076955159,"tis_abs":0.018604476004838943,"tis_clipfrac":0.0,"train_rollout_kl":0.0007514981552958488},"nonzero_advantages":null,"rollout_id":1,"submitted_groups":0,"time_unix":1791057292.9470909,"tool_wait_seconds":124.1331004190015,"trained_groups":6,"trained_sample_ids_sha256":"8e834f43960dba58578bec39225d36a09d4364f0ed2425da968ee27727e3d4ec","trained_samples":24} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1791057292.9475033} -{"action_tokens":30673,"active_groups":6,"applied_lr":5e-06,"applied_lrs":[5e-06],"base_policy_version":1,"cancelled_groups":1,"clip_fraction":0.0,"completed_groups":6,"completed_trajectories":24,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":1,"dynamic_filter_generated_groups":7,"dynamic_filter_replacement_attempts":1,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.09851840883493423,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":80.95116212499988,"time_unix":1791057293.0197885,"tool_wait_seconds":124.1331004190015,"train_seconds":67.39810137899985,"train_step":null,"zero_variance_group_ratio":1.0} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1791057293.0278099} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:cdd2b3d46b7f49f907b5ffadcca39e5dcc2d0eeb53ce3234bf07e575a1985ea4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":4718592,"sync/publication_payload_hash":"94dbf0df16079717d0536ab8960a987f3c093d02a3f61e68fb22a28af2cb2c59","time_unix":1791057293.4093134} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":2,"time_unix":1791057293.4098613} -{"event":"rl_learner_finalized","island_id":0,"time_unix":1791057298.5279248} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-decoupled2/verify_peft.log b/openspec/changes/rl-engine-ports/evidence/2026-09-29-decoupled2/verify_peft.log deleted file mode 100644 index 728053dc0..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-decoupled2/verify_peft.log +++ /dev/null @@ -1,242 +0,0 @@ -[transformers] `torch_dtype` is deprecated! Use `dtype` instead! -{ - "island0": { - "n_events": 132, - "event_kinds": [ - "rl_driver_phase", - "rl_driver_start", - "rl_engine_selected", - "rl_fragment_bcast", - "rl_fragment_push", - "rl_local_round", - "rl_policy_apply", - "rl_policy_snapshot", - "rl_publication", - "rl_sync_hook" - ], - "last_apply": { - "event": "rl_policy_apply", - "island_id": 0, - "local_step": 8, - "optimizer": "preserve", - "partial_fragment_apply": true, - "policy_version": 8, - "sync/apply_seconds": 0.906831529999863, - "sync/global_policy_hash": "41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60", - "time_unix": 1790657068.283698 - }, - "tail": [ - { - "base_version": 8, - "c_steps": 8, - "c_tokens": 59480, - "delta_l2_norm": 0.17140410840511322, - "event": "rl_fragment_push", - "fragment_id": 7, - "global_step": 16, - "island_id": 0, - "payload_bytes": 5046272, - "pull_to_push_seconds": 0.028142118000005212, - "realized_h": 8, - "round_attempt": 1, - "time_unix": 1790657067.3766575 - }, - { - "event": "rl_driver_phase", - "island_id": 0, - "optimizer": "preserve", - "phase": "apply", - "policy_version": 8, - "time_unix": 1790657067.376718 - }, - { - "event": "rl_policy_apply", - "island_id": 0, - "local_step": 8, - "optimizer": "preserve", - "partial_fragment_apply": true, - "policy_version": 8, - "sync/apply_seconds": 0.906831529999863, - "sync/global_policy_hash": "41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60", - "time_unix": 1790657068.283698 - }, - { - "event": "rl_policy_snapshot", - "island_id": 0, - "rl/canonical_layout_hash": "f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3", - "rl/fragment_versions": [ - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16 - ], - "rl/mixed_version_group_count": 0, - "rl/policy_hash": "41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60", - "rl/policy_token": "yeto:8:41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60", - "rl/rollout_id": 8, - "rl/sync_layout_fingerprint": "16a4d88fc98c2525fbee672648d6c151c5eeb324a3ccd8ef6caeac5f17383d12", - "time_unix": 1790657068.4894679 - }, - { - "event": "rl_sync_hook", - "island_id": 0, - "rl/rollout_id": 7, - "sync/finalization": true, - "sync/hook_seconds": 6.537137942000072, - "sync/remote_quorum_wait_seconds": 0.0, - "time_unix": 1790657068.641325 - }, - { - "event": "rl_driver_phase", - "island_id": 0, - "phase": "publish", - "policy_version": 8, - "time_unix": 1790657068.7204113 - }, - { - "event": "rl_publication", - "island_id": 0, - "policy_version": 8, - "rl/policy_token": "yeto:8:41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60", - "sync/publication_members": [ - "engine:inference-engine-all-0-0-00000" - ], - "sync/publication_payload_bytes": 40370176, - "sync/publication_payload_hash": "ee1a0457706ee7d52ef631d7d5aca82f0c86d569c4a40b0cdc81da5e704d91c9", - "time_unix": 1790657069.7397597 - }, - { - "event": "rl_driver_phase", - "island_id": 0, - "phase": "finish", - "rollout_id": 8, - "time_unix": 1790657069.7402756 - } - ] - }, - "island1": { - "n_events": 132, - "event_kinds": [ - "rl_driver_phase", - "rl_driver_start", - "rl_engine_selected", - "rl_fragment_bcast", - "rl_fragment_push", - "rl_local_round", - "rl_policy_apply", - "rl_policy_snapshot", - "rl_publication", - "rl_sync_hook" - ], - "last_apply": { - "event": "rl_policy_apply", - "island_id": 1, - "local_step": 8, - "optimizer": "preserve", - "partial_fragment_apply": true, - "policy_version": 8, - "sync/apply_seconds": 1.454939548999846, - "sync/global_policy_hash": "41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60", - "time_unix": 1790657068.9525995 - }, - "tail": [ - { - "base_version": 8, - "c_steps": 8, - "c_tokens": 59977, - "delta_l2_norm": 0.17341628670692444, - "event": "rl_fragment_push", - "fragment_id": 7, - "global_step": 16, - "island_id": 1, - "payload_bytes": 5046272, - "pull_to_push_seconds": 0.010652423999999883, - "realized_h": 8, - "round_attempt": 1, - "time_unix": 1790657067.497421 - }, - { - "event": "rl_driver_phase", - "island_id": 1, - "optimizer": "preserve", - "phase": "apply", - "policy_version": 8, - "time_unix": 1790657067.4974859 - }, - { - "event": "rl_policy_apply", - "island_id": 1, - "local_step": 8, - "optimizer": "preserve", - "partial_fragment_apply": true, - "policy_version": 8, - "sync/apply_seconds": 1.454939548999846, - "sync/global_policy_hash": "41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60", - "time_unix": 1790657068.9525995 - }, - { - "event": "rl_policy_snapshot", - "island_id": 1, - "rl/canonical_layout_hash": "f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3", - "rl/fragment_versions": [ - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16 - ], - "rl/mixed_version_group_count": 0, - "rl/policy_hash": "41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60", - "rl/policy_token": "yeto:8:41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60", - "rl/rollout_id": 8, - "rl/sync_layout_fingerprint": "16a4d88fc98c2525fbee672648d6c151c5eeb324a3ccd8ef6caeac5f17383d12", - "time_unix": 1790657069.027938 - }, - { - "event": "rl_sync_hook", - "island_id": 1, - "rl/rollout_id": 7, - "sync/finalization": true, - "sync/hook_seconds": 14.7205672099999, - "sync/remote_quorum_wait_seconds": 0.0, - "time_unix": 1790657069.0748827 - }, - { - "event": "rl_driver_phase", - "island_id": 1, - "phase": "publish", - "policy_version": 8, - "time_unix": 1790657069.1338925 - }, - { - "event": "rl_publication", - "island_id": 1, - "policy_version": 8, - "rl/policy_token": "yeto:8:41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60", - "sync/publication_members": [ - "engine:inference-engine-all-0-0-00000" - ], - "sync/publication_payload_bytes": 40370176, - "sync/publication_payload_hash": "ee1a0457706ee7d52ef631d7d5aca82f0c86d569c4a40b0cdc81da5e704d91c9", - "time_unix": 1790657070.1687844 - }, - { - "event": "rl_driver_phase", - "island_id": 1, - "phase": "finish", - "rollout_id": 8, - "time_unix": 1790657070.1692758 - } - ] - } -} - Loading weights: 0%| | 0/311 [00:00 padded vocab (size: 151936) with 0 dummy tokens (new size: 151936) -[2026-09-29 11:25:09.587 main] arguments.py:40 - --tokenizer-model not set, use --hf-checkpoint as tokenizer model. -[2026-09-29 11:25:09.587 main] arguments.py:2668 - MoE-expert LoRA layout: per-expert (--experts-shared-outer-loras). -[2026-09-29 11:25:09.590 main] arguments.py:2816 - Colocate mode: defaulting --sglang-cuda-graph-backend-prefill=disabled to avoid NVLS OOM. Set --sglang-cuda-graph-backend-prefill explicitly to override. -[2026-09-29 11:25:09.590 main] arguments.py:2826 - rollout_num_gpus None != actor_num_gpus_per_node 1 * actor_num_nodes 1, overriding rollout_num_gpus to match actor_num_gpus_per_node * actor_num_nodes. -[2026-09-29 11:25:09.591 main] arguments.py:2910 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 1024 as default value. -[2026-09-29 11:25:09.591 main] arguments.py:2918 - args.rollout_max_prompt_len is not set. Use args.rollout_max_context_len - 1 (1024 - 1) as default value so that there is at least one generated token to compute loss. -using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 -Number of virtual stages per pipeline stage: None -using torch.bfloat16 for parameters ... ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - activation_func_clamp_shared_expert ............. True - activation_func_clamp_value ..................... None - actor_num_gpus_per_node ......................... 1 - actor_num_nodes ................................. 1 - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.999 - adam_eps ........................................ 1e-08 - add_bias_linear ................................. False - add_position_embedding .......................... True - add_qkv_bias .................................... False - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - advantage_estimator ............................. grpo - align_grad_reduce ............................... True - align_param_gather .............................. False - allgather_cp .................................... False - allow_ambiguous_pad_tokens ...................... False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_chat_template ............................. True - apply_chat_template_kwargs ...................... {'enable_thinking': False} - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - apply_wd_to_qk_layernorm ........................ False - async_max_concurrent_samples .................... None - async_save ...................................... False - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.unfused - attention_dropout ............................... 0.1 - attention_output_gate ........................... False - attention_softmax_in_fp32 ....................... True - auto_detect_ckpt_format ......................... False - balance_data .................................... True - barrier_with_L1_time ............................ True - batch_invariant_mode ............................ False - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - beta_fast ....................................... 32 - beta_slow ....................................... 1 - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - buffer_filter_path .............................. None - cache_mla_latents ............................... False - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - chat_template_path .............................. None - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_lora_weight_equal ......................... False - check_weight_hash_across_dp_replicas_interval ... None - check_weight_update_allow_quant_error ........... False - check_weight_update_equal ....................... False - check_weight_update_selector .................... all - check_weight_update_skip_list ................... None - ci_check_model_hash ............................. False - ci_disable_kl_checker ........................... False - ci_disable_logprobs_checker ..................... False - ci_disable_weight_update_checker ................ False - ci_enable_metrics_capture ....................... False - ci_ft_test_actions .............................. None - ci_inject_rollout_data_min_match_ratio .......... 0.9 - ci_inject_rollout_data_path ..................... None - ci_inject_rollout_data_start_rollout_id ......... None - ci_load_grad_norm ............................... None - ci_metric_checker_key ........................... None - ci_metric_checker_threshold ..................... None - ci_save_grad_norm ............................... None - ci_save_model_hash .............................. False - ci_test ......................................... False - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch_dist - ckpt_fully_parallel_load ........................ False - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - colocate ........................................ True - compress_ratios ................................. None - compute_advantages_and_returns .................. True - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - control_server_port ............................. 0 - cp_comm_type .................................... ['p2p'] - cpu_offloading_num_layers ....................... 0 - create_all_gather_group ......................... False - create_attention_mask_in_dataloader ............. True - critic_load ..................................... /root/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca - critic_lr ....................................... 1e-05 - critic_lr_warmup_iters .......................... 0 - critic_num_gpus_per_node ........................ 1 - critic_num_nodes ................................ 1 - critic_save ..................................... None - cross_entropy_fusion_impl ....................... native - cross_entropy_loss_fusion ....................... False - cuda_graph_impl ................................. none - cuda_graph_scope ................................ [] - cuda_graph_warmup_steps ......................... 3 - custom_config_path .............................. None - custom_convert_samples_to_train_data_path ....... None - custom_eval_rollout_log_function_path ........... None - custom_generate_function_path ................... None - custom_loss_function_path ....................... None - custom_megatron_before_log_prob_hook_path ....... None - custom_megatron_before_train_step_hook_path ..... None - custom_megatron_init_path ....................... None - custom_megatron_post_save_hook_path ............. None - custom_model_provider_path ...................... None - custom_pg_loss_reducer_function_path ............ None - custom_reward_post_process_path ................. None -... [truncated: 1711 of 2111 lines removed; full evidence archived outside repo] ... -(MegatronTrainRayActor pid=5849) /usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. -(MegatronTrainRayActor pid=5849) return func(*args, **kwargs) -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"apply_finished","policy_version":1} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_begin","policy_version":1} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_finished","policy_version":1} -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:10.054 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before TrainRayActor.clear_memory: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 70.04, 'used_GB': 9.14, 'allocated_GB': 1.3, 'reserved_GB': 4.56} -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:10.722 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after TrainRayActor.clear_memory: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 73.11, 'used_GB': 6.07, 'allocated_GB': 1.3, 'reserved_GB': 1.49} -(SGLangEngine pid=4474) [2026-09-29 11:30:10] Cache flushed successfully! -(SGLangEngine pid=4474) [2026-09-29 11:30:10] Start unload Lora adapter. Lora name=miles_lora -(SGLangEngine pid=4474) [2026-09-29 11:30:10] LoRA adapter unloading starts: LoRARef(lora_id=be26b7df6e2d4d8f8da606da472092dd, lora_name=miles_lora). avail mem=71.94 GB -(SGLangEngine pid=4474) [2026-09-29 11:30:10] LoRA adapter unloading completes: LoRARef(lora_id=be26b7df6e2d4d8f8da606da472092dd, lora_name=miles_lora). avail mem=71.94 GB -(SGLangEngine pid=4474) [2026-09-29 11:30:10] Start load Lora adapter from tensors. Lora name=miles_lora -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:10.805 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=update_weights phase=start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:10.805 actor_cell0_rank0] timer.py:31 - Timer update_weights start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:10.805 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.92, 'used_GB': 7.26, 'allocated_GB': 1.3, 'reserved_GB': 1.49} -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:10.890 actor_cell0_rank0] update_weight_from_tensor.py:461 - Packed 392 LoRA tensors (20185088 bytes) into 1 flattened buckets (max 268435456 bytes/1024 tensors) -(SGLangEngine pid=4474) [2026-09-29 11:30:10] LoRA adapter loading from tensors starts: LoRARef(lora_id=070d65a66a9146f896c11a26eafc80c0, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(SGLangEngine pid=4474) [2026-09-29 11:30:10] LoRA adapter loading from tensors completes: LoRARef(lora_id=070d65a66a9146f896c11a26eafc80c0, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(RolloutManager pid=1876) Rollout generation: 0%| | 0/32 [00:00 33096.0 MB (freed 0.0 MB) -(SGLangEngine pid=4474) [2026-09-29 11:30:11] LoRA adapter '070d65a66a9146f896c11a26eafc80c0': loaded weights for target modules ['down_proj', 'gate_up_proj', 'o_proj', 'qkv_proj']. -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:11.565 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.95, 'used_GB': 7.23, 'allocated_GB': 1.3, 'reserved_GB': 1.49} -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:11.565 actor_cell0_rank0] timer.py:42 - Timer update_weights end (elapsed: 0.8s) -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:11.565 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.8 -(SGLangEngine pid=4474) [2026-09-29 11:30:13] Prefill batch, #new-seq: 1, #new-token: 47, #cached-token: 0, token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 1.41 -(SGLangEngine pid=4474) [2026-09-29 11:30:13] Prefill batch, #new-seq: 7, #new-token: 329, #cached-token: 0, token usage: 0.00, #running-req: 1, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 80323.17 -(SGLangEngine pid=4474) [2026-09-29 11:30:13] Decode batch, #running-req: 8, #token: 375, token usage: 0.00, cuda graph: True, gen throughput (token/s): 9.71, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:13] Decode batch, #running-req: 8, #token: 695, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1750.02, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:13] Prefill batch, #new-seq: 1, #new-token: 83, #cached-token: 3, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 151.57 -(SGLangEngine pid=4474) [2026-09-29 11:30:14] Decode batch, #running-req: 7, #token: 617, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1268.23, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:14] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 11.17 -(SGLangEngine pid=4474) [2026-09-29 11:30:14] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 8, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 32.18 -(SGLangEngine pid=4474) [2026-09-29 11:30:14] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 9, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 300.60 -(SGLangEngine pid=4474) [2026-09-29 11:30:14] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 8.65 -(SGLangEngine pid=4474) [2026-09-29 11:30:14] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 21.14 -(SGLangEngine pid=4474) [2026-09-29 11:30:14] Decode batch, #running-req: 8, #token: 664, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1010.46, #queue-req: 0 -(RolloutManager pid=1876) [2026-09-29 11:30:14.512 rollout_manager] sglang_rollout.py:473 - First rollout sample: ["<|im_start|>user\nJames writes a 3-page letter to 2 different friends twice a week. How many pages does he write a year?\nPut the final numeric answer in \\boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nJames writes a **3-page letter** to **both friends** **twice a week**. Let's calculate how many pages he writes in a year:\n\n### Step-by-step:\n\n- **Number of letters per week**: 2 \n- **Pages per letter**: 3 \n- **Weeks in a year**: 52\n\n### Total pages per year:\n$$\n2 \\text{ letters/week} \\times 3 \\text{ pages/letter per week} \\times 52 \\text{ weeks/year} = 312 \\text{ pages}\n$$\n\n### Final Answer:\n$$\n\\boxed{312}\n$$<|im_end|>"], label: 624, reward: 0.0 -(RolloutManager pid=1876) Rollout generation: 25%|██▌ | 8/32 [00:02<00:08, 2.74it/s] -(SGLangEngine pid=4474) [2026-09-29 11:30:14] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 4.56 -(SGLangEngine pid=4474) [2026-09-29 11:30:14] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 18.11 -(SGLangEngine pid=4474) [2026-09-29 11:30:14] Decode batch, #running-req: 8, #token: 571, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1259.60, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:14] Decode batch, #running-req: 8, #token: 891, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1696.21, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:15] Decode batch, #running-req: 6, #token: 1051, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1448.40, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:15] Prefill batch, #new-seq: 2, #new-token: 150, #cached-token: 6, token usage: 0.00, #running-req: 6, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 319.13 -(SGLangEngine pid=4474) [2026-09-29 11:30:15] Decode batch, #running-req: 8, #token: 1304, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1681.78, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:15] Decode batch, #running-req: 8, #token: 1624, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1685.35, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:15] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 2.34 -(SGLangEngine pid=4474) [2026-09-29 11:30:15] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 19.32 -(SGLangEngine pid=4474) [2026-09-29 11:30:15] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 20.77 -(SGLangEngine pid=4474) [2026-09-29 11:30:15] Decode batch, #running-req: 8, #token: 1540, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1168.34, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:15] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 3.99 -(SGLangEngine pid=4474) [2026-09-29 11:30:15] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 11.54 -(SGLangEngine pid=4474) [2026-09-29 11:30:15] Decode batch, #running-req: 8, #token: 1241, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1259.10, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:15] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 14.82 -(SGLangEngine pid=4474) [2026-09-29 11:30:16] Prefill batch, #new-seq: 1, #new-token: 124, #cached-token: 3, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 859.37 -(SGLangEngine pid=4474) [2026-09-29 11:30:16] Decode batch, #running-req: 8, #token: 1593, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1271.25, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:16] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 126, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 7.17 -(SGLangEngine pid=4474) [2026-09-29 11:30:16] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 126, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 11.39 -(SGLangEngine pid=4474) [2026-09-29 11:30:16] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 126, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 17.27 -(SGLangEngine pid=4474) [2026-09-29 11:30:16] Decode batch, #running-req: 8, #token: 1017, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1126.91, #queue-req: 0 -(RolloutManager pid=1876) Rollout generation: 50%|█████ | 16/32 [00:04<00:04, 3.33it/s] -(SGLangEngine pid=4474) [2026-09-29 11:30:16] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 126, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 8.39 -(SGLangEngine pid=4474) [2026-09-29 11:30:16] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 126, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 11.46 -(RolloutManager pid=1876) Rollout generation: 75%|███████▌ | 24/32 [00:05<00:01, 5.88it/s] -(SGLangEngine pid=4474) [2026-09-29 11:30:16] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 126, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 13.93 -(SGLangEngine pid=4474) [2026-09-29 11:30:16] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 126, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 20.74 -(SGLangEngine pid=4474) [2026-09-29 11:30:16] Decode batch, #running-req: 8, #token: 494, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1024.61, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:16] Decode batch, #running-req: 8, #token: 814, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1718.77, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:17] Decode batch, #running-req: 7, #token: 981, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1658.07, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:17] Decode batch, #running-req: 5, #token: 870, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1186.94, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:17] Decode batch, #running-req: 3, #token: 726, token usage: 0.00, cuda graph: True, gen throughput (token/s): 812.29, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:17] Decode batch, #running-req: 3, #token: 846, token usage: 0.00, cuda graph: True, gen throughput (token/s): 643.05, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:17] Decode batch, #running-req: 1, #token: 402, token usage: 0.00, cuda graph: True, gen throughput (token/s): 537.28, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:18] Decode batch, #running-req: 1, #token: 442, token usage: 0.00, cuda graph: True, gen throughput (token/s): 222.24, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:18] Decode batch, #running-req: 1, #token: 482, token usage: 0.00, cuda graph: True, gen throughput (token/s): 222.34, #queue-req: 0 -(RolloutManager pid=1876) Rollout generation: 100%|██████████| 32/32 [00:06<00:00, 5.35it/s] Rollout generation: 100%|██████████| 32/32 [00:06<00:00, 4.72it/s] -(RolloutManager pid=1876) [2026-09-29 11:30:18.377 rollout_manager] sglang_rollout.py:494 - Finish rollout: ["<|im_start|>user\nKen created a care package to send to his brother, who was away at boarding school. Ken placed a box on a scale, and then he poured into the box enough jelly beans to bring the weight to 2 pounds. Then, he added enough brownies to cause the weight to triple. Next, he added another 2 pounds of jelly beans. And finally, he added enough gummy worms to double the weight once again. What was the final weight of the box of goodies, in pounds?\nPut the final numeric answer in \\boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nWe are given that ken **finally** added **gummy worms**, **tripling the weight**.\n\nLet's track the weight step by step.\n\n---\n\n### Step 1: Start with the box weight = 0 pounds.\n\nThen:\n- Added **2 pounds of jelly beans**, making the total weight = **2 pounds**.\n\nNext:\n- Added **2 pounds of jelly beans**, making the total weight = **4 pounds**.\n\nThen:\n- Added **2 pounds of jelly beans**, making the total weight = **6 pounds**.\n\nFinally:\n- Added **x pounds of gummy worms**, and the weight is now **12** pounds.\n\nSo:\n\n$$\n6 + x = 12 \\Rightarrow x = 6 \\text{ pounds}\n$$\n\n---\n\n### Final Answer:\n\n$$\n\\boxed{12} \\text{ pounds}\n$$<|im_end|>"], label: 16, reward: 0.0 -(RolloutManager pid=1876) [2026-09-29 11:30:18.379 rollout_manager] sglang_rollout.py:391 - Abort request for ['http://172.20.28.65:21100'] -(RolloutManager pid=1876) [2026-09-29 11:30:18.384 rollout_manager] rollout_data_conversion.py:42 - Final collected 32 samples from rollout to train -(RolloutManager pid=1876) [2026-09-29 11:30:18.384 rollout_manager] metrics.py:79 - perf 1: {'rollout/response_len/mean': 180.46875, 'rollout/response_len/median': 142.0, 'rollout/response_len/max': 384, 'rollout/response_len/min': 71, 'rollout/zero_std/count_0.0': 1, 'rollout/zero_std/count_1.0': 1, 'rollout/zero_std/all_zero_percentage': 0.25, 'rollout/zero_std/all_one_percentage': 0.25, 'rollout/prefix_cache_hit_rate': 0.7215236686390533, 'rollout/avg_cached_tokens_per_sample': 60.96875, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.09375, 'perf/rollout_time': 6.790853023529053, 'perf/tokens_per_gpu_per_sec': 850.4086276040272, 'perf/longest_sample_tokens_per_sec': 56.546651601722324, 'perf/effective_tokens_per_gpu_per_sec': 850.4086276040272, 'perf/longest_effective_sample_tokens_per_sec': 56.546651601722324} -(SGLangEngine pid=4474) [2026-09-29 11:30:18] Cache flushed successfully! -(SGLangEngine pid=4474) [2026-09-29 11:30:18] Cache flushed successfully! -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:18.848 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=train phase=start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:18.848 actor_cell0_rank0] timer.py:31 - Timer data_preprocess start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:18.853 actor_cell0_rank0] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:18.854 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=train_actor phase=start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:18.854 actor_cell0_rank0] timer.py:42 - Timer train_wait end (elapsed: 10.7s) -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:18.854 actor_cell0_rank0] timer.py:31 - Timer train start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:18.878 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:18.878 actor_cell0_rank0] timer.py:31 - Timer log_probs start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:23.024 actor_cell0_rank0] timer.py:42 - Timer log_probs end (elapsed: 4.1s) -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:23.024 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=end ok=true elapsed_s=4.1 -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:23.031 actor_cell0_rank0] log_utils.py:42 - ft op=cross_cell phase=start kind=log_gather rank=0 -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:23.032 actor_cell0_rank0] log_utils.py:48 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:23.032 actor_cell0_rank0] log_utils.py:66 - rollout 1: {'rollout/response_lengths': 180.46875, 'rollout/rewards': 9.313225746154785e-10, 'rollout/truncated': 0.09375, 'rollout/rollout_log_probs': -0.39379236102104187, 'rollout/raw_reward': 0.34375, 'rollout/total_lengths': 264.96875, 'rollout/log_probs': -0.39432212710380554, 'rollout/advantages': -9.313225746154785e-10, 'rollout/returns': -9.313225746154785e-10} -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:23.032 actor_cell0_rank0] timer.py:31 - Timer actor_train start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:30.938 actor_cell0_rank0] model.py:601 - ft op=train_step rollout=1 step=0 attempt=0 outcome=NORMAL valid_step=true -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:30.939 actor_cell0_rank0] log_utils.py:460 - step 1: {'train/loss': 9.313225746154785e-10, 'train/pg_loss': 9.313225746154785e-10, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.015055712312459946, 'train/train_rollout_kl': 0.0007043661316856742, 'train/grad_norm': 0.3819052786927497, 'train/lr-pg_0': 5e-06, 'train/step': 1} -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:30.939 actor_cell0_rank0] model.py:814 - step 1: {'train/loss': 9.313225746154785e-10, 'train/pg_loss': 9.313225746154785e-10, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.015055712312459946, 'train/train_rollout_kl': 0.0007043661316856742, 'train/grad_norm': 0.3819052786927497, 'train/lr-pg_0': 5e-06, 'train/step': 1} -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:30.940 actor_cell0_rank0] timer.py:42 - Timer actor_train end (elapsed: 7.9s) -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:30.940 actor_cell0_rank0] timer.py:42 - Timer train end (elapsed: 12.1s) -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:30.940 actor_cell0_rank0] timer.py:31 - Timer train_wait start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:30.964 actor_cell0_rank0] train_metric_utils.py:50 - perf 1: {'perf/update_weights_time': 0.7602176666259766, 'perf/data_preprocess_time': 0.005410432815551758, 'perf/train_wait_time': 10.743270874023438, 'perf/log_probs_time': 4.146162509918213, 'perf/actor_train_time': 7.907407999038696, 'perf/train_time': 12.085911273956299, 'perf/log_probs_tflops': 2.193705373097741, 'perf/actor_train_tflops': 3.450735934018986, 'perf/actor_train_tok_per_s': 1072.2856340574297, 'perf/step_time': 22.829182147979736, 'perf/wait_time_ratio': 0.4705937691672481} -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:30.964 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train_actor phase=end ok=true elapsed_s=12.1 -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:30.964 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=12.1 -(SGLangEngine pid=4474) [2026-09-29 11:30:10] INFO: 172.20.28.65:58931 - "POST /update_weight_version HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:10] INFO: 172.20.28.65:52351 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:10] INFO: 172.20.28.65:30510 - "POST /pause_generation HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:10] INFO: 172.20.28.65:29445 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:10] INFO: 172.20.28.65:26146 - "POST /unload_lora_adapter HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:10] INFO: 172.20.28.65:34661 - "POST /load_lora_adapter_from_tensors HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:11] INFO: 172.20.28.65:51484 - "POST /continue_generation HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:11] INFO: 172.20.28.65:47421 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:13] INFO: 172.20.28.65:25172 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:14] INFO: 172.20.28.65:55778 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:14] INFO: 172.20.28.65:25404 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:14] INFO: 172.20.28.65:34419 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:14] INFO: 172.20.28.65:28338 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:14] INFO: 172.20.28.65:17581 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:14] INFO: 172.20.28.65:23126 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:14] INFO: 172.20.28.65:43991 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:14] INFO: 172.20.28.65:34419 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:14] INFO: 172.20.28.65:28338 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:15] INFO: 172.20.28.65:17581 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:15] INFO: 172.20.28.65:34419 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:15] INFO: 172.20.28.65:28338 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:15] INFO: 172.20.28.65:55778 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:15] INFO: 172.20.28.65:43991 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:15] INFO: 172.20.28.65:34419 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:16] INFO: 172.20.28.65:28338 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:16] INFO: 172.20.28.65:17581 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:16] INFO: 172.20.28.65:25172 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:16] INFO: 172.20.28.65:25404 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:16] INFO: 172.20.28.65:43991 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:16] INFO: 172.20.28.65:23126 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:16] INFO: 172.20.28.65:34419 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:16] INFO: 172.20.28.65:55778 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:17] INFO: 172.20.28.65:17581 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:17] INFO: 172.20.28.65:28338 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:17] INFO: 172.20.28.65:25404 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:17] INFO: 172.20.28.65:55778 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:17] INFO: 172.20.28.65:34419 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:17] INFO: 172.20.28.65:23126 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:17] INFO: 172.20.28.65:25172 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:18] INFO: 172.20.28.65:43991 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:18] INFO: 172.20.28.65:35405 - "POST /abort_request HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:18] INFO: 172.20.28.65:21202 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=4474) [2026-09-29 11:30:18] INFO: 172.20.28.65:22362 - "POST /release_memory_occupation HTTP/1.1" 200 OK -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"begin","policy_version":2} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"apply_finished","policy_version":2} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_begin","policy_version":2} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_finished","policy_version":2} -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:32.651 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before TrainRayActor.clear_memory: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 70.1, 'used_GB': 9.08, 'allocated_GB': 1.3, 'reserved_GB': 4.5} -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:33.298 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after TrainRayActor.clear_memory: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 73.11, 'used_GB': 6.07, 'allocated_GB': 1.3, 'reserved_GB': 1.49} -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:33.381 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=update_weights phase=start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:33.381 actor_cell0_rank0] timer.py:31 - Timer update_weights start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:33.382 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.92, 'used_GB': 7.26, 'allocated_GB': 1.3, 'reserved_GB': 1.49} -(SGLangEngine pid=4474) [2026-09-29 11:30:33] Cache flushed successfully! -(SGLangEngine pid=4474) [2026-09-29 11:30:33] Start unload Lora adapter. Lora name=miles_lora -(SGLangEngine pid=4474) [2026-09-29 11:30:33] LoRA adapter unloading starts: LoRARef(lora_id=070d65a66a9146f896c11a26eafc80c0, lora_name=miles_lora). avail mem=71.95 GB -(SGLangEngine pid=4474) [2026-09-29 11:30:33] LoRA adapter unloading completes: LoRARef(lora_id=070d65a66a9146f896c11a26eafc80c0, lora_name=miles_lora). avail mem=71.95 GB -(SGLangEngine pid=4474) [2026-09-29 11:30:33] Start load Lora adapter from tensors. Lora name=miles_lora -(SGLangEngine pid=4474) [2026-09-29 11:30:33] LoRA adapter loading from tensors starts: LoRARef(lora_id=f4c7d10565bc4ff087129c7d1820aed3, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(SGLangEngine pid=4474) [2026-09-29 11:30:33] LoRA adapter loading from tensors completes: LoRARef(lora_id=f4c7d10565bc4ff087129c7d1820aed3, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:33.461 actor_cell0_rank0] update_weight_from_tensor.py:461 - Packed 392 LoRA tensors (20185088 bytes) into 1 flattened buckets (max 268435456 bytes/1024 tensors) -(RolloutManager pid=1876) Rollout generation: 0%| | 0/32 [00:00 33096.0 MB (freed 0.0 MB) -(SGLangEngine pid=4474) [2026-09-29 11:30:34] LoRA adapter 'f4c7d10565bc4ff087129c7d1820aed3': loaded weights for target modules ['down_proj', 'gate_up_proj', 'o_proj', 'qkv_proj']. -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:34.117 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.95, 'used_GB': 7.23, 'allocated_GB': 1.3, 'reserved_GB': 1.49} -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:34.117 actor_cell0_rank0] timer.py:42 - Timer update_weights end (elapsed: 0.7s) -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:34.117 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.7 -(SGLangEngine pid=4474) [2026-09-29 11:30:39] Prefill batch, #new-seq: 1, #new-token: 131, #cached-token: 0, token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 5.67 -(SGLangEngine pid=4474) [2026-09-29 11:30:39] Prefill batch, #new-seq: 7, #new-token: 917, #cached-token: 0, token usage: 0.00, #running-req: 1, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 222680.16 -(SGLangEngine pid=4474) [2026-09-29 11:30:40] Decode batch, #running-req: 8, #token: 459, token usage: 0.00, cuda graph: True, gen throughput (token/s): 14.70, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:40] Decode batch, #running-req: 8, #token: 779, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1716.67, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:40] Decode batch, #running-req: 8, #token: 1099, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1709.05, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:40] Decode batch, #running-req: 8, #token: 1419, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1807.57, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:40] Prefill batch, #new-seq: 1, #new-token: 86, #cached-token: 3, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 107.96 -(SGLangEngine pid=4474) [2026-09-29 11:30:40] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 88, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 10.94 -(SGLangEngine pid=4474) [2026-09-29 11:30:40] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 88, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 18.63 -(SGLangEngine pid=4474) [2026-09-29 11:30:40] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 88, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 18.61 -(SGLangEngine pid=4474) [2026-09-29 11:30:40] Decode batch, #running-req: 8, #token: 1107, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1021.45, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:40] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 88, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 9.60 -(SGLangEngine pid=4474) [2026-09-29 11:30:41] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 88, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 13.09 -(SGLangEngine pid=4474) [2026-09-29 11:30:41] Decode batch, #running-req: 8, #token: 764, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1307.92, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:41] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 88, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 6.72 -(SGLangEngine pid=4474) [2026-09-29 11:30:41] Decode batch, #running-req: 8, #token: 1083, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1450.25, #queue-req: 0 -(RolloutManager pid=1876) [2026-09-29 11:30:41.470 rollout_manager] sglang_rollout.py:473 - First rollout sample: ['<|im_start|>user\nAlexis is applying for a new job and bought a new set of business clothes to wear to the interview. She went to a department store with a budget of $200 and spent $30 on a button-up shirt, $46 on suit pants, $38 on a suit coat, $11 on socks, and $18 on a belt. She also purchased a pair of shoes, but lost the receipt for them. She has $16 left from her budget. How much did Alexis pay for the shoes?\nPut the final numeric answer in \\boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nWe are given the following information:\n\n- Total budget: **$200**\n- Spent on shirt: **$30**\n- Spent on suit pants: **$46**\n- Spent on suit coat: **$38**\n- Spent on socks: **$11**\n- Spent on belt: **$18**\n- Remaining amount: **$16**\n\nLet’s calculate the total amount spent so far:\n\n$$\n30 + 46 + 38 + 11 + 18 = 133\n$$\n\nNow, the remaining amount is **$16**, so the cost of the **shoes** must be:\n\n$$\n200 - 133 = 67\n$$\n\nSo, Alexis paid **$67** for the shoes. ✅\n\n$$\n\\boxed{67}\n$$<|im_end|>'], label: 41, reward: 0.0 -(RolloutManager pid=1876) Rollout generation: 25%|██▌ | 8/32 [00:07<00:21, 1.09it/s] -(SGLangEngine pid=4474) [2026-09-29 11:30:41] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 88, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 2.76 -(SGLangEngine pid=4474) [2026-09-29 11:30:41] Decode batch, #running-req: 8, #token: 963, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1446.51, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:41] Decode batch, #running-req: 8, #token: 1283, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1758.00, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:41] Prefill batch, #new-seq: 1, #new-token: 94, #cached-token: 3, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 323.79 -(SGLangEngine pid=4474) [2026-09-29 11:30:41] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 96, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 20.24 -(SGLangEngine pid=4474) [2026-09-29 11:30:41] Decode batch, #running-req: 8, #token: 1300, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1275.33, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:42] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 96, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 4.10 -(SGLangEngine pid=4474) [2026-09-29 11:30:42] Decode batch, #running-req: 8, #token: 1444, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1439.41, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:42] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 96, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 3.87 -(SGLangEngine pid=4474) [2026-09-29 11:30:42] Decode batch, #running-req: 8, #token: 1482, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1453.33, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:42] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 96, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 4.53 -(SGLangEngine pid=4474) [2026-09-29 11:30:42] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 96, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 15.76 -(SGLangEngine pid=4474) [2026-09-29 11:30:42] Decode batch, #running-req: 8, #token: 1176, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1277.48, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:42] Decode batch, #running-req: 8, #token: 1496, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1719.23, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:43] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 96, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 2.61 -(SGLangEngine pid=4474) [2026-09-29 11:30:43] Decode batch, #running-req: 8, #token: 1443, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1440.62, #queue-req: 0 -(RolloutManager pid=1876) Rollout generation: 50%|█████ | 16/32 [00:09<00:08, 1.99it/s] -(SGLangEngine pid=4474) [2026-09-29 11:30:43] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 96, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 5.07 -(SGLangEngine pid=4474) [2026-09-29 11:30:43] Decode batch, #running-req: 8, #token: 1365, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1444.01, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:43] Decode batch, #running-req: 8, #token: 1685, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1686.23, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:43] Decode batch, #running-req: 8, #token: 2005, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1671.85, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:43] Prefill batch, #new-seq: 1, #new-token: 116, #cached-token: 4, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 176.85 -(SGLangEngine pid=4474) [2026-09-29 11:30:43] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 119, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 20.54 -(SGLangEngine pid=4474) [2026-09-29 11:30:43] Decode batch, #running-req: 8, #token: 1670, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1248.48, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:44] Decode batch, #running-req: 8, #token: 1990, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1683.07, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:44] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 119, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 4.07 -(SGLangEngine pid=4474) [2026-09-29 11:30:44] Decode batch, #running-req: 8, #token: 1924, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1434.51, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:44] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 119, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 3.85 -(SGLangEngine pid=4474) [2026-09-29 11:30:44] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 119, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 10.76 -(SGLangEngine pid=4474) [2026-09-29 11:30:44] Decode batch, #running-req: 8, #token: 1507, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1251.91, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:44] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 119, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 6.11 -(SGLangEngine pid=4474) [2026-09-29 11:30:44] Decode batch, #running-req: 8, #token: 1441, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1437.62, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:45] Decode batch, #running-req: 8, #token: 1761, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1679.46, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:45] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 119, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 2.38 -(RolloutManager pid=1876) Rollout generation: 75%|███████▌ | 24/32 [00:11<00:03, 2.55it/s] -(SGLangEngine pid=4474) [2026-09-29 11:30:45] Decode batch, #running-req: 8, #token: 1695, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1442.47, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:45] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 119, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 5.12 -(SGLangEngine pid=4474) [2026-09-29 11:30:45] Decode batch, #running-req: 7, #token: 1298, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1467.62, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:45] Decode batch, #running-req: 7, #token: 1578, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1465.34, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:45] Decode batch, #running-req: 6, #token: 1496, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1299.14, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:46] Decode batch, #running-req: 5, #token: 1330, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1146.35, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:46] Decode batch, #running-req: 5, #token: 1530, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1050.52, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:46] Decode batch, #running-req: 4, #token: 1341, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1042.07, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:46] Decode batch, #running-req: 2, #token: 697, token usage: 0.00, cuda graph: True, gen throughput (token/s): 679.06, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:46] Decode batch, #running-req: 2, #token: 777, token usage: 0.00, cuda graph: True, gen throughput (token/s): 433.62, #queue-req: 0 -(SGLangEngine pid=4474) [2026-09-29 11:30:46] Decode batch, #running-req: 1, #token: 471, token usage: 0.00, cuda graph: True, gen throughput (token/s): 432.43, #queue-req: 0 -(RolloutManager pid=1876) Rollout generation: 100%|██████████| 32/32 [00:12<00:00, 3.05it/s] Rollout generation: 100%|██████████| 32/32 [00:12<00:00, 2.47it/s] -(RolloutManager pid=1876) [2026-09-29 11:30:47.095 rollout_manager] sglang_rollout.py:494 - Finish rollout: ["<|im_start|>user\nTobias is buying a new pair of shoes that costs $95. He has been saving up his money each month for the past three months. He gets a $5 allowance a month. He also mows lawns and shovels driveways. He charges $15 to mow a lawn and $7 to shovel. After buying the shoes, he has $15 in change. If he mows 4 lawns, how many driveways did he shovel?\nPut the final numeric answer in \\boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nLet's solve the problem step by step:\n\nWe are given the following information:\n\n- Tobias earns $5 a month.\n- He mows 4 lawns for $15 each.\n- He shovels driveways for $7 each.\n- He saves $95 to buy the shoes.\n- After buying the shoes and getting the $15 in change, his total amount is $15.\n\nLet's find out how much money Tobias actually has.\n\n**Total money after buying shoes:**\n- The shoes cost $95.\n- He has $15 in change.\n\nSo his total earnings before buying the shoes would be $95 + $15 = $110.\n\nNow, let's calculate his **weekly earnings**:\n\n- He receives $5 per month, so over 3 months: \n $5 × 3 = $15.\n\nHe also mows 4 lawns and shovels driveways, paying for:\n- 4 lawns × $15 = $60\n- $7 per driveway × ? ≈ ?\n\nSo total cost of:\n- Lawns: $60\n- Shovels: $7 × x\n\nWhere x is the number of driveways he shovels.\n\nHe earns $15 in total income from allowance + mowing and shoveling.\n\nSo:\n$$\n15 + (60 + 7x) = 110\n$$\n\nLet's solve for x:\n\n$$\n15 + 60 + 7x = 110 \\\\\n75 + 7x = 110 \\\\\n7x = 110 - 75 \\\\\n7x = 35 \\\\\nx = 5\n$$\n\n### ✅ Final Answer: \nHe shovel **5 driveways**. \n\n$$\n\\boxed{5}\n$$"], label: 5, reward: 1.0 -(RolloutManager pid=1876) [2026-09-29 11:30:47.097 rollout_manager] sglang_rollout.py:391 - Abort request for ['http://172.20.28.65:21100'] -(RolloutManager pid=1876) [2026-09-29 11:30:47.102 rollout_manager] rollout_data_conversion.py:42 - Final collected 32 samples from rollout to train -(RolloutManager pid=1876) [2026-09-29 11:30:47.102 rollout_manager] metrics.py:79 - perf 2: {'rollout/response_len/mean': 304.15625, 'rollout/response_len/median': 322.5, 'rollout/response_len/max': 384, 'rollout/response_len/min': 164, 'rollout/zero_std/count_0.0': 2, 'rollout/zero_std/all_zero_percentage': 0.5, 'rollout/zero_std/all_one_percentage': 0.0, 'rollout/prefix_cache_hit_rate': 0.6095537757437071, 'rollout/avg_cached_tokens_per_sample': 66.59375, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.40625, 'perf/rollout_time': 12.952808380126953, 'perf/tokens_per_gpu_per_sec': 751.4200561272107, 'perf/longest_sample_tokens_per_sec': 29.646080504762036, 'perf/effective_tokens_per_gpu_per_sec': 751.4200561272107, 'perf/longest_effective_sample_tokens_per_sec': 29.646080504762036} -(SGLangEngine pid=4474) [2026-09-29 11:30:47] Cache flushed successfully! -(SGLangEngine pid=4474) [2026-09-29 11:30:47] Cache flushed successfully! -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:47.547 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=train phase=start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:47.547 actor_cell0_rank0] timer.py:31 - Timer data_preprocess start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:47.553 actor_cell0_rank0] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:47.553 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=train_actor phase=start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:47.553 actor_cell0_rank0] timer.py:42 - Timer train_wait end (elapsed: 16.6s) -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:47.553 actor_cell0_rank0] timer.py:31 - Timer train start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:47.577 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=start -(MegatronTrainRayActor pid=5849) [2026-09-29 11:30:47.577 actor_cell0_rank0] timer.py:31 - Timer log_probs start diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/miles-restart.log b/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/miles-restart.log deleted file mode 100644 index 914a35f9e..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/miles-restart.log +++ /dev/null @@ -1,401 +0,0 @@ -Unable to import `torchao` Tensor objects. This may affect loading checkpoints serialized with `torchao` -/usr/local/lib/python3.12/dist-packages/modelopt/torch/__init__.py:51: UserWarning: transformers 5.12.1 is not tested with current version of modelopt and may cause issues. Please install recommended version with `pip install -U nvidia-modelopt[hf]` if working with HF models. - _warnings.warn( -Loading a model with trust_remote_code=True allows arbitrary code execution from the model repository. Only use this with models you trust. -[transformers] `torch_dtype` is deprecated! Use `dtype` instead! -/root/Megatron-LM/megatron/core/transformer/transformer_config.py:1811: UserWarning: full scope is deprecated. Use empty cuda_graph_scope to capture the whole layer. - warnings.warn( -[ERROR] `cache_position` is part of Qwen3ASRThinkerTextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /usr/local/lib/python3.12/dist-packages/megatron/bridge/models/qwen3_asr/hf_qwen3_asr/modeling_qwen3_asr.py. -[ERROR] `cache_position` is part of Qwen3ASRThinkerForConditionalGeneration.forward's signature, but not documented. Make sure to add it to the docstring of the function in /usr/local/lib/python3.12/dist-packages/megatron/bridge/models/qwen3_asr/hf_qwen3_asr/modeling_qwen3_asr.py. -LANG_RL Log directory: None - > padded vocab (size: 151936) with 0 dummy tokens (new size: 151936) -[2026-09-29 11:31:35.326 main] arguments.py:40 - --tokenizer-model not set, use --hf-checkpoint as tokenizer model. -[2026-09-29 11:31:35.326 main] arguments.py:2668 - MoE-expert LoRA layout: per-expert (--experts-shared-outer-loras). -[2026-09-29 11:31:35.327 main] arguments.py:2816 - Colocate mode: defaulting --sglang-cuda-graph-backend-prefill=disabled to avoid NVLS OOM. Set --sglang-cuda-graph-backend-prefill explicitly to override. -[2026-09-29 11:31:35.327 main] arguments.py:2826 - rollout_num_gpus None != actor_num_gpus_per_node 1 * actor_num_nodes 1, overriding rollout_num_gpus to match actor_num_gpus_per_node * actor_num_nodes. -[2026-09-29 11:31:35.327 main] arguments.py:2910 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 1024 as default value. -[2026-09-29 11:31:35.327 main] arguments.py:2918 - args.rollout_max_prompt_len is not set. Use args.rollout_max_context_len - 1 (1024 - 1) as default value so that there is at least one generated token to compute loss. -using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 -Number of virtual stages per pipeline stage: None -using torch.bfloat16 for parameters ... ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - activation_func_clamp_shared_expert ............. True - activation_func_clamp_value ..................... None - actor_num_gpus_per_node ......................... 1 - actor_num_nodes ................................. 1 - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.999 - adam_eps ........................................ 1e-08 - add_bias_linear ................................. False - add_position_embedding .......................... True - add_qkv_bias .................................... False - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - advantage_estimator ............................. grpo - align_grad_reduce ............................... True - align_param_gather .............................. False - allgather_cp .................................... False - allow_ambiguous_pad_tokens ...................... False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_chat_template ............................. True - apply_chat_template_kwargs ...................... {'enable_thinking': False} - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - apply_wd_to_qk_layernorm ........................ False - async_max_concurrent_samples .................... None - async_save ...................................... False - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.unfused - attention_dropout ............................... 0.1 - attention_output_gate ........................... False - attention_softmax_in_fp32 ....................... True - auto_detect_ckpt_format ......................... False - balance_data .................................... True - barrier_with_L1_time ............................ True - batch_invariant_mode ............................ False - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - beta_fast ....................................... 32 - beta_slow ....................................... 1 - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - buffer_filter_path .............................. None - cache_mla_latents ............................... False - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - chat_template_path .............................. None - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_lora_weight_equal ......................... False - check_weight_hash_across_dp_replicas_interval ... None - check_weight_update_allow_quant_error ........... False - check_weight_update_equal ....................... False - check_weight_update_selector .................... all - check_weight_update_skip_list ................... None - ci_check_model_hash ............................. False - ci_disable_kl_checker ........................... False - ci_disable_logprobs_checker ..................... False - ci_disable_weight_update_checker ................ False - ci_enable_metrics_capture ....................... False - ci_ft_test_actions .............................. None - ci_inject_rollout_data_min_match_ratio .......... 0.9 - ci_inject_rollout_data_path ..................... None - ci_inject_rollout_data_start_rollout_id ......... None - ci_load_grad_norm ............................... None - ci_metric_checker_key ........................... None - ci_metric_checker_threshold ..................... None - ci_save_grad_norm ............................... None - ci_save_model_hash .............................. False - ci_test ......................................... False - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch_dist - ckpt_fully_parallel_load ........................ False - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - colocate ........................................ True - compress_ratios ................................. None - compute_advantages_and_returns .................. True - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - control_server_port ............................. 0 - cp_comm_type .................................... ['p2p'] - cpu_offloading_num_layers ....................... 0 - create_all_gather_group ......................... False - create_attention_mask_in_dataloader ............. True - critic_load ..................................... /root/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca - critic_lr ....................................... 1e-05 - critic_lr_warmup_iters .......................... 0 - critic_num_gpus_per_node ........................ 1 - critic_num_nodes ................................ 1 - critic_save ..................................... None - cross_entropy_fusion_impl ....................... native - cross_entropy_loss_fusion ....................... False - cuda_graph_impl ................................. none - cuda_graph_scope ................................ [] - cuda_graph_warmup_steps ......................... 3 - custom_config_path .............................. None - custom_convert_samples_to_train_data_path ....... None - custom_eval_rollout_log_function_path ........... None - custom_generate_function_path ................... None - custom_loss_function_path ....................... None - custom_megatron_before_log_prob_hook_path ....... None - custom_megatron_before_train_step_hook_path ..... None - custom_megatron_init_path ....................... None - custom_megatron_post_save_hook_path ............. None - custom_model_provider_path ...................... None - custom_pg_loss_reducer_function_path ............ None - custom_reward_post_process_path ................. None - custom_rm_path .................................. gsm8k_reward.score - custom_rollout_log_function_path ................ None -... [truncated: 1630 of 2030 lines removed; full evidence archived outside repo] ... -(RolloutManager pid=1869) Rollout generation: 100%|██████████| 32/32 [00:05<00:00, 6.22it/s] Rollout generation: 100%|██████████| 32/32 [00:05<00:00, 5.61it/s] -(RolloutManager pid=1869) [2026-09-29 11:34:38.968 rollout_manager] sglang_rollout.py:494 - Finish rollout: ["<|im_start|>user\nJulie is reading a 120-page book. Yesterday, she was able to read 12 pages and today, she read twice as many pages as yesterday. If she wants to read half of the remaining pages tomorrow, how many pages should she read?\nPut the final numeric answer in \\boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nJulie is reading a **120-page book**. Let's go step by step.\n\n---\n\n### Step 1: Yesterday she read **12 pages**\n\nSo the remaining pages are:\n\n$$\n120 - 12 = 108 \\text{ pages}\n$$\n\n---\n\n### Step 2: Today, she read **twice as many pages as yesterday**\n\nThat means today she read:\n\n$$\n2 \\times 12 = 24 \\text{ pages}\n$$\n\nSo the total pages read so far:\n\n$$\n12 + 24 = 36 \\text{ pages}\n$$\n\nRemaining pages after today:\n\n$$\n108 - 24 = 84 \\text{ pages}\n$$\n\n---\n\n### Step 3: She wants to **read half of the remaining pages tomorrow**\n\nLet's calculate that half:\n\n$$\n\\frac{84}{2} = 42 \\text{ pages}\n$$\n\n---\n\n### ✅ Final Answer:\n\n$$\n\\boxed{42}\n$$<|im_end|>"], label: 42, reward: 1.0 -(RolloutManager pid=1869) [2026-09-29 11:34:38.969 rollout_manager] sglang_rollout.py:391 - Abort request for ['http://172.20.28.65:21100'] -(RolloutManager pid=1869) [2026-09-29 11:34:38.975 rollout_manager] rollout_data_conversion.py:42 - Final collected 32 samples from rollout to train -(RolloutManager pid=1869) [2026-09-29 11:34:38.975 rollout_manager] metrics.py:79 - perf 2: {'rollout/response_len/mean': 164.0, 'rollout/response_len/median': 154.0, 'rollout/response_len/max': 384, 'rollout/response_len/min': 82, 'rollout/zero_std/count_1.0': 3, 'rollout/zero_std/all_zero_percentage': 0.0, 'rollout/zero_std/all_one_percentage': 0.75, 'rollout/prefix_cache_hit_rate': 0.6519516728624535, 'rollout/avg_cached_tokens_per_sample': 43.84375, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.03125, 'perf/rollout_time': 6.159783124923706, 'perf/tokens_per_gpu_per_sec': 851.9780475331265, 'perf/longest_sample_tokens_per_sec': 62.33985713657024, 'perf/effective_tokens_per_gpu_per_sec': 851.9780475331265, 'perf/longest_effective_sample_tokens_per_sec': 62.33985713657024} -(SGLangEngine pid=7734) [2026-09-29 11:34:38] Cache flushed successfully! -(SGLangEngine pid=7734) [2026-09-29 11:34:39] Cache flushed successfully! -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:39.530 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=train phase=start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:39.530 actor_cell0_rank0] timer.py:31 - Timer data_preprocess start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:39.534 actor_cell0_rank0] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:39.534 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=train_actor phase=start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:39.534 actor_cell0_rank0] timer.py:42 - Timer train_wait end (elapsed: 9.0s) -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:39.534 actor_cell0_rank0] timer.py:31 - Timer train start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:39.558 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:39.558 actor_cell0_rank0] timer.py:31 - Timer log_probs start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:49.005 actor_cell0_rank0] timer.py:42 - Timer log_probs end (elapsed: 9.4s) -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:49.006 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=end ok=true elapsed_s=9.4 -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:49.035 actor_cell0_rank0] log_utils.py:42 - ft op=cross_cell phase=start kind=log_gather rank=0 -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:49.036 actor_cell0_rank0] log_utils.py:48 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:49.037 actor_cell0_rank0] log_utils.py:66 - rollout 2: {'rollout/response_lengths': 164.0, 'rollout/rewards': -3.725290298461914e-09, 'rollout/truncated': 0.03125, 'rollout/rollout_log_probs': -0.3132147490978241, 'rollout/raw_reward': 0.90625, 'rollout/total_lengths': 231.25, 'rollout/log_probs': -0.3139707148075104, 'rollout/advantages': -1.1175870895385742e-08, 'rollout/returns': -1.1175870895385742e-08} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:34:49.037 actor_cell0_rank0] timer.py:31 - Timer actor_train start -(MegatronTrainRayActor pid=8402) /root/Megatron-LM/megatron/core/pipeline_parallel/schedules.py:190: UserWarning: The AccumulateGrad node's stream does not match the stream of the node that produced the incoming gradient. This may incur unnecessary synchronization and break CUDA graph capture if the AccumulateGrad node's stream is the default stream. This mismatch is caused by an AccumulateGrad node created prior to the current iteration being kept alive. This can happen if the autograd graph is still being kept alive by tensors such as the loss, or if you are using DDP, which will stash a reference to the node. To resolve the mismatch, delete all references to the autograd graph or ensure that DDP initialization is performed under the same stream as subsequent forwards. If the mismatch is intentional, you can use torch.autograd.graph.set_warn_on_accumulate_grad_stream_mismatch(False) to suppress this warning. (Triggered internally at /pytorch/torch/csrc/autograd/input_buffer.cpp:240.) -(MegatronTrainRayActor pid=8402) Variable._execution_engine.run_backward( -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:00.137 actor_cell0_rank0] model.py:601 - ft op=train_step rollout=2 step=0 attempt=0 outcome=NORMAL valid_step=true -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:00.138 actor_cell0_rank0] log_utils.py:460 - step 2: {'train/loss': 1.1175870895385742e-08, 'train/pg_loss': 1.1175870895385742e-08, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.012934734113514423, 'train/train_rollout_kl': 0.0006191637367010117, 'train/grad_norm': 0.21108577073643137, 'train/lr-pg_0': 2.5e-06, 'train/step': 2} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:00.139 actor_cell0_rank0] model.py:814 - step 2: {'train/loss': 1.1175870895385742e-08, 'train/pg_loss': 1.1175870895385742e-08, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.012934734113514423, 'train/train_rollout_kl': 0.0006191637367010117, 'train/grad_norm': 0.21108577073643137, 'train/lr-pg_0': 2.5e-06, 'train/step': 2} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:00.139 actor_cell0_rank0] timer.py:42 - Timer actor_train end (elapsed: 11.1s) -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:00.139 actor_cell0_rank0] timer.py:42 - Timer train end (elapsed: 20.6s) -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:00.139 actor_cell0_rank0] timer.py:31 - Timer train_wait start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:00.163 actor_cell0_rank0] train_metric_utils.py:50 - perf 2: {'perf/update_weights_time': 0.6623046398162842, 'perf/data_preprocess_time': 0.003518819808959961, 'perf/train_wait_time': 9.03148603439331, 'perf/log_probs_time': 9.447287797927856, 'perf/actor_train_time': 11.101924657821655, 'perf/train_time': 20.604564666748047, 'perf/log_probs_tflops': 0.8351607158671055, 'perf/actor_train_tflops': 2.1320637322361695, 'perf/actor_train_tok_per_s': 666.5510916421566, 'perf/step_time': 29.636050701141357, 'perf/wait_time_ratio': 0.3047466116679806} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:00.163 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train_actor phase=end ok=true elapsed_s=20.6 -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:00.163 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=20.6 -(SGLangEngine pid=7734) [2026-09-29 11:34:31] INFO: 172.20.28.65:61748 - "POST /update_weight_version HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:31] INFO: 172.20.28.65:30323 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:31] INFO: 172.20.28.65:31420 - "POST /pause_generation HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:31] INFO: 172.20.28.65:44872 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:32] INFO: 172.20.28.65:29581 - "POST /load_lora_adapter_from_tensors HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:32] INFO: 172.20.28.65:55431 - "POST /continue_generation HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:32] INFO: 172.20.28.65:20385 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:35] INFO: 172.20.28.65:30248 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:35] INFO: 172.20.28.65:30262 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:35] INFO: 172.20.28.65:59511 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:35] INFO: 172.20.28.65:51750 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:35] INFO: 172.20.28.65:32929 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:35] INFO: 172.20.28.65:63976 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:35] INFO: 172.20.28.65:30846 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:35] INFO: 172.20.28.65:54815 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:35] INFO: 172.20.28.65:30262 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:35] INFO: 172.20.28.65:30248 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:35] INFO: 172.20.28.65:54815 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:35] INFO: 172.20.28.65:51750 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:36] INFO: 172.20.28.65:63976 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:36] INFO: 172.20.28.65:32929 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:36] INFO: 172.20.28.65:59511 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:36] INFO: 172.20.28.65:30846 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:36] INFO: 172.20.28.65:51750 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:37] INFO: 172.20.28.65:32929 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:37] INFO: 172.20.28.65:30248 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:37] INFO: 172.20.28.65:59511 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:37] INFO: 172.20.28.65:30262 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:37] INFO: 172.20.28.65:30846 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:38] INFO: 172.20.28.65:54815 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:38] INFO: 172.20.28.65:32929 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:38] INFO: 172.20.28.65:51750 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:38] INFO: 172.20.28.65:63976 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:38] INFO: 172.20.28.65:30248 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:38] INFO: 172.20.28.65:30262 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:38] INFO: 172.20.28.65:59511 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:38] INFO: 172.20.28.65:30846 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:38] INFO: 172.20.28.65:54815 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:38] INFO: 172.20.28.65:32929 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:38] INFO: 172.20.28.65:20860 - "POST /abort_request HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:38] INFO: 172.20.28.65:42257 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:34:39] INFO: 172.20.28.65:27652 - "POST /release_memory_occupation HTTP/1.1" 200 OK -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"begin","policy_version":3} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"apply_finished","policy_version":3} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_begin","policy_version":3} -(MegatronTrainRayActor pid=8402) /usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. -(MegatronTrainRayActor pid=8402) return func(*args, **kwargs) -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_finished","policy_version":3} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:02.055 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before TrainRayActor.clear_memory: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 70.26, 'used_GB': 8.92, 'allocated_GB': 1.3, 'reserved_GB': 4.34} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:03.110 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after TrainRayActor.clear_memory: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 73.05, 'used_GB': 6.13, 'allocated_GB': 1.3, 'reserved_GB': 1.55} -(SGLangEngine pid=7734) [2026-09-29 11:35:03] Cache flushed successfully! -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:03.433 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=update_weights phase=start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:03.433 actor_cell0_rank0] timer.py:31 - Timer update_weights start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:03.434 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.86, 'used_GB': 7.32, 'allocated_GB': 1.3, 'reserved_GB': 1.55} -(SGLangEngine pid=7734) [2026-09-29 11:35:03] Start unload Lora adapter. Lora name=miles_lora -(SGLangEngine pid=7734) [2026-09-29 11:35:03] LoRA adapter unloading starts: LoRARef(lora_id=747f683afd0e40a8a38092330dac11b6, lora_name=miles_lora). avail mem=71.88 GB -(SGLangEngine pid=7734) [2026-09-29 11:35:03] LoRA adapter unloading completes: LoRARef(lora_id=747f683afd0e40a8a38092330dac11b6, lora_name=miles_lora). avail mem=71.88 GB -(SGLangEngine pid=7734) [2026-09-29 11:35:03] Start load Lora adapter from tensors. Lora name=miles_lora -(SGLangEngine pid=7734) [2026-09-29 11:35:03] LoRA adapter loading from tensors starts: LoRARef(lora_id=f4d2d007e44a4ce8bc411ecba6be06b6, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(SGLangEngine pid=7734) [2026-09-29 11:35:03] LoRA adapter loading from tensors completes: LoRARef(lora_id=f4d2d007e44a4ce8bc411ecba6be06b6, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:03.538 actor_cell0_rank0] update_weight_from_tensor.py:461 - Packed 392 LoRA tensors (20185088 bytes) into 1 flattened buckets (max 268435456 bytes/1024 tensors) -(RolloutManager pid=1869) Rollout generation: 0%| | 0/32 [00:00 33096.0 MB (freed 0.0 MB) -(SGLangEngine pid=7734) [2026-09-29 11:35:04] LoRA adapter 'f4d2d007e44a4ce8bc411ecba6be06b6': loaded weights for target modules ['down_proj', 'gate_up_proj', 'o_proj', 'qkv_proj']. -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:04.182 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.88, 'used_GB': 7.29, 'allocated_GB': 1.3, 'reserved_GB': 1.55} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:04.183 actor_cell0_rank0] timer.py:42 - Timer update_weights end (elapsed: 0.7s) -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:04.183 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.7 -(SGLangEngine pid=7734) [2026-09-29 11:35:04] Prefill batch, #new-seq: 1, #new-token: 47, #cached-token: 0, token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 1.80 -(SGLangEngine pid=7734) [2026-09-29 11:35:04] Prefill batch, #new-seq: 7, #new-token: 329, #cached-token: 0, token usage: 0.00, #running-req: 1, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 70669.77 -(SGLangEngine pid=7734) [2026-09-29 11:35:04] Decode batch, #running-req: 8, #token: 375, token usage: 0.00, cuda graph: True, gen throughput (token/s): 12.46, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:04] Decode batch, #running-req: 8, #token: 695, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1723.61, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:04] Prefill batch, #new-seq: 1, #new-token: 83, #cached-token: 3, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 147.60 -(SGLangEngine pid=7734) [2026-09-29 11:35:04] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 12.96 -(SGLangEngine pid=7734) [2026-09-29 11:35:04] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 8, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 359.26 -(SGLangEngine pid=7734) [2026-09-29 11:35:04] Decode batch, #running-req: 8, #token: 748, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1123.18, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:05] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 17.04 -(SGLangEngine pid=7734) [2026-09-29 11:35:05] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 8.32 -(SGLangEngine pid=7734) [2026-09-29 11:35:05] Decode batch, #running-req: 8, #token: 640, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1271.60, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:05] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 8.08 -(RolloutManager pid=1869) [2026-09-29 11:35:05.425 rollout_manager] sglang_rollout.py:473 - First rollout sample: ["<|im_start|>user\nJames writes a 3-page letter to 2 different friends twice a week. How many pages does he write a year?\nPut the final numeric answer in \\boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nJames writes a **3-page letter** to **both friends** **twice a week**. \n\nLet's break it down week by week:\n\n### Week:\n- 3 pages *twice* = 3 * 2 = **6 pages per week**\n\n### Multiply by the number of weeks in a year:\n- 52 weeks in a year\n\n$$\n6 \\text{ pages/week} \\times 52 \\text{ weeks} = \\boxed{312} \\text{ pages}\n$$\n\n**James writes 312 pages a year.**<|im_end|>"], label: 624, reward: 0.0 -(RolloutManager pid=1869) Rollout generation: 25%|██▌ | 8/32 [00:01<00:03, 6.61it/s] -(SGLangEngine pid=7734) [2026-09-29 11:35:05] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 6.31 -(SGLangEngine pid=7734) [2026-09-29 11:35:05] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 85, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 18.68 -(SGLangEngine pid=7734) [2026-09-29 11:35:05] Decode batch, #running-req: 8, #token: 529, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1119.61, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:05] Decode batch, #running-req: 8, #token: 849, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1786.92, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:05] Decode batch, #running-req: 8, #token: 1169, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1702.35, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:05] Prefill batch, #new-seq: 1, #new-token: 75, #cached-token: 3, token usage: 0.00, #running-req: 6, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 158.99 -(SGLangEngine pid=7734) [2026-09-29 11:35:05] Prefill batch, #new-seq: 1, #new-token: 75, #cached-token: 3, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 20439.64 -(SGLangEngine pid=7734) [2026-09-29 11:35:06] Decode batch, #running-req: 8, #token: 1263, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1241.57, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:06] Decode batch, #running-req: 8, #token: 1583, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1688.62, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:06] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 2.28 -(SGLangEngine pid=7734) [2026-09-29 11:35:06] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 15.33 -(SGLangEngine pid=7734) [2026-09-29 11:35:06] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 9.82 -(SGLangEngine pid=7734) [2026-09-29 11:35:06] Decode batch, #running-req: 8, #token: 1495, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1106.66, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:06] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 6.22 -(SGLangEngine pid=7734) [2026-09-29 11:35:06] Decode batch, #running-req: 8, #token: 1514, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1427.53, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:06] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 5.59 -(SGLangEngine pid=7734) [2026-09-29 11:35:07] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 6.63 -(SGLangEngine pid=7734) [2026-09-29 11:35:07] Prefill batch, #new-seq: 1, #new-token: 124, #cached-token: 3, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 3114.81 -(SGLangEngine pid=7734) [2026-09-29 11:35:07] Decode batch, #running-req: 8, #token: 1520, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1106.12, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:07] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 126, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 8.99 -(SGLangEngine pid=7734) [2026-09-29 11:35:07] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 126, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 17.00 -(SGLangEngine pid=7734) [2026-09-29 11:35:07] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 126, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 20.48 -(SGLangEngine pid=7734) [2026-09-29 11:35:07] Decode batch, #running-req: 8, #token: 944, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1129.07, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:07] Prefill batch, #new-seq: 2, #new-token: 2, #cached-token: 252, token usage: 0.00, #running-req: 6, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 9.55 -(RolloutManager pid=1869) Rollout generation: 50%|█████ | 16/32 [00:03<00:03, 4.43it/s] -(SGLangEngine pid=7734) [2026-09-29 11:35:07] Decode batch, #running-req: 8, #token: 1047, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1467.13, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:07] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 126, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 5.33 -(SGLangEngine pid=7734) [2026-09-29 11:35:07] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 126, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 12.12 -(SGLangEngine pid=7734) [2026-09-29 11:35:07] Decode batch, #running-req: 8, #token: 711, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1257.45, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:08] Decode batch, #running-req: 8, #token: 1031, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1692.05, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:08] Decode batch, #running-req: 8, #token: 1351, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1685.24, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:08] Decode batch, #running-req: 6, #token: 1249, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1494.80, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:08] Decode batch, #running-req: 3, #token: 606, token usage: 0.00, cuda graph: True, gen throughput (token/s): 828.74, #queue-req: 0 -(RolloutManager pid=1869) Rollout generation: 100%|██████████| 32/32 [00:04<00:00, 7.77it/s] Rollout generation: 100%|██████████| 32/32 [00:04<00:00, 7.01it/s] -(RolloutManager pid=1869) [2026-09-29 11:35:08.783 rollout_manager] sglang_rollout.py:494 - Finish rollout: ["<|im_start|>user\nKen created a care package to send to his brother, who was away at boarding school. Ken placed a box on a scale, and then he poured into the box enough jelly beans to bring the weight to 2 pounds. Then, he added enough brownies to cause the weight to triple. Next, he added another 2 pounds of jelly beans. And finally, he added enough gummy worms to double the weight once again. What was the final weight of the box of goodies, in pounds?\nPut the final numeric answer in \\boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nWe are given that ken **finally** added **gummy worms**, **tripling the weight again**, and we need to find the **final weight** of the box of goodies.\n\nWe'll break down the total addition step by step:\n\n1. **Initial weight = 2 pounds**\n2. **First action:** added jelly beans to make the weight **triply** → becomes $2 \\times 3 = 6$ pounds.\n3. **Second action:** added 2 pounds of jelly beans to get a final weight of $6 + 2 = 8$ pounds.\n4. **Third action:** added gummy worms (by doubling the weight again). But the problem didn't specify how much gummy worms were added, only that it was done to **doubling**.\n\nSince we are **only** told the final weight after one operation, and that was doubling (to 2 balls, total weight 8 pounds), then the **final weight** is:\n\n$$\n\\boxed{8} \\text{ pounds}\n$$<|im_end|>"], label: 16, reward: 0.0 -(RolloutManager pid=1869) [2026-09-29 11:35:08.785 rollout_manager] sglang_rollout.py:391 - Abort request for ['http://172.20.28.65:21100'] -(RolloutManager pid=1869) [2026-09-29 11:35:08.789 rollout_manager] rollout_data_conversion.py:42 - Final collected 32 samples from rollout to train -(RolloutManager pid=1869) [2026-09-29 11:35:08.790 rollout_manager] metrics.py:79 - perf 3: {'rollout/response_len/mean': 183.8125, 'rollout/response_len/median': 159.5, 'rollout/response_len/max': 384, 'rollout/response_len/min': 71, 'rollout/zero_std/count_0.0': 1, 'rollout/zero_std/count_1.0': 1, 'rollout/zero_std/all_zero_percentage': 0.25, 'rollout/zero_std/all_one_percentage': 0.25, 'rollout/prefix_cache_hit_rate': 0.7215236686390533, 'rollout/avg_cached_tokens_per_sample': 60.96875, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.09375, 'perf/rollout_time': 4.575300693511963, 'perf/tokens_per_gpu_per_sec': 1285.5985636836965, 'perf/longest_sample_tokens_per_sec': 83.92890997187001, 'perf/effective_tokens_per_gpu_per_sec': 1285.5985636836965, 'perf/longest_effective_sample_tokens_per_sec': 83.92890997187001} -(SGLangEngine pid=7734) [2026-09-29 11:35:08] Decode batch, #running-req: 1, #token: 385, token usage: 0.00, cuda graph: True, gen throughput (token/s): 261.53, #queue-req: 0 -(SGLangEngine pid=7734) [2026-09-29 11:35:08] Cache flushed successfully! -(SGLangEngine pid=7734) [2026-09-29 11:35:08] Cache flushed successfully! -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:09.503 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=train phase=start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:09.503 actor_cell0_rank0] timer.py:31 - Timer data_preprocess start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:09.508 actor_cell0_rank0] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:09.508 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=train_actor phase=start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:09.508 actor_cell0_rank0] timer.py:42 - Timer train_wait end (elapsed: 9.4s) -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:09.508 actor_cell0_rank0] timer.py:31 - Timer train start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:09.532 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:09.532 actor_cell0_rank0] timer.py:31 - Timer log_probs start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:13.824 actor_cell0_rank0] timer.py:42 - Timer log_probs end (elapsed: 4.3s) -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:13.824 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=end ok=true elapsed_s=4.3 -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:13.830 actor_cell0_rank0] log_utils.py:42 - ft op=cross_cell phase=start kind=log_gather rank=0 -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:13.831 actor_cell0_rank0] log_utils.py:48 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:13.831 actor_cell0_rank0] log_utils.py:66 - rollout 3: {'rollout/response_lengths': 183.8125, 'rollout/rewards': 0.0, 'rollout/truncated': 0.09375, 'rollout/rollout_log_probs': -0.4074595272541046, 'rollout/raw_reward': 0.375, 'rollout/total_lengths': 268.3125, 'rollout/log_probs': -0.4081624448299408, 'rollout/advantages': 0.0, 'rollout/returns': 0.0} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:13.832 actor_cell0_rank0] timer.py:31 - Timer actor_train start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:21.898 actor_cell0_rank0] model.py:601 - ft op=train_step rollout=3 step=0 attempt=0 outcome=NORMAL valid_step=true -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:21.899 actor_cell0_rank0] log_utils.py:460 - step 3: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.014970701187849045, 'train/train_rollout_kl': 0.0007022398058325052, 'train/grad_norm': 0.31632452957322144, 'train/lr-pg_0': 0.0, 'train/step': 3} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:21.899 actor_cell0_rank0] model.py:814 - step 3: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.014970701187849045, 'train/train_rollout_kl': 0.0007022398058325052, 'train/grad_norm': 0.31632452957322144, 'train/lr-pg_0': 0.0, 'train/step': 3} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:21.899 actor_cell0_rank0] timer.py:42 - Timer actor_train end (elapsed: 8.1s) -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:21.899 actor_cell0_rank0] timer.py:42 - Timer train end (elapsed: 12.4s) -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:21.899 actor_cell0_rank0] timer.py:31 - Timer train_wait start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:21.923 actor_cell0_rank0] train_metric_utils.py:50 - perf 3: {'perf/update_weights_time': 0.7494399547576904, 'perf/data_preprocess_time': 0.004262208938598633, 'perf/train_wait_time': 9.369312047958374, 'perf/log_probs_time': 4.291313648223877, 'perf/actor_train_time': 8.06747817993164, 'perf/train_time': 12.39091444015503, 'perf/log_probs_tflops': 2.146196185363405, 'perf/actor_train_tflops': 3.424862432820627, 'perf/actor_train_tok_per_s': 1064.273098544997, 'perf/step_time': 21.760226488113403, 'perf/wait_time_ratio': 0.43057052062745726} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:21.923 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train_actor phase=end ok=true elapsed_s=12.4 -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:21.924 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=12.4 -(SGLangEngine pid=7734) [2026-09-29 11:35:02] INFO: 172.20.28.65:33086 - "POST /update_weight_version HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:03] INFO: 172.20.28.65:61140 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:03] INFO: 172.20.28.65:20086 - "POST /pause_generation HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:03] INFO: 172.20.28.65:48486 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:03] INFO: 172.20.28.65:23418 - "POST /unload_lora_adapter HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:03] INFO: 172.20.28.65:63969 - "POST /load_lora_adapter_from_tensors HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:04] INFO: 172.20.28.65:41142 - "POST /continue_generation HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:04] INFO: 172.20.28.65:23682 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:04] INFO: 172.20.28.65:38173 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:04] INFO: 172.20.28.65:17537 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:04] INFO: 172.20.28.65:64022 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:04] INFO: 172.20.28.65:31929 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:05] INFO: 172.20.28.65:54279 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:05] INFO: 172.20.28.65:64497 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:05] INFO: 172.20.28.65:44099 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:05] INFO: 172.20.28.65:45735 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:05] INFO: 172.20.28.65:31929 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:05] INFO: 172.20.28.65:54279 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:06] INFO: 172.20.28.65:31929 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:06] INFO: 172.20.28.65:54279 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:06] INFO: 172.20.28.65:64497 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:06] INFO: 172.20.28.65:17537 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:06] INFO: 172.20.28.65:54279 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:06] INFO: 172.20.28.65:64497 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:07] INFO: 172.20.28.65:45735 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:07] INFO: 172.20.28.65:31929 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:07] INFO: 172.20.28.65:38173 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:07] INFO: 172.20.28.65:64022 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:07] INFO: 172.20.28.65:54279 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:07] INFO: 172.20.28.65:17537 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:07] INFO: 172.20.28.65:44099 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:07] INFO: 172.20.28.65:64497 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:08] INFO: 172.20.28.65:45735 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:08] INFO: 172.20.28.65:54279 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:08] INFO: 172.20.28.65:31929 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:08] INFO: 172.20.28.65:44099 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:08] INFO: 172.20.28.65:64022 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:08] INFO: 172.20.28.65:64497 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:08] INFO: 172.20.28.65:38173 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:08] INFO: 172.20.28.65:17537 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:08] INFO: 172.20.28.65:32563 - "POST /abort_request HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:08] INFO: 172.20.28.65:29062 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:09] INFO: 172.20.28.65:55881 - "POST /release_memory_occupation HTTP/1.1" 200 OK -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"begin","policy_version":4} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"apply_finished","policy_version":4} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_begin","policy_version":4} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_finished","policy_version":4} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:23.539 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before TrainRayActor.clear_memory: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 70.11, 'used_GB': 9.07, 'allocated_GB': 1.3, 'reserved_GB': 4.49} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:24.252 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after TrainRayActor.clear_memory: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 73.05, 'used_GB': 6.13, 'allocated_GB': 1.3, 'reserved_GB': 1.55} -(SGLangEngine pid=7734) [2026-09-29 11:35:24] Cache flushed successfully! -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:24.346 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=update_weights phase=start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:24.346 actor_cell0_rank0] timer.py:31 - Timer update_weights start -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:24.347 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.86, 'used_GB': 7.32, 'allocated_GB': 1.3, 'reserved_GB': 1.55} -(SGLangEngine pid=7734) [2026-09-29 11:35:24] Start unload Lora adapter. Lora name=miles_lora -(SGLangEngine pid=7734) [2026-09-29 11:35:24] LoRA adapter unloading starts: LoRARef(lora_id=f4d2d007e44a4ce8bc411ecba6be06b6, lora_name=miles_lora). avail mem=71.88 GB -(SGLangEngine pid=7734) [2026-09-29 11:35:24] LoRA adapter unloading completes: LoRARef(lora_id=f4d2d007e44a4ce8bc411ecba6be06b6, lora_name=miles_lora). avail mem=71.88 GB -(SGLangEngine pid=7734) [2026-09-29 11:35:24] Start load Lora adapter from tensors. Lora name=miles_lora -(SGLangEngine pid=7734) [2026-09-29 11:35:24] LoRA adapter loading from tensors starts: LoRARef(lora_id=9af492b9c4c34c77ab40bf3712f848b9, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(SGLangEngine pid=7734) [2026-09-29 11:35:24] LoRA adapter loading from tensors completes: LoRARef(lora_id=9af492b9c4c34c77ab40bf3712f848b9, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:24.405 actor_cell0_rank0] update_weight_from_tensor.py:461 - Packed 392 LoRA tensors (20185088 bytes) into 1 flattened buckets (max 268435456 bytes/1024 tensors) -(SGLangEngine pid=7734) [2026-09-29 11:35:25] [continue_generation] torch.cuda.empty_cache() called: reserved 33096.0 MB -> 33096.0 MB (freed 0.0 MB) -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:25.037 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.88, 'used_GB': 7.3, 'allocated_GB': 1.3, 'reserved_GB': 1.55} -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:25.037 actor_cell0_rank0] timer.py:42 - Timer update_weights end (elapsed: 0.7s) -(MegatronTrainRayActor pid=8402) [2026-09-29 11:35:25.037 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.7 -(SGLangEngine pid=7734) [2026-09-29 11:35:23] INFO: 172.20.28.65:18627 - "POST /update_weight_version HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:24] INFO: 172.20.28.65:40330 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:24] INFO: 172.20.28.65:48280 - "POST /pause_generation HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:24] INFO: 172.20.28.65:46769 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:24] INFO: 172.20.28.65:31309 - "POST /unload_lora_adapter HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:24] INFO: 172.20.28.65:57020 - "POST /load_lora_adapter_from_tensors HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:25] INFO: 172.20.28.65:19032 - "POST /continue_generation HTTP/1.1" 200 OK -(SGLangEngine pid=7734) [2026-09-29 11:35:25] INFO: 172.20.28.65:22818 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -[rl] learner 0 finalized diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/run.log b/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/run.log deleted file mode 100644 index d537c0271..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-legacy/run.log +++ /dev/null @@ -1,7 +0,0 @@ - Fetching 10 files: 0%| | 0/10 [00:00 failed for pad_packed_seq_alignment: typing.Union[int, typing.Literal['max'], NoneType]. -Deferring to attribute metadata. If the metadata is incomplete, 'parser.add_argument()' may fail. -Original failure: Unions not supported by argparse: typing.Union[int, typing.Literal['max'], NoneType] -using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 -Number of virtual stages per pipeline stage: None -using torch.bfloat16 for parameters ... ------------------------- arguments ------------------------ - _is_global_batch_size_explicitly_specified ...... True - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - activation_func_clamp_shared_expert ............. True - activation_func_clamp_value ..................... None - activation_offload_fraction ..................... 1.0 - actor_num_gpus_per_node ......................... 1 - actor_num_nodes ................................. 1 - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.999 - adam_eps ........................................ 1e-08 - add_bias_linear ................................. False - add_position_embedding .......................... True - add_qkv_bias .................................... False - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - advantage_estimator ............................. grpo - align_grad_reduce ............................... True - align_param_gather .............................. False - allgather_cp .................................... False - allow_ambiguous_pad_tokens ...................... False - allow_partial_train_step ........................ False - api_server_host ................................. 127.0.0.1 - api_server_port ................................. 0 - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_chat_template ............................. True - apply_chat_template_kwargs ...................... {'enable_thinking': False} - apply_dsa_kernel_fusion ......................... None - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - apply_wd_to_qk_layernorm ........................ False - async_ckpt_cpu_priority ......................... 10 - async_ckpt_io_priority .......................... 3 - async_ckpt_use_cpu_shm .......................... False - async_data_buffer_capacity_factor ............... 2.0 - async_max_concurrent_samples .................... None - async_save ...................................... False - async_strategy .................................. mcore - async_unused_samples_handler .................... drop - attention_backend ............................... AttnBackend.unfused - attention_dropout ............................... 0.1 - attention_output_gate ........................... False - attention_softmax_in_fp32 ....................... True - auto_detect_ckpt_format ......................... False - balance_by_flops ................................ False - balance_data .................................... True - barrier_with_L1_time ............................ True - batch_invariant_mode ............................ False - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - beta_fast ....................................... 32 - beta_slow ....................................... 1 - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - buffer_filter_path .............................. yeto.rl.engine.miles_adapter.rollout_meta_hook.policy_buffer_filter - cache_mla_latents ............................... False - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - chat_template ................................... None - chat_template_path .............................. None - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_lora_weight_equal ......................... False - check_rematerialize_param_from_master_weight .... False - check_weight_hash_across_dp_replicas_interval ... None - check_weight_update_allow_quant_error ........... False - check_weight_update_equal ....................... False - check_weight_update_selector .................... all - check_weight_update_skip_list ................... None - chunked_optimizer_state_offload ................. False - ci_check_model_hash ............................. False - ci_disable_kl_checker ........................... False - ci_disable_logprobs_checker ..................... False - ci_disable_weight_update_checker ................ False - ci_enable_metrics_capture ....................... False - ci_ft_test_actions .............................. None - ci_ft_test_actions_path ......................... None - ci_inject_rollout_data_min_match_ratio .......... 0.9 - ci_inject_rollout_data_path ..................... None - ci_inject_rollout_data_start_rollout_id ......... None - ci_load_grad_norm ............................... None - ci_metric_checker_expect_num .................... None - ci_metric_checker_key ........................... None - ci_metric_checker_threshold ..................... None - ci_save_grad_norm ............................... None - ci_save_model_hash .............................. False - ci_test ......................................... False - ci_tito_special_token_count_threshold ........... 0.0 - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch_dist - ckpt_fully_parallel_load ........................ False - ckpt_fully_parallel_load_exchange_algo .......... broadcast - ckpt_fully_parallel_load_process_group .......... dp - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_fully_parallel_save_process_group .......... dp - ckpt_load_validate_sharding_integrity ........... True - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clear_quantized_weight_workspaces_on_offload .... True - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - cluster_backend ................................. ray - colocate ........................................ True - colocate_memory_peak_device ..................... cpu - compress_ratios ................................. None - compute_advantages_and_returns .................. True - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 -... [truncated: 2313 of 2713 lines removed; full evidence archived outside repo] ... -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:11.593 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.2 -(InferenceController pid=4542) [2026-09-29 11:33:11.635 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/resume_memory_occupation "HTTP/1.1 200 OK" -(InferenceController pid=4542) [2026-09-29 11:33:11.680 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/weights_checker "HTTP/1.1 200 OK" -(TrainerController pid=4534) [2026-09-29 11:33:11.688 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=sleep n_actors=1 -(CommandActor pid=4544) [2026-09-29 11:33:11] Weight version changed. old_version='2' new_version='yeto:1:f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a' num_recorded=0 -(CommandActor pid=4544) [2026-09-29 11:33:11] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(CommandActor pid=4544) [2026-09-29 11:33:11] [WeightChecker] checksum computed for 226 tensors in 0.021s -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:11.689 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:11.689 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:12.470 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 41.94, 'used_GB': 37.24, 'allocated_GB': 1.38, 'reserved_GB': 1.48} -(RolloutExecutor pid=4536) Rollout generation: 0%| | 0/32 [00:00user\nAlbert is wondering how much pizza he can eat in one day. He buys 2 large pizzas and 2 small pizzas. A large pizza has 16 slices and a small pizza has 8 slices. If he eats it all, how many pieces does he eat that day?\nPut the final numeric answer in \boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nAlbert buys:\n\n- 2 large pizzas, each with **16 slices**\n- 2 small pizzas, each with **8 slices**\n\nSo the total number of pizza slices he eats is:\n\n$$\n2 \times 16 + 2 \t..., label=48, reward_summary=1.0 -(RolloutExecutor pid=4536) Rollout generation: 25%|██▌ | 8/32 [00:05<00:17, 1.35it/s] -(CommandActor pid=4544) [2026-09-29 11:33:19] Decode batch, #running-req: 19, #token: 3313, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4065.09, #queue-req: 0 -(RolloutExecutor pid=4536) Rollout generation: 50%|█████ | 16/32 [00:06<00:05, 3.00it/s] -(CommandActor pid=4544) [2026-09-29 11:33:19] Decode batch, #running-req: 9, #token: 2019, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2884.69, #queue-req: 0 -(CommandActor pid=4544) [2026-09-29 11:33:19] Decode batch, #running-req: 8, #token: 2138, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1891.74, #queue-req: 0 -(CommandActor pid=4544) [2026-09-29 11:33:19] Decode batch, #running-req: 7, #token: 1896, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1525.02, #queue-req: 0 -(RolloutExecutor pid=4536) Rollout generation: 75%|███████▌ | 24/32 [00:06<00:01, 4.79it/s] -(CommandActor pid=4544) [2026-09-29 11:33:19] Decode batch, #running-req: 4, #token: 1370, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1023.83, #queue-req: 0 -(CommandActor pid=4544) [2026-09-29 11:33:20] Decode batch, #running-req: 1, #token: 447, token usage: 0.00, cuda graph: True, gen throughput (token/s): 633.48, #queue-req: 0 -(RolloutExecutor pid=4536) Rollout generation: 100%|██████████| 32/32 [00:07<00:00, 6.87it/s] Rollout generation: 100%|██████████| 32/32 [00:07<00:00, 4.45it/s] -(RolloutExecutor pid=4536) [2026-09-29 11:33:20.117 rollout_executor] inference_rollout_train.py:161 - Finish rollout: text_preview=<|im_start|>user\nMark has a garden with flowers. He planted plants of three different colors in it. Ten of them are yellow, and there are 80% more of those in purple. There are only 25% as many green flowers as there are yellow and purple flowers. How many flowers does Mark have in his garden?\nPut the final numeric answer in \boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nLet's solve the problem step by step.\n\n---\n\n### Step 1: Represent the number of yellow, purple, and green flower..., label=35, reward_summary=1.0 -(RolloutExecutor pid=4536) [2026-09-29 11:33:20.120 rollout_executor] inference_rollout_train.py:33 - Abort request for ['http://172.20.8.111:21100'] -(RolloutExecutor pid=4536) [2026-09-29 11:33:20.128 rollout_executor] rollout_data_conversion.py:39 - Final collected 32 samples from rollout to train -(RolloutExecutor pid=4536) [2026-09-29 11:33:20.128 rollout_executor] metrics.py:93 - perf 1: {'rollout/raw_reward_unfiltered': 0.375, 'rollout/num_training_samples': 32, 'rollout/episode_raw_reward': 0.375, 'rollout/episode_response_length/mean': 188.1875, 'rollout/episode_response_length/median': 173.5, 'rollout/episode_response_length/max': 384, 'rollout/episode_response_length/min': 71, 'rollout/episode_total_response_length/mean': 188.1875, 'rollout/response_len/mean': 188.1875, 'rollout/response_len/median': 173.5, 'rollout/response_len/max': 384, 'rollout/response_len/min': 71, 'rollout/zero_std/count_1.0': 1, 'rollout/zero_std/all_zero_percentage': 0.0, 'rollout/zero_std/all_one_percentage': 0.25, 'rollout/prefix_cache_hit_rate': 0.0, 'rollout/avg_cached_tokens_per_sample': 0.0, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.03125, 'perf/rollout_time': 7.203626871109009, 'perf/tokens_per_gpu_per_sec': 835.9677850822532, 'perf/longest_sample_tokens_per_sec': 53.30648114772256, 'perf/effective_tokens_per_gpu_per_sec': 835.9677850822532, 'perf/longest_effective_sample_tokens_per_sec': 53.30648114772256} -(RolloutExecutor pid=4536) [2026-09-29 11:33:20.130 rollout_executor] train_data_conversion.py:334 - Rollout-side DP schedule: num_samples=32, num_rollouts=[32], num_microbatches=[32] -(InferenceController pid=4542) [2026-09-29 11:33:20.145 inference_controller] _client.py:1740 - HTTP Request: GET http://172.20.8.111:21100/flush_cache "HTTP/1.1 200 OK" -(CommandActor pid=4544) [2026-09-29 11:33:20] Cache flushed successfully! -(CommandActor pid=4544) [2026-09-29 11:33:20] Cache flushed successfully! -(InferenceController pid=4542) [2026-09-29 11:33:20.691 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/release_memory_occupation "HTTP/1.1 200 OK" -(TrainerController pid=4534) [2026-09-29 11:33:20.699 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=wake_up n_actors=1 -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:20.700 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:20.700 actor_cell00000_rank00000] timer.py:31 - Timer wake_up start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:20.701 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 74.61, 'used_GB': 4.57, 'allocated_GB': 1.38, 'reserved_GB': 1.48} -(TrainerController pid=4534) [2026-09-29 11:33:21.574 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=wake_up ok=true elapsed_s=0.9 -(TrainerController pid=4534) [2026-09-29 11:33:21.578 trainer_controller_actor] group.py:190 - ft op=train phase=start rollout=1 attempt=0 -(TrainerController pid=4534) [2026-09-29 11:33:21.578 trainer_controller_actor] group.py:534 - ft op=refresh phase=start rollout=1 alive=0 healing= all_states="(0, 'StateAllocatedAlive')" quorum=0 -(TrainerController pid=4534) [2026-09-29 11:33:21.578 trainer_controller_actor] group.py:557 - ft op=refresh phase=decision rollout=1 needs_reconfigure=false reason=alive_config_unchanged,no_healing quorum=0 -(TrainerController pid=4534) [2026-09-29 11:33:21.578 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=train n_actors=1 -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:21.572 actor_cell00000_rank00000] reloadable_process_group.py:151 - Reloading 0 process groups in pid 4943 -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:21.573 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 73.15, 'used_GB': 6.03, 'allocated_GB': 1.38, 'reserved_GB': 1.48} -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:21.573 actor_cell00000_rank00000] timer.py:42 - Timer wake_up end (elapsed: 0.9s) -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:21.573 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=0.9 -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:21.580 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:21.580 actor_cell00000_rank00000] timer.py:31 - Timer data_preprocess start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:21.585 actor_cell00000_rank00000] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:21.585 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:21.585 actor_cell00000_rank00000] timer.py:42 - Timer train_wait end (elapsed: 68.8s) -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:21.586 actor_cell00000_rank00000] timer.py:31 - Timer train start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:21.586 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:21.586 actor_cell00000_rank00000] timer.py:31 - Timer log_probs start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:26.632 actor_cell00000_rank00000] timer.py:42 - Timer log_probs end (elapsed: 5.0s) -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:26.633 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=5.0 -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:26.642 actor_cell00000_rank00000] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=0 -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:26.643 actor_cell00000_rank00000] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:26.643 actor_cell00000_rank00000] log_utils.py:126 - rollout 1: {'rollout/response_lengths': 188.1875, 'rollout/rewards': 1.862645149230957e-09, 'rollout/truncated': 0.03125, 'rollout/rollout_log_probs': -0.36941587924957275, 'rollout/raw_reward': 0.375, 'rollout/total_lengths': 272.6875, 'rollout/log_probs': -0.3697364926338196, 'rollout/advantages': 1.862645149230957e-08, 'rollout/returns': 1.862645149230957e-08} -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:26.644 actor_cell00000_rank00000] timer.py:31 - Timer actor_train start -(MegatronTrainRayActor pid=4943) /opt/sglang/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.reduce_scatter_tensor` is deprecated. Please use `torch.distributed.reduce_scatter_single` instead. -(MegatronTrainRayActor pid=4943) return func(*args, **kwargs) -(TrainerController pid=4534) [2026-09-29 11:33:36.533 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=train ok=true elapsed_s=15.0 -(TrainerController pid=4534) [2026-09-29 11:33:36.533 trainer_controller_actor] group.py:290 - ft op=check errored= discarded= normal=0 decision=no_retry reason="survivors normal, gradients valid" -(TrainerController pid=4534) [2026-09-29 11:33:36.536 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=4534) [2026-09-29 11:33:36.538 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(TrainerController pid=4534) [2026-09-29 11:33:36.542 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(MegatronTrainRayActor pid=4943) /opt/sglang/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead. -(MegatronTrainRayActor pid=4943) return func(*args, **kwargs) -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:36.527 actor_cell00000_rank00000] model.py:637 - train op=train_step rollout=1 step=0 attempt=0 outcome=NORMAL valid_step=true -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:36.528 actor_cell00000_rank00000] log_utils.py:554 - step 1: {'train/loss': -1.862645149230957e-08, 'train/pg_loss': -1.862645149230957e-08, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.01430277619510889, 'train/train_rollout_kl': 0.0006510338862426579, 'train/grad_norm': 0.4057203531265259, 'train/lr-pg_0': 5e-06, 'train/step': 1} -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:36.529 actor_cell00000_rank00000] model.py:847 - step 1: {'train/loss': -1.862645149230957e-08, 'train/pg_loss': -1.862645149230957e-08, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.01430277619510889, 'train/train_rollout_kl': 0.0006510338862426579, 'train/grad_norm': 0.4057203531265259, 'train/lr-pg_0': 5e-06, 'train/step': 1} -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:36.529 actor_cell00000_rank00000] timer.py:42 - Timer actor_train end (elapsed: 9.9s) -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:36.529 actor_cell00000_rank00000] timer.py:42 - Timer train end (elapsed: 14.9s) -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:36.529 actor_cell00000_rank00000] timer.py:31 - Timer train_wait start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:36.529 actor_cell00000_rank00000] train_metric_utils.py:56 - perf 1: {'perf/update_weights_implementation_time': 0.07333660125732422, 'perf/finalize_and_resume_engines_time': 0.07778143882751465, 'perf/update_weights_time': 0.19381403923034668, 'perf/sleep_time': 1.2257049083709717, 'perf/wake_up_time': 0.8726959228515625, 'perf/data_preprocess_time': 0.005101680755615234, 'perf/train_wait_time': 68.79093098640442, 'perf/log_probs_time': 5.046548128128052, 'perf/actor_train_time': 9.885117053985596, 'perf/train_time': 14.943270683288574, 'perf/log_probs_tflops': 1.8548072693578181, 'perf/actor_train_tflops': 2.840747591180615, 'perf/actor_train_tok_per_s': 882.7411908573961, 'perf/mfu_peak_tflops': 989.0, 'perf/actor_train_mfu': 0.0028723433682311577, 'perf/step_time': 83.734201669693, 'perf/wait_time_ratio': 0.8215392230974456} -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:36.529 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=14.9 -(MegatronTrainRayActor pid=4943) [2026-09-29 11:33:36.530 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=15.0 -(TrainerController pid=4534) [2026-09-29 11:33:36.828 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.3 -(TrainerController pid=4534) [2026-09-29 11:34:01.616 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=4534) [2026-09-29 11:34:24.793 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=23.2 -(TrainerController pid=4534) [2026-09-29 11:34:24.848 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=4534) [2026-09-29 11:34:25.144 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.3 -(TrainerController pid=4534) [2026-09-29 11:34:25.561 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=4534) [2026-09-29 11:34:25.849 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.3 -(InferenceController pid=4542) [2026-09-29 11:34:26.478 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/resume_memory_occupation "HTTP/1.1 200 OK" -(TrainerController pid=4534) [2026-09-29 11:34:26.484 trainer_controller_actor] group.py:392 - ft op=update_weights phase=start rollout= -(TrainerController pid=4534) [2026-09-29 11:34:26.485 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=update_weights n_actors=1 -(CommandActor pid=4544) [2026-09-29 11:34:26] Cache flush skipped: KV cache memory is released. -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.486 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.486 actor_cell00000_rank00000] timer.py:31 - Timer update_weights start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.489 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 69.39, 'used_GB': 9.79, 'allocated_GB': 1.38, 'reserved_GB': 4.06} -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.493 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/pause_generation "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.496 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: GET http://172.20.8.111:21100/flush_cache "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.500 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/begin_weight_update "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.500 actor_cell00000_rank00000] timer.py:31 - Timer update_weights_implementation start -(MegatronTrainRayActor pid=4943) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(CommandActor pid=4544) [2026-09-29 11:34:26] LoRA adapter loading from tensors starts: LoRARef(lora_id=5c0e30df52f143e2bfc9e74f3579825c, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(CommandActor pid=4544) [2026-09-29 11:34:26] LoRA adapter '5c0e30df52f143e2bfc9e74f3579825c': loaded weights for target modules ['down_proj', 'gate_up_proj', 'o_proj', 'qkv_proj']. -(CommandActor pid=4544) [2026-09-29 11:34:26] LoRA adapter loading from tensors completes: LoRARef(lora_id=5c0e30df52f143e2bfc9e74f3579825c, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(CommandActor pid=4544) [2026-09-29 11:34:26] Weight version changed. old_version='yeto:1:f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a' new_version='3' num_recorded=0 -(CommandActor pid=4544) [2026-09-29 11:34:26] [continue_generation] torch.cuda.empty_cache() called: reserved 32306.0 MB -> 32266.0 MB (freed 40.0 MB) -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.578 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/update_weights_from_tensor "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.579 actor_cell00000_rank00000] timer.py:42 - Timer update_weights_implementation end (elapsed: 0.1s) -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.579 actor_cell00000_rank00000] timer.py:31 - Timer finalize_and_resume_engines start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.634 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/end_weight_update "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.637 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/update_weight_version "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.640 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/continue_generation "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.640 actor_cell00000_rank00000] timer.py:42 - Timer finalize_and_resume_engines end (elapsed: 0.1s) -(MegatronTrainRayActor pid=4943) [miles-colocate] Update weights: 1it [00:00, 5.65it/s] -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.679 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.96, 'used_GB': 7.22, 'allocated_GB': 1.38, 'reserved_GB': 1.51} -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.680 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 0.2s) -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.680 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.2 -(TrainerController pid=4534) [2026-09-29 11:34:26.681 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=update_weights ok=true elapsed_s=0.2 -(InferenceController pid=4542) [2026-09-29 11:34:26.741 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/resume_memory_occupation "HTTP/1.1 200 OK" -(InferenceController pid=4542) [2026-09-29 11:34:26.783 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/weights_checker "HTTP/1.1 200 OK" -(CommandActor pid=4544) [2026-09-29 11:34:26] Weight version changed. old_version='3' new_version='yeto:2:a7f6ab7b8c7c26eef71d9ff5e3a821ac9050b5250fcef17d082eedcff387af3b' num_recorded=0 -(CommandActor pid=4544) [2026-09-29 11:34:26] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(CommandActor pid=4544) [2026-09-29 11:34:26] [WeightChecker] checksum computed for 226 tensors in 0.021s -(TrainerController pid=4534) [2026-09-29 11:34:26.791 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=sleep n_actors=1 -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.792 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:26.792 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:27.544 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 41.93, 'used_GB': 37.25, 'allocated_GB': 1.38, 'reserved_GB': 1.48} -(RolloutExecutor pid=4536) Rollout generation: 0%| | 0/32 [00:00user\nAlexis is applying for a new job and bought a new set of business clothes to wear to the interview. She went to a department store with a budget of $200 and spent $30 on a button-up shirt, $46 on suit pants, $38 on a suit coat, $11 on socks, and $18 on a belt. She also purchased a pair of shoes, but lost the receipt for them. She has $16 left from her budget. How much did Alexis pay for the shoes?\nPut the final numeric answer in \boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n, label=41, reward_summary=0.0 -(RolloutExecutor pid=4536) Rollout generation: 25%|██▌ | 8/32 [00:06<00:18, 1.31it/s] -(CommandActor pid=4544) [2026-09-29 11:34:34] Decode batch, #running-req: 22, #token: 5601, token usage: 0.02, cuda graph: True, gen throughput (token/s): 4584.22, #queue-req: 0 - -(CommandActor pid=4544) [2026-09-29 11:33:11] INFO: 172.20.8.111:56654 - "POST /update_weight_version HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:11] INFO: 172.20.8.111:56654 - "POST /continue_generation HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:11] INFO: 172.20.8.111:58954 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:11] INFO: 172.20.8.111:42426 - "POST /update_weight_version HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:11] INFO: 172.20.8.111:42426 - "GET /model_info HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:11] INFO: 172.20.8.111:58954 - "POST /weights_checker HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:18] INFO: 172.20.8.111:16553 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:18] INFO: 172.20.8.111:27510 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:18] INFO: 172.20.8.111:59973 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:18] INFO: 172.20.8.111:44833 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:18] INFO: 172.20.8.111:20405 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:18] INFO: 172.20.8.111:44743 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:18] INFO: 172.20.8.111:43644 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:18] INFO: 172.20.8.111:53620 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:18] INFO: 172.20.8.111:65413 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:18] INFO: 172.20.8.111:63761 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:18] INFO: 172.20.8.111:44548 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:18] INFO: 172.20.8.111:44320 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:49492 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:32281 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:42726 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:63837 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:46239 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:60432 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:45386 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:42165 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:18423 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:55006 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:31823 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:25340 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:37739 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:42784 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:55239 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:63587 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:42956 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:55445 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:19] INFO: 172.20.8.111:60919 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:20] INFO: 172.20.8.111:52867 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:20] INFO: 172.20.8.111:16800 - "POST /abort_request HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:20] INFO: 172.20.8.111:48497 - "GET /flush_cache HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:20] INFO: 172.20.8.111:48497 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:33:33] INFO: 172.20.8.111:27860 - "GET /health HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:26] INFO: 172.20.8.111:18663 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:26] INFO: 172.20.8.111:22123 - "POST /pause_generation HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:26] INFO: 172.20.8.111:22123 - "GET /flush_cache HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:26] INFO: 172.20.8.111:22123 - "POST /begin_weight_update HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:26] INFO: 172.20.8.111:22123 - "POST /update_weights_from_tensor HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:26] INFO: 172.20.8.111:22123 - "POST /end_weight_update HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:26] INFO: 172.20.8.111:22123 - "POST /update_weight_version HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:26] INFO: 172.20.8.111:22123 - "POST /continue_generation HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:26] INFO: 172.20.8.111:18663 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:26] INFO: 172.20.8.111:53641 - "POST /update_weight_version HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:26] INFO: 172.20.8.111:53641 - "GET /model_info HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:26] INFO: 172.20.8.111:18663 - "POST /weights_checker HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:33] INFO: 172.20.8.111:27068 - "GET /health HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:34] INFO: 172.20.8.111:45149 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:34] INFO: 172.20.8.111:62227 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:34] INFO: 172.20.8.111:29766 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:34] INFO: 172.20.8.111:33761 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:34] INFO: 172.20.8.111:22893 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:34] INFO: 172.20.8.111:40195 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:34] INFO: 172.20.8.111:32462 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:34] INFO: 172.20.8.111:50221 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:34] INFO: 172.20.8.111:44169 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:34] INFO: 172.20.8.111:24712 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:34] INFO: 172.20.8.111:44078 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:34] INFO: 172.20.8.111:64278 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=4544) [2026-09-29 11:34:34] INFO: 172.20.8.111:35751 - "POST /generate HTTP/1.1" 200 OK(CommandActor pid=4544) [2026-09-29 11:34:34] Decode batch, #running-req: 21, #token: 5919, token usage: 0.02, cuda graph: True, gen throughput (token/s): 4121.45, #queue-req: 0 -(RolloutExecutor pid=4536) Rollout generation: 50%|█████ | 16/32 [00:06<00:05, 2.83it/s] -(CommandActor pid=4544) [2026-09-29 11:34:34] Decode batch, #running-req: 16, #token: 4708, token usage: 0.02, cuda graph: True, gen throughput (token/s): 3366.66, #queue-req: 0 -(CommandActor pid=4544) [2026-09-29 11:34:35] Decode batch, #running-req: 13, #token: 4907, token usage: 0.02, cuda graph: True, gen throughput (token/s): 2606.40, #queue-req: 0 -(RolloutExecutor pid=4536) Rollout generation: 75%|███████▌ | 24/32 [00:06<00:01, 4.74it/s] Rollout generation: 100%|██████████| 32/32 [00:06<00:00, 4.59it/s] -(RolloutExecutor pid=4536) [2026-09-29 11:34:35.231 rollout_executor] inference_rollout_train.py:161 - Finish rollout: text_preview=<|im_start|>user\nA deep-sea monster rises from the waters once every hundred years to feast on a ship and sate its hunger. Over three hundred years, it has consumed 847 people. Ships have been built larger over time, so each new ship has twice as many people as the last ship. How many people were on the ship the monster ate in the first hundred years?\nPut the final numeric answer in \boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nWe are told:\n\n- The monster **consumes 847 people in 30..., label=121, reward_summary=0.0 -(RolloutExecutor pid=4536) [2026-09-29 11:34:35.233 rollout_executor] inference_rollout_train.py:33 - Abort request for ['http://172.20.8.111:21100'] -(RolloutExecutor pid=4536) [2026-09-29 11:34:35.240 rollout_executor] rollout_data_conversion.py:39 - Final collected 32 samples from rollout to train -(RolloutExecutor pid=4536) [2026-09-29 11:34:35.241 rollout_executor] metrics.py:93 - perf 2: {'rollout/raw_reward_unfiltered': 0.09375, 'rollout/num_training_samples': 32, 'rollout/episode_raw_reward': 0.09375, 'rollout/episode_response_length/mean': 300.21875, 'rollout/episode_response_length/median': 316.0, 'rollout/episode_response_length/max': 384, 'rollout/episode_response_length/min': 177, 'rollout/episode_total_response_length/mean': 300.21875, 'rollout/response_len/mean': 300.21875, 'rollout/response_len/median': 316.0, 'rollout/response_len/max': 384, 'rollout/response_len/min': 177, 'rollout/zero_std/count_0.0': 2, 'rollout/zero_std/all_zero_percentage': 0.5, 'rollout/zero_std/all_one_percentage': 0.0, 'rollout/prefix_cache_hit_rate': 0.0, 'rollout/avg_cached_tokens_per_sample': 0.0, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.34375, 'perf/rollout_time': 6.977983236312866, 'perf/tokens_per_gpu_per_sec': 1376.7588248143018, 'perf/longest_sample_tokens_per_sec': 55.03022678554095, 'perf/effective_tokens_per_gpu_per_sec': 1376.7588248143018, 'perf/longest_effective_sample_tokens_per_sec': 55.03022678554095} -(RolloutExecutor pid=4536) [2026-09-29 11:34:35.243 rollout_executor] train_data_conversion.py:334 - Rollout-side DP schedule: num_samples=32, num_rollouts=[32], num_microbatches=[32] -(InferenceController pid=4542) [2026-09-29 11:34:35.255 inference_controller] _client.py:1740 - HTTP Request: GET http://172.20.8.111:21100/flush_cache "HTTP/1.1 200 OK" -(CommandActor pid=4544) [2026-09-29 11:34:35] Cache flushed successfully! -(CommandActor pid=4544) [2026-09-29 11:34:35] Cache flushed successfully! -(InferenceController pid=4542) [2026-09-29 11:34:35.616 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/release_memory_occupation "HTTP/1.1 200 OK" -(TrainerController pid=4534) [2026-09-29 11:34:35.623 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=wake_up n_actors=1 -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:35.624 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:35.624 actor_cell00000_rank00000] timer.py:31 - Timer wake_up start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:35.625 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 74.61, 'used_GB': 4.57, 'allocated_GB': 1.38, 'reserved_GB': 1.48} -(TrainerController pid=4534) [2026-09-29 11:34:36.757 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=wake_up ok=true elapsed_s=1.1 -(TrainerController pid=4534) [2026-09-29 11:34:36.761 trainer_controller_actor] group.py:190 - ft op=train phase=start rollout=2 attempt=0 -(TrainerController pid=4534) [2026-09-29 11:34:36.761 trainer_controller_actor] group.py:534 - ft op=refresh phase=start rollout=2 alive=0 healing= all_states="(0, 'StateAllocatedAlive')" quorum=0 -(TrainerController pid=4534) [2026-09-29 11:34:36.761 trainer_controller_actor] group.py:557 - ft op=refresh phase=decision rollout=2 needs_reconfigure=false reason=alive_config_unchanged,no_healing quorum=0 -(TrainerController pid=4534) [2026-09-29 11:34:36.762 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=train n_actors=1 -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:36.754 actor_cell00000_rank00000] reloadable_process_group.py:151 - Reloading 0 process groups in pid 4943 -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:36.755 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 73.15, 'used_GB': 6.03, 'allocated_GB': 1.38, 'reserved_GB': 1.48} -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:36.755 actor_cell00000_rank00000] timer.py:42 - Timer wake_up end (elapsed: 1.1s) -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:36.756 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.1 -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:36.763 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:36.764 actor_cell00000_rank00000] timer.py:31 - Timer data_preprocess start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:36.769 actor_cell00000_rank00000] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:36.770 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:36.770 actor_cell00000_rank00000] timer.py:42 - Timer train_wait end (elapsed: 60.2s) -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:36.770 actor_cell00000_rank00000] timer.py:31 - Timer train start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:36.770 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(MegatronTrainRayActor pid=4943) [2026-09-29 11:34:36.770 actor_cell00000_rank00000] timer.py:31 - Timer log_probs start diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/miles-restart.log b/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/miles-restart.log deleted file mode 100644 index 61c3be71f..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/miles-restart.log +++ /dev/null @@ -1,401 +0,0 @@ -/opt/sglang/lib/python3.12/site-packages/transformer_engine/__init__.py:103: RuntimeWarning: Detected a Jax installation but could not find the shared object file for the Transformer Engine Jax extension library. If this is not intentional, please reinstall Transformer Engine with `pip install transformer_engine[jax]` or build from source with `NVTE_FRAMEWORK=jax`. - warnings.warn( -Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -WARNING:megatron.bridge.models.conversion.auto_bridge:Loading a model with trust_remote_code=True allows arbitrary code execution from the model repository. Only use this with models you trust. -/opt/sglang/lib/python3.12/site-packages/megatron/bridge/models/qwen/qwen3_bridge.py:45: FutureWarning: Qwen3Bridge.provider_bridge() and AutoBridge.to_megatron_provider() are deprecated for builder-backed models. Use AutoBridge.get_model_config() for training recipes or AutoBridge.get_model() to construct a model. Legacy provider support will be removed in a future release. - provider = super().provider_bridge(hf_pretrained) -[transformers] `torch_dtype` is deprecated! Use `dtype` instead! -/root/Megatron-LM/megatron/core/transformer/transformer_config.py:3249: UserWarning: full scope is deprecated. Use empty cuda_graph_modules to capture the whole layer. - warnings.warn( -[2026-09-29 11:35:27.366 main] arguments.py:49 - --tokenizer-model not set, use --hf-checkpoint as tokenizer model. -[2026-09-29 11:35:27.410 main] arguments.py:152 - MoE-expert LoRA layout: per-expert (--experts-shared-outer-loras). -[2026-09-29 11:35:27.410 main] arguments.py:3616 - Colocate mode: defaulting --sglang-cuda-graph-backend-prefill=disabled to avoid NVLS OOM. Set --sglang-cuda-graph-backend-prefill explicitly to override. -[2026-09-29 11:35:27.411 main] arguments.py:3626 - rollout_num_gpus None != actor_num_gpus_per_node 1 * actor_num_nodes 1, overriding rollout_num_gpus to match actor_num_gpus_per_node * actor_num_nodes. -[2026-09-29 11:35:27.411 main] arguments.py:3858 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 1024 as default value. -[2026-09-29 11:35:27.411 main] arguments.py:3866 - args.rollout_max_prompt_len is not set. Use args.rollout_max_context_len - 1 (1024 - 1) as default value so that there is at least one generated token to compute loss. -/root/Megatron-LM/megatron/core/_rank_utils.py:39: UserWarning: Could not determine rank from torch.distributed, RANK, or SLURM_PROCID. Defaulting to rank 0. - warnings.warn( -WARNING: Inferring the appropriate argparse argument type from failed for pad_packed_seq_alignment: typing.Union[int, typing.Literal['max'], NoneType]. -Deferring to attribute metadata. If the metadata is incomplete, 'parser.add_argument()' may fail. -Original failure: Unions not supported by argparse: typing.Union[int, typing.Literal['max'], NoneType] -using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 -Number of virtual stages per pipeline stage: None -using torch.bfloat16 for parameters ... ------------------------- arguments ------------------------ - _is_global_batch_size_explicitly_specified ...... True - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - activation_func_clamp_shared_expert ............. True - activation_func_clamp_value ..................... None - activation_offload_fraction ..................... 1.0 - actor_num_gpus_per_node ......................... 1 - actor_num_nodes ................................. 1 - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.999 - adam_eps ........................................ 1e-08 - add_bias_linear ................................. False - add_position_embedding .......................... True - add_qkv_bias .................................... False - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - advantage_estimator ............................. grpo - align_grad_reduce ............................... True - align_param_gather .............................. False - allgather_cp .................................... False - allow_ambiguous_pad_tokens ...................... False - allow_partial_train_step ........................ False - api_server_host ................................. 127.0.0.1 - api_server_port ................................. 0 - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_chat_template ............................. True - apply_chat_template_kwargs ...................... {'enable_thinking': False} - apply_dsa_kernel_fusion ......................... None - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - apply_wd_to_qk_layernorm ........................ False - async_ckpt_cpu_priority ......................... 10 - async_ckpt_io_priority .......................... 3 - async_ckpt_use_cpu_shm .......................... False - async_data_buffer_capacity_factor ............... 2.0 - async_max_concurrent_samples .................... None - async_save ...................................... False - async_strategy .................................. mcore - async_unused_samples_handler .................... drop - attention_backend ............................... AttnBackend.unfused - attention_dropout ............................... 0.1 - attention_output_gate ........................... False - attention_softmax_in_fp32 ....................... True - auto_detect_ckpt_format ......................... False - balance_by_flops ................................ False - balance_data .................................... True - barrier_with_L1_time ............................ True - batch_invariant_mode ............................ False - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - beta_fast ....................................... 32 - beta_slow ....................................... 1 - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - buffer_filter_path .............................. yeto.rl.engine.miles_adapter.rollout_meta_hook.policy_buffer_filter - cache_mla_latents ............................... False - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - chat_template ................................... None - chat_template_path .............................. None - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_lora_weight_equal ......................... False - check_rematerialize_param_from_master_weight .... False - check_weight_hash_across_dp_replicas_interval ... None - check_weight_update_allow_quant_error ........... False - check_weight_update_equal ....................... False - check_weight_update_selector .................... all - check_weight_update_skip_list ................... None - chunked_optimizer_state_offload ................. False - ci_check_model_hash ............................. False - ci_disable_kl_checker ........................... False - ci_disable_logprobs_checker ..................... False - ci_disable_weight_update_checker ................ False - ci_enable_metrics_capture ....................... False - ci_ft_test_actions .............................. None - ci_ft_test_actions_path ......................... None - ci_inject_rollout_data_min_match_ratio .......... 0.9 - ci_inject_rollout_data_path ..................... None - ci_inject_rollout_data_start_rollout_id ......... None - ci_load_grad_norm ............................... None - ci_metric_checker_expect_num .................... None - ci_metric_checker_key ........................... None - ci_metric_checker_threshold ..................... None - ci_save_grad_norm ............................... None - ci_save_model_hash .............................. False - ci_test ......................................... False - ci_tito_special_token_count_threshold ........... 0.0 - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch_dist - ckpt_fully_parallel_load ........................ False - ckpt_fully_parallel_load_exchange_algo .......... broadcast - ckpt_fully_parallel_load_process_group .......... dp - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_fully_parallel_save_process_group .......... dp - ckpt_load_validate_sharding_integrity ........... True - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clear_quantized_weight_workspaces_on_offload .... True - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - cluster_backend ................................. ray - colocate ........................................ True - colocate_memory_peak_device ..................... cpu - compress_ratios ................................. None - compute_advantages_and_returns .................. True - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 -... [truncated: 2231 of 2631 lines removed; full evidence archived outside repo] ... -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:36633 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:47730 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:16987 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:37370 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:57139 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:52466 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:56202 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:55293 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:42592 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:21319 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:52114 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:38427 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:48810 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:48491 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:55562 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:57256 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:26] INFO: 172.20.8.111:18120 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:27] INFO: 172.20.8.111:21962 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:27] INFO: 172.20.8.111:38234 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:27] INFO: 172.20.8.111:43138 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:27] INFO: 172.20.8.111:31293 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:27] INFO: 172.20.8.111:51331 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:27] INFO: 172.20.8.111:50661 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:27] INFO: 172.20.8.111:48892 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:27] INFO: 172.20.8.111:17877 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:27] INFO: 172.20.8.111:29532 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:27] INFO: 172.20.8.111:62505 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:27] INFO: 172.20.8.111:36635 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:27] INFO: 172.20.8.111:41532 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:28] INFO: 172.20.8.111:51507 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:28] INFO: 172.20.8.111:18967 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:28] INFO: 172.20.8.111:29790 - "POST /abort_request HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:28] INFO: 172.20.8.111:61037 - "GET /flush_cache HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:39:28] INFO: 172.20.8.111:61037 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:05] INFO: 172.20.8.111:40448 - "GET /health HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:46] INFO: 172.20.8.111:37342 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:46] INFO: 172.20.8.111:37074 - "POST /pause_generation HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:46] INFO: 172.20.8.111:37074 - "GET /flush_cache HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:46] INFO: 172.20.8.111:37074 - "POST /begin_weight_update HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:46] INFO: 172.20.8.111:37074 - "POST /update_weights_from_tensor HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:46] INFO: 172.20.8.111:37074 - "POST /end_weight_update HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:46] INFO: 172.20.8.111:37074 - "POST /update_weight_version HTTP/1.1" 200 OK(CommandActor pid=9893) [2026-09-29 11:40:46] LoRA adapter '9540a0ee7ae1473385c24d5be93cb221': loaded weights for target modules ['down_proj', 'gate_up_proj', 'o_proj', 'qkv_proj']. -(CommandActor pid=9893) [2026-09-29 11:40:46] LoRA adapter loading from tensors completes: LoRARef(lora_id=9540a0ee7ae1473385c24d5be93cb221, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(CommandActor pid=9893) [2026-09-29 11:40:46] Weight version changed. old_version='yeto:2:a7f6ab7b8c7c26eef71d9ff5e3a821ac9050b5250fcef17d082eedcff387af3b' new_version='2' num_recorded=0 -(CommandActor pid=9893) [2026-09-29 11:40:46] [continue_generation] torch.cuda.empty_cache() called: reserved 32306.0 MB -> 32266.0 MB (freed 40.0 MB) -(CommandActor pid=9893) [2026-09-29 11:40:46] Weight version changed. old_version='2' new_version='yeto:3:65107a5aecf20f43c1055315bc037632b55026d42f255aade8b5993cd7af0030' num_recorded=0 -(CommandActor pid=9893) [2026-09-29 11:40:46] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(TrainerController pid=9896) [2026-09-29 11:40:46.485 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=update_weights ok=true elapsed_s=0.2 -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:46.460 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/end_weight_update "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:46.463 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/update_weight_version "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:46.466 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/continue_generation "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:46.466 actor_cell00000_rank00000] timer.py:42 - Timer finalize_and_resume_engines end (elapsed: 0.1s) -(MegatronTrainRayActor pid=10245) [miles-colocate] Update weights: 1it [00:00, 5.69it/s] -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:46.483 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.96, 'used_GB': 7.22, 'allocated_GB': 1.38, 'reserved_GB': 1.51} -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:46.484 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 0.2s) -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:46.484 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.2 -(InferenceController pid=9891) [2026-09-29 11:40:46.558 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/weights_checker "HTTP/1.1 200 OK" -(CommandActor pid=9893) [2026-09-29 11:40:46] [WeightChecker] checksum computed for 226 tensors in 0.021s -(TrainerController pid=9896) [2026-09-29 11:40:46.565 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=sleep n_actors=1 -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:46.566 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:46.566 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:47.154 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 41.93, 'used_GB': 37.25, 'allocated_GB': 1.38, 'reserved_GB': 1.49} -(RolloutExecutor pid=9889) Rollout generation: 0%| | 0/32 [00:00user\nAlbert is wondering how much pizza he can eat in one day. He buys 2 large pizzas and 2 small pizzas. A large pizza has 16 slices and a small pizza has 8 slices. If he eats it all, how many pieces does he eat that day?\nPut the final numeric answer in \boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nAlbert buys:\n\n- 2 large pizzas × 16 slices = $2 \times 16 = 32$ slices \n- 2 small pizzas × 8 slices = $2 \times 8 = 16$ slices\n\nTotal slices eaten **in one day** = 32 + 1..., label=48, reward_summary=1.0 -(RolloutExecutor pid=9889) Rollout generation: 25%|██▌ | 8/32 [00:04<00:14, 1.61it/s] -(CommandActor pid=9893) [2026-09-29 11:40:52] Decode batch, #running-req: 24, #token: 3112, token usage: 0.01, cuda graph: True, gen throughput (token/s): 5272.11, #queue-req: 0 -(CommandActor pid=9893) [2026-09-29 11:40:52] Decode batch, #running-req: 15, #token: 2669, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3771.73, #queue-req: 0 -(RolloutExecutor pid=9889) Rollout generation: 50%|█████ | 16/32 [00:05<00:04, 3.60it/s] -(CommandActor pid=9893) [2026-09-29 11:40:52] Decode batch, #running-req: 11, #token: 2421, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2929.32, #queue-req: 0 -(CommandActor pid=9893) [2026-09-29 11:40:53] Decode batch, #running-req: 9, #token: 2379, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2026.85, #queue-req: 0 -(CommandActor pid=9893) [2026-09-29 11:40:53] Decode batch, #running-req: 7, #token: 1896, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1553.33, #queue-req: 0 -(CommandActor pid=9893) [2026-09-29 11:40:53] Decode batch, #running-req: 4, #token: 1494, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1009.63, #queue-req: 0 -(CommandActor pid=9893) [2026-09-29 11:40:53] Decode batch, #running-req: 3, #token: 1293, token usage: 0.00, cuda graph: True, gen throughput (token/s): 777.15, #queue-req: 0 -(RolloutExecutor pid=9889) Rollout generation: 75%|███████▌ | 24/32 [00:06<00:01, 4.92it/s] Rollout generation: 100%|██████████| 32/32 [00:06<00:00, 5.15it/s] -(RolloutExecutor pid=9889) [2026-09-29 11:40:53.774 rollout_executor] inference_rollout_train.py:161 - Finish rollout: text_preview=<|im_start|>user\nMark has a garden with flowers. He planted plants of three different colors in it. Ten of them are yellow, and there are 80% more of those in purple. There are only 25% as many green flowers as there are yellow and purple flowers. How many flowers does Mark have in his garden?\nPut the final numeric answer in \boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nLet's solve the problem step by step.\n\n---\n\n### Step 1: Represent the flowers in terms of variables \n- Let the..., label=35, reward_summary=0.0 -(RolloutExecutor pid=9889) [2026-09-29 11:40:53.777 rollout_executor] inference_rollout_train.py:33 - Abort request for ['http://172.20.8.111:21100'] -(RolloutExecutor pid=9889) [2026-09-29 11:40:53.785 rollout_executor] rollout_data_conversion.py:39 - Final collected 32 samples from rollout to train -(RolloutExecutor pid=9889) [2026-09-29 11:40:53.786 rollout_executor] metrics.py:93 - perf 3: {'rollout/raw_reward_unfiltered': 0.375, 'rollout/num_training_samples': 32, 'rollout/episode_raw_reward': 0.375, 'rollout/episode_response_length/mean': 189.5625, 'rollout/episode_response_length/median': 154.0, 'rollout/episode_response_length/max': 384, 'rollout/episode_response_length/min': 71, 'rollout/episode_total_response_length/mean': 189.5625, 'rollout/response_len/mean': 189.5625, 'rollout/response_len/median': 154.0, 'rollout/response_len/max': 384, 'rollout/response_len/min': 71, 'rollout/zero_std/count_0.0': 1, 'rollout/zero_std/count_1.0': 1, 'rollout/zero_std/all_zero_percentage': 0.25, 'rollout/zero_std/all_one_percentage': 0.25, 'rollout/prefix_cache_hit_rate': 0.0, 'rollout/avg_cached_tokens_per_sample': 0.0, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.0625, 'perf/rollout_time': 6.221613883972168, 'perf/tokens_per_gpu_per_sec': 974.9881804184196, 'perf/longest_sample_tokens_per_sec': 61.720320026487485, 'perf/effective_tokens_per_gpu_per_sec': 974.9881804184196, 'perf/longest_effective_sample_tokens_per_sec': 61.720320026487485} -(RolloutExecutor pid=9889) [2026-09-29 11:40:53.788 rollout_executor] train_data_conversion.py:334 - Rollout-side DP schedule: num_samples=32, num_rollouts=[32], num_microbatches=[32] -(InferenceController pid=9891) [2026-09-29 11:40:53.825 inference_controller] _client.py:1740 - HTTP Request: GET http://172.20.8.111:21100/flush_cache "HTTP/1.1 200 OK" -(CommandActor pid=9893) [2026-09-29 11:40:53] Cache flushed successfully! -(CommandActor pid=9893) [2026-09-29 11:40:53] Cache flushed successfully! -(InferenceController pid=9891) [2026-09-29 11:40:54.451 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/release_memory_occupation "HTTP/1.1 200 OK" -(TrainerController pid=9896) [2026-09-29 11:40:54.458 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=wake_up n_actors=1 -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:54.459 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:54.459 actor_cell00000_rank00000] timer.py:31 - Timer wake_up start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:54.460 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 74.61, 'used_GB': 4.57, 'allocated_GB': 1.38, 'reserved_GB': 1.49} -(TrainerController pid=9896) [2026-09-29 11:40:55.195 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=wake_up ok=true elapsed_s=0.7 -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:55.192 actor_cell00000_rank00000] reloadable_process_group.py:151 - Reloading 0 process groups in pid 10245 -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:55.193 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 73.14, 'used_GB': 6.04, 'allocated_GB': 1.38, 'reserved_GB': 1.49} -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:55.193 actor_cell00000_rank00000] timer.py:42 - Timer wake_up end (elapsed: 0.7s) -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:55.193 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=0.7 -(TrainerController pid=9896) [2026-09-29 11:40:55.198 trainer_controller_actor] group.py:190 - ft op=train phase=start rollout=3 attempt=0 -(TrainerController pid=9896) [2026-09-29 11:40:55.199 trainer_controller_actor] group.py:534 - ft op=refresh phase=start rollout=3 alive=0 healing= all_states="(0, 'StateAllocatedAlive')" quorum=0 -(TrainerController pid=9896) [2026-09-29 11:40:55.199 trainer_controller_actor] group.py:557 - ft op=refresh phase=decision rollout=3 needs_reconfigure=false reason=alive_config_unchanged,no_healing quorum=0 -(TrainerController pid=9896) [2026-09-29 11:40:55.199 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=train n_actors=1 -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:55.200 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:55.200 actor_cell00000_rank00000] timer.py:31 - Timer data_preprocess start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:55.205 actor_cell00000_rank00000] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:55.205 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:55.205 actor_cell00000_rank00000] timer.py:42 - Timer train_wait end (elapsed: 64.2s) -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:55.206 actor_cell00000_rank00000] timer.py:31 - Timer train start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:55.206 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:55.206 actor_cell00000_rank00000] timer.py:31 - Timer log_probs start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:59.834 actor_cell00000_rank00000] timer.py:42 - Timer log_probs end (elapsed: 4.6s) -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:59.834 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.6 -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:59.840 actor_cell00000_rank00000] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=0 -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:59.841 actor_cell00000_rank00000] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:59.841 actor_cell00000_rank00000] log_utils.py:126 - rollout 3: {'rollout/response_lengths': 189.5625, 'rollout/rewards': 0.0, 'rollout/truncated': 0.0625, 'rollout/rollout_log_probs': -0.4027121961116791, 'rollout/raw_reward': 0.375, 'rollout/total_lengths': 274.0625, 'rollout/log_probs': -0.4031979739665985, 'rollout/advantages': 0.0, 'rollout/returns': 0.0} -(MegatronTrainRayActor pid=10245) [2026-09-29 11:40:59.841 actor_cell00000_rank00000] timer.py:31 - Timer actor_train start -(RayWorkerManager pid=9621) [2026-09-29 11:41:04.063 worker_manager] reporter.py:90 - audit op=env_report_summary hostname=modal versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-4.19.0-gvisor-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.0.0.dev18354+g9f29303be\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"miles\":\"0394715083c91182b5eb0c526eeee4196ac694b9\",\"sglang\":\"9f29303bef1eea38eb613e5f454a52db1326422d\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=98 packages_probed=true stored=false -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:08.865 actor_cell00000_rank00000] model.py:637 - train op=train_step rollout=3 step=0 attempt=0 outcome=NORMAL valid_step=true -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:08.866 actor_cell00000_rank00000] log_utils.py:554 - step 3: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.01489894837141037, 'train/train_rollout_kl': 0.0006911580567248166, 'train/grad_norm': 0.31687822937965393, 'train/lr-pg_0': 0.0, 'train/step': 3} -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:08.866 actor_cell00000_rank00000] model.py:847 - step 3: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.01489894837141037, 'train/train_rollout_kl': 0.0006911580567248166, 'train/grad_norm': 0.31687822937965393, 'train/lr-pg_0': 0.0, 'train/step': 3} -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:08.866 actor_cell00000_rank00000] timer.py:42 - Timer actor_train end (elapsed: 9.0s) -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:08.866 actor_cell00000_rank00000] timer.py:42 - Timer train end (elapsed: 13.7s) -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:08.866 actor_cell00000_rank00000] timer.py:31 - Timer train_wait start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:08.867 actor_cell00000_rank00000] train_metric_utils.py:56 - perf 3: {'perf/update_weights_implementation_time': 0.09208822250366211, 'perf/finalize_and_resume_engines_time': 0.06747865676879883, 'perf/update_weights_time': 0.1876850128173828, 'perf/sleep_time': 0.9899134635925293, 'perf/wake_up_time': 0.7334136962890625, 'perf/data_preprocess_time': 0.0049784183502197266, 'perf/train_wait_time': 64.1993670463562, 'perf/log_probs_time': 4.628071546554565, 'perf/actor_train_time': 9.025370836257935, 'perf/train_time': 13.660853624343872, 'perf/log_probs_tflops': 2.0354881761439363, 'perf/actor_train_tflops': 3.1313012226096553, 'perf/actor_train_tok_per_s': 971.7052250936854, 'perf/mfu_peak_tflops': 989.0, 'perf/actor_train_mfu': 0.0031661286376235137, 'perf/step_time': 77.86022067070007, 'perf/wait_time_ratio': 0.8245464306847945} -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:08.867 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=13.7 -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:08.867 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=13.7 -(TrainerController pid=9896) [2026-09-29 11:41:08.869 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=train ok=true elapsed_s=13.7 -(TrainerController pid=9896) [2026-09-29 11:41:08.869 trainer_controller_actor] group.py:290 - ft op=check errored= discarded= normal=0 decision=no_retry reason="survivors normal, gradients valid" -(TrainerController pid=9896) [2026-09-29 11:41:08.873 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=9896) [2026-09-29 11:41:08.875 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(TrainerController pid=9896) [2026-09-29 11:41:08.877 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=9896) [2026-09-29 11:41:09.143 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.3 -(TrainerController pid=9896) [2026-09-29 11:41:31.964 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(MegatronTrainRayActor pid=10245) /opt/sglang/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. -(MegatronTrainRayActor pid=10245) return func(*args, **kwargs) -(TrainerController pid=9896) [2026-09-29 11:41:54.299 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=22.3 -(TrainerController pid=9896) [2026-09-29 11:41:54.369 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=9896) [2026-09-29 11:41:54.682 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.3 -(TrainerController pid=9896) [2026-09-29 11:41:55.141 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(RolloutExecutor pid=9889) [2026-09-29 11:41:55.274 rollout_executor] reporter.py:90 - audit op=env_report_summary hostname=modal versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-4.19.0-gvisor-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.0.0.dev18354+g9f29303be\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"miles\":\"0394715083c91182b5eb0c526eeee4196ac694b9\",\"sglang\":\"9f29303bef1eea38eb613e5f454a52db1326422d\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=98 packages_probed=true stored=false -(TrainerController pid=9896) [2026-09-29 11:41:55.433 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.3 -(InferenceController pid=9891) [2026-09-29 11:41:55.904 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/resume_memory_occupation "HTTP/1.1 200 OK" -(CommandActor pid=9893) [2026-09-29 11:41:55] Cache flush skipped: KV cache memory is released. -(TrainerController pid=9896) [2026-09-29 11:41:55.911 trainer_controller_actor] group.py:392 - ft op=update_weights phase=start rollout= -(TrainerController pid=9896) [2026-09-29 11:41:55.911 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=update_weights n_actors=1 -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:55.913 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:55.913 actor_cell00000_rank00000] timer.py:31 - Timer update_weights start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:55.914 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 69.19, 'used_GB': 9.99, 'allocated_GB': 1.38, 'reserved_GB': 4.26} -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:55.918 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/pause_generation "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:55.921 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: GET http://172.20.8.111:21100/flush_cache "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:55.924 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/begin_weight_update "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:55.925 actor_cell00000_rank00000] timer.py:31 - Timer update_weights_implementation start -(MegatronTrainRayActor pid=10245) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(CommandActor pid=9893) [2026-09-29 11:41:55] LoRA adapter loading from tensors starts: LoRARef(lora_id=9540a0ee7ae1473385c24d5be93cb221, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(CommandActor pid=9893) [2026-09-29 11:41:56] LoRA adapter '9540a0ee7ae1473385c24d5be93cb221': loaded weights for target modules ['down_proj', 'gate_up_proj', 'o_proj', 'qkv_proj']. -(CommandActor pid=9893) [2026-09-29 11:41:56] LoRA adapter loading from tensors completes: LoRARef(lora_id=9540a0ee7ae1473385c24d5be93cb221, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(CommandActor pid=9893) [2026-09-29 11:41:56] Weight version changed. old_version='yeto:3:65107a5aecf20f43c1055315bc037632b55026d42f255aade8b5993cd7af0030' new_version='3' num_recorded=0 -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:55.992 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/update_weights_from_tensor "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:55.993 actor_cell00000_rank00000] timer.py:42 - Timer update_weights_implementation end (elapsed: 0.1s) -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:55.993 actor_cell00000_rank00000] timer.py:31 - Timer finalize_and_resume_engines start -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:56.041 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/end_weight_update "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:56.044 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/update_weight_version "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:56.047 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/continue_generation "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:56.048 actor_cell00000_rank00000] timer.py:42 - Timer finalize_and_resume_engines end (elapsed: 0.1s) -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:56.052 actor_cell00000_rank00000] reporter.py:90 - audit op=env_report_summary hostname=modal versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-4.19.0-gvisor-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.0.0.dev18354+g9f29303be\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"miles\":\"0394715083c91182b5eb0c526eeee4196ac694b9\",\"sglang\":\"9f29303bef1eea38eb613e5f454a52db1326422d\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=119 packages_probed=true stored=false -(CommandActor pid=9893) [2026-09-29 11:41:56] [continue_generation] torch.cuda.empty_cache() called: reserved 32306.0 MB -> 32266.0 MB (freed 40.0 MB) -(TrainerController pid=9896) [2026-09-29 11:41:56.149 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=update_weights ok=true elapsed_s=0.2 -(MegatronTrainRayActor pid=10245) [miles-colocate] Update weights: 1it [00:00, 4.55it/s] -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:56.146 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.95, 'used_GB': 7.22, 'allocated_GB': 1.38, 'reserved_GB': 1.51} -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:56.148 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 0.2s) -(MegatronTrainRayActor pid=10245) [2026-09-29 11:41:56.148 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.2 -(InferenceController pid=9891) [2026-09-29 11:41:56.255 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/resume_memory_occupation "HTTP/1.1 200 OK" -(CommandActor pid=9893) [2026-09-29 11:41:56] Weight version changed. old_version='3' new_version='yeto:4:dd7b6da08bac2f25de6eeba21ba64944858e597fe2b00417a583640e23afb109' num_recorded=0 -(CommandActor pid=9893) [2026-09-29 11:41:56] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(CommandActor pid=9893) [2026-09-29 11:41:56] [WeightChecker] checksum computed for 226 tensors in 0.019s -(InferenceController pid=9891) [2026-09-29 11:41:56.292 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.8.111:21100/weights_checker "HTTP/1.1 200 OK" -(RayWorkerManager pid=9621) [2026-09-29 11:41:56.439 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='rollout-executor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b4c71be59e0>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.rollout_executor.RolloutExecutor', ctor_kwargs=. at 0x2b4c71be6c00>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=ServeWorkerSpec(name='rollout-executor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b4c71be59e0>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.rollout_executor.RolloutExecutor', ctor_kwargs=. at 0x2b4c71be6c00>, concurrency_groups=None), actor_handle=Actor(RolloutExecutor, 89c831028934321559b2aa9003000000), self_addrs={'rpc': HostAndPort(host='172.20.8.111', port=21137, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(RayWorkerManager pid=9621) [2026-09-29 11:41:56.440 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='inference-controller', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b4c71be6d40>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.inference_controller.InferenceController', ctor_kwargs=. at 0x2b4ca3a0ed40>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=ServeWorkerSpec(name='inference-controller', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b4c71be6d40>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.inference_controller.InferenceController', ctor_kwargs=. at 0x2b4ca3a0ed40>, concurrency_groups=None), actor_handle=Actor(InferenceController, 71dc9c6519eee21ec8d5671903000000), self_addrs={'rpc': HostAndPort(host='172.20.8.111', port=21136, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(RayWorkerManager pid=9621) [2026-09-29 11:41:56.441 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-controller-actor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b4ca3a0f380>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=False), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.train.group.TrainerController', ctor_kwargs=. at 0x2b4ca3a0f4c0>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-controller-actor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b4ca3a0f380>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=False), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.train.group.TrainerController', ctor_kwargs=. at 0x2b4ca3a0f4c0>, concurrency_groups=None), actor_handle=Actor(TrainerController, 9bde7ecb4cde2716886cc35503000000), self_addrs={'rpc': HostAndPort(host='172.20.8.111', port=21140, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(RayWorkerManager pid=9621) [2026-09-29 11:41:56.442 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b4ca3a0f560>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=1, pg_name='actor', pg_slot_offset=0, pin_to_head=False), meta=. at 0x2b4ca3a40a40>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x2b4ca3a41620>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b4ca3a0f560>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=1, pg_name='actor', pg_slot_offset=0, pin_to_head=False), meta=. at 0x2b4ca3a40a40>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x2b4ca3a41620>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 711f378bb066d8b89696f3fa03000000), self_addrs={'master': HostAndPort(host='172.20.8.111', port=21138, external_host=None), 'rpc': HostAndPort(host='172.20.8.111', port=21139, external_host=None)}, local_gpu_ids=[0], gpu_slot_index=0) -(InferenceController pid=9891) [2026-09-29 11:41:56.433 inference_controller] rollout_server.py:132 - Killing server cell_id='inference-engine-all-0-0-00000'... -(InferenceController pid=9891) [2026-09-29 11:41:56.435 inference_controller] _client.py:1740 - HTTP Request: GET http://172.20.8.111:21000/workers "HTTP/1.1 200 OK" -(InferenceController pid=9891) [2026-09-29 11:41:56.436 inference_controller] _client.py:1740 - HTTP Request: DELETE http://172.20.8.111:21000/workers/e9bfcd19-e926-4e89-80ab-90f5aca7d99b "HTTP/1.1 202 Accepted" -(InferenceController pid=9891) [2026-09-29 11:41:56.437 inference_controller] server_cell.py:268 - Cell inference-engine-all-0-0-00000 dispose start old=addr_info=CellAddrInfo(server_url='http://172.20.8.111:21100', bootstrap_port=None, gate_url='http://172.20.8.111:21135') -(InferenceController pid=9891) [2026-09-29 11:41:56.437 inference_controller] server_cell.py:271 - Cell inference-engine-all-0-0-00000 dispose end new= -(CommandActor pid=9893) [2026-09-29 11:41:56] SIGTERM received. signum=None frame=None. Draining requests and shutting down... -(RayWorkerManager pid=9621) [2026-09-29 11:41:57.044 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_CommandActorManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=CommandWorkerSpec(name='inference-router-0', category=None, port_infos=[PortInfo(name='primary', static_port=21000, mode='per_worker', allow_dynamic=False, num_consecutive=1, offset_by_cell=False), PortInfo(name='prometheus', static_port=9000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b4ca3a0ede0>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x2b4ca3a0ee80>), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=CommandWorkerSpec(name='inference-router-0', category=None, port_infos=[PortInfo(name='primary', static_port=21000, mode='per_worker', allow_dynamic=False, num_consecutive=1, offset_by_cell=False), PortInfo(name='prometheus', static_port=9000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b4ca3a0ede0>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x2b4ca3a0ee80>), actor_handle=Actor(CommandActor, cf08beec0bc0830ddc496b8303000000), self_addrs={'primary': HostAndPort(host='172.20.8.111', port=21000, external_host=None), 'prometheus': HostAndPort(host='172.20.8.111', port=21101, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(CommandActor pid=9893) [2026-09-29 11:41:57] Subprocess detokenizer (pid=11463) crashed with exit code -15. Triggering SIGQUIT for cleanup... -(CommandActor pid=9893) [2026-09-29 11:41:57] SIGQUIT received. signum=None, frame=None. It usually means one child failed. -(CommandActor pid=9893) [2026-09-29 11:41:57] Sleeping 5 seconds before crash diagnostics to let GPU activity settle. -(CommandActor pid=9893) WARNING:miles.utils.workers.process_utils:Process 10976 did not exit within 5.0s after SIGTERM; escalating to SIGKILL - -(CommandActor pid=9893) [2026-09-29 11:40:46] INFO: 172.20.8.111:37074 - "POST /continue_generation HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:46] INFO: 172.20.8.111:37342 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:46] INFO: 172.20.8.111:22811 - "POST /update_weight_version HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:46] INFO: 172.20.8.111:22811 - "GET /model_info HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:46] INFO: 172.20.8.111:37342 - "POST /weights_checker HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:36409 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:25044 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:19721 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:49850 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:24600 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:61096 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:33322 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:46347 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:17323 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:17496 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:34183 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:22159 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:48886 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:51060 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:26274 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:47969 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:35035 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:28800 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:28852 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:33499 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:59667 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:29907 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:52] INFO: 172.20.8.111:50613 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:53] INFO: 172.20.8.111:63216 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:53] INFO: 172.20.8.111:54055 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:53] INFO: 172.20.8.111:65417 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:53] INFO: 172.20.8.111:29974 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:53] INFO: 172.20.8.111:36161 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:53] INFO: 172.20.8.111:30283 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:53] INFO: 172.20.8.111:57080 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:53] INFO: 172.20.8.111:23855 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:53] INFO: 172.20.8.111:32794 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:53] INFO: 172.20.8.111:39301 - "POST /abort_request HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:53] INFO: 172.20.8.111:59483 - "GET /flush_cache HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:40:54] INFO: 172.20.8.111:59483 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:41:05] INFO: 172.20.8.111:23544 - "GET /health HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:41:55] INFO: 172.20.8.111:20523 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:41:55] INFO: 172.20.8.111:61463 - "POST /pause_generation HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:41:55] INFO: 172.20.8.111:61463 - "GET /flush_cache HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:41:55] INFO: 172.20.8.111:61463 - "POST /begin_weight_update HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:41:55] INFO: 172.20.8.111:61463 - "POST /update_weights_from_tensor HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:41:56] INFO: 172.20.8.111:61463 - "POST /end_weight_update HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:41:56] INFO: 172.20.8.111:61463 - "POST /update_weight_version HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:41:56] INFO: 172.20.8.111:61463 - "POST /continue_generation HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:41:56] INFO: 172.20.8.111:20523 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:41:56] INFO: 172.20.8.111:27938 - "POST /update_weight_version HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:41:56] INFO: 172.20.8.111:27938 - "GET /model_info HTTP/1.1" 200 OK -(CommandActor pid=9893) [2026-09-29 11:41:56] INFO: 172.20.8.111:20523 - "POST /weights_checker HTTP/1.1" 200 OK -[rl] learner 0 finalized (rl_engine=ports) -(RayWorkerManager pid=9621) [2026-09-29 11:42:01.505 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_CommandActorManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=CommandWorkerSpec(name='inference-engine-all-0-0', category='inference_engine', port_infos=[PortInfo(name='primary', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='dist_init', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=31, offset_by_cell=False), PortInfo(name='nccl', static_port=10000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='engine_info_bootstrap', static_port=12000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='gate', static_port=13000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b4ca3a0f060>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.2, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=1, num_gpus_per_node=1, pg_name='rollout', pg_slot_offset=0, pin_to_head=False), meta=. at 0x2b4ca3a0f1a0>, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x2b4ca3a0f2e0>), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=CommandWorkerSpec(name='inference-engine-all-0-0', category='inference_engine', port_infos=[PortInfo(name='primary', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='dist_init', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=31, offset_by_cell=False), PortInfo(name='nccl', static_port=10000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='engine_info_bootstrap', static_port=12000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='gate', static_port=13000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b4ca3a0f060>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.2, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=1, num_gpus_per_node=1, pg_name='rollout', pg_slot_offset=0, pin_to_head=False), meta=. at 0x2b4ca3a0f1a0>, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x2b4ca3a0f2e0>), actor_handle=Actor(CommandActor, 8f36da5563a65169c7a7bdce03000000), self_addrs={'primary': HostAndPort(host='172.20.8.111', port=21100, external_host=None), 'dist_init': HostAndPort(host='172.20.8.111', port=21102, external_host=None), 'nccl': HostAndPort(host='172.20.8.111', port=21133, external_host=None), 'engine_info_bootstrap': HostAndPort(host='172.20.8.111', port=21134, external_host=None), 'gate': HostAndPort(host='172.20.8.111', port=21135, external_host=None)}, local_gpu_ids=[0], gpu_slot_index=0) diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/run.log b/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/run.log deleted file mode 100644 index 9dbfad172..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-kill44-legacy-vs-ports/kill44-ports/run.log +++ /dev/null @@ -1,7 +0,0 @@ - Fetching 10 files: 0%| | 0/10 [00:00 src/iso_worker.rs:116:12 - | -108 | impl IsoBackendConfig { - | --------------------- method in this implementation -... -116 | pub fn start_pool(&self, devices: Vec, queue_capacity: usize) -> Result { - | ^^^^^^^^^^ - | - = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default - -warning: associated items `start_pool` and `flatten` are never used - --> src/iso_worker.rs:158:12 - | -134 | impl IsoBackend { - | --------------- associated items in this implementation -... -158 | pub fn start_pool( - | ^^^^^^^^^^ -... -182 | pub fn flatten(&mut self, matrix: &mut [f32], rows: usize, cols: usize) -> Result<()> { - | ^^^^^^^ - -warning: associated function `start` is never used - --> src/iso_worker.rs:518:8 - | -517 | impl TorchIsoPool { - | ----------------- associated function in this implementation -518 | fn start(python: PathBuf, devices: Vec, queue_capacity: usize) -> Result { - | ^^^^^ - -warning: function `merge_iso` is never used - --> src/merge.rs:105:8 - | -105 | pub fn merge_iso(deltas: &[&[f32]], weights: &[f64], rows: usize, cols: usize, out: &mut [f32]) { - | ^^^^^^^^^ - -warning: function `next_committable_round` is never used - --> src/server.rs:2768:4 - | -2768 | fn next_committable_round(rounds: &[Round], now: Instant) -> Option { - | ^^^^^^^^^^^^^^^^^^^^^^ - -warning: function `fragment_available` is never used - --> src/server.rs:2776:4 - | -2776 | fn fragment_available(rounds: &[Round], fragment_id: usize) -> bool { - | ^^^^^^^^^^^^^^^^^^ - -warning: function `broadcast_updated_fragment` is never used - --> src/server.rs:3463:10 - | -3463 | async fn broadcast_updated_fragment(st: &GlobalState, registry: &Registry, p: usize) -> Result<()> { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: methods `fid`, `base_version`, and `finish_inline` are never used - --> src/state.rs:190:12 - | -189 | impl PreparedMerge { - | ------------------ methods in this implementation -190 | pub fn fid(&self) -> usize { - | ^^^ -... -194 | pub fn base_version(&self) -> u64 { - | ^^^^^^^^^^^^ -... -241 | fn finish_inline(self) -> Result { - | ^^^^^^^^^^^^^ - -warning: associated items `new_with_layout_fingerprint` and `merge_and_step` are never used - --> src/state.rs:259:12 - | -253 | impl GlobalState { - | ---------------- associated items in this implementation -... -259 | pub fn new_with_layout_fingerprint( - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -... -589 | pub fn merge_and_step( - | ^^^^^^^^^^^^^^ - -yeto-federated-m2: 2 island(s) x 1 GPU, 4 groups/round -PLAN_JSON {"arms": [{"benchmark_islands": 2, "fragments": 1, "gpus_per_island": 1, "groups_per_gpu_per_round": 4.0, "groups_per_round": 4, "islands": 2, "kind": "federated", "local_horizon": 1, "name": "yeto-federated-m2", "pipeline": 1, "prompt_groups": 24, "total_gpus": 2, "trajectories": 192}], "expert_parallel": 1, "fairness": {"paired_prompt_streams": true, "same_expert_parallel": true, "same_per_rank_batch": true, "same_pipeline_parallel": true, "same_prompt_groups": true, "same_total_gpus": true, "same_trajectories": true}, "model": "Qwen/Qwen3-0.6B", "pipeline_parallel": 1, "seeds": [17]} - Fetching 10 files: 0%| | 0/10 [00:00 padded vocab (size: 151936) with 0 dummy tokens (new size: 151936) -[2026-09-29 04:27:15.133 main] arguments.py:40 - --tokenizer-model not set, use --hf-checkpoint as tokenizer model. -[2026-09-29 04:27:15.133 main] arguments.py:2668 - MoE-expert LoRA layout: per-expert (--experts-shared-outer-loras). -[2026-09-29 04:27:15.135 main] arguments.py:2816 - Colocate mode: defaulting --sglang-cuda-graph-backend-prefill=disabled to avoid NVLS OOM. Set --sglang-cuda-graph-backend-prefill explicitly to override. -[2026-09-29 04:27:15.135 main] arguments.py:2826 - rollout_num_gpus None != actor_num_gpus_per_node 1 * actor_num_nodes 1, overriding rollout_num_gpus to match actor_num_gpus_per_node * actor_num_nodes. -[2026-09-29 04:27:15.136 main] arguments.py:2910 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 1024 as default value. -[2026-09-29 04:27:15.136 main] arguments.py:2918 - args.rollout_max_prompt_len is not set. Use args.rollout_max_context_len - 1 (1024 - 1) as default value so that there is at least one generated token to compute loss. -using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 -Number of virtual stages per pipeline stage: None -using torch.bfloat16 for parameters ... ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - activation_func_clamp_shared_expert ............. True - activation_func_clamp_value ..................... None - actor_num_gpus_per_node ......................... 1 - actor_num_nodes ................................. 1 - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.999 - adam_eps ........................................ 1e-08 - add_bias_linear ................................. False - add_position_embedding .......................... True - add_qkv_bias .................................... False - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - advantage_estimator ............................. grpo - align_grad_reduce ............................... True - align_param_gather .............................. False - allgather_cp .................................... False - allow_ambiguous_pad_tokens ...................... False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_chat_template ............................. True - apply_chat_template_kwargs ...................... {'enable_thinking': False} - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - apply_wd_to_qk_layernorm ........................ False - async_max_concurrent_samples .................... None - async_save ...................................... False - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.unfused - attention_dropout ............................... 0.1 - attention_output_gate ........................... False - attention_softmax_in_fp32 ....................... True - auto_detect_ckpt_format ......................... False - balance_data .................................... True - barrier_with_L1_time ............................ True - batch_invariant_mode ............................ False - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - beta_fast ....................................... 32 - beta_slow ....................................... 1 - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - buffer_filter_path .............................. None - cache_mla_latents ............................... False - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - chat_template_path .............................. None - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_lora_weight_equal ......................... False - check_weight_hash_across_dp_replicas_interval ... None - check_weight_update_allow_quant_error ........... False - check_weight_update_equal ....................... False - check_weight_update_selector .................... all - check_weight_update_skip_list ................... None - ci_check_model_hash ............................. False - ci_disable_kl_checker ........................... False - ci_disable_logprobs_checker ..................... False - ci_disable_weight_update_checker ................ False - ci_enable_metrics_capture ....................... False - ci_ft_test_actions .............................. None - ci_inject_rollout_data_min_match_ratio .......... 0.9 - ci_inject_rollout_data_path ..................... None - ci_inject_rollout_data_start_rollout_id ......... None - ci_load_grad_norm ............................... None - ci_metric_checker_key ........................... None - ci_metric_checker_threshold ..................... None - ci_save_grad_norm ............................... None - ci_save_model_hash .............................. False - ci_test ......................................... False - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch_dist - ckpt_fully_parallel_load ........................ False - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - colocate ........................................ True - compress_ratios ................................. None - compute_advantages_and_returns .................. True - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - control_server_port ............................. 0 - cp_comm_type .................................... ['p2p'] - cpu_offloading_num_layers ....................... 0 - create_all_gather_group ......................... False - create_attention_mask_in_dataloader ............. True - critic_load ..................................... /root/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca - critic_lr ....................................... 0.0001 - critic_lr_warmup_iters .......................... 0 - critic_num_gpus_per_node ........................ 1 - critic_num_nodes ................................ 1 - critic_save ..................................... None - cross_entropy_fusion_impl ....................... native - cross_entropy_loss_fusion ....................... False - cuda_graph_impl ................................. none - cuda_graph_scope ................................ [] - cuda_graph_warmup_steps ......................... 3 - custom_config_path .............................. None - custom_convert_samples_to_train_data_path ....... None - custom_eval_rollout_log_function_path ........... None - custom_generate_function_path ................... None - custom_loss_function_path ....................... None - custom_megatron_before_log_prob_hook_path ....... None - custom_megatron_before_train_step_hook_path ..... None - custom_megatron_init_path ....................... None - custom_megatron_post_save_hook_path ............. None - custom_model_provider_path ...................... None - custom_pg_loss_reducer_function_path ............ None - custom_reward_post_process_path ................. None -... [truncated: 1816 of 2216 lines removed; full evidence archived outside repo] ... -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:18.266 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:18.266 actor_cell0_rank0] timer.py:31 - Timer log_probs start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:23.830 actor_cell0_rank0] timer.py:42 - Timer log_probs end (elapsed: 5.6s) -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:23.830 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=end ok=true elapsed_s=5.6 -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:23.837 actor_cell0_rank0] log_utils.py:42 - ft op=cross_cell phase=start kind=log_gather rank=0 -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:23.838 actor_cell0_rank0] log_utils.py:48 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:23.838 actor_cell0_rank0] log_utils.py:66 - rollout 1: {'rollout/response_lengths': 326.0625, 'rollout/rewards': 0.0, 'rollout/truncated': 0.46875, 'rollout/rollout_log_probs': -0.465812623500824, 'rollout/raw_reward': 0.0, 'rollout/total_lengths': 435.3125, 'rollout/log_probs': -0.4668426215648651, 'rollout/advantages': 0.0, 'rollout/returns': 0.0} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:23.838 actor_cell0_rank0] timer.py:31 - Timer actor_train start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:33.820 actor_cell0_rank0] model.py:601 - ft op=train_step rollout=1 step=0 attempt=0 outcome=NORMAL valid_step=true -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:33.821 actor_cell0_rank0] log_utils.py:460 - step 1: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.015803804621100426, 'train/train_rollout_kl': 0.0007186654256656766, 'train/grad_norm': 0.0, 'train/lr-pg_0': 3.333333333333334e-05, 'train/step': 1} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:33.821 actor_cell0_rank0] model.py:814 - step 1: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.015803804621100426, 'train/train_rollout_kl': 0.0007186654256656766, 'train/grad_norm': 0.0, 'train/lr-pg_0': 3.333333333333334e-05, 'train/step': 1} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:33.821 actor_cell0_rank0] timer.py:42 - Timer actor_train end (elapsed: 10.0s) -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:33.821 actor_cell0_rank0] timer.py:42 - Timer train end (elapsed: 15.6s) -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:33.821 actor_cell0_rank0] timer.py:31 - Timer train_wait start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:33.845 actor_cell0_rank0] train_metric_utils.py:50 - perf 1: {'perf/update_weights_time': 0.7955546379089355, 'perf/data_preprocess_time': 0.00583338737487793, 'perf/train_wait_time': 18.516362190246582, 'perf/log_probs_time': 5.563802242279053, 'perf/actor_train_time': 9.982789278030396, 'perf/train_time': 15.579174280166626, 'perf/log_probs_tflops': 2.7363172826796567, 'perf/actor_train_tflops': 4.5751726723707105, 'perf/actor_train_tok_per_s': 1395.4015868747647, 'perf/step_time': 34.09553647041321, 'perf/wait_time_ratio': 0.5430729094500204} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:33.845 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train_actor phase=end ok=true elapsed_s=15.6 -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:33.846 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=15.6 -(SGLangEngine pid=5257) [2026-09-29 04:33:01] INFO: 172.20.13.80:39389 - "POST /update_weight_version HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:02] INFO: 172.20.13.80:63733 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:02] INFO: 172.20.13.80:42282 - "POST /pause_generation HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:02] INFO: 172.20.13.80:39249 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:02] INFO: 172.20.13.80:40920 - "POST /unload_lora_adapter HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:02] INFO: 172.20.13.80:59233 - "POST /load_lora_adapter_from_tensors HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:03] INFO: 172.20.13.80:30152 - "POST /continue_generation HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:03] INFO: 172.20.13.80:54266 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:10] INFO: 172.20.13.80:53259 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:10] INFO: 172.20.13.80:20778 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:10] INFO: 172.20.13.80:17691 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:10] INFO: 172.20.13.80:61536 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:10] INFO: 172.20.13.80:23774 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:11] INFO: 172.20.13.80:42177 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:11] INFO: 172.20.13.80:46251 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:11] INFO: 172.20.13.80:16188 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:12] INFO: 172.20.13.80:23774 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:12] INFO: 172.20.13.80:61536 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:12] INFO: 172.20.13.80:53259 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:12] INFO: 172.20.13.80:20778 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:12] INFO: 172.20.13.80:17691 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:13] INFO: 172.20.13.80:42177 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:13] INFO: 172.20.13.80:46251 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:13] INFO: 172.20.13.80:17691 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:13] INFO: 172.20.13.80:23774 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:13] INFO: 172.20.13.80:16188 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:14] INFO: 172.20.13.80:61536 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:14] INFO: 172.20.13.80:53259 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:14] INFO: 172.20.13.80:20778 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:15] INFO: 172.20.13.80:42177 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:15] INFO: 172.20.13.80:46251 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:15] INFO: 172.20.13.80:41792 - "GET /health HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:15] INFO: 172.20.13.80:17691 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:16] INFO: 172.20.13.80:23774 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:16] INFO: 172.20.13.80:16188 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:16] INFO: 172.20.13.80:61536 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:16] INFO: 172.20.13.80:53259 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:16] INFO: 172.20.13.80:20778 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:17] INFO: 172.20.13.80:42177 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:17] INFO: 172.20.13.80:46251 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:17] INFO: 172.20.13.80:17691 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:17] INFO: 172.20.13.80:25905 - "POST /abort_request HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:17] INFO: 172.20.13.80:62327 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:18] INFO: 172.20.13.80:59970 - "POST /release_memory_occupation HTTP/1.1" 200 OK -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"begin","policy_version":2} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"apply_finished","policy_version":2} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_begin","policy_version":2} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_finished","policy_version":2} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:35.666 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before TrainRayActor.clear_memory: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 69.34, 'used_GB': 9.84, 'allocated_GB': 1.3, 'reserved_GB': 5.26} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:36.338 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after TrainRayActor.clear_memory: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 73.1, 'used_GB': 6.08, 'allocated_GB': 1.3, 'reserved_GB': 1.49} -(SGLangEngine pid=5257) [2026-09-29 04:33:36] Cache flushed successfully! -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:36.519 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=update_weights phase=start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:36.519 actor_cell0_rank0] timer.py:31 - Timer update_weights start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:36.520 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 71.92, 'used_GB': 7.26, 'allocated_GB': 1.3, 'reserved_GB': 1.49} -(SGLangEngine pid=5257) [2026-09-29 04:33:36] Start unload Lora adapter. Lora name=miles_lora -(SGLangEngine pid=5257) [2026-09-29 04:33:36] LoRA adapter unloading starts: LoRARef(lora_id=48b69e43183c4caa987bc581640c3553, lora_name=miles_lora). avail mem=71.94 GB -(SGLangEngine pid=5257) [2026-09-29 04:33:36] LoRA adapter unloading completes: LoRARef(lora_id=48b69e43183c4caa987bc581640c3553, lora_name=miles_lora). avail mem=71.94 GB -(SGLangEngine pid=5257) [2026-09-29 04:33:36] Start load Lora adapter from tensors. Lora name=miles_lora -(SGLangEngine pid=5257) [2026-09-29 04:33:36] LoRA adapter loading from tensors starts: LoRARef(lora_id=4a38b99852224715a0987973ce46a7fe, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(SGLangEngine pid=5257) [2026-09-29 04:33:36] LoRA adapter loading from tensors completes: LoRARef(lora_id=4a38b99852224715a0987973ce46a7fe, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:36.617 actor_cell0_rank0] update_weight_from_tensor.py:461 - Packed 392 LoRA tensors (20185088 bytes) into 1 flattened buckets (max 268435456 bytes/1024 tensors) -(RolloutManager pid=1909) Rollout generation: 0%| | 0/32 [00:00 33096.0 MB (freed 0.0 MB) -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:37.247 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 71.94, 'used_GB': 7.24, 'allocated_GB': 1.3, 'reserved_GB': 1.49} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:37.247 actor_cell0_rank0] timer.py:42 - Timer update_weights end (elapsed: 0.7s) -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:37.247 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.7 -(SGLangEngine pid=5257) [2026-09-29 04:33:37] LoRA adapter '4a38b99852224715a0987973ce46a7fe': loaded weights for target modules ['down_proj', 'gate_up_proj', 'o_proj', 'qkv_proj']. -(SGLangEngine pid=5257) [2026-09-29 04:33:43] Prefill batch, #new-seq: 1, #new-token: 77, #cached-token: 0, token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 2.78 -(SGLangEngine pid=5257) [2026-09-29 04:33:43] Prefill batch, #new-seq: 7, #new-token: 539, #cached-token: 0, token usage: 0.00, #running-req: 1, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 117749.68 -(SGLangEngine pid=5257) [2026-09-29 04:33:43] Decode batch, #running-req: 8, #token: 405, token usage: 0.00, cuda graph: True, gen throughput (token/s): 12.24, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:44] Decode batch, #running-req: 8, #token: 725, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1782.98, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:44] Decode batch, #running-req: 8, #token: 1045, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1697.69, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:44] Decode batch, #running-req: 8, #token: 1365, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1708.16, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:44] Decode batch, #running-req: 8, #token: 1685, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1678.66, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:44] Prefill batch, #new-seq: 1, #new-token: 98, #cached-token: 3, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 90.72 -(SGLangEngine pid=5257) [2026-09-29 04:33:44] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 100, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 9.60 -(SGLangEngine pid=5257) [2026-09-29 04:33:44] Decode batch, #running-req: 8, #token: 1641, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1181.71, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:44] Prefill batch, #new-seq: 2, #new-token: 2, #cached-token: 200, token usage: 0.00, #running-req: 6, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 24.23 -(SGLangEngine pid=5257) [2026-09-29 04:33:44] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 100, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 14.08 -(SGLangEngine pid=5257) [2026-09-29 04:33:45] Decode batch, #running-req: 8, #token: 932, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1171.04, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:45] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 100, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 5.68 -(SGLangEngine pid=5257) [2026-09-29 04:33:45] Decode batch, #running-req: 8, #token: 1250, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1431.24, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:45] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 100, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 4.24 -(RolloutManager pid=1909) [2026-09-29 04:33:45.464 rollout_manager] sglang_rollout.py:473 - First rollout sample: ["<|im_start|>user\nThe profit from a business transaction is shared among 2 business partners, Mike and Johnson in the ratio 2:5 respectively. If Johnson got $2500, how much will Mike have after spending some of his share on a shirt that costs $200?\nPut the final numeric answer in \\boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nWe are given that the profit is shared **in the ratio 2:5 between Mike and Johnson**. The total profit is split as:\n\n- Mike gets **2 parts**,\n- Johnson gets **5 parts**,\n\nand the ratio of their shares is **2:5**. Since they share a total profit **2500**, the value of each part is:\n\n$$\n\\frac{2500}{2 + 5} = \\frac{2500}{7}\n$$\n\nNow, let's calculate Mike's share:\n\n$$\n\\text{Mike's share} = 2 \\times \\frac{2500}{7} = \\frac{5000}{7} \\approx 714.29\n$$\n\n### Problem Statement:\nAfter spending **$200** on a shirt, Mike will have **$714.29 - $200 = $514.29** left.\n\n### Final Answer:\n\n$$\n\\boxed{514.29}\n$$<|im_end|>"], label: 800, reward: 0.0 -(RolloutManager pid=1909) Rollout generation: 25%|██▌ | 8/32 [00:08<00:24, 1.02s/it] -(SGLangEngine pid=5257) [2026-09-29 04:33:45] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 100, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 8.58 -(SGLangEngine pid=5257) [2026-09-29 04:33:45] Decode batch, #running-req: 8, #token: 827, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1210.96, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:45] Decode batch, #running-req: 8, #token: 1147, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1689.23, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:45] Decode batch, #running-req: 8, #token: 1467, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1681.13, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:46] Decode batch, #running-req: 8, #token: 1787, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1697.94, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:46] Prefill batch, #new-seq: 1, #new-token: 82, #cached-token: 3, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 111.78 -(SGLangEngine pid=5257) [2026-09-29 04:33:46] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 13.92 -(SGLangEngine pid=5257) [2026-09-29 04:33:46] Decode batch, #running-req: 8, #token: 1673, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1176.26, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:46] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 5.74 -(SGLangEngine pid=5257) [2026-09-29 04:33:46] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 7.44 -(SGLangEngine pid=5257) [2026-09-29 04:33:46] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 14.97 -(SGLangEngine pid=5257) [2026-09-29 04:33:46] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 21.40 -(SGLangEngine pid=5257) [2026-09-29 04:33:46] Decode batch, #running-req: 8, #token: 916, token usage: 0.00, cuda graph: True, gen throughput (token/s): 914.63, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:46] Decode batch, #running-req: 8, #token: 1236, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1676.51, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:47] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 3.11 -(RolloutManager pid=1909) Rollout generation: 50%|█████ | 16/32 [00:09<00:08, 1.84it/s] -(SGLangEngine pid=5257) [2026-09-29 04:33:47] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 7.40 -(SGLangEngine pid=5257) [2026-09-29 04:33:47] Decode batch, #running-req: 8, #token: 802, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1169.51, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:47] Prefill batch, #new-seq: 1, #new-token: 82, #cached-token: 3, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 407.03 -(SGLangEngine pid=5257) [2026-09-29 04:33:47] Decode batch, #running-req: 8, #token: 1027, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1382.76, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:47] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 3.48 -(SGLangEngine pid=5257) [2026-09-29 04:33:47] Decode batch, #running-req: 8, #token: 1180, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1380.27, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:47] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 9.89 -(SGLangEngine pid=5257) [2026-09-29 04:33:47] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 11.80 -(SGLangEngine pid=5257) [2026-09-29 04:33:47] Decode batch, #running-req: 8, #token: 1049, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1183.71, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:48] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 6.11 -(SGLangEngine pid=5257) [2026-09-29 04:33:48] Decode batch, #running-req: 8, #token: 1155, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1423.91, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:48] Decode batch, #running-req: 8, #token: 1475, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1681.95, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:48] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 2.11 -(SGLangEngine pid=5257) [2026-09-29 04:33:48] Decode batch, #running-req: 8, #token: 1480, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1385.27, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:48] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 3.16 -(SGLangEngine pid=5257) [2026-09-29 04:33:48] Decode batch, #running-req: 8, #token: 1494, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1371.95, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:49] Decode batch, #running-req: 8, #token: 1814, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1667.72, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:49] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 84, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 2.47 -(SGLangEngine pid=5257) [2026-09-29 04:33:49] Decode batch, #running-req: 8, #token: 1858, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1367.56, #queue-req: 0 -(RolloutManager pid=1909) Rollout generation: 75%|███████▌ | 24/32 [00:11<00:03, 2.40it/s] -(SGLangEngine pid=5257) [2026-09-29 04:33:49] Decode batch, #running-req: 4, #token: 732, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1195.67, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:49] Decode batch, #running-req: 4, #token: 553, token usage: 0.00, cuda graph: True, gen throughput (token/s): 846.85, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:49] Decode batch, #running-req: 2, #token: 413, token usage: 0.00, cuda graph: True, gen throughput (token/s): 456.17, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:50] Decode batch, #running-req: 2, #token: 493, token usage: 0.00, cuda graph: True, gen throughput (token/s): 428.61, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:50] Decode batch, #running-req: 1, #token: 291, token usage: 0.00, cuda graph: True, gen throughput (token/s): 387.88, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:50] Decode batch, #running-req: 1, #token: 331, token usage: 0.00, cuda graph: True, gen throughput (token/s): 220.14, #queue-req: 0 -(SGLangEngine pid=5257) [2026-09-29 04:33:50] Decode batch, #running-req: 1, #token: 371, token usage: 0.00, cuda graph: True, gen throughput (token/s): 220.14, #queue-req: 0 -(RolloutManager pid=1909) Rollout generation: 100%|██████████| 32/32 [00:13<00:00, 3.08it/s] Rollout generation: 100%|██████████| 32/32 [00:13<00:00, 2.38it/s] -(RolloutManager pid=1909) [2026-09-29 04:33:50.736 rollout_manager] sglang_rollout.py:494 - Finish rollout: ['<|im_start|>user\nTim rides his bike back and forth to work for each of his 5 workdays. His work is 20 miles away. He also goes for a weekend bike ride of 200 miles. If he can bike at 25 mph how much time does he spend biking a week?\nPut the final numeric answer in \\boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nWe are told the following information:\n\n- Tim rides his bike **back and forth** every **5 workdays**.\n- His **work distance** is **20 miles per day**.\n- He also goes on a **weekend bike ride** of **200 miles per day**.\n- His biking speed is **25 miles per hour**.\n\n---\n\n### Step 1: Find total distance biking per day\n\nTim travels:\n- 20 miles/day (work trip)\n- 200 miles (weekend trip)\n\nSo, **total biking distance per day is**:\n\n$$\n20 \\text{ miles} + 200 \\text{ miles} = 220 \\text{ miles per day}\n$$\n\n---\n\n### Step 2: Find total biking distance per week\n\nHe travels **5 workdays** and one weekend day. So:\n\n$$\n220 \\text{ miles/day} \\times 5 \\text{ workdays} + 200 \\text{ miles} = 1100 \\text{ miles (work trips)} + 200 \\text{ miles (weekend ride)} = 1300 \\text{ miles}\n$$\n\n---\n\n### Step 3: Calculate time biking in hours\n\nWe know speed is 25 mph, so time = distance / speed:\n\n$$\n\\text{Time} = \\frac{1300 \\text{ miles}}{25 \\text{ mph}} = 52 \\text{ hours}\n$$\n\n---\n\n### ✅ Final Answer:\n\n$$\n\\boxed{52} \\text{ hours}\n$$<|im_end|>'], label: 16, reward: 0.0 -(RolloutManager pid=1909) [2026-09-29 04:33:50.738 rollout_manager] sglang_rollout.py:391 - Abort request for ['http://172.20.13.80:21100'] -(SGLangEngine pid=5257) [2026-09-29 04:33:50] Decode batch, #running-req: 1, #token: 0, token usage: 0.00, cuda graph: True, gen throughput (token/s): 219.28, #queue-req: 0 -(RolloutManager pid=1909) [2026-09-29 04:33:50.743 rollout_manager] rollout_data_conversion.py:42 - Final collected 32 samples from rollout to train -(RolloutManager pid=1909) [2026-09-29 04:33:50.744 rollout_manager] debug_data.py:107 - Save debug rollout data to /work/out/legacy-0/work/seed-17/yeto-federated-m2/rollouts/island-0/2.pt -(RolloutManager pid=1909) [2026-09-29 04:33:50.757 rollout_manager] metrics.py:79 - perf 2: {'rollout/response_len/mean': 271.71875, 'rollout/response_len/median': 270.5, 'rollout/response_len/max': 384, 'rollout/response_len/min': 165, 'rollout/zero_std/count_0.0': 1, 'rollout/zero_std/all_zero_percentage': 0.25, 'rollout/zero_std/all_one_percentage': 0.0, 'rollout/prefix_cache_hit_rate': 0.6770833333333334, 'rollout/avg_cached_tokens_per_sample': 58.90625, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.03125, 'perf/rollout_time': 13.452865362167358, 'perf/tokens_per_gpu_per_sec': 646.3307084342342, 'perf/longest_sample_tokens_per_sec': 28.544104892322707, 'perf/effective_tokens_per_gpu_per_sec': 646.3307084342342, 'perf/longest_effective_sample_tokens_per_sec': 28.544104892322707} -(SGLangEngine pid=5257) [2026-09-29 04:33:50] Cache flushed successfully! -(SGLangEngine pid=5257) [2026-09-29 04:33:50] Cache flushed successfully! -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:51.178 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=train phase=start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:51.178 actor_cell0_rank0] timer.py:31 - Timer data_preprocess start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:51.184 actor_cell0_rank0] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:51.184 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=train_actor phase=start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:51.184 actor_cell0_rank0] timer.py:42 - Timer train_wait end (elapsed: 17.4s) -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:51.184 actor_cell0_rank0] timer.py:31 - Timer train start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:51.208 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:51.208 actor_cell0_rank0] timer.py:31 - Timer log_probs start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:56.561 actor_cell0_rank0] timer.py:42 - Timer log_probs end (elapsed: 5.4s) -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:56.561 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=end ok=true elapsed_s=5.4 -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:56.568 actor_cell0_rank0] log_utils.py:42 - ft op=cross_cell phase=start kind=log_gather rank=0 -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:56.569 actor_cell0_rank0] log_utils.py:48 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:56.569 actor_cell0_rank0] log_utils.py:66 - rollout 2: {'rollout/response_lengths': 271.71875, 'rollout/rewards': 4.6566128730773926e-09, 'rollout/truncated': 0.03125, 'rollout/rollout_log_probs': -0.39077457785606384, 'rollout/raw_reward': 0.25, 'rollout/total_lengths': 358.71875, 'rollout/log_probs': -0.3916918933391571, 'rollout/advantages': -2.7939677238464355e-09, 'rollout/returns': -2.7939677238464355e-09} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:33:56.569 actor_cell0_rank0] timer.py:31 - Timer actor_train start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:06.340 actor_cell0_rank0] model.py:601 - ft op=train_step rollout=2 step=0 attempt=0 outcome=NORMAL valid_step=true -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:06.341 actor_cell0_rank0] log_utils.py:460 - step 2: {'train/loss': 2.7939677238464355e-09, 'train/pg_loss': 2.7939677238464355e-09, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.013932695612311363, 'train/train_rollout_kl': 0.0006400272832252085, 'train/grad_norm': 0.30387297597703, 'train/lr-pg_0': 0.0, 'train/step': 2} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:06.341 actor_cell0_rank0] model.py:814 - step 2: {'train/loss': 2.7939677238464355e-09, 'train/pg_loss': 2.7939677238464355e-09, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.013932695612311363, 'train/train_rollout_kl': 0.0006400272832252085, 'train/grad_norm': 0.30387297597703, 'train/lr-pg_0': 0.0, 'train/step': 2} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:06.341 actor_cell0_rank0] timer.py:42 - Timer actor_train end (elapsed: 9.8s) -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:06.341 actor_cell0_rank0] timer.py:42 - Timer train end (elapsed: 15.2s) -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:06.341 actor_cell0_rank0] timer.py:31 - Timer train_wait start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:06.365 actor_cell0_rank0] train_metric_utils.py:50 - perf 2: {'perf/update_weights_time': 0.7279658317565918, 'perf/data_preprocess_time': 0.005347013473510742, 'perf/train_wait_time': 17.36267375946045, 'perf/log_probs_time': 5.352491140365601, 'perf/actor_train_time': 9.771608114242554, 'perf/train_time': 15.1568763256073, 'perf/log_probs_tflops': 2.3187909904426762, 'perf/actor_train_tflops': 3.810419356036382, 'perf/actor_train_tok_per_s': 1174.7298771907201, 'perf/step_time': 32.51955008506775, 'perf/wait_time_ratio': 0.5339149439042516} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:06.365 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train_actor phase=end ok=true elapsed_s=15.2 -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:06.366 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=15.2 -(SGLangEngine pid=5257) [2026-09-29 04:33:35] INFO: 172.20.13.80:24400 - "POST /update_weight_version HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:36] INFO: 172.20.13.80:60393 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:36] INFO: 172.20.13.80:31494 - "POST /pause_generation HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:36] INFO: 172.20.13.80:17852 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:36] INFO: 172.20.13.80:57283 - "POST /unload_lora_adapter HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:36] INFO: 172.20.13.80:58555 - "POST /load_lora_adapter_from_tensors HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:37] INFO: 172.20.13.80:60864 - "POST /continue_generation HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:37] INFO: 172.20.13.80:28275 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:44] INFO: 172.20.13.80:46524 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:44] INFO: 172.20.13.80:44090 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:44] INFO: 172.20.13.80:26086 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:44] INFO: 172.20.13.80:36688 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:44] INFO: 172.20.13.80:28372 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:45] INFO: 172.20.13.80:48593 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:45] INFO: 172.20.13.80:39706 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:45] INFO: 172.20.13.80:33702 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:46] INFO: 172.20.13.80:26086 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:46] INFO: 172.20.13.80:46524 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:46] INFO: 172.20.13.80:44090 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:46] INFO: 172.20.13.80:28372 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:46] INFO: 172.20.13.80:39706 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:46] INFO: 172.20.13.80:48593 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:47] INFO: 172.20.13.80:33702 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:47] INFO: 172.20.13.80:36688 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:47] INFO: 172.20.13.80:26086 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:47] INFO: 172.20.13.80:39706 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:47] INFO: 172.20.13.80:46524 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:47] INFO: 172.20.13.80:44090 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:47] INFO: 172.20.13.80:48593 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:48] INFO: 172.20.13.80:28372 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:48] INFO: 172.20.13.80:33702 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:49] INFO: 172.20.13.80:39706 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:49] INFO: 172.20.13.80:36688 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:49] INFO: 172.20.13.80:26086 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:49] INFO: 172.20.13.80:46524 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:49] INFO: 172.20.13.80:48593 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:49] INFO: 172.20.13.80:44090 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:49] INFO: 172.20.13.80:28372 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:50] INFO: 172.20.13.80:33702 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:50] INFO: 172.20.13.80:39706 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:50] INFO: 172.20.13.80:46202 - "POST /abort_request HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:50] INFO: 172.20.13.80:22167 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:33:51] INFO: 172.20.13.80:45642 - "POST /release_memory_occupation HTTP/1.1" 200 OK -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"begin","policy_version":3} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"apply_finished","policy_version":3} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_begin","policy_version":3} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_finished","policy_version":3} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:08.097 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before TrainRayActor.clear_memory: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 70.06, 'used_GB': 9.12, 'allocated_GB': 1.3, 'reserved_GB': 4.54} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:08.728 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after TrainRayActor.clear_memory: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 73.11, 'used_GB': 6.07, 'allocated_GB': 1.3, 'reserved_GB': 1.49} -(SGLangEngine pid=5257) [2026-09-29 04:34:08] Cache flushed successfully! -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:08.839 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=update_weights phase=start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:08.839 actor_cell0_rank0] timer.py:31 - Timer update_weights start -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:08.840 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 71.92, 'used_GB': 7.26, 'allocated_GB': 1.3, 'reserved_GB': 1.49} -(SGLangEngine pid=5257) [2026-09-29 04:34:08] Start unload Lora adapter. Lora name=miles_lora -(SGLangEngine pid=5257) [2026-09-29 04:34:08] LoRA adapter unloading starts: LoRARef(lora_id=4a38b99852224715a0987973ce46a7fe, lora_name=miles_lora). avail mem=71.94 GB -(SGLangEngine pid=5257) [2026-09-29 04:34:08] LoRA adapter unloading completes: LoRARef(lora_id=4a38b99852224715a0987973ce46a7fe, lora_name=miles_lora). avail mem=71.94 GB -(SGLangEngine pid=5257) [2026-09-29 04:34:08] Start load Lora adapter from tensors. Lora name=miles_lora -(SGLangEngine pid=5257) [2026-09-29 04:34:08] LoRA adapter loading from tensors starts: LoRARef(lora_id=865c6faf428746a7b8a506af6e10b562, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(SGLangEngine pid=5257) [2026-09-29 04:34:08] LoRA adapter loading from tensors completes: LoRARef(lora_id=865c6faf428746a7b8a506af6e10b562, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:08.911 actor_cell0_rank0] update_weight_from_tensor.py:461 - Packed 392 LoRA tensors (20185088 bytes) into 1 flattened buckets (max 268435456 bytes/1024 tensors) -(SGLangEngine pid=5257) [2026-09-29 04:34:09] [continue_generation] torch.cuda.empty_cache() called: reserved 33096.0 MB -> 33096.0 MB (freed 0.0 MB) -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:09.449 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 71.94, 'used_GB': 7.24, 'allocated_GB': 1.3, 'reserved_GB': 1.49} -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:09.449 actor_cell0_rank0] timer.py:42 - Timer update_weights end (elapsed: 0.6s) -(MegatronTrainRayActor pid=8038) [2026-09-29 04:34:09.449 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.6 -(SGLangEngine pid=5257) [2026-09-29 04:34:08] INFO: 172.20.13.80:43703 - "POST /update_weight_version HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:34:08] INFO: 172.20.13.80:40721 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:34:08] INFO: 172.20.13.80:61740 - "POST /pause_generation HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:34:08] INFO: 172.20.13.80:55811 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:34:08] INFO: 172.20.13.80:61422 - "POST /unload_lora_adapter HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:34:08] INFO: 172.20.13.80:35931 - "POST /load_lora_adapter_from_tensors HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:34:09] INFO: 172.20.13.80:40429 - "POST /continue_generation HTTP/1.1" 200 OK -(SGLangEngine pid=5257) [2026-09-29 04:34:09] INFO: 172.20.13.80:58347 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -[rl] learner 0 finalized diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-1/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-1/events.jsonl deleted file mode 100644 index 73ab55e43..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-1/events.jsonl +++ /dev/null @@ -1,23 +0,0 @@ -{"event":"rl_policy_apply_progress","island_id":1,"phase":"begin","policy_version":0,"time_unix":1790656328.9160192} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"apply_finished","policy_version":0,"time_unix":1790656329.246588} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"hash_begin","policy_version":0,"time_unix":1790656329.2470427} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"hash_finished","policy_version":0,"time_unix":1790656329.6396937} -{"event":"rl_policy_apply","island_id":1,"optimizer_reset_count":1,"policy_version":0,"reset_parameter_count":392,"rl/global_policy_version":0,"rl/optimizer_reset_count":1,"sync/global_policy_hash":"62f3b8f69c1cb9f2db1ab500f19da0dd76a5323536e6f38b0ee81bf1a486eaa6","time_unix":1790656329.6624675} -{"action_tokens":6145,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.2333943396806717,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":13.631327551500021,"group_p95_seconds":14.796674439299998,"group_p99_seconds":14.945654249459999,"island_id":1,"local_round_id":1,"loss":null,"lr":null,"mean_kl":0.0007255167001858354,"nonzero_advantage_count":24,"pass_rate":null,"pg_loss":null,"reward_mean":0.4375,"reward_std":0.49607837082461076,"rl/action_tokens":6145,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0007255167001858354,"rl/ess_ratio":1.0,"rl/global_policy_version":0,"rl/group_p50_seconds":13.631327551500021,"rl/group_p95_seconds":14.796674439299998,"rl/group_p99_seconds":14.945654249459999,"rl/local_delta_norm":0.2333943396806717,"rl/mixed_version_group_count":0,"rl/reward_mean":0.4375,"rl/reward_std":0.49607837082461076,"rl/rollout_policy_version":0,"rl/rollout_seconds":15.416142742999966,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":15.416142742999966,"sync/bytes_sent":40370224,"time_unix":1790656378.9985523,"tool_wait_seconds":0.0,"train_seconds":17.268849756999998,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"begin","policy_version":1,"time_unix":1790656380.9655342} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"apply_finished","policy_version":1,"time_unix":1790656381.268826} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"hash_begin","policy_version":1,"time_unix":1790656381.2693107} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"hash_finished","policy_version":1,"time_unix":1790656381.7521532} -{"event":"rl_policy_apply","island_id":1,"optimizer_reset_count":2,"policy_version":1,"reset_parameter_count":392,"rl/global_policy_version":1,"rl/optimizer_reset_count":2,"sync/global_policy_hash":"a4bac44b623f4e11c79353a73fbc1800a1728371053c77ba6b45885e1f5ac0f0","time_unix":1790656381.7621002} -{"action_tokens":8079,"active_groups":4,"base_policy_version":1,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.14705266058444977,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":4.065228561499993,"group_p95_seconds":6.1381318926999615,"group_p99_seconds":6.346702054539958,"island_id":1,"local_round_id":2,"loss":null,"lr":null,"mean_kl":0.0004483942757360637,"nonzero_advantage_count":16,"pass_rate":null,"pg_loss":null,"reward_mean":0.59375,"reward_std":0.4911323014219285,"rl/action_tokens":8079,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0004483942757360637,"rl/ess_ratio":1.0,"rl/global_policy_version":1,"rl/group_p50_seconds":4.065228561499993,"rl/group_p95_seconds":6.1381318926999615,"rl/group_p99_seconds":6.346702054539958,"rl/local_delta_norm":0.14705266058444977,"rl/mixed_version_group_count":0,"rl/reward_mean":0.59375,"rl/reward_std":0.4911323014219285,"rl/rollout_policy_version":1,"rl/rollout_seconds":6.407340986000008,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.5,"rollout_seconds":6.407340986000008,"sync/bytes_sent":40370224,"time_unix":1790656405.8404112,"tool_wait_seconds":0.0,"train_seconds":9.949453756000025,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"begin","policy_version":2,"time_unix":1790656414.9522343} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"apply_finished","policy_version":2,"time_unix":1790656415.1803212} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"hash_begin","policy_version":2,"time_unix":1790656415.18075} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"hash_finished","policy_version":2,"time_unix":1790656415.5813534} -{"event":"rl_policy_apply","island_id":1,"optimizer_reset_count":3,"policy_version":2,"reset_parameter_count":392,"rl/global_policy_version":2,"rl/optimizer_reset_count":3,"sync/global_policy_hash":"8211764dbc4e101fdfb11e956b96685b22c117359a2b8683d0a57f3e7337cf51","time_unix":1790656415.591482} -{"action_tokens":8283,"active_groups":4,"base_policy_version":2,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.06198877841234207,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":6.5207023175000245,"group_p95_seconds":8.346004892000007,"group_p99_seconds":8.457454364000002,"island_id":1,"local_round_id":3,"loss":null,"lr":null,"mean_kl":0.0005430057644844055,"nonzero_advantage_count":16,"pass_rate":null,"pg_loss":null,"reward_mean":0.6875,"reward_std":0.46351240544347894,"rl/action_tokens":8283,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0005430057644844055,"rl/ess_ratio":1.0,"rl/global_policy_version":2,"rl/group_p50_seconds":6.5207023175000245,"rl/group_p95_seconds":8.346004892000007,"rl/group_p99_seconds":8.457454364000002,"rl/local_delta_norm":0.06198877841234207,"rl/mixed_version_group_count":0,"rl/reward_mean":0.6875,"rl/reward_std":0.46351240544347894,"rl/rollout_policy_version":2,"rl/rollout_seconds":8.493302468000024,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.5,"rollout_seconds":8.493302468000024,"sync/bytes_sent":40370224,"time_unix":1790656441.9629796,"tool_wait_seconds":0.0,"train_seconds":9.830465680999964,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"begin","policy_version":3,"time_unix":1790656447.452848} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"apply_finished","policy_version":3,"time_unix":1790656447.6832607} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"hash_begin","policy_version":3,"time_unix":1790656447.6837478} -{"event":"rl_policy_apply_progress","island_id":1,"phase":"hash_finished","policy_version":3,"time_unix":1790656448.10875} -{"event":"rl_policy_apply","island_id":1,"optimizer_reset_count":4,"policy_version":3,"reset_parameter_count":392,"rl/global_policy_version":3,"rl/optimizer_reset_count":4,"sync/global_policy_hash":"6c1703848d95068e089eda92e134efaa9a58857da83d8c4248b02fad6235e5e1","time_unix":1790656448.1186445} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-1/miles.log b/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-1/miles.log deleted file mode 100644 index c0c630891..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-1/miles.log +++ /dev/null @@ -1,401 +0,0 @@ -/usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' - """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). -Unable to import `torchao` Tensor objects. This may affect loading checkpoints serialized with `torchao` -/usr/local/lib/python3.12/dist-packages/modelopt/torch/__init__.py:51: UserWarning: transformers 5.12.1 is not tested with current version of modelopt and may cause issues. Please install recommended version with `pip install -U nvidia-modelopt[hf]` if working with HF models. - _warnings.warn( -Loading a model with trust_remote_code=True allows arbitrary code execution from the model repository. Only use this with models you trust. -[transformers] `torch_dtype` is deprecated! Use `dtype` instead! -/root/Megatron-LM/megatron/core/transformer/transformer_config.py:1811: UserWarning: full scope is deprecated. Use empty cuda_graph_scope to capture the whole layer. - warnings.warn( -[ERROR] `cache_position` is part of Qwen3ASRThinkerTextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /usr/local/lib/python3.12/dist-packages/megatron/bridge/models/qwen3_asr/hf_qwen3_asr/modeling_qwen3_asr.py. -[ERROR] `cache_position` is part of Qwen3ASRThinkerForConditionalGeneration.forward's signature, but not documented. Make sure to add it to the docstring of the function in /usr/local/lib/python3.12/dist-packages/megatron/bridge/models/qwen3_asr/hf_qwen3_asr/modeling_qwen3_asr.py. -LANG_RL Log directory: None - > padded vocab (size: 151936) with 0 dummy tokens (new size: 151936) -[2026-09-29 04:27:15.133 main] arguments.py:40 - --tokenizer-model not set, use --hf-checkpoint as tokenizer model. -[2026-09-29 04:27:15.133 main] arguments.py:2668 - MoE-expert LoRA layout: per-expert (--experts-shared-outer-loras). -[2026-09-29 04:27:15.135 main] arguments.py:2816 - Colocate mode: defaulting --sglang-cuda-graph-backend-prefill=disabled to avoid NVLS OOM. Set --sglang-cuda-graph-backend-prefill explicitly to override. -[2026-09-29 04:27:15.136 main] arguments.py:2826 - rollout_num_gpus None != actor_num_gpus_per_node 1 * actor_num_nodes 1, overriding rollout_num_gpus to match actor_num_gpus_per_node * actor_num_nodes. -[2026-09-29 04:27:15.136 main] arguments.py:2910 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 1024 as default value. -[2026-09-29 04:27:15.136 main] arguments.py:2918 - args.rollout_max_prompt_len is not set. Use args.rollout_max_context_len - 1 (1024 - 1) as default value so that there is at least one generated token to compute loss. -using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 -Number of virtual stages per pipeline stage: None -using torch.bfloat16 for parameters ... ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - activation_func_clamp_shared_expert ............. True - activation_func_clamp_value ..................... None - actor_num_gpus_per_node ......................... 1 - actor_num_nodes ................................. 1 - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.999 - adam_eps ........................................ 1e-08 - add_bias_linear ................................. False - add_position_embedding .......................... True - add_qkv_bias .................................... False - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - advantage_estimator ............................. grpo - align_grad_reduce ............................... True - align_param_gather .............................. False - allgather_cp .................................... False - allow_ambiguous_pad_tokens ...................... False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_chat_template ............................. True - apply_chat_template_kwargs ...................... {'enable_thinking': False} - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - apply_wd_to_qk_layernorm ........................ False - async_max_concurrent_samples .................... None - async_save ...................................... False - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.unfused - attention_dropout ............................... 0.1 - attention_output_gate ........................... False - attention_softmax_in_fp32 ....................... True - auto_detect_ckpt_format ......................... False - balance_data .................................... True - barrier_with_L1_time ............................ True - batch_invariant_mode ............................ False - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - beta_fast ....................................... 32 - beta_slow ....................................... 1 - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - buffer_filter_path .............................. None - cache_mla_latents ............................... False - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - chat_template_path .............................. None - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_lora_weight_equal ......................... False - check_weight_hash_across_dp_replicas_interval ... None - check_weight_update_allow_quant_error ........... False - check_weight_update_equal ....................... False - check_weight_update_selector .................... all - check_weight_update_skip_list ................... None - ci_check_model_hash ............................. False - ci_disable_kl_checker ........................... False - ci_disable_logprobs_checker ..................... False - ci_disable_weight_update_checker ................ False - ci_enable_metrics_capture ....................... False - ci_ft_test_actions .............................. None - ci_inject_rollout_data_min_match_ratio .......... 0.9 - ci_inject_rollout_data_path ..................... None - ci_inject_rollout_data_start_rollout_id ......... None - ci_load_grad_norm ............................... None - ci_metric_checker_key ........................... None - ci_metric_checker_threshold ..................... None - ci_save_grad_norm ............................... None - ci_save_model_hash .............................. False - ci_test ......................................... False - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch_dist - ckpt_fully_parallel_load ........................ False - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - colocate ........................................ True - compress_ratios ................................. None - compute_advantages_and_returns .................. True - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - control_server_port ............................. 0 - cp_comm_type .................................... ['p2p'] - cpu_offloading_num_layers ....................... 0 - create_all_gather_group ......................... False - create_attention_mask_in_dataloader ............. True - critic_load ..................................... /root/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca - critic_lr ....................................... 0.0001 - critic_lr_warmup_iters .......................... 0 - critic_num_gpus_per_node ........................ 1 - critic_num_nodes ................................ 1 - critic_save ..................................... None - cross_entropy_fusion_impl ....................... native - cross_entropy_loss_fusion ....................... False - cuda_graph_impl ................................. none - cuda_graph_scope ................................ [] - cuda_graph_warmup_steps ......................... 3 - custom_config_path .............................. None - custom_convert_samples_to_train_data_path ....... None - custom_eval_rollout_log_function_path ........... None - custom_generate_function_path ................... None - custom_loss_function_path ....................... None - custom_megatron_before_log_prob_hook_path ....... None - custom_megatron_before_train_step_hook_path ..... None - custom_megatron_init_path ....................... None - custom_megatron_post_save_hook_path ............. None - custom_model_provider_path ...................... None - custom_pg_loss_reducer_function_path ............ None - custom_reward_post_process_path ................. None -... [truncated: 1805 of 2205 lines removed; full evidence archived outside repo] ... -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:10.032 actor_cell0_rank0] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:10.032 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=train_actor phase=start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:10.032 actor_cell0_rank0] timer.py:42 - Timer train_wait end (elapsed: 11.7s) -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:10.032 actor_cell0_rank0] timer.py:31 - Timer train start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:10.056 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:10.056 actor_cell0_rank0] timer.py:31 - Timer log_probs start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:15.283 actor_cell0_rank0] timer.py:42 - Timer log_probs end (elapsed: 5.2s) -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:15.283 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=end ok=true elapsed_s=5.2 -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:15.290 actor_cell0_rank0] log_utils.py:42 - ft op=cross_cell phase=start kind=log_gather rank=0 -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:15.291 actor_cell0_rank0] log_utils.py:48 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:15.291 actor_cell0_rank0] log_utils.py:66 - rollout 1: {'rollout/response_lengths': 252.46875, 'rollout/rewards': -9.313225746154785e-10, 'rollout/truncated': 0.15625, 'rollout/rollout_log_probs': -0.2559622824192047, 'rollout/raw_reward': 0.59375, 'rollout/total_lengths': 325.46875, 'rollout/log_probs': -0.25694024562835693, 'rollout/advantages': 1.862645149230957e-09, 'rollout/returns': 1.862645149230957e-09} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:15.291 actor_cell0_rank0] timer.py:31 - Timer actor_train start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:25.240 actor_cell0_rank0] model.py:601 - ft op=train_step rollout=1 step=0 attempt=0 outcome=NORMAL valid_step=true -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:25.241 actor_cell0_rank0] log_utils.py:460 - step 1: {'train/loss': -1.862645149230957e-09, 'train/pg_loss': -1.862645149230957e-09, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.010266735218465328, 'train/train_rollout_kl': 0.0004483942757360637, 'train/grad_norm': 0.29781785367734875, 'train/lr-pg_0': 3.333333333333334e-05, 'train/step': 1} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:25.241 actor_cell0_rank0] model.py:814 - step 1: {'train/loss': -1.862645149230957e-09, 'train/pg_loss': -1.862645149230957e-09, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.010266735218465328, 'train/train_rollout_kl': 0.0004483942757360637, 'train/grad_norm': 0.29781785367734875, 'train/lr-pg_0': 3.333333333333334e-05, 'train/step': 1} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:25.241 actor_cell0_rank0] timer.py:42 - Timer actor_train end (elapsed: 9.9s) -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:25.241 actor_cell0_rank0] timer.py:42 - Timer train end (elapsed: 15.2s) -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:25.241 actor_cell0_rank0] timer.py:31 - Timer train_wait start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:25.265 actor_cell0_rank0] train_metric_utils.py:50 - perf 1: {'perf/update_weights_time': 0.6558315753936768, 'perf/data_preprocess_time': 0.005410194396972656, 'perf/train_wait_time': 11.719826221466064, 'perf/log_probs_time': 5.2262115478515625, 'perf/actor_train_time': 9.94955062866211, 'perf/train_time': 15.208785772323608, 'perf/log_probs_tflops': 2.1496932706098515, 'perf/actor_train_tflops': 3.3875153404925307, 'perf/actor_train_tok_per_s': 1046.7809440556089, 'perf/step_time': 26.928611993789673, 'perf/wait_time_ratio': 0.43521835526349867} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:25.265 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train_actor phase=end ok=true elapsed_s=15.2 -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:25.266 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=15.2 -(SGLangEngine pid=5256) [2026-09-29 04:33:01] INFO: 172.20.13.80:41740 - "POST /update_weight_version HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:02] INFO: 172.20.13.80:20806 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:02] INFO: 172.20.13.80:54783 - "POST /pause_generation HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:02] INFO: 172.20.13.80:25546 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:02] INFO: 172.20.13.80:48799 - "POST /unload_lora_adapter HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:02] INFO: 172.20.13.80:17550 - "POST /load_lora_adapter_from_tensors HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:03] INFO: 172.20.13.80:57082 - "POST /continue_generation HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:03] INFO: 172.20.13.80:44445 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:04] INFO: 172.20.13.80:37181 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:04] INFO: 172.20.13.80:64073 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:04] INFO: 172.20.13.80:48913 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:04] INFO: 172.20.13.80:53297 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:04] INFO: 172.20.13.80:21045 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:04] INFO: 172.20.13.80:53160 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:04] INFO: 172.20.13.80:39217 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:04] INFO: 172.20.13.80:22414 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:04] INFO: 172.20.13.80:64073 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:05] INFO: 172.20.13.80:22414 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:05] INFO: 172.20.13.80:37181 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:05] INFO: 172.20.13.80:48913 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:05] INFO: 172.20.13.80:53297 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:05] INFO: 172.20.13.80:53160 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:06] INFO: 172.20.13.80:64073 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:06] INFO: 172.20.13.80:39217 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:06] INFO: 172.20.13.80:22414 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:06] INFO: 172.20.13.80:21045 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:06] INFO: 172.20.13.80:37181 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:07] INFO: 172.20.13.80:53297 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:07] INFO: 172.20.13.80:48913 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:07] INFO: 172.20.13.80:64073 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:07] INFO: 172.20.13.80:53160 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:07] INFO: 172.20.13.80:39217 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:08] INFO: 172.20.13.80:21045 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:08] INFO: 172.20.13.80:53297 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:08] INFO: 172.20.13.80:22414 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:08] INFO: 172.20.13.80:37181 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:09] INFO: 172.20.13.80:48913 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:09] INFO: 172.20.13.80:53160 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:09] INFO: 172.20.13.80:64073 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:09] INFO: 172.20.13.80:39217 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:09] INFO: 172.20.13.80:43465 - "POST /abort_request HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:09] INFO: 172.20.13.80:57349 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:10] INFO: 172.20.13.80:23313 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:15] INFO: 172.20.13.80:24437 - "GET /health HTTP/1.1" 200 OK -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"begin","policy_version":2} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"apply_finished","policy_version":2} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_begin","policy_version":2} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_finished","policy_version":2} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:35.594 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before TrainRayActor.clear_memory: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 70.04, 'used_GB': 9.14, 'allocated_GB': 1.3, 'reserved_GB': 4.56} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:36.288 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after TrainRayActor.clear_memory: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 73.11, 'used_GB': 6.06, 'allocated_GB': 1.3, 'reserved_GB': 1.48} -(SGLangEngine pid=5256) [2026-09-29 04:33:36] Cache flushed successfully! -(SGLangEngine pid=5256) [2026-09-29 04:33:36] Start unload Lora adapter. Lora name=miles_lora -(SGLangEngine pid=5256) [2026-09-29 04:33:36] LoRA adapter unloading starts: LoRARef(lora_id=9825b6be2fd8437899e12ac6e3cd55ee, lora_name=miles_lora). avail mem=71.95 GB -(SGLangEngine pid=5256) [2026-09-29 04:33:36] LoRA adapter unloading completes: LoRARef(lora_id=9825b6be2fd8437899e12ac6e3cd55ee, lora_name=miles_lora). avail mem=71.95 GB -(SGLangEngine pid=5256) [2026-09-29 04:33:36] Start load Lora adapter from tensors. Lora name=miles_lora -(SGLangEngine pid=5256) [2026-09-29 04:33:36] LoRA adapter loading from tensors starts: LoRARef(lora_id=676d35978b9c4b4994fa2482df9241d7, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:36.472 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=update_weights phase=start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:36.472 actor_cell0_rank0] timer.py:31 - Timer update_weights start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:36.473 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.93, 'used_GB': 7.25, 'allocated_GB': 1.3, 'reserved_GB': 1.48} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:36.543 actor_cell0_rank0] update_weight_from_tensor.py:461 - Packed 392 LoRA tensors (20185088 bytes) into 1 flattened buckets (max 268435456 bytes/1024 tensors) -(SGLangEngine pid=5256) [2026-09-29 04:33:36] LoRA adapter loading from tensors completes: LoRARef(lora_id=676d35978b9c4b4994fa2482df9241d7, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(RolloutManager pid=1912) Rollout generation: 0%| | 0/32 [00:00 33096.0 MB (freed 0.0 MB) -(SGLangEngine pid=5256) [2026-09-29 04:33:37] LoRA adapter '676d35978b9c4b4994fa2482df9241d7': loaded weights for target modules ['down_proj', 'gate_up_proj', 'o_proj', 'qkv_proj']. -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:37.135 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.95, 'used_GB': 7.23, 'allocated_GB': 1.3, 'reserved_GB': 1.48} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:37.135 actor_cell0_rank0] timer.py:42 - Timer update_weights end (elapsed: 0.7s) -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:37.136 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.7 -(SGLangEngine pid=5256) [2026-09-29 04:33:39] Prefill batch, #new-seq: 1, #new-token: 94, #cached-token: 0, token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 2.99 -(SGLangEngine pid=5256) [2026-09-29 04:33:39] Prefill batch, #new-seq: 7, #new-token: 658, #cached-token: 0, token usage: 0.00, #running-req: 1, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 117103.19 -(SGLangEngine pid=5256) [2026-09-29 04:33:39] Decode batch, #running-req: 8, #token: 422, token usage: 0.00, cuda graph: True, gen throughput (token/s): 10.62, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:39] Decode batch, #running-req: 8, #token: 742, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1763.84, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:39] Decode batch, #running-req: 8, #token: 1062, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1712.96, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:40] Prefill batch, #new-seq: 1, #new-token: 69, #cached-token: 3, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 88.56 -(SGLangEngine pid=5256) [2026-09-29 04:33:40] Decode batch, #running-req: 8, #token: 1292, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1394.42, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:40] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 71, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 16.25 -(SGLangEngine pid=5256) [2026-09-29 04:33:40] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 71, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 9.69 -(SGLangEngine pid=5256) [2026-09-29 04:33:40] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 71, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 14.27 -(SGLangEngine pid=5256) [2026-09-29 04:33:40] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 71, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 15.59 -(SGLangEngine pid=5256) [2026-09-29 04:33:40] Decode batch, #running-req: 8, #token: 894, token usage: 0.00, cuda graph: True, gen throughput (token/s): 925.34, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:40] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 71, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 6.54 -(SGLangEngine pid=5256) [2026-09-29 04:33:40] Decode batch, #running-req: 8, #token: 998, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1426.87, #queue-req: 0 -(RolloutManager pid=1912) [2026-09-29 04:33:40.939 rollout_manager] sglang_rollout.py:473 - First rollout sample: ["<|im_start|>user\nBella bought stamps at the post office. Some of the stamps had a snowflake design, some had a truck design, and some had a rose design. Bella bought 11 snowflake stamps. She bought 9 more truck stamps than snowflake stamps, and 13 fewer rose stamps than truck stamps. How many stamps did Bella buy in all?\nPut the final numeric answer in \\boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nLet's break down the information given:\n\n- Bella bought **11** snowflake stamps.\n- She bought **9 more truck stamps** than snowflake stamps. \n So, the **number of truck stamps** = 11 + 9 = **20**.\n- She bought **13 fewer rose stamps** than truck stamps. \n So, the **number of rose stamps** = 20 - 13 = **7**.\n\nNow, add the total number of stamps:\n\n- Snowflake: 11 \n- Truck: 20 \n- Rose: 7 \nTotal = $11 + 20 + 7 = 38$\n\n### ✅ Final Answer: $\\boxed{38}$ stamps.<|im_end|>"], label: 38, reward: 1.0 -(RolloutManager pid=1912) Rollout generation: 25%|██▌ | 8/32 [00:03<00:11, 2.12it/s] -(SGLangEngine pid=5256) [2026-09-29 04:33:40] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 71, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 4.47 -(SGLangEngine pid=5256) [2026-09-29 04:33:40] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 71, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 9.00 -(SGLangEngine pid=5256) [2026-09-29 04:33:41] Decode batch, #running-req: 8, #token: 699, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1216.53, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:41] Decode batch, #running-req: 8, #token: 1019, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1690.98, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:41] Decode batch, #running-req: 8, #token: 1339, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1824.56, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:41] Prefill batch, #new-seq: 1, #new-token: 77, #cached-token: 3, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 142.64 -(SGLangEngine pid=5256) [2026-09-29 04:33:41] Decode batch, #running-req: 8, #token: 1532, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1426.58, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:41] Decode batch, #running-req: 8, #token: 1852, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1717.19, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:41] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 79, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 2.94 -(SGLangEngine pid=5256) [2026-09-29 04:33:42] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 79, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 5.84 -(SGLangEngine pid=5256) [2026-09-29 04:33:42] Decode batch, #running-req: 8, #token: 1573, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1199.02, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:42] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 79, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 4.90 -(SGLangEngine pid=5256) [2026-09-29 04:33:42] Decode batch, #running-req: 8, #token: 1599, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1392.55, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:42] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 79, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 4.56 -(SGLangEngine pid=5256) [2026-09-29 04:33:42] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 79, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 16.43 -(SGLangEngine pid=5256) [2026-09-29 04:33:42] Decode batch, #running-req: 8, #token: 1281, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1189.17, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:42] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 79, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 11.18 -(RolloutManager pid=1912) Rollout generation: 50%|█████ | 16/32 [00:05<00:05, 3.12it/s] -(SGLangEngine pid=5256) [2026-09-29 04:33:42] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 79, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 10.65 -(SGLangEngine pid=5256) [2026-09-29 04:33:42] Decode batch, #running-req: 8, #token: 833, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1215.04, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:43] Decode batch, #running-req: 8, #token: 1153, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1703.08, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:43] Decode batch, #running-req: 8, #token: 1473, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1683.07, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:43] Decode batch, #running-req: 8, #token: 1793, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1673.79, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:43] Decode batch, #running-req: 8, #token: 2113, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1667.64, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:43] Prefill batch, #new-seq: 1, #new-token: 75, #cached-token: 3, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 79.11 -(SGLangEngine pid=5256) [2026-09-29 04:33:43] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.01, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 17.55 -(SGLangEngine pid=5256) [2026-09-29 04:33:43] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 17.77 -(SGLangEngine pid=5256) [2026-09-29 04:33:43] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 8.38 -(SGLangEngine pid=5256) [2026-09-29 04:33:43] Decode batch, #running-req: 8, #token: 1357, token usage: 0.00, cuda graph: True, gen throughput (token/s): 911.13, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:44] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 4.35 -(SGLangEngine pid=5256) [2026-09-29 04:33:44] Decode batch, #running-req: 8, #token: 1291, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1383.43, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:44] Decode batch, #running-req: 8, #token: 1611, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1673.27, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:44] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.00, #running-req: 6, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 2.71 -(SGLangEngine pid=5256) [2026-09-29 04:33:44] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 260.08 -(SGLangEngine pid=5256) [2026-09-29 04:33:44] Decode batch, #running-req: 8, #token: 1227, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1192.78, #queue-req: 0 -(RolloutManager pid=1912) Rollout generation: 75%|███████▌ | 24/32 [00:07<00:02, 3.42it/s] -(SGLangEngine pid=5256) [2026-09-29 04:33:44] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 77, token usage: 0.00, #running-req: 7, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 3.44 -(SGLangEngine pid=5256) [2026-09-29 04:33:44] Decode batch, #running-req: 6, #token: 712, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1316.92, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:45] Decode batch, #running-req: 5, #token: 719, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1083.28, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:45] Decode batch, #running-req: 5, #token: 919, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1054.83, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:45] Decode batch, #running-req: 3, #token: 583, token usage: 0.00, cuda graph: True, gen throughput (token/s): 824.10, #queue-req: 0 -(SGLangEngine pid=5256) [2026-09-29 04:33:45] Decode batch, #running-req: 1, #token: 251, token usage: 0.00, cuda graph: True, gen throughput (token/s): 523.61, #queue-req: 0 -(RolloutManager pid=1912) Rollout generation: 100%|██████████| 32/32 [00:08<00:00, 4.49it/s] Rollout generation: 100%|██████████| 32/32 [00:08<00:00, 3.77it/s] -(RolloutManager pid=1912) [2026-09-29 04:33:45.657 rollout_manager] sglang_rollout.py:494 - Finish rollout: ["<|im_start|>user\nAnn's favorite store was having a summer clearance. For $75 she bought 5 pairs of shorts for $7 each and 2 pairs of shoes for $10 each. She also bought 4 tops, all at the same price. How much did each top cost?\nPut the final numeric answer in \\boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nLet's break down the problem step by step:\n\n### Given:\n- $75 was spent.\n- She bought:\n - **5 pairs of shorts at $7 each**: $5 × 7 = $35.\n - **2 pairs of shoes at $10 each**: $2 × 10 = $20.\n - **4 tops at the same price**: Let the cost of each top be $t. So, total for tops is $4t.\n\n### Total cost = sum of short + shoes + tops:\n\n$$\n35 + 20 + 4t = 75\n$$\n\n$$\n55 + 4t = 75\n$$\n$$\n4t = 75 - 55 = 20\n$$\n$$\nt = 5\n$$\n\n### Final answer:\n\n$$\n\\boxed{5} \\text{ dollars each top}\n$$<|im_end|>"], label: 5, reward: 1.0 -(RolloutManager pid=1912) [2026-09-29 04:33:45.658 rollout_manager] sglang_rollout.py:391 - Abort request for ['http://172.20.13.80:22100'] -(RolloutManager pid=1912) [2026-09-29 04:33:45.663 rollout_manager] rollout_data_conversion.py:42 - Final collected 32 samples from rollout to train -(RolloutManager pid=1912) [2026-09-29 04:33:45.663 rollout_manager] debug_data.py:107 - Save debug rollout data to /work/out/legacy-0/work/seed-17/yeto-federated-m2/rollouts/island-1/2.pt -(RolloutManager pid=1912) [2026-09-29 04:33:45.675 rollout_manager] metrics.py:79 - perf 2: {'rollout/response_len/mean': 258.84375, 'rollout/response_len/median': 246.0, 'rollout/response_len/max': 384, 'rollout/response_len/min': 157, 'rollout/zero_std/count_1.0': 1, 'rollout/zero_std/count_0.0': 1, 'rollout/zero_std/all_zero_percentage': 0.25, 'rollout/zero_std/all_one_percentage': 0.25, 'rollout/prefix_cache_hit_rate': 0.6165123456790124, 'rollout/avg_cached_tokens_per_sample': 49.9375, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.09375, 'perf/rollout_time': 8.505682468414307, 'perf/tokens_per_gpu_per_sec': 973.8195648331297, 'perf/longest_sample_tokens_per_sec': 45.14628913387924, 'perf/effective_tokens_per_gpu_per_sec': 973.8195648331297, 'perf/longest_effective_sample_tokens_per_sec': 45.14628913387924} -(SGLangEngine pid=5256) [2026-09-29 04:33:45] Cache flushed successfully! -(SGLangEngine pid=5256) [2026-09-29 04:33:45] Cache flushed successfully! -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:46.119 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=train phase=start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:46.120 actor_cell0_rank0] timer.py:31 - Timer data_preprocess start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:46.125 actor_cell0_rank0] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:46.125 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=train_actor phase=start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:46.125 actor_cell0_rank0] timer.py:42 - Timer train_wait end (elapsed: 20.9s) -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:46.125 actor_cell0_rank0] timer.py:31 - Timer train start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:46.149 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:46.149 actor_cell0_rank0] timer.py:31 - Timer log_probs start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:51.530 actor_cell0_rank0] timer.py:42 - Timer log_probs end (elapsed: 5.4s) -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:51.530 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=compute_log_prob phase=end ok=true elapsed_s=5.4 -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:51.537 actor_cell0_rank0] log_utils.py:42 - ft op=cross_cell phase=start kind=log_gather rank=0 -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:51.538 actor_cell0_rank0] log_utils.py:48 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:51.538 actor_cell0_rank0] log_utils.py:66 - rollout 2: {'rollout/response_lengths': 258.84375, 'rollout/rewards': -1.862645149230957e-09, 'rollout/truncated': 0.09375, 'rollout/rollout_log_probs': -0.3315589129924774, 'rollout/raw_reward': 0.6875, 'rollout/total_lengths': 339.84375, 'rollout/log_probs': -0.331943541765213, 'rollout/advantages': -5.587935447692871e-09, 'rollout/returns': -5.587935447692871e-09} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:33:51.538 actor_cell0_rank0] timer.py:31 - Timer actor_train start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:01.368 actor_cell0_rank0] model.py:601 - ft op=train_step rollout=2 step=0 attempt=0 outcome=NORMAL valid_step=true -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:01.369 actor_cell0_rank0] log_utils.py:460 - step 2: {'train/loss': 5.587935447692871e-09, 'train/pg_loss': 5.587935447692871e-09, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.011952925473451614, 'train/train_rollout_kl': 0.0005430057644844055, 'train/grad_norm': 0.2661688650334445, 'train/lr-pg_0': 0.0, 'train/step': 2} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:01.369 actor_cell0_rank0] model.py:814 - step 2: {'train/loss': 5.587935447692871e-09, 'train/pg_loss': 5.587935447692871e-09, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.011952925473451614, 'train/train_rollout_kl': 0.0005430057644844055, 'train/grad_norm': 0.2661688650334445, 'train/lr-pg_0': 0.0, 'train/step': 2} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:01.369 actor_cell0_rank0] timer.py:42 - Timer actor_train end (elapsed: 9.8s) -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:01.369 actor_cell0_rank0] timer.py:42 - Timer train end (elapsed: 15.2s) -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:01.369 actor_cell0_rank0] timer.py:31 - Timer train_wait start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:01.393 actor_cell0_rank0] train_metric_utils.py:50 - perf 2: {'perf/update_weights_time': 0.6632890701293945, 'perf/data_preprocess_time': 0.005283355712890625, 'perf/train_wait_time': 20.88415002822876, 'perf/log_probs_time': 5.380279064178467, 'perf/actor_train_time': 9.830567598342896, 'perf/train_time': 15.243664503097534, 'perf/log_probs_tflops': 2.181836894773125, 'perf/actor_train_tflops': 3.5823642680750556, 'perf/actor_train_tok_per_s': 1106.2433467049411, 'perf/step_time': 36.127814531326294, 'perf/wait_time_ratio': 0.5780629218554084} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:01.393 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train_actor phase=end ok=true elapsed_s=15.3 -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:01.394 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=15.3 -(SGLangEngine pid=5256) [2026-09-29 04:33:35] INFO: 172.20.13.80:31608 - "POST /update_weight_version HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:36] INFO: 172.20.13.80:26814 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:36] INFO: 172.20.13.80:50540 - "POST /pause_generation HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:36] INFO: 172.20.13.80:47356 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:36] INFO: 172.20.13.80:34931 - "POST /unload_lora_adapter HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:36] INFO: 172.20.13.80:63196 - "POST /load_lora_adapter_from_tensors HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:37] INFO: 172.20.13.80:36889 - "POST /continue_generation HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:37] INFO: 172.20.13.80:25461 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:40] INFO: 172.20.13.80:19959 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:40] INFO: 172.20.13.80:38035 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:40] INFO: 172.20.13.80:53405 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:40] INFO: 172.20.13.80:58516 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:40] INFO: 172.20.13.80:28773 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:40] INFO: 172.20.13.80:26234 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:40] INFO: 172.20.13.80:43418 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:40] INFO: 172.20.13.80:57703 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:41] INFO: 172.20.13.80:53405 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:41] INFO: 172.20.13.80:38035 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:41] INFO: 172.20.13.80:19959 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:42] INFO: 172.20.13.80:26234 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:42] INFO: 172.20.13.80:28773 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:42] INFO: 172.20.13.80:57703 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:42] INFO: 172.20.13.80:58516 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:42] INFO: 172.20.13.80:43418 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:43] INFO: 172.20.13.80:53405 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:43] INFO: 172.20.13.80:28773 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:43] INFO: 172.20.13.80:19959 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:43] INFO: 172.20.13.80:57703 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:44] INFO: 172.20.13.80:38035 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:44] INFO: 172.20.13.80:43418 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:44] INFO: 172.20.13.80:26234 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:44] INFO: 172.20.13.80:58516 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:44] INFO: 172.20.13.80:28773 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:44] INFO: 172.20.13.80:19959 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:44] INFO: 172.20.13.80:53405 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:45] INFO: 172.20.13.80:57703 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:45] INFO: 172.20.13.80:38035 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:45] INFO: 172.20.13.80:26234 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:45] INFO: 172.20.13.80:43418 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:45] INFO: 172.20.13.80:58516 - "POST /generate HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:45] INFO: 172.20.13.80:31967 - "POST /abort_request HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:45] INFO: 172.20.13.80:33450 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:33:46] INFO: 172.20.13.80:22030 - "POST /release_memory_occupation HTTP/1.1" 200 OK -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"begin","policy_version":3} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"apply_finished","policy_version":3} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_begin","policy_version":3} -[yeto-rl-policy-apply-progress] {"event":"rl_policy_apply_progress","phase":"hash_finished","policy_version":3} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:08.122 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before TrainRayActor.clear_memory: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 70.32, 'used_GB': 8.86, 'allocated_GB': 1.3, 'reserved_GB': 4.28} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:08.740 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after TrainRayActor.clear_memory: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 73.11, 'used_GB': 6.06, 'allocated_GB': 1.3, 'reserved_GB': 1.48} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:08.849 actor_cell0_rank0] structured_log.py:23 - ft cls=MegatronTrainRayActor fn=update_weights phase=start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:08.849 actor_cell0_rank0] timer.py:31 - Timer update_weights start -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:08.849 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.93, 'used_GB': 7.25, 'allocated_GB': 1.3, 'reserved_GB': 1.48} -(SGLangEngine pid=5256) [2026-09-29 04:34:08] Cache flushed successfully! -(SGLangEngine pid=5256) [2026-09-29 04:34:08] Start unload Lora adapter. Lora name=miles_lora -(SGLangEngine pid=5256) [2026-09-29 04:34:08] LoRA adapter unloading starts: LoRARef(lora_id=676d35978b9c4b4994fa2482df9241d7, lora_name=miles_lora). avail mem=71.95 GB -(SGLangEngine pid=5256) [2026-09-29 04:34:08] LoRA adapter unloading completes: LoRARef(lora_id=676d35978b9c4b4994fa2482df9241d7, lora_name=miles_lora). avail mem=71.95 GB -(SGLangEngine pid=5256) [2026-09-29 04:34:08] Start load Lora adapter from tensors. Lora name=miles_lora -(SGLangEngine pid=5256) [2026-09-29 04:34:08] LoRA adapter loading from tensors starts: LoRARef(lora_id=d4ef4c4ac73b4220a0403a5f82aaff34, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(SGLangEngine pid=5256) [2026-09-29 04:34:08] LoRA adapter loading from tensors completes: LoRARef(lora_id=d4ef4c4ac73b4220a0403a5f82aaff34, lora_name=miles_lora, lora_path=__tensor__, pinned=False). -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:08.920 actor_cell0_rank0] update_weight_from_tensor.py:461 - Packed 392 LoRA tensors (20185088 bytes) into 1 flattened buckets (max 268435456 bytes/1024 tensors) -(SGLangEngine pid=5256) [2026-09-29 04:34:09] [continue_generation] torch.cuda.empty_cache() called: reserved 33096.0 MB -> 33096.0 MB (freed 0.0 MB) -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:09.466 actor_cell0_rank0] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.95, 'used_GB': 7.23, 'allocated_GB': 1.3, 'reserved_GB': 1.48} -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:09.466 actor_cell0_rank0] timer.py:42 - Timer update_weights end (elapsed: 0.6s) -(MegatronTrainRayActor pid=8037) [2026-09-29 04:34:09.466 actor_cell0_rank0] structured_log.py:27 - ft cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.6 -(SGLangEngine pid=5256) [2026-09-29 04:34:08] INFO: 172.20.13.80:42735 - "POST /update_weight_version HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:34:08] INFO: 172.20.13.80:63774 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:34:08] INFO: 172.20.13.80:50042 - "POST /pause_generation HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:34:08] INFO: 172.20.13.80:44921 - "GET /flush_cache HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:34:08] INFO: 172.20.13.80:35182 - "POST /unload_lora_adapter HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:34:08] INFO: 172.20.13.80:39360 - "POST /load_lora_adapter_from_tensors HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:34:09] INFO: 172.20.13.80:22885 - "POST /continue_generation HTTP/1.1" 200 OK -(SGLangEngine pid=5256) [2026-09-29 04:34:09] INFO: 172.20.13.80:63125 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -[rl] learner 1 finalized diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/syncer.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/syncer.jsonl deleted file mode 100644 index 05630b040..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/syncer.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","sync/base_version":0,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.373,"sync/global_delta_norm":0.16508958732762427,"step":1,"fragment":0,"launch_base_version":0,"attempt":1,"gnorm":0.16508958732762427,"ms":52062,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":11672731523954912505},{"id":1,"generation":13211740740372596908}],"responded":[0,1],"responded_members":[{"id":0,"generation":11672731523954912505},{"id":1,"generation":13211740740372596908}],"missed_grace":[],"missed_members":[],"quorum_ms":51666,"grace_ms":0,"sync_ms":373,"responders":[{"id":0,"generation":11672731523954912505,"base_version":0,"staleness":0,"c_steps":1,"c_tokens":1,"weight":1,"contribution":0.5},{"id":1,"generation":13211740740372596908,"base_version":0,"staleness":0,"c_steps":1,"c_tokens":1,"weight":1,"contribution":0.5}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","sync/base_version":1,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.351,"sync/global_delta_norm":0.07335663551080543,"step":2,"fragment":0,"launch_base_version":1,"attempt":1,"gnorm":0.07335663551080543,"ms":33978,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":11672731523954912505},{"id":1,"generation":13211740740372596908}],"responded":[0,1],"responded_members":[{"id":0,"generation":11672731523954912505},{"id":1,"generation":13211740740372596908}],"missed_grace":[],"missed_members":[],"quorum_ms":33605,"grace_ms":0,"sync_ms":351,"responders":[{"id":0,"generation":11672731523954912505,"base_version":1,"staleness":0,"c_steps":1,"c_tokens":1,"weight":1,"contribution":0.5},{"id":1,"generation":13211740740372596908,"base_version":1,"staleness":0,"c_steps":1,"c_tokens":1,"weight":1,"contribution":0.5}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","sync/base_version":2,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.248,"sync/global_delta_norm":0.04354769062367688,"step":3,"fragment":0,"launch_base_version":2,"attempt":1,"gnorm":0.04354769062367688,"ms":32436,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":11672731523954912505},{"id":1,"generation":13211740740372596908}],"responded":[0,1],"responded_members":[{"id":0,"generation":11672731523954912505},{"id":1,"generation":13211740740372596908}],"missed_grace":[],"missed_members":[],"quorum_ms":32164,"grace_ms":0,"sync_ms":248,"responders":[{"id":0,"generation":11672731523954912505,"base_version":2,"staleness":0,"c_steps":1,"c_tokens":1,"weight":1,"contribution":0.5},{"id":1,"generation":13211740740372596908,"base_version":2,"staleness":0,"c_steps":1,"c_tokens":1,"weight":1,"contribution":0.5}]} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/syncer.log b/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/syncer.log deleted file mode 100644 index f811a1f98..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/syncer.log +++ /dev/null @@ -1,28 +0,0 @@ -2026-09-29T04:26:22.160751Z INFO yeto_syncer::server: syncer listening port=19408 semantic_profile_sha256="1e3a092fbeb8c06c79dcc3d771088ab76debcdc692d0507904484411c0864cd6" require_profile_binding=false -2026-09-29T04:26:22.160864Z INFO yeto_syncer::server: waiting for learners and INIT_PARAMS expected=2 -2026-09-29T04:26:22.245251Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:40635 -2026-09-29T04:32:08.439298Z INFO yeto_syncer::server: learner connected (layout: 1 fragments) learner_id=0 generation=11672731523954912505 num_streams=4 -2026-09-29T04:32:08.439370Z INFO yeto_syncer::server: initialized global state iso_backend=scalar -2026-09-29T04:32:08.521239Z INFO yeto_syncer::server: global parameters initialized -2026-09-29T04:32:08.544178Z INFO yeto_syncer::server: learner connected (layout: 1 fragments) learner_id=1 generation=13211740740372596908 num_streams=4 -2026-09-29T04:32:08.749468Z INFO yeto_syncer::server: checkpoint committed step=0 path=/work/out/legacy-0/work/seed-17/yeto-federated-m2/state.ckpt -2026-09-29T04:33:00.809252Z INFO yeto_syncer::server: checkpoint committed step=1 path=/work/out/legacy-0/work/seed-17/yeto-federated-m2/state.ckpt -2026-09-29T04:33:00.905500Z INFO yeto_syncer::server: outer step step=1 fragment=0 responders=[Member { learner_id: 0, generation: 11672731523954912505 }, Member { learner_id: 1, generation: 13211740740372596908 }] gnorm="0.1651" ms=52062 -2026-09-29T04:33:34.788137Z INFO yeto_syncer::server: checkpoint committed step=2 path=/work/out/legacy-0/work/seed-17/yeto-federated-m2/state.ckpt -2026-09-29T04:33:34.884309Z INFO yeto_syncer::server: outer step step=2 fragment=0 responders=[Member { learner_id: 0, generation: 11672731523954912505 }, Member { learner_id: 1, generation: 13211740740372596908 }] gnorm="0.0734" ms=33978 -2026-09-29T04:34:07.316105Z INFO yeto_syncer::server: checkpoint committed step=3 path=/work/out/legacy-0/work/seed-17/yeto-federated-m2/state.ckpt -2026-09-29T04:34:07.321810Z INFO yeto_syncer::server: outer step step=3 fragment=0 responders=[Member { learner_id: 0, generation: 11672731523954912505 }, Member { learner_id: 1, generation: 13211740740372596908 }] gnorm="0.0435" ms=32436 -2026-09-29T04:34:09.486232Z INFO yeto_syncer::server: learner finalized authoritative parameters learner_id=0 generation=11672731523954912505 step=3 remaining=1 -2026-09-29T04:34:09.508879Z INFO yeto_syncer::server: learner finalized authoritative parameters learner_id=1 generation=13211740740372596908 step=3 remaining=0 -2026-09-29T04:34:09.508943Z INFO yeto_syncer::server: all learners acknowledged final cut learners=2 -2026-09-29T04:34:09.508968Z INFO yeto_syncer::server: training complete after 3 outer steps -2026-09-29T04:34:09.629087Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:29719 -2026-09-29T04:34:09.631131Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:53020 -2026-09-29T04:34:09.631457Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:54429 -2026-09-29T04:34:09.631569Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:59729 -2026-09-29T04:34:09.631625Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:49319 -2026-09-29T04:34:09.647429Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:62378 -2026-09-29T04:34:09.650290Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:28979 -2026-09-29T04:34:09.650442Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:50288 -2026-09-29T04:34:09.650673Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:53887 -2026-09-29T04:34:09.650963Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:20011 diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-1/work/seed-17/yeto-federated-m2/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-1/work/seed-17/yeto-federated-m2/island-0/events.jsonl deleted file mode 100644 index cdbd1735e..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-1/work/seed-17/yeto-federated-m2/island-0/events.jsonl +++ /dev/null @@ -1,23 +0,0 @@ -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":0,"time_unix":1790656838.8473728} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":0,"time_unix":1790656839.0741708} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":0,"time_unix":1790656839.074739} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":0,"time_unix":1790656839.4250329} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":1,"policy_version":0,"reset_parameter_count":392,"rl/global_policy_version":0,"rl/optimizer_reset_count":1,"sync/global_policy_hash":"62f3b8f69c1cb9f2db1ab500f19da0dd76a5323536e6f38b0ee81bf1a486eaa6","time_unix":1790656839.446681} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.23342479765415192,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":4.582833003000019,"group_p95_seconds":6.170002610000074,"group_p99_seconds":6.245041216400086,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":0.0005554890958592296,"nonzero_advantage_count":24,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5227,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0005554890958592296,"rl/ess_ratio":1.0,"rl/global_policy_version":0,"rl/group_p50_seconds":4.582833003000019,"rl/group_p95_seconds":6.170002610000074,"rl/group_p99_seconds":6.245041216400086,"rl/local_delta_norm":0.23342479765415192,"rl/mixed_version_group_count":0,"rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_policy_version":0,"rl/rollout_seconds":6.711111673000005,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":6.711111673000005,"sync/bytes_sent":40370224,"time_unix":1790656872.154757,"tool_wait_seconds":0.0,"train_seconds":12.788346484000044,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":1,"time_unix":1790656873.7883508} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":1,"time_unix":1790656874.0882225} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":1,"time_unix":1790656874.0887327} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":1,"time_unix":1790656874.6069045} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":2,"policy_version":1,"reset_parameter_count":392,"rl/global_policy_version":1,"rl/optimizer_reset_count":2,"sync/global_policy_hash":"a4bac44b623f4e11c79353a73fbc1800a1728371053c77ba6b45885e1f5ac0f0","time_unix":1790656874.616712} -{"action_tokens":10434,"active_groups":4,"base_policy_version":1,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":5.255987343844026e-05,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":5.244702322999956,"group_p95_seconds":7.75007958664998,"group_p99_seconds":7.969439590129981,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":0.0007186654256656766,"nonzero_advantage_count":0,"pass_rate":null,"pg_loss":null,"reward_mean":0.0,"reward_std":0.0,"rl/action_tokens":10434,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0007186654256656766,"rl/ess_ratio":1.0,"rl/global_policy_version":1,"rl/group_p50_seconds":5.244702322999956,"rl/group_p95_seconds":7.75007958664998,"rl/group_p99_seconds":7.969439590129981,"rl/local_delta_norm":5.255987343844026e-05,"rl/mixed_version_group_count":0,"rl/reward_mean":0.0,"rl/reward_std":0.0,"rl/rollout_policy_version":1,"rl/rollout_seconds":8.032547635000014,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":1.0,"rollout_seconds":8.032547635000014,"sync/bytes_sent":40370224,"time_unix":1790656901.1234617,"tool_wait_seconds":0.0,"train_seconds":10.169746420000024,"train_step":null,"zero_variance_group_ratio":1.0} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":2,"time_unix":1790656901.6252182} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":2,"time_unix":1790656901.867444} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":2,"time_unix":1790656901.867843} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":2,"time_unix":1790656902.2842772} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":3,"policy_version":2,"reset_parameter_count":392,"rl/global_policy_version":2,"rl/optimizer_reset_count":3,"sync/global_policy_hash":"8211764dbc4e101fdfb11e956b96685b22c117359a2b8683d0a57f3e7337cf51","time_unix":1790656902.2940488} -{"action_tokens":8695,"active_groups":4,"base_policy_version":2,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.06244878098368645,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":4.676976443499939,"group_p95_seconds":6.954246981749918,"group_p99_seconds":7.127173984349917,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":0.0006400272832252085,"nonzero_advantage_count":24,"pass_rate":null,"pg_loss":null,"reward_mean":0.25,"reward_std":0.4330127018922193,"rl/action_tokens":8695,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0006400272832252085,"rl/ess_ratio":1.0,"rl/global_policy_version":2,"rl/group_p50_seconds":4.676976443499939,"rl/group_p95_seconds":6.954246981749918,"rl/group_p99_seconds":7.127173984349917,"rl/local_delta_norm":0.06244878098368645,"rl/mixed_version_group_count":0,"rl/reward_mean":0.25,"rl/reward_std":0.4330127018922193,"rl/rollout_policy_version":2,"rl/rollout_seconds":7.179332556000077,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":7.179332556000077,"sync/bytes_sent":40370224,"time_unix":1790656926.8776329,"tool_wait_seconds":0.0,"train_seconds":9.695245537999995,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":3,"time_unix":1790656927.3752522} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":3,"time_unix":1790656927.6150174} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":3,"time_unix":1790656927.615468} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":3,"time_unix":1790656928.0036974} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":4,"policy_version":3,"reset_parameter_count":392,"rl/global_policy_version":3,"rl/optimizer_reset_count":4,"sync/global_policy_hash":"6c1703848d95068e089eda92e134efaa9a58857da83d8c4248b02fad6235e5e1","time_unix":1790656928.0129807} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-2/work/seed-17/yeto-federated-m2/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-2/work/seed-17/yeto-federated-m2/island-0/events.jsonl deleted file mode 100644 index 7f6890121..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline-v2/legacy-2/work/seed-17/yeto-federated-m2/island-0/events.jsonl +++ /dev/null @@ -1,23 +0,0 @@ -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":0,"time_unix":1790657308.9514534} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":0,"time_unix":1790657309.1981323} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":0,"time_unix":1790657309.1985428} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":0,"time_unix":1790657309.6559272} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":1,"policy_version":0,"reset_parameter_count":392,"rl/global_policy_version":0,"rl/optimizer_reset_count":1,"sync/global_policy_hash":"62f3b8f69c1cb9f2db1ab500f19da0dd76a5323536e6f38b0ee81bf1a486eaa6","time_unix":1790657309.6779938} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.23342479765415192,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":4.452385038500097,"group_p95_seconds":6.058433104550147,"group_p99_seconds":6.134925859310158,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":0.0005554890958592296,"nonzero_advantage_count":24,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5227,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0005554890958592296,"rl/ess_ratio":1.0,"rl/global_policy_version":0,"rl/group_p50_seconds":4.452385038500097,"rl/group_p95_seconds":6.058433104550147,"rl/group_p99_seconds":6.134925859310158,"rl/local_delta_norm":0.23342479765415192,"rl/mixed_version_group_count":0,"rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_policy_version":0,"rl/rollout_seconds":6.619237706000149,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":6.619237706000149,"sync/bytes_sent":40370224,"time_unix":1790657338.2238123,"tool_wait_seconds":0.0,"train_seconds":11.795361677999836,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":1,"time_unix":1790657339.690251} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":1,"time_unix":1790657339.9883} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":1,"time_unix":1790657339.988788} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":1,"time_unix":1790657340.5261505} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":2,"policy_version":1,"reset_parameter_count":392,"rl/global_policy_version":1,"rl/optimizer_reset_count":2,"sync/global_policy_hash":"a4bac44b623f4e11c79353a73fbc1800a1728371053c77ba6b45885e1f5ac0f0","time_unix":1790657340.5356913} -{"action_tokens":10434,"active_groups":4,"base_policy_version":1,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":5.255987343844026e-05,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":5.258988534499963,"group_p95_seconds":7.80001016389998,"group_p99_seconds":8.02319408317999,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":0.0007186654256656766,"nonzero_advantage_count":0,"pass_rate":null,"pg_loss":null,"reward_mean":0.0,"reward_std":0.0,"rl/action_tokens":10434,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0007186654256656766,"rl/ess_ratio":1.0,"rl/global_policy_version":1,"rl/group_p50_seconds":5.258988534499963,"rl/group_p95_seconds":7.80001016389998,"rl/group_p99_seconds":8.02319408317999,"rl/local_delta_norm":5.255987343844026e-05,"rl/mixed_version_group_count":0,"rl/reward_mean":0.0,"rl/reward_std":0.0,"rl/rollout_policy_version":1,"rl/rollout_seconds":8.087335144000008,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":1.0,"rollout_seconds":8.087335144000008,"sync/bytes_sent":40370224,"time_unix":1790657366.949091,"tool_wait_seconds":0.0,"train_seconds":10.238627040999972,"train_step":null,"zero_variance_group_ratio":1.0} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":2,"time_unix":1790657367.4510496} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":2,"time_unix":1790657367.679691} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":2,"time_unix":1790657367.680213} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":2,"time_unix":1790657368.0917637} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":3,"policy_version":2,"reset_parameter_count":392,"rl/global_policy_version":2,"rl/optimizer_reset_count":3,"sync/global_policy_hash":"8211764dbc4e101fdfb11e956b96685b22c117359a2b8683d0a57f3e7337cf51","time_unix":1790657368.1033132} -{"action_tokens":8695,"active_groups":4,"base_policy_version":2,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.06244878098368645,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":4.919646138999951,"group_p95_seconds":7.30516994969995,"group_p99_seconds":7.485838220339958,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":0.0006400272832252085,"nonzero_advantage_count":24,"pass_rate":null,"pg_loss":null,"reward_mean":0.25,"reward_std":0.4330127018922193,"rl/action_tokens":8695,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0006400272832252085,"rl/ess_ratio":1.0,"rl/global_policy_version":2,"rl/group_p50_seconds":4.919646138999951,"rl/group_p95_seconds":7.30516994969995,"rl/group_p99_seconds":7.485838220339958,"rl/local_delta_norm":0.06244878098368645,"rl/mixed_version_group_count":0,"rl/reward_mean":0.25,"rl/reward_std":0.4330127018922193,"rl/rollout_policy_version":2,"rl/rollout_seconds":7.540405873000054,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":7.540405873000054,"sync/bytes_sent":40370224,"time_unix":1790657392.9227774,"tool_wait_seconds":0.0,"train_seconds":9.568956209999897,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":3,"time_unix":1790657393.398638} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":3,"time_unix":1790657393.6285799} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":3,"time_unix":1790657393.6291096} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":3,"time_unix":1790657394.0426667} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":4,"policy_version":3,"reset_parameter_count":392,"rl/global_policy_version":3,"rl/optimizer_reset_count":4,"sync/global_policy_hash":"6c1703848d95068e089eda92e134efaa9a58857da83d8c4248b02fad6235e5e1","time_unix":1790657394.0524035} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-0/work/seed-17/yeto-federated-m2/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-0/work/seed-17/yeto-federated-m2/island-0/events.jsonl deleted file mode 100644 index 51924e446..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-0/work/seed-17/yeto-federated-m2/island-0/events.jsonl +++ /dev/null @@ -1,23 +0,0 @@ -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":0,"time_unix":1790654053.6954086} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":0,"time_unix":1790654054.0714593} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":0,"time_unix":1790654054.0722394} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":0,"time_unix":1790654054.4615808} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":1,"policy_version":0,"reset_parameter_count":392,"rl/global_policy_version":0,"rl/optimizer_reset_count":1,"sync/global_policy_hash":"62f3b8f69c1cb9f2db1ab500f19da0dd76a5323536e6f38b0ee81bf1a486eaa6","time_unix":1790654054.487641} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":7.879927579779178e-05,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":16.021886450000068,"group_p95_seconds":17.66816700205003,"group_p99_seconds":17.744499956410024,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":0.0005554890958592296,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5227,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0005554890958592296,"rl/ess_ratio":1.0,"rl/global_policy_version":0,"rl/group_p50_seconds":16.021886450000068,"rl/group_p95_seconds":17.66816700205003,"rl/group_p99_seconds":17.744499956410024,"rl/local_delta_norm":7.879927579779178e-05,"rl/mixed_version_group_count":0,"rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_policy_version":0,"rl/rollout_seconds":18.234425749000025,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":18.234425749000025,"sync/bytes_sent":40370224,"time_unix":1790654110.9695013,"tool_wait_seconds":0.0,"train_seconds":20.237144615000034,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":1,"time_unix":1790654111.4892838} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":1,"time_unix":1790654111.8656793} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":1,"time_unix":1790654111.8663905} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":1,"time_unix":1790654112.5008895} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":2,"policy_version":1,"reset_parameter_count":392,"rl/global_policy_version":1,"rl/optimizer_reset_count":2,"sync/global_policy_hash":"c6e252269a4df367e1a7b8a0f90339e99c4b9dc2468c8c932223b951b397b414","time_unix":1790654112.5154755} -{"action_tokens":9325,"active_groups":4,"base_policy_version":1,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":5.255987343844026e-05,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":11.582322723000061,"group_p95_seconds":14.096321992500043,"group_p99_seconds":14.321643732900037,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":0.0006805541925132275,"pass_rate":null,"pg_loss":null,"reward_mean":0.25,"reward_std":0.4330127018922193,"rl/action_tokens":9325,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0006805541925132275,"rl/ess_ratio":1.0,"rl/global_policy_version":1,"rl/group_p50_seconds":11.582322723000061,"rl/group_p95_seconds":14.096321992500043,"rl/group_p99_seconds":14.321643732900037,"rl/local_delta_norm":5.255987343844026e-05,"rl/mixed_version_group_count":0,"rl/reward_mean":0.25,"rl/reward_std":0.4330127018922193,"rl/rollout_policy_version":1,"rl/rollout_seconds":14.387731841000004,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.5,"rollout_seconds":14.387731841000004,"sync/bytes_sent":40370224,"time_unix":1790654148.0394201,"tool_wait_seconds":0.0,"train_seconds":11.540224419999959,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":2,"time_unix":1790654148.551303} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":2,"time_unix":1790654148.9101825} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":2,"time_unix":1790654148.910849} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":2,"time_unix":1790654149.402238} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":3,"policy_version":2,"reset_parameter_count":392,"rl/global_policy_version":2,"rl/optimizer_reset_count":3,"sync/global_policy_hash":"3a7fa6b5831539b46532f5517828a202a10b2bd12fc1e0ebb8f7a6731e8bdbba","time_unix":1790654149.4149458} -{"action_tokens":7898,"active_groups":4,"base_policy_version":2,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":2.6147527023567818e-05,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":11.602486010000007,"group_p95_seconds":13.8215080855,"group_p99_seconds":14.024075274699998,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":0.0006458177813328803,"pass_rate":null,"pg_loss":null,"reward_mean":0.3125,"reward_std":0.46351240544347894,"rl/action_tokens":7898,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0006458177813328803,"rl/ess_ratio":1.0,"rl/global_policy_version":2,"rl/group_p50_seconds":11.602486010000007,"rl/group_p95_seconds":13.8215080855,"rl/group_p99_seconds":14.024075274699998,"rl/local_delta_norm":2.6147527023567818e-05,"rl/mixed_version_group_count":0,"rl/reward_mean":0.3125,"rl/reward_std":0.46351240544347894,"rl/rollout_policy_version":2,"rl/rollout_seconds":14.086005646999979,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":14.086005646999979,"sync/bytes_sent":40370224,"time_unix":1790654184.1763399,"tool_wait_seconds":0.0,"train_seconds":11.216469434999908,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":3,"time_unix":1790654184.6860774} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":3,"time_unix":1790654184.9808636} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":3,"time_unix":1790654184.9814508} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":3,"time_unix":1790654185.4326808} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":4,"policy_version":3,"reset_parameter_count":392,"rl/global_policy_version":3,"rl/optimizer_reset_count":4,"sync/global_policy_hash":"b473f51a351f5efb09e4e86afe4ea41702ee093a4131023215078c7bda17ff97","time_unix":1790654185.4443557} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-1/work/seed-17/yeto-federated-m2/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-1/work/seed-17/yeto-federated-m2/island-0/events.jsonl deleted file mode 100644 index d74e6da7d..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-1/work/seed-17/yeto-federated-m2/island-0/events.jsonl +++ /dev/null @@ -1,23 +0,0 @@ -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":0,"time_unix":1790654697.8941975} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":0,"time_unix":1790654698.2295082} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":0,"time_unix":1790654698.2299855} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":0,"time_unix":1790654698.7379508} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":1,"policy_version":0,"reset_parameter_count":392,"rl/global_policy_version":0,"rl/optimizer_reset_count":1,"sync/global_policy_hash":"62f3b8f69c1cb9f2db1ab500f19da0dd76a5323536e6f38b0ee81bf1a486eaa6","time_unix":1790654698.7615213} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":7.879927579779178e-05,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":5.006038565500035,"group_p95_seconds":6.64905858550004,"group_p99_seconds":6.724837917100038,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":0.0005554890958592296,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5227,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0005554890958592296,"rl/ess_ratio":1.0,"rl/global_policy_version":0,"rl/group_p50_seconds":5.006038565500035,"rl/group_p95_seconds":6.64905858550004,"rl/group_p99_seconds":6.724837917100038,"rl/local_delta_norm":7.879927579779178e-05,"rl/mixed_version_group_count":0,"rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_policy_version":0,"rl/rollout_seconds":7.223724451999942,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":7.223724451999942,"sync/bytes_sent":40370224,"time_unix":1790654736.571071,"tool_wait_seconds":0.0,"train_seconds":14.689180244999989,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":1,"time_unix":1790654737.4456966} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":1,"time_unix":1790654737.803503} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":1,"time_unix":1790654737.803944} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":1,"time_unix":1790654738.434456} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":2,"policy_version":1,"reset_parameter_count":392,"rl/global_policy_version":1,"rl/optimizer_reset_count":2,"sync/global_policy_hash":"c6e252269a4df367e1a7b8a0f90339e99c4b9dc2468c8c932223b951b397b414","time_unix":1790654738.4459343} -{"action_tokens":9325,"active_groups":4,"base_policy_version":1,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":5.255987343844026e-05,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":4.51177974999996,"group_p95_seconds":7.013383160250031,"group_p99_seconds":7.237001816850042,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":0.0006805541925132275,"pass_rate":null,"pg_loss":null,"reward_mean":0.25,"reward_std":0.4330127018922193,"rl/action_tokens":9325,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0006805541925132275,"rl/ess_ratio":1.0,"rl/global_policy_version":1,"rl/group_p50_seconds":4.51177974999996,"rl/group_p95_seconds":7.013383160250031,"rl/group_p99_seconds":7.237001816850042,"rl/local_delta_norm":5.255987343844026e-05,"rl/mixed_version_group_count":0,"rl/reward_mean":0.25,"rl/reward_std":0.4330127018922193,"rl/rollout_policy_version":1,"rl/rollout_seconds":7.302017234999994,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.5,"rollout_seconds":7.302017234999994,"sync/bytes_sent":40370224,"time_unix":1790654766.3620481,"tool_wait_seconds":0.0,"train_seconds":11.343177235999974,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":2,"time_unix":1790654766.8205655} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":2,"time_unix":1790654767.102261} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":2,"time_unix":1790654767.102806} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":2,"time_unix":1790654767.6851556} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":3,"policy_version":2,"reset_parameter_count":392,"rl/global_policy_version":2,"rl/optimizer_reset_count":3,"sync/global_policy_hash":"3a7fa6b5831539b46532f5517828a202a10b2bd12fc1e0ebb8f7a6731e8bdbba","time_unix":1790654767.6987982} -{"action_tokens":7898,"active_groups":4,"base_policy_version":2,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":2.6147527023567818e-05,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":4.322214040999825,"group_p95_seconds":6.505389649799872,"group_p99_seconds":6.702276362759876,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":0.0006458177813328803,"pass_rate":null,"pg_loss":null,"reward_mean":0.3125,"reward_std":0.46351240544347894,"rl/action_tokens":7898,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0006458177813328803,"rl/ess_ratio":1.0,"rl/global_policy_version":2,"rl/group_p50_seconds":4.322214040999825,"rl/group_p95_seconds":6.505389649799872,"rl/group_p99_seconds":6.702276362759876,"rl/local_delta_norm":2.6147527023567818e-05,"rl/mixed_version_group_count":0,"rl/reward_mean":0.3125,"rl/reward_std":0.46351240544347894,"rl/rollout_policy_version":2,"rl/rollout_seconds":6.760769797999956,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":6.760769797999956,"sync/bytes_sent":40370224,"time_unix":1790654794.5167391,"tool_wait_seconds":0.0,"train_seconds":11.066808130000027,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":3,"time_unix":1790654795.025934} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":3,"time_unix":1790654795.3186858} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":3,"time_unix":1790654795.3194234} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":3,"time_unix":1790654795.7690508} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":4,"policy_version":3,"reset_parameter_count":392,"rl/global_policy_version":3,"rl/optimizer_reset_count":4,"sync/global_policy_hash":"b473f51a351f5efb09e4e86afe4ea41702ee093a4131023215078c7bda17ff97","time_unix":1790654795.7809527} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-2/work/seed-17/yeto-federated-m2/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-2/work/seed-17/yeto-federated-m2/island-0/events.jsonl deleted file mode 100644 index 1b573db22..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-legacy-baseline/legacy-2/work/seed-17/yeto-federated-m2/island-0/events.jsonl +++ /dev/null @@ -1,23 +0,0 @@ -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":0,"time_unix":1790655301.796785} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":0,"time_unix":1790655302.2217433} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":0,"time_unix":1790655302.2222936} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":0,"time_unix":1790655302.6470468} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":1,"policy_version":0,"reset_parameter_count":392,"rl/global_policy_version":0,"rl/optimizer_reset_count":1,"sync/global_policy_hash":"62f3b8f69c1cb9f2db1ab500f19da0dd76a5323536e6f38b0ee81bf1a486eaa6","time_unix":1790655302.670006} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":7.879927579779178e-05,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":4.7189319014999,"group_p95_seconds":6.339700582549881,"group_p99_seconds":6.41398208770986,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":0.0005554890958592296,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5227,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0005554890958592296,"rl/ess_ratio":1.0,"rl/global_policy_version":0,"rl/group_p50_seconds":4.7189319014999,"rl/group_p95_seconds":6.339700582549881,"rl/group_p99_seconds":6.41398208770986,"rl/local_delta_norm":7.879927579779178e-05,"rl/mixed_version_group_count":0,"rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_policy_version":0,"rl/rollout_seconds":6.921478041000228,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":6.921478041000228,"sync/bytes_sent":40370224,"time_unix":1790655334.654608,"tool_wait_seconds":0.0,"train_seconds":13.331173118000152,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":1,"time_unix":1790655335.7822435} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":1,"time_unix":1790655336.1455321} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":1,"time_unix":1790655336.1461544} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":1,"time_unix":1790655336.6989174} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":2,"policy_version":1,"reset_parameter_count":392,"rl/global_policy_version":1,"rl/optimizer_reset_count":2,"sync/global_policy_hash":"c6e252269a4df367e1a7b8a0f90339e99c4b9dc2468c8c932223b951b397b414","time_unix":1790655336.7109444} -{"action_tokens":9325,"active_groups":4,"base_policy_version":1,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":5.255987343844026e-05,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":4.411955459000183,"group_p95_seconds":6.877420190100041,"group_p99_seconds":7.097202274020023,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":0.0006805541925132275,"pass_rate":null,"pg_loss":null,"reward_mean":0.25,"reward_std":0.4330127018922193,"rl/action_tokens":9325,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0006805541925132275,"rl/ess_ratio":1.0,"rl/global_policy_version":1,"rl/group_p50_seconds":4.411955459000183,"rl/group_p95_seconds":6.877420190100041,"rl/group_p99_seconds":7.097202274020023,"rl/local_delta_norm":5.255987343844026e-05,"rl/mixed_version_group_count":0,"rl/reward_mean":0.25,"rl/reward_std":0.4330127018922193,"rl/rollout_policy_version":1,"rl/rollout_seconds":7.161613553999814,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.5,"rollout_seconds":7.161613553999814,"sync/bytes_sent":40370224,"time_unix":1790655363.825981,"tool_wait_seconds":0.0,"train_seconds":10.899924001999807,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":2,"time_unix":1790655364.3424082} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":2,"time_unix":1790655364.642091} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":2,"time_unix":1790655364.6426854} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":2,"time_unix":1790655365.0906665} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":3,"policy_version":2,"reset_parameter_count":392,"rl/global_policy_version":2,"rl/optimizer_reset_count":3,"sync/global_policy_hash":"3a7fa6b5831539b46532f5517828a202a10b2bd12fc1e0ebb8f7a6731e8bdbba","time_unix":1790655365.1015377} -{"action_tokens":7898,"active_groups":4,"base_policy_version":2,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":2.6147527023567818e-05,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":4.317380347999915,"group_p95_seconds":6.531009911699993,"group_p99_seconds":6.732253187940014,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":0.0006458177813328803,"pass_rate":null,"pg_loss":null,"reward_mean":0.3125,"reward_std":0.46351240544347894,"rl/action_tokens":7898,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0006458177813328803,"rl/ess_ratio":1.0,"rl/global_policy_version":2,"rl/group_p50_seconds":4.317380347999915,"rl/group_p95_seconds":6.531009911699993,"rl/group_p99_seconds":6.732253187940014,"rl/local_delta_norm":2.6147527023567818e-05,"rl/mixed_version_group_count":0,"rl/reward_mean":0.3125,"rl/reward_std":0.46351240544347894,"rl/rollout_policy_version":2,"rl/rollout_seconds":6.792250227000295,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":6.792250227000295,"sync/bytes_sent":40370224,"time_unix":1790655391.3456233,"tool_wait_seconds":0.0,"train_seconds":10.630589547999989,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":3,"time_unix":1790655391.851286} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":3,"time_unix":1790655392.1357138} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":3,"time_unix":1790655392.136135} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":3,"time_unix":1790655392.5718088} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":4,"policy_version":3,"reset_parameter_count":392,"rl/global_policy_version":3,"rl/optimizer_reset_count":4,"sync/global_policy_hash":"b473f51a351f5efb09e4e86afe4ea41702ee093a4131023215078c7bda17ff97","time_unix":1790655392.5833178} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-decoupled/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-decoupled/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl deleted file mode 100644 index 3d0098c2a..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-decoupled/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl +++ /dev/null @@ -1,132 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790661661.975874} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:4003728e7d0d05e41853b0515baf2d3f0e5794476e99f7ec3de85994cf857afa","time_unix":1790661805.4349666} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":0,"time_unix":1790661809.66459} -{"event":"rl_policy_apply","island_id":0,"local_step":0,"optimizer":"reset","partial_fragment_apply":false,"policy_version":0,"sync/apply_seconds":6.778917376999971,"sync/global_policy_hash":"a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","time_unix":1790661816.4439187} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[0,0,0,0,0,0,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","rl/rollout_id":0,"rl/sync_layout_fingerprint":"16a4d88fc98c2525fbee672648d6c151c5eeb324a3ccd8ef6caeac5f17383d12","time_unix":1790661816.6011372} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790661816.7363265} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790661915.3284502} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790661915.3286586} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790661915.334346} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790661928.0014367} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790661929.0141466} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790661947.5885653} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_bcast","rollout_id":0,"time_unix":1790661948.079877} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_pull","rollout_id":0,"time_unix":1790661948.2272117} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6349465847015381,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5227,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/fragment_versions":[0,0,0,0,0,0,0,0],"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.0,"rl/mixed_version_group_count":0,"rl/policy_hash":"a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_id":0,"rl/rollout_seconds":12.671328970000104,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":12.671328970000104,"sync/applied_fragments":[],"sync/fragment_payload_bytes_received":0,"sync/fragment_payload_bytes_sent":0,"sync/submitted_fragments":[],"time_unix":1790661948.2762108,"tool_wait_seconds":0.0,"train_seconds":18.574063585999966,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[0,0,0,0,0,0,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"fd964dc2c456732bdaea1a74591656a835db05dff436a0c8ee10f1dc2bfdf245","rl/policy_token":"yeto:1:fd964dc2c456732bdaea1a74591656a835db05dff436a0c8ee10f1dc2bfdf245","rl/rollout_id":1,"rl/sync_layout_fingerprint":"16a4d88fc98c2525fbee672648d6c151c5eeb324a3ccd8ef6caeac5f17383d12","time_unix":1790661948.3508017} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":0,"sync/finalization":false,"sync/hook_seconds":0.8106617799999185,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790661948.3994422} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790661948.4659374} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:fd964dc2c456732bdaea1a74591656a835db05dff436a0c8ee10f1dc2bfdf245","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"304694beceb332a372e1c8fac3fabea923e76d3afd27bffb2f10dad8dd92a26d","time_unix":1790661949.4347904} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790661949.4351056} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790661950.7776377} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1790661960.876884} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790661961.714262} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790661976.0414054} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_bcast","rollout_id":1,"time_unix":1790661976.5407505} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_pull","rollout_id":1,"time_unix":1790661976.698289} -{"action_tokens":10958,"active_groups":4,"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.28957298398017883,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.15625,"reward_std":0.3630921887069453,"rl/action_tokens":10958,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/fragment_versions":[0,0,0,0,0,0,0,0],"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.0,"rl/mixed_version_group_count":0,"rl/policy_hash":"fd964dc2c456732bdaea1a74591656a835db05dff436a0c8ee10f1dc2bfdf245","rl/reward_mean":0.15625,"rl/reward_std":0.3630921887069453,"rl/rollout_id":1,"rl/rollout_seconds":11.440864308000073,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":11.440864308000073,"sync/applied_fragments":[],"sync/fragment_payload_bytes_received":0,"sync/fragment_payload_bytes_sent":0,"sync/submitted_fragments":[],"time_unix":1790661976.7578568,"tool_wait_seconds":0.0,"train_seconds":14.32674668300001,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[0,0,0,0,0,0,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"47f28aae136c432d4670d9583b9960ba00980ecb8835e60e0e49acb458b42b21","rl/policy_token":"yeto:2:47f28aae136c432d4670d9583b9960ba00980ecb8835e60e0e49acb458b42b21","rl/rollout_id":2,"rl/sync_layout_fingerprint":"16a4d88fc98c2525fbee672648d6c151c5eeb324a3ccd8ef6caeac5f17383d12","time_unix":1790661976.8401008} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":1,"sync/finalization":false,"sync/hook_seconds":0.8547311390000232,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790661976.8995492} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790661976.977728} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:47f28aae136c432d4670d9583b9960ba00980ecb8835e60e0e49acb458b42b21","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"8cdf46fa4eb073b831d9ae7bbfc35e04b7b768a35b2828c200db9e52237471aa","time_unix":1790661977.9632325} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790661977.9640262} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790661979.3155112} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790661984.6882503} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790661985.5102785} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790661999.2737904} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_bcast","rollout_id":2,"time_unix":1790661999.7425785} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_pull","rollout_id":2,"time_unix":1790661999.844004} -{"action_tokens":8696,"active_groups":4,"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.288250207901001,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.1875,"reward_std":0.3903123748998999,"rl/action_tokens":8696,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/fragment_versions":[0,0,0,0,0,0,0,0],"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.0,"rl/mixed_version_group_count":0,"rl/policy_hash":"47f28aae136c432d4670d9583b9960ba00980ecb8835e60e0e49acb458b42b21","rl/reward_mean":0.1875,"rl/reward_std":0.3903123748998999,"rl/rollout_id":2,"rl/rollout_seconds":6.722818243999882,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":6.722818243999882,"sync/applied_fragments":[],"sync/fragment_payload_bytes_received":0,"sync/fragment_payload_bytes_sent":0,"sync/submitted_fragments":[],"time_unix":1790661999.8937206,"tool_wait_seconds":0.0,"train_seconds":13.763063203000002,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[0,0,0,0,0,0,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"43ae8b6c0985c32b460d614a5ecfe799b73fb7cda9569c215da7b56989fb35d5","rl/policy_token":"yeto:3:43ae8b6c0985c32b460d614a5ecfe799b73fb7cda9569c215da7b56989fb35d5","rl/rollout_id":3,"rl/sync_layout_fingerprint":"16a4d88fc98c2525fbee672648d6c151c5eeb324a3ccd8ef6caeac5f17383d12","time_unix":1790661999.9651814} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":2,"sync/finalization":false,"sync/hook_seconds":0.7372495060001256,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790662000.0112374} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790662000.0785716} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:43ae8b6c0985c32b460d614a5ecfe799b73fb7cda9569c215da7b56989fb35d5","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"8649980f5c64d04236d2a81b9e4d1e1f50f0e36f908b4739e643c641a0f7f088","time_unix":1790662001.0350392} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":3,"time_unix":1790662001.0357816} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":3,"rollout_id":3,"time_unix":1790662002.554324} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":3,"time_unix":1790662007.2170777} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":3,"time_unix":1790662007.9845703} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":3,"time_unix":1790662021.694227} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_bcast","rollout_id":3,"time_unix":1790662022.160217} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_pull","rollout_id":3,"time_unix":1790662022.251542} -{"action_tokens":6850,"active_groups":4,"base_policy_version":3,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.26714133142730256,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.2023099660873413,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":4,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":6850,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/fragment_versions":[0,0,0,0,0,0,0,0],"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.26714133142730256,"rl/mixed_version_group_count":0,"rl/policy_hash":"43ae8b6c0985c32b460d614a5ecfe799b73fb7cda9569c215da7b56989fb35d5","rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_id":3,"rl/rollout_seconds":6.180401087000064,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.75,"rollout_seconds":6.180401087000064,"sync/applied_fragments":[],"sync/fragment_payload_bytes_received":0,"sync/fragment_payload_bytes_sent":10092544,"sync/submitted_fragments":[0,1],"time_unix":1790662022.3212,"tool_wait_seconds":0.0,"train_seconds":13.70896151199986,"train_step":null,"zero_variance_group_ratio":0.75} -{"base_version":0,"c_steps":4,"c_tokens":31731,"delta_l2_norm":0.18051262199878693,"event":"rl_fragment_push","fragment_id":0,"global_step":1,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":212.8136930390001,"realized_h":4,"round_attempt":1,"time_unix":1790662022.3214731} -{"base_version":0,"c_steps":4,"c_tokens":31731,"delta_l2_norm":0.1969255805015564,"event":"rl_fragment_push","fragment_id":1,"global_step":2,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":212.82039076299998,"realized_h":4,"round_attempt":1,"time_unix":1790662022.321694} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[0,0,0,0,0,0,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"34a8cfa72fb049f0170a012e6d5d30b9825a4a2ec30a46c62c825f6f01b2296a","rl/policy_token":"yeto:4:34a8cfa72fb049f0170a012e6d5d30b9825a4a2ec30a46c62c825f6f01b2296a","rl/rollout_id":4,"rl/sync_layout_fingerprint":"16a4d88fc98c2525fbee672648d6c151c5eeb324a3ccd8ef6caeac5f17383d12","time_unix":1790662022.3917906} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":3,"sync/finalization":false,"sync/hook_seconds":0.7454866770001445,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790662022.4400158} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":4,"time_unix":1790662022.509044} -{"event":"rl_publication","island_id":0,"policy_version":4,"rl/policy_token":"yeto:4:34a8cfa72fb049f0170a012e6d5d30b9825a4a2ec30a46c62c825f6f01b2296a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"0bab614735efdb4d15c7cdc14d91f596b34eaac69a43862f73d1f4e61a3a6570","time_unix":1790662023.4591072} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":4,"time_unix":1790662023.459821} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":4,"rollout_id":4,"time_unix":1790662024.8755822} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":4,"time_unix":1790662027.4229136} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":4,"time_unix":1790662028.2963185} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":4,"time_unix":1790662041.5997849} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_bcast","rollout_id":4,"time_unix":1790662042.0828052} -{"event":"rl_driver_phase","island_id":0,"optimizer":"preserve","phase":"apply","policy_version":5,"time_unix":1790662042.1850083} -{"event":"rl_policy_apply","island_id":0,"local_step":5,"optimizer":"preserve","partial_fragment_apply":true,"policy_version":5,"sync/apply_seconds":0.9563614319999942,"sync/global_policy_hash":"332f63726660f7d4a3c0d39b584ea00dece0e06e65f899b8dc342760394a2f19","time_unix":1790662043.1419306} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_pull","rollout_id":4,"time_unix":1790662043.143025} -{"action_tokens":6160,"active_groups":4,"base_policy_version":4,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.2926517637907931,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.27874892950057983,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":5,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.84375,"reward_std":0.3630921887069453,"rl/action_tokens":6160,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/fragment_versions":[0,0,0,0,0,0,0,0],"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.2926517637907931,"rl/mixed_version_group_count":0,"rl/policy_hash":"34a8cfa72fb049f0170a012e6d5d30b9825a4a2ec30a46c62c825f6f01b2296a","rl/reward_mean":0.84375,"rl/reward_std":0.3630921887069453,"rl/rollout_id":4,"rl/rollout_seconds":3.9621566959999655,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.5,"rollout_seconds":3.9621566959999655,"sync/applied_fragments":[0,1],"sync/fragment_payload_bytes_received":10092544,"sync/fragment_payload_bytes_sent":10092544,"sync/submitted_fragments":[2,3],"time_unix":1790662043.206009,"tool_wait_seconds":0.0,"train_seconds":13.303142322999975,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_fragment_bcast","fragment_id":0,"island_id":0,"payload_bytes":5046272,"queue_seconds":19.443843270000116,"time_unix":1790662043.2063131,"version":1} -{"event":"rl_fragment_bcast","fragment_id":1,"island_id":0,"payload_bytes":5046272,"queue_seconds":19.049302517000115,"time_unix":1790662043.2065535,"version":2} -{"base_version":0,"c_steps":5,"c_tokens":37891,"delta_l2_norm":0.20194990932941437,"event":"rl_fragment_push","fragment_id":2,"global_step":3,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":20.512727367000025,"realized_h":5,"round_attempt":1,"time_unix":1790662043.2067385} -{"base_version":0,"c_steps":5,"c_tokens":37891,"delta_l2_norm":0.2118048369884491,"event":"rl_fragment_push","fragment_id":3,"global_step":4,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":20.12323509199996,"realized_h":5,"round_attempt":1,"time_unix":1790662043.20686} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[1,2,0,0,0,0,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"332f63726660f7d4a3c0d39b584ea00dece0e06e65f899b8dc342760394a2f19","rl/policy_token":"yeto:5:332f63726660f7d4a3c0d39b584ea00dece0e06e65f899b8dc342760394a2f19","rl/rollout_id":5,"rl/sync_layout_fingerprint":"16a4d88fc98c2525fbee672648d6c151c5eeb324a3ccd8ef6caeac5f17383d12","time_unix":1790662043.281812} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":4,"sync/finalization":false,"sync/hook_seconds":1.729585244999953,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790662043.3295865} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":5,"time_unix":1790662043.3982131} -{"event":"rl_publication","island_id":0,"policy_version":5,"rl/policy_token":"yeto:5:332f63726660f7d4a3c0d39b584ea00dece0e06e65f899b8dc342760394a2f19","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"96cead5682d6755cfb2c5dab71302d3a5a63b0395cb70c877c3f4e2c78881c29","time_unix":1790662044.5838895} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":5,"time_unix":1790662044.5845823} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":5,"rollout_id":5,"time_unix":1790662045.9485977} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":5,"time_unix":1790662050.6793902} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":5,"time_unix":1790662051.49549} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":5,"time_unix":1790662065.1941664} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_bcast","rollout_id":5,"time_unix":1790662065.5974762} -{"event":"rl_driver_phase","island_id":0,"optimizer":"preserve","phase":"apply","policy_version":6,"time_unix":1790662065.6872663} -{"event":"rl_policy_apply","island_id":0,"local_step":6,"optimizer":"preserve","partial_fragment_apply":true,"policy_version":6,"sync/apply_seconds":0.8635948769999686,"sync/global_policy_hash":"066651fdc660ecffd0f85820cefddc71da71f83ea29d3d517a53cede26d11165","time_unix":1790662066.551143} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_pull","rollout_id":5,"time_unix":1790662066.5522773} -{"action_tokens":7097,"active_groups":4,"base_policy_version":5,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.30898117866978375,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.32454198598861694,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":6,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.53125,"reward_std":0.4990224819584785,"rl/action_tokens":7097,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/fragment_versions":[1,2,0,0,0,0,0,0],"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.30898117866978375,"rl/mixed_version_group_count":0,"rl/policy_hash":"332f63726660f7d4a3c0d39b584ea00dece0e06e65f899b8dc342760394a2f19","rl/reward_mean":0.53125,"rl/reward_std":0.4990224819584785,"rl/rollout_id":5,"rl/rollout_seconds":6.093840952999926,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":6.093840952999926,"sync/applied_fragments":[2,3],"sync/fragment_payload_bytes_received":10092544,"sync/fragment_payload_bytes_sent":10092544,"sync/submitted_fragments":[4,5],"time_unix":1790662066.6107192,"tool_wait_seconds":0.0,"train_seconds":13.698256077999986,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_fragment_bcast","fragment_id":2,"island_id":0,"payload_bytes":5046272,"queue_seconds":22.094626294000136,"time_unix":1790662066.611191,"version":3} -{"event":"rl_fragment_bcast","fragment_id":3,"island_id":0,"payload_bytes":5046272,"queue_seconds":21.73545278000006,"time_unix":1790662066.6114156,"version":4} -{"base_version":0,"c_steps":6,"c_tokens":44988,"delta_l2_norm":0.20895451307296753,"event":"rl_fragment_push","fragment_id":4,"global_step":5,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":23.05711911399999,"realized_h":6,"round_attempt":1,"time_unix":1790662066.6115549} -{"base_version":0,"c_steps":6,"c_tokens":44988,"delta_l2_norm":0.22761234641075134,"event":"rl_fragment_push","fragment_id":5,"global_step":6,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":12.802919532000033,"realized_h":6,"round_attempt":1,"time_unix":1790662066.6116722} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[1,2,3,4,0,0,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"066651fdc660ecffd0f85820cefddc71da71f83ea29d3d517a53cede26d11165","rl/policy_token":"yeto:6:066651fdc660ecffd0f85820cefddc71da71f83ea29d3d517a53cede26d11165","rl/rollout_id":6,"rl/sync_layout_fingerprint":"16a4d88fc98c2525fbee672648d6c151c5eeb324a3ccd8ef6caeac5f17383d12","time_unix":1790662066.6834242} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":5,"sync/finalization":false,"sync/hook_seconds":1.5391894280000997,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790662066.7335715} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":6,"time_unix":1790662066.8039494} -{"event":"rl_publication","island_id":0,"policy_version":6,"rl/policy_token":"yeto:6:066651fdc660ecffd0f85820cefddc71da71f83ea29d3d517a53cede26d11165","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"bb626d0154b7877825ad06c089690d8cbbfc2721c42e9cf1a96e89a5c9825ea1","time_unix":1790662067.630931} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":6,"time_unix":1790662067.6315618} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":6,"rollout_id":6,"time_unix":1790662069.1741652} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":6,"time_unix":1790662071.7594516} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":6,"time_unix":1790662072.567845} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":6,"time_unix":1790662086.224285} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_bcast","rollout_id":6,"time_unix":1790662086.6732843} -{"event":"rl_driver_phase","island_id":0,"optimizer":"preserve","phase":"apply","policy_version":7,"time_unix":1790662086.8046348} -{"event":"rl_policy_apply","island_id":0,"local_step":7,"optimizer":"preserve","partial_fragment_apply":true,"policy_version":7,"sync/apply_seconds":0.8987144079999325,"sync/global_policy_hash":"3584b366d21c0f50793b4c79e0a95209d4acd5ddca33e18cbe8230967d17f435","time_unix":1790662087.7037473} -{"event":"rl_driver_phase","island_id":0,"phase":"drain_pull","rollout_id":6,"time_unix":1790662087.704985} -{"action_tokens":7190,"active_groups":4,"base_policy_version":6,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.3133593014546144,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.25402066111564636,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":7,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.65625,"reward_std":0.47495887979908324,"rl/action_tokens":7190,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/fragment_versions":[1,2,3,4,0,0,0,0],"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.3133593014546144,"rl/mixed_version_group_count":0,"rl/policy_hash":"066651fdc660ecffd0f85820cefddc71da71f83ea29d3d517a53cede26d11165","rl/reward_mean":0.65625,"rl/reward_std":0.47495887979908324,"rl/rollout_id":6,"rl/rollout_seconds":4.126914657000043,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.75,"rollout_seconds":4.126914657000043,"sync/applied_fragments":[4,5],"sync/fragment_payload_bytes_received":10092544,"sync/fragment_payload_bytes_sent":10092544,"sync/submitted_fragments":[6,7],"time_unix":1790662087.7810366,"tool_wait_seconds":0.0,"train_seconds":13.6561185999999,"train_step":null,"zero_variance_group_ratio":0.75} -{"event":"rl_fragment_bcast","fragment_id":4,"island_id":0,"payload_bytes":5046272,"queue_seconds":19.747510511999963,"time_unix":1790662087.7814207,"version":5} -{"event":"rl_fragment_bcast","fragment_id":5,"island_id":0,"payload_bytes":5046272,"queue_seconds":19.376736529000027,"time_unix":1790662087.7818384,"version":6} -{"base_version":0,"c_steps":7,"c_tokens":52178,"delta_l2_norm":0.20997501909732819,"event":"rl_fragment_push","fragment_id":6,"global_step":7,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":20.78637611299996,"realized_h":7,"round_attempt":1,"time_unix":1790662087.7822418} -{"base_version":0,"c_steps":7,"c_tokens":52178,"delta_l2_norm":0.23260383307933807,"event":"rl_fragment_push","fragment_id":7,"global_step":8,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":9.89908876200002,"realized_h":7,"round_attempt":1,"time_unix":1790662087.782669} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[1,2,3,4,5,6,0,0],"rl/mixed_version_group_count":0,"rl/policy_hash":"3584b366d21c0f50793b4c79e0a95209d4acd5ddca33e18cbe8230967d17f435","rl/policy_token":"yeto:7:3584b366d21c0f50793b4c79e0a95209d4acd5ddca33e18cbe8230967d17f435","rl/rollout_id":7,"rl/sync_layout_fingerprint":"16a4d88fc98c2525fbee672648d6c151c5eeb324a3ccd8ef6caeac5f17383d12","time_unix":1790662087.8772635} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":6,"sync/finalization":false,"sync/hook_seconds":1.7282741179999448,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790662087.9528596} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":7,"time_unix":1790662088.032796} -{"event":"rl_publication","island_id":0,"policy_version":7,"rl/policy_token":"yeto:7:3584b366d21c0f50793b4c79e0a95209d4acd5ddca33e18cbe8230967d17f435","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"4ddbf0c03a541dbaa9f4131a396b0b6acab52824b06262f698ba0ec82664f1e1","time_unix":1790662089.0763216} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":7,"time_unix":1790662089.0775568} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":7,"rollout_id":7,"time_unix":1790662090.4491973} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":7,"time_unix":1790662095.3102226} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":7,"time_unix":1790662096.071774} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":7,"time_unix":1790662109.7230885} -{"action_tokens":7302,"active_groups":4,"base_policy_version":7,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.2355746642350831,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.27034592628479004,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":8,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.78125,"reward_std":0.41339864235384227,"rl/action_tokens":7302,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/fragment_versions":[1,2,3,4,5,6,0,0],"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.2355746642350831,"rl/mixed_version_group_count":0,"rl/policy_hash":"3584b366d21c0f50793b4c79e0a95209d4acd5ddca33e18cbe8230967d17f435","rl/reward_mean":0.78125,"rl/reward_std":0.41339864235384227,"rl/rollout_id":7,"rl/rollout_seconds":6.231811139999991,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.5,"rollout_seconds":6.231811139999991,"sync/applied_fragments":[],"sync/fragment_payload_bytes_received":116064256,"sync/fragment_payload_bytes_sent":40370176,"sync/submitted_fragments":[0,1,2,3,4,5,6,7],"time_unix":1790662115.294347,"tool_wait_seconds":0.0,"train_seconds":13.650986310000008,"train_step":null,"zero_variance_group_ratio":0.5} -{"base_version":1,"c_steps":8,"c_tokens":59480,"delta_l2_norm":0.03198469802737236,"event":"rl_fragment_push","fragment_id":0,"global_step":9,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":0.03618706299994301,"realized_h":8,"round_attempt":1,"time_unix":1790662115.294799} -{"base_version":2,"c_steps":8,"c_tokens":59480,"delta_l2_norm":0.03160163760185242,"event":"rl_fragment_push","fragment_id":1,"global_step":10,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":0.015402579000010519,"realized_h":8,"round_attempt":1,"time_unix":1790662115.2950153} -{"base_version":3,"c_steps":8,"c_tokens":59480,"delta_l2_norm":0.015392216853797436,"event":"rl_fragment_push","fragment_id":2,"global_step":11,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":0.02160272299988719,"realized_h":8,"round_attempt":1,"time_unix":1790662115.2951565} -{"base_version":4,"c_steps":8,"c_tokens":59480,"delta_l2_norm":0.015319818630814552,"event":"rl_fragment_push","fragment_id":3,"global_step":12,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":0.013909099000102287,"realized_h":8,"round_attempt":1,"time_unix":1790662115.2952864} -{"base_version":5,"c_steps":8,"c_tokens":59480,"delta_l2_norm":0.005148519296199083,"event":"rl_fragment_push","fragment_id":4,"global_step":13,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":0.01808013300001221,"realized_h":8,"round_attempt":1,"time_unix":1790662115.2954025} -{"base_version":6,"c_steps":8,"c_tokens":59480,"delta_l2_norm":0.00498935068026185,"event":"rl_fragment_push","fragment_id":5,"global_step":14,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":0.005694376000064949,"realized_h":8,"round_attempt":1,"time_unix":1790662115.2955062} -{"base_version":7,"c_steps":8,"c_tokens":59480,"delta_l2_norm":0.15352964401245117,"event":"rl_fragment_push","fragment_id":6,"global_step":15,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":0.012678095999945072,"realized_h":8,"round_attempt":1,"time_unix":1790662115.2955935} -{"base_version":8,"c_steps":8,"c_tokens":59480,"delta_l2_norm":0.17140410840511322,"event":"rl_fragment_push","fragment_id":7,"global_step":16,"island_id":0,"payload_bytes":5046272,"pull_to_push_seconds":0.01689290300009816,"realized_h":8,"round_attempt":1,"time_unix":1790662115.2956688} -{"event":"rl_driver_phase","island_id":0,"optimizer":"preserve","phase":"apply","policy_version":8,"time_unix":1790662115.2957792} -{"event":"rl_policy_apply","island_id":0,"local_step":8,"optimizer":"preserve","partial_fragment_apply":true,"policy_version":8,"sync/apply_seconds":0.9005988700000671,"sync/global_policy_hash":"41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60","time_unix":1790662116.1967976} -{"event":"rl_policy_snapshot","island_id":0,"rl/canonical_layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","rl/fragment_versions":[9,10,11,12,13,14,15,16],"rl/mixed_version_group_count":0,"rl/policy_hash":"41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60","rl/policy_token":"yeto:8:41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60","rl/rollout_id":8,"rl/sync_layout_fingerprint":"16a4d88fc98c2525fbee672648d6c151c5eeb324a3ccd8ef6caeac5f17383d12","time_unix":1790662116.2694628} -{"event":"rl_sync_hook","island_id":0,"rl/rollout_id":7,"sync/finalization":true,"sync/hook_seconds":6.594787904999976,"sync/remote_quorum_wait_seconds":0.0,"time_unix":1790662116.3181975} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":8,"time_unix":1790662116.3856063} -{"event":"rl_publication","island_id":0,"policy_version":8,"rl/policy_token":"yeto:8:41cccefa43adb187d13e003b9bc14fc9ce12951fbef9bc2106c13c0412857e60","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"ee1a0457706ee7d52ef631d7d5aca82f0c86d569c4a40b0cdc81da5e704d91c9","time_unix":1790662117.3242657} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":8,"time_unix":1790662117.3251202} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-kill44/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-kill44/island-0/events.jsonl deleted file mode 100644 index 8cf51a851..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-kill44/island-0/events.jsonl +++ /dev/null @@ -1,65 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790662207.4961364} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:65459e0ca221ab029cddd395952dbec09af5baee2a613cfeca5f5dbee40cff1f","time_unix":1790662331.433197} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":0,"time_unix":1790662334.6695364} -{"event":"rl_policy_apply","island_id":0,"local_step":0,"optimizer":"reset","partial_fragment_apply":false,"policy_version":0,"sync/apply_seconds":110.05826830199999,"sync/global_policy_hash":"a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","time_unix":1790662444.7283957} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790662444.7987933} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790662508.956398} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790662508.956604} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790662508.962987} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790662522.6238184} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790662523.3909926} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790662545.1685913} -{"event":"rl_driver_phase","island_id":0,"phase":"export_push","rollout_id":0,"time_unix":1790662545.168788} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.023378854617476463,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6349465847015381,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5227,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":0,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.023378854617476463,"rl/mixed_version_group_count":0,"rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_policy_version":0,"rl/rollout_seconds":13.666169958999944,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":13.666169958999944,"sync/bytes_sent":40370224,"time_unix":1790662546.1433687,"tool_wait_seconds":0.0,"train_seconds":21.777085086999932,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"wait_global","policy_version":1,"time_unix":1790662546.1925056} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":1,"time_unix":1790662546.6043897} -{"event":"rl_policy_apply","island_id":0,"local_step":1,"optimizer":"reset","partial_fragment_apply":false,"policy_version":1,"sync/apply_seconds":58.43521483199993,"sync/global_policy_hash":"f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a","time_unix":1790662605.0401735} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790662605.1736767} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"9b28cafc782f0959717e92aef89a30a92c3384a708cb818791ccb23937a34b9c","time_unix":1790662606.5676436} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790662606.5681338} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790662608.204752} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1790662618.256215} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790662619.2402315} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790662636.5306635} -{"event":"rl_driver_phase","island_id":0,"phase":"export_push","rollout_id":1,"time_unix":1790662636.5310614} -{"action_tokens":6022,"active_groups":4,"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.020606035366654396,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.4057203531265259,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.375,"reward_std":0.4841229182759271,"rl/action_tokens":6022,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":1,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.020606035366654396,"rl/mixed_version_group_count":0,"rl/reward_mean":0.375,"rl/reward_std":0.4841229182759271,"rl/rollout_policy_version":1,"rl/rollout_seconds":11.68633121699986,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":11.68633121699986,"sync/bytes_sent":40370224,"time_unix":1790662637.7560706,"tool_wait_seconds":0.0,"train_seconds":17.289577602999998,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"wait_global","policy_version":2,"time_unix":1790662637.826045} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":2,"time_unix":1790662638.489505} -{"event":"rl_policy_apply","island_id":0,"local_step":2,"optimizer":"reset","partial_fragment_apply":false,"policy_version":2,"sync/apply_seconds":54.98461539300001,"sync/global_policy_hash":"a7f6ab7b8c7c26eef71d9ff5e3a821ac9050b5250fcef17d082eedcff387af3b","time_unix":1790662693.4746585} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790662693.5477712} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:a7f6ab7b8c7c26eef71d9ff5e3a821ac9050b5250fcef17d082eedcff387af3b","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"9d1ff53c0f4af7950a2f13dac6d86a11b779bb5cd686eb7370a7dea3b7e492f8","time_unix":1790662694.680788} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790662694.6814787} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790662696.056905} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790662703.7853558} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790662704.6508791} -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790662762.993801} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:65459e0ca221ab029cddd395952dbec09af5baee2a613cfeca5f5dbee40cff1f","time_unix":1790662891.8512304} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":2,"time_unix":1790662894.613282} -{"event":"rl_policy_apply","island_id":0,"local_step":2,"optimizer":"reset","partial_fragment_apply":false,"policy_version":2,"sync/apply_seconds":72.72510173299997,"sync/global_policy_hash":"a7f6ab7b8c7c26eef71d9ff5e3a821ac9050b5250fcef17d082eedcff387af3b","time_unix":1790662967.338877} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790662967.414779} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:a7f6ab7b8c7c26eef71d9ff5e3a821ac9050b5250fcef17d082eedcff387af3b","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"9d1ff53c0f4af7950a2f13dac6d86a11b779bb5cd686eb7370a7dea3b7e492f8","time_unix":1790663034.7451336} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790663034.7454534} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790663034.7529602} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790663047.5473266} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790663048.2983708} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790663072.6010125} -{"event":"rl_driver_phase","island_id":0,"phase":"export_push","rollout_id":2,"time_unix":1790663072.6027026} -{"action_tokens":5398,"active_groups":4,"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.013470642268657684,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.23495541512966156,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.875,"reward_std":0.33071891388307384,"rl/action_tokens":5398,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":2,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.013470642268657684,"rl/mixed_version_group_count":0,"rl/reward_mean":0.875,"rl/reward_std":0.33071891388307384,"rl/rollout_policy_version":2,"rl/rollout_seconds":12.8009665579998,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.5,"rollout_seconds":12.8009665579998,"sync/bytes_sent":40370224,"time_unix":1790663073.6320696,"tool_wait_seconds":0.0,"train_seconds":24.30142296599979,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"wait_global","policy_version":3,"time_unix":1790663073.6806943} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":3,"time_unix":1790663074.251179} -{"event":"rl_policy_apply","island_id":0,"local_step":3,"optimizer":"reset","partial_fragment_apply":false,"policy_version":3,"sync/apply_seconds":72.23386172599976,"sync/global_policy_hash":"65107a5aecf20f43c1055315bc037632b55026d42f255aade8b5993cd7af0030","time_unix":1790663146.4858012} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790663146.5603397} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:65107a5aecf20f43c1055315bc037632b55026d42f255aade8b5993cd7af0030","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"920299bb2ff71bff3e009ae265c851a741878bc5a2ea06b7a84f815f28c9c6d2","time_unix":1790663147.8401535} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":3,"time_unix":1790663147.8405952} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":3,"rollout_id":3,"time_unix":1790663149.1171358} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":3,"time_unix":1790663156.6488094} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":3,"time_unix":1790663157.4970546} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":3,"time_unix":1790663172.8486276} -{"event":"rl_driver_phase","island_id":0,"phase":"export_push","rollout_id":3,"time_unix":1790663172.848831} -{"action_tokens":6066,"active_groups":4,"base_policy_version":3,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0069558643735945225,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.31687822937965393,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":4,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.375,"reward_std":0.4841229182759271,"rl/action_tokens":6066,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":3,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.0069558643735945225,"rl/mixed_version_group_count":0,"rl/reward_mean":0.375,"rl/reward_std":0.4841229182759271,"rl/rollout_policy_version":3,"rl/rollout_seconds":8.80661007000026,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.5,"rollout_seconds":8.80661007000026,"sync/bytes_sent":40370224,"time_unix":1790663173.830754,"tool_wait_seconds":0.0,"train_seconds":15.351141071999791,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"wait_global","policy_version":4,"time_unix":1790663173.8757517} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":4,"time_unix":1790663174.388725} -{"event":"rl_policy_apply","island_id":0,"local_step":4,"optimizer":"reset","partial_fragment_apply":false,"policy_version":4,"sync/apply_seconds":49.02041566200023,"sync/global_policy_hash":"dd7b6da08bac2f25de6eeba21ba64944858e597fe2b00417a583640e23afb109","time_unix":1790663223.409577} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":4,"time_unix":1790663223.47914} -{"event":"rl_publication","island_id":0,"policy_version":4,"rl/policy_token":"yeto:4:dd7b6da08bac2f25de6eeba21ba64944858e597fe2b00417a583640e23afb109","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"446f9b7e5ac6e5e5b6eae2604a23e20c4869ff094f9a78fdf005743ffc087c65","time_unix":1790663224.659809} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":4,"time_unix":1790663224.660517} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-badtoken/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-badtoken/island-0/events.jsonl deleted file mode 100644 index 85e29ed78..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-badtoken/island-0/events.jsonl +++ /dev/null @@ -1,14 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790661230.1878252} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:f2ba0999a5d458ea3e89a2b762b548987d9308c3868d03604e4b3f2f9887a074","time_unix":1790661358.7389414} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790661361.783984} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790661460.2278163} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790661460.228006} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790661460.2345388} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790661473.026894} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790661473.7669528} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790661495.0820715} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6349465847015381,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rollout_seconds":12.798782419000077,"time_unix":1790661495.6773376,"tool_wait_seconds":0.0,"train_seconds":21.314716335999947,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790661495.750921} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"9b28cafc782f0959717e92aef89a30a92c3384a708cb818791ccb23937a34b9c","time_unix":1790661497.01806} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790661497.0183842} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790661498.4125168} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-normal/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-normal/island-0/events.jsonl deleted file mode 100644 index ad22f273e..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-normal/island-0/events.jsonl +++ /dev/null @@ -1,29 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790660784.062864} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:9f9a0d07c72038c04c7ebf78417a9ada540c3976fec93e7ac685ddf58aa06b01","time_unix":1790660914.025611} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790660917.2202675} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790661055.3718283} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790661055.372021} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790661055.378886} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790661072.718549} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790661073.4916792} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790661108.4249067} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6349465847015381,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rollout_seconds":17.346452447000047,"time_unix":1790661109.0282042,"tool_wait_seconds":0.0,"train_seconds":34.932813904,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790661109.1035717} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"9b28cafc782f0959717e92aef89a30a92c3384a708cb818791ccb23937a34b9c","time_unix":1790661110.4918935} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790661114.2183723} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790661115.6668513} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1790661123.0622134} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790661124.0597794} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790661139.764844} -{"action_tokens":6022,"active_groups":4,"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.4057203531265259,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.375,"reward_std":0.4841229182759271,"rollout_seconds":8.843925323000008,"time_unix":1790661140.1862457,"tool_wait_seconds":0.0,"train_seconds":15.704690583000001,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790661140.259472} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:372a77f480c439518a3226df373c0aa5a4639a8e5e7a85d6a6ca7a51fcb1c212","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"8d4d0c338efef8b48de7e7cb6e84ceb058119d1868eec507393116e8307ace3b","time_unix":1790661141.4054449} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790661141.4061847} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790661142.7315474} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790661150.0619414} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790661151.060569} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790661167.6473494} -{"action_tokens":9426,"active_groups":4,"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.26242998242378235,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.125,"reward_std":0.3307189138830738,"rollout_seconds":8.655774463999933,"time_unix":1790661168.074638,"tool_wait_seconds":0.0,"train_seconds":16.58640750400002,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790661168.1488793} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:635ac0788eff8f0da32c7712fe7a03c22859322a8f2f8dc7e8521b894ff02c50","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"3dd463fca284b832fc751fd3bdedd72a8a057ef94c81117441180036bcc4f4ac","time_unix":1790661169.266169} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":3,"time_unix":1790661169.2668014} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-normal/probe.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-normal/probe.jsonl deleted file mode 100644 index 3adff5c90..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-normal/probe.jsonl +++ /dev/null @@ -1,13 +0,0 @@ -{"probe": "installed", "t": 1790660721.6751816, "mode": "normal"} -{"probe": "publication", "t": 1790661055.3678768, "policy_version": 0, "manifest": {"publication_mode": "full", "base_policy_version": null, "target_policy_version": 0, "target_policy_hash": "a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4", "target_manifest_hash": "8643a174f895fdbb1b759d87eec9269141fcd6683e9d9f1c2965644f7e6c9018", "payload_hash": "373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f", "payload_bytes": 40370176, "complete": true}, "members": ["engine:inference-engine-all-0-0-00000"], "engine_checksum_digest": {"http://172.20.27.255:20103": "39bcef4e942c896982a0c10ca6f5b58f0f1a4173ec483a5d0ab8d9db81ce7759"}, "engines_agree": true, "manifest_hash_matches_engine_checksums": true, "checksum_sample": {"http://172.20.27.255:20103": {"rank0/model.embed_tokens.weight": "e3eed1375025f42d", "rank0/model.layers.0.input_layernorm.weight": "b5d50b7139f03ecb", "rank0/model.layers.0.mlp.down_proj.weight": "5f6f15bcba4c13dd"}}} -{"probe": "rollout_handle", "t": 1790661072.7180068, "rollout_id": 0, "payload_type": "RolloutDataPack", "payload_repr": "RolloutDataPack(sample_indices=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], data_ref=[_RayStoreObjectRef(backend='ray', payload=ObjectRef(00a7a58e80412ec617cdc7c6644e120fbd62d2d40200000001e1f505))])", "payload_scan": {"tensors": 0, "tensor_bytes": 0, "types": ["RolloutDataPack"]}, "groups": 4, "group_scan": {"tensors": 0, "tensor_bytes": 0, "types": ["GroupMetadata", "float", "int", "str", "tuple"]}, "policy_hash": "a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4", "expected_token": "yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4", "group_tokens": ["yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4"], "rewards": [[1.0, 0.0], [0.875, 0.33071891388307384], [0.5, 0.5], [0.875, 0.33071891388307384]]} -{"probe": "receipt", "t": 1790661108.424309, "rollout_id": 0, "receipt": {"algorithm": "grpo", "learner_id": 0, "learner_generation": 0, "base_policy_version": 0, "base_policy_hash": "a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4", "input_batch_hash": "02b530a7af08118af8282c9cb193357f0480ce7415b5208d27755e792dc14ffa", "trajectory_ids": ["r0:g0:s0", "r0:g0:s1", "r0:g0:s2", "r0:g0:s3", "r0:g0:s4", "r0:g0:s5", "r0:g0:s6", "r0:g0:s7", "r0:g1:s8", "r0:g1:s9", "r0:g1:s10", "r0:g1:s11", "r0:g1:s12", "r0:g1:s13", "r0:g1:s14", "r0:g1:s15", "r0:g2:s16", "r0:g2:s17", "r0:g2:s18", "r0:g2:s19", "r0:g2:s20", "r0:g2:s21", "r0:g2:s22", "r0:g2:s23", "r0:g3:s24", "r0:g3:s25", "r0:g3:s26", "r0:g3:s27", "r0:g3:s28", "r0:g3:s29", "r0:g3:s30", "r0:g3:s31"], "trained_tokens": 5227, "optimizer_steps": 1, "optimizer_step_succeeded": true, "parameter_layout_hash": "f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3"}, "empty_fields": [], "grad_norm": 0.6349465847015381} -{"probe": "publication", "t": 1790661110.4885345, "policy_version": 1, "manifest": {"publication_mode": "full", "base_policy_version": null, "target_policy_version": 1, "target_policy_hash": "f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a", "target_manifest_hash": "ca1ee74b0485ef7f60f80964272e539d3f8b6df2b00a36e13655b18bf21c17ca", "payload_hash": "9b28cafc782f0959717e92aef89a30a92c3384a708cb818791ccb23937a34b9c", "payload_bytes": 40370176, "complete": true}, "members": ["engine:inference-engine-all-0-0-00000"], "engine_checksum_digest": {"http://172.20.27.255:20103": "39bcef4e942c896982a0c10ca6f5b58f0f1a4173ec483a5d0ab8d9db81ce7759"}, "engines_agree": true, "manifest_hash_matches_engine_checksums": true, "checksum_sample": {"http://172.20.27.255:20103": {"rank0/model.embed_tokens.weight": "e3eed1375025f42d", "rank0/model.layers.0.input_layernorm.weight": "b5d50b7139f03ecb", "rank0/model.layers.0.mlp.down_proj.weight": "5f6f15bcba4c13dd"}}} -{"probe": "raw_lora_names", "t": 1790661110.503521, "names": [["module.module.decoder.layers.0.self_attention.linear_proj.adapter.linear_in.weight", "module.module.decoder.layers.0.self_attention.linear_proj.adapter.linear_out.weight", "module.module.decoder.layers.0.self_attention.linear_qkv.adapter.adapter_q.linear_in.weight", "module.module.decoder.layers.0.self_attention.linear_qkv.adapter.adapter_q.linear_out.weight", "module.module.decoder.layers.0.self_attention.linear_qkv.adapter.adapter_k.linear_in.weight", "module.module.decoder.layers.0.self_attention.linear_qkv.adapter.adapter_k.linear_out.weight", 392]]} -{"probe": "export_apply_export", "t": 1790661114.2142007, "h1": "f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a", "h_preserve": "f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a", "h2": "f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a", "equal": true, "boundary_hash": "f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a", "n_tensors": 392, "names_sample": ["base_model.model.model.layers.0.mlp.down_proj.lora_A.weight", "base_model.model.model.layers.0.mlp.down_proj.lora_B.weight", "base_model.model.model.layers.0.mlp.gate_proj.lora_A.weight", "base_model.model.model.layers.0.mlp.gate_proj.lora_B.weight"], "canonical_prefix": true, "moments_before": [{"exp_avg": 77.81868102028966, "exp_avg_sq": 0.00040315198273788155, "n_state": 392, "steps": [1.0], "scheduler_num_steps": 32}], "moments_after_preserve": [{"exp_avg": 77.81868102028966, "exp_avg_sq": 0.00040315198273788155, "n_state": 392, "steps": [1.0], "scheduler_num_steps": 32}], "moments_after_reset": [{"exp_avg": 0.0, "exp_avg_sq": 0.0, "n_state": 392, "steps": [0.0], "scheduler_num_steps": 32}]} -{"probe": "rollout_handle", "t": 1790661123.0617294, "rollout_id": 1, "payload_type": "RolloutDataPack", "payload_repr": "RolloutDataPack(sample_indices=[32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63], data_ref=[_RayStoreObjectRef(backend='ray', payload=ObjectRef(00a7a58e80412ec617cdc7c6644e120fbd62d2d40200000002e1f505))])", "payload_scan": {"tensors": 0, "tensor_bytes": 0, "types": ["RolloutDataPack"]}, "groups": 4, "group_scan": {"tensors": 0, "tensor_bytes": 0, "types": ["GroupMetadata", "float", "int", "str", "tuple"]}, "policy_hash": "f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a", "expected_token": "yeto:1:f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a", "group_tokens": ["yeto:1:f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a"], "rewards": [[0.125, 0.33071891388307384], [0.125, 0.33071891388307384], [1.0, 0.0], [0.25, 0.4330127018922193]]} -{"probe": "receipt", "t": 1790661139.7641683, "rollout_id": 1, "receipt": {"algorithm": "grpo", "learner_id": 0, "learner_generation": 0, "base_policy_version": 1, "base_policy_hash": "f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a", "input_batch_hash": "4ee76d3f1508608cdd806fd8adb4602523599d12b306df360df51081a3483cb4", "trajectory_ids": ["r1:g4:s32", "r1:g4:s33", "r1:g4:s34", "r1:g4:s35", "r1:g4:s36", "r1:g4:s37", "r1:g4:s38", "r1:g4:s39", "r1:g5:s40", "r1:g5:s41", "r1:g5:s42", "r1:g5:s43", "r1:g5:s44", "r1:g5:s45", "r1:g5:s46", "r1:g5:s47", "r1:g6:s48", "r1:g6:s49", "r1:g6:s50", "r1:g6:s51", "r1:g6:s52", "r1:g6:s53", "r1:g6:s54", "r1:g6:s55", "r1:g7:s56", "r1:g7:s57", "r1:g7:s58", "r1:g7:s59", "r1:g7:s60", "r1:g7:s61", "r1:g7:s62", "r1:g7:s63"], "trained_tokens": 6022, "optimizer_steps": 1, "optimizer_step_succeeded": true, "parameter_layout_hash": "f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3"}, "empty_fields": [], "grad_norm": 0.4057203531265259} -{"probe": "publication", "t": 1790661141.401667, "policy_version": 2, "manifest": {"publication_mode": "full", "base_policy_version": null, "target_policy_version": 2, "target_policy_hash": "372a77f480c439518a3226df373c0aa5a4639a8e5e7a85d6a6ca7a51fcb1c212", "target_manifest_hash": "8e1e22419e4ed6779b440be908ac13d3afa002d6770ea00f694a6d208134d173", "payload_hash": "8d4d0c338efef8b48de7e7cb6e84ceb058119d1868eec507393116e8307ace3b", "payload_bytes": 40370176, "complete": true}, "members": ["engine:inference-engine-all-0-0-00000"], "engine_checksum_digest": {"http://172.20.27.255:20103": "39bcef4e942c896982a0c10ca6f5b58f0f1a4173ec483a5d0ab8d9db81ce7759"}, "engines_agree": true, "manifest_hash_matches_engine_checksums": true, "checksum_sample": {"http://172.20.27.255:20103": {"rank0/model.embed_tokens.weight": "e3eed1375025f42d", "rank0/model.layers.0.input_layernorm.weight": "b5d50b7139f03ecb", "rank0/model.layers.0.mlp.down_proj.weight": "5f6f15bcba4c13dd"}}} -{"probe": "rollout_handle", "t": 1790661150.061181, "rollout_id": 2, "payload_type": "RolloutDataPack", "payload_repr": "RolloutDataPack(sample_indices=[64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95], data_ref=[_RayStoreObjectRef(backend='ray', payload=ObjectRef(00a7a58e80412ec617cdc7c6644e120fbd62d2d40200000003e1f505))])", "payload_scan": {"tensors": 0, "tensor_bytes": 0, "types": ["RolloutDataPack"]}, "groups": 4, "group_scan": {"tensors": 0, "tensor_bytes": 0, "types": ["GroupMetadata", "float", "int", "str", "tuple"]}, "policy_hash": "372a77f480c439518a3226df373c0aa5a4639a8e5e7a85d6a6ca7a51fcb1c212", "expected_token": "yeto:2:372a77f480c439518a3226df373c0aa5a4639a8e5e7a85d6a6ca7a51fcb1c212", "group_tokens": ["yeto:2:372a77f480c439518a3226df373c0aa5a4639a8e5e7a85d6a6ca7a51fcb1c212"], "rewards": [[0.25, 0.4330127018922193], [0.25, 0.4330127018922193], [0.0, 0.0], [0.0, 0.0]]} -{"probe": "receipt", "t": 1790661167.6467092, "rollout_id": 2, "receipt": {"algorithm": "grpo", "learner_id": 0, "learner_generation": 0, "base_policy_version": 2, "base_policy_hash": "372a77f480c439518a3226df373c0aa5a4639a8e5e7a85d6a6ca7a51fcb1c212", "input_batch_hash": "d6cd107fbee78934954a491d9a17512b381ae36684dcd6de7911ea99c66b6dfe", "trajectory_ids": ["r2:g10:s80", "r2:g10:s81", "r2:g10:s82", "r2:g10:s83", "r2:g10:s84", "r2:g10:s85", "r2:g10:s86", "r2:g10:s87", "r2:g11:s88", "r2:g11:s89", "r2:g11:s90", "r2:g11:s91", "r2:g11:s92", "r2:g11:s93", "r2:g11:s94", "r2:g11:s95", "r2:g8:s64", "r2:g8:s65", "r2:g8:s66", "r2:g8:s67", "r2:g8:s68", "r2:g8:s69", "r2:g8:s70", "r2:g8:s71", "r2:g9:s72", "r2:g9:s73", "r2:g9:s74", "r2:g9:s75", "r2:g9:s76", "r2:g9:s77", "r2:g9:s78", "r2:g9:s79"], "trained_tokens": 9426, "optimizer_steps": 1, "optimizer_step_succeeded": true, "parameter_layout_hash": "f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3"}, "empty_fields": [], "grad_norm": 0.26242998242378235} -{"probe": "publication", "t": 1790661169.2625446, "policy_version": 3, "manifest": {"publication_mode": "full", "base_policy_version": null, "target_policy_version": 3, "target_policy_hash": "635ac0788eff8f0da32c7712fe7a03c22859322a8f2f8dc7e8521b894ff02c50", "target_manifest_hash": "661416ab1dfcab0dc73854fa844d3b1d213ddd39cc2fa1a0f0638e2c6624a837", "payload_hash": "3dd463fca284b832fc751fd3bdedd72a8a057ef94c81117441180036bcc4f4ac", "payload_bytes": 40370176, "complete": true}, "members": ["engine:inference-engine-all-0-0-00000"], "engine_checksum_digest": {"http://172.20.27.255:20103": "39bcef4e942c896982a0c10ca6f5b58f0f1a4173ec483a5d0ab8d9db81ce7759"}, "engines_agree": true, "manifest_hash_matches_engine_checksums": true, "checksum_sample": {"http://172.20.27.255:20103": {"rank0/model.embed_tokens.weight": "e3eed1375025f42d", "rank0/model.layers.0.input_layernorm.weight": "b5d50b7139f03ecb", "rank0/model.layers.0.mlp.down_proj.weight": "5f6f15bcba4c13dd"}}} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-pubfail/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-pubfail/island-0/events.jsonl deleted file mode 100644 index b7dbacade..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-pubfail/island-0/events.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790661880.573514} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:02bbb4dd2e1fd21d04686c4aeb3895395d59bb4c014856d799d415bf7240b8ff","time_unix":1790662006.3756373} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790662009.2597973} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790662107.7871892} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790662107.7874901} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790662107.7932708} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790662121.5920117} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790662122.354597} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790662143.7880049} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6349465847015381,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rollout_seconds":13.804474047999975,"time_unix":1790662144.356121,"tool_wait_seconds":0.0,"train_seconds":21.43292616100007,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790662144.427762} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-zerograd/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-zerograd/island-0/events.jsonl deleted file mode 100644 index 3f02d941e..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-smoke-zerograd/island-0/events.jsonl +++ /dev/null @@ -1,9 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790661564.9442315} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:d291053645e3cf28b3afe9f15a98c111120065a4276a867a79b5ebf4c9f40fd0","time_unix":1790661689.5006535} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790661692.4617925} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790661785.8899815} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790661785.8902948} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790661785.8963113} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790661798.3900774} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790661799.137105} -{"error":"StrictRlInvariantError: rollout 0: non-zero advantages produced grad_norm 0; adapter gradients are not flowing","event":"rl_strict_failure","island_id":0,"metric":"zero_grad_norm_with_nonzero_advantages","time_unix":1790661819.7570882,"value":1} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-strict2/work/seed-17/yeto-federated-m2/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-strict2/work/seed-17/yeto-federated-m2/island-0/events.jsonl deleted file mode 100644 index 5ce3f802a..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-rerun-strict2/work/seed-17/yeto-federated-m2/island-0/events.jsonl +++ /dev/null @@ -1,43 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790660782.5601907} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:3701e6eee6c39b64d280e2fc824b5f3164fbc4885f5f205dd435e2204dee597a","time_unix":1790660926.0351243} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":0,"time_unix":1790660929.910868} -{"event":"rl_policy_apply","island_id":0,"local_step":0,"optimizer":"reset","partial_fragment_apply":false,"policy_version":0,"sync/apply_seconds":139.38732623699997,"sync/global_policy_hash":"a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","time_unix":1790661069.2985883} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790661069.3719997} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790661177.7268105} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790661177.7270353} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790661177.73493} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790661194.7002866} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790661195.4851089} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790661227.7978098} -{"event":"rl_driver_phase","island_id":0,"phase":"export_push","rollout_id":0,"time_unix":1790661227.7980087} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.2334245890378952,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6349465847015381,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5227,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":0,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.2334245890378952,"rl/mixed_version_group_count":0,"rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_policy_version":0,"rl/rollout_seconds":16.972505877000003,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":16.972505877000003,"sync/bytes_sent":40370224,"time_unix":1790661229.021104,"tool_wait_seconds":0.0,"train_seconds":32.31232855299993,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"wait_global","policy_version":1,"time_unix":1790661229.0963883} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":1,"time_unix":1790661229.8860953} -{"event":"rl_policy_apply","island_id":0,"local_step":1,"optimizer":"reset","partial_fragment_apply":false,"policy_version":1,"sync/apply_seconds":77.69501998500004,"sync/global_policy_hash":"42a3838531e620dc7029bf0a51c979f4e65ea8a052cc700e4552485c95a0cae3","time_unix":1790661307.581614} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790661307.6672778} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:42a3838531e620dc7029bf0a51c979f4e65ea8a052cc700e4552485c95a0cae3","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"b7468b340cbd6e544958f986bf3af2f0313b1962ea08bd15dd798f1ddfe594a8","time_unix":1790661308.8661437} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790661308.8664086} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790661310.600612} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1790661320.8835323} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790661321.780678} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790661336.2215614} -{"event":"rl_driver_phase","island_id":0,"phase":"export_push","rollout_id":1,"time_unix":1790661336.2217393} -{"action_tokens":10542,"active_groups":4,"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.1988547444343567,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.2512449324131012,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.09375,"reward_std":0.2914805954090255,"rl/action_tokens":10542,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":1,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.1988547444343567,"rl/mixed_version_group_count":0,"rl/reward_mean":0.09375,"rl/reward_std":0.2914805954090255,"rl/rollout_policy_version":1,"rl/rollout_seconds":12.01547340500008,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.5,"rollout_seconds":12.01547340500008,"sync/bytes_sent":40370224,"time_unix":1790661337.1354995,"tool_wait_seconds":0.0,"train_seconds":14.440565410999966,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"wait_global","policy_version":2,"time_unix":1790661337.1839821} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":2,"time_unix":1790661337.7594075} -{"event":"rl_policy_apply","island_id":0,"local_step":2,"optimizer":"reset","partial_fragment_apply":false,"policy_version":2,"sync/apply_seconds":74.15404232900005,"sync/global_policy_hash":"63b0317bb2ce36b7906fce2e3e8ceb45e4d87a67e8617a607fd5f6d5c886f3d5","time_unix":1790661411.9138918} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790661412.0432477} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:63b0317bb2ce36b7906fce2e3e8ceb45e4d87a67e8617a607fd5f6d5c886f3d5","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"2b25fc9ebfc90f83413464c9bba8abb9676b031bda2493d9c1001ada30ad5991","time_unix":1790661413.7074804} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790661413.7081587} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790661415.053981} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790661419.8531573} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790661420.6865163} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790661435.1236897} -{"event":"rl_driver_phase","island_id":0,"phase":"export_push","rollout_id":2,"time_unix":1790661435.1238806} -{"action_tokens":8736,"active_groups":4,"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.10093832015991211,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.34883660078048706,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.28125,"reward_std":0.4496092053105675,"rl/action_tokens":8736,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":2,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.10093832015991211,"rl/mixed_version_group_count":0,"rl/reward_mean":0.28125,"rl/reward_std":0.4496092053105675,"rl/rollout_policy_version":2,"rl/rollout_seconds":6.143785610000009,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":6.143785610000009,"sync/bytes_sent":40370224,"time_unix":1790661436.0688066,"tool_wait_seconds":0.0,"train_seconds":14.436819485000001,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"wait_global","policy_version":3,"time_unix":1790661436.111968} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":3,"time_unix":1790661437.0901494} -{"event":"rl_policy_apply","island_id":0,"local_step":3,"optimizer":"reset","partial_fragment_apply":false,"policy_version":3,"sync/apply_seconds":50.97697193700003,"sync/global_policy_hash":"2d30c85bbf38b14b9a93fa5a5f4ddb04a896235c2c81b9b62e27d8a83c40327a","time_unix":1790661488.0675921} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790661488.1712172} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:2d30c85bbf38b14b9a93fa5a5f4ddb04a896235c2c81b9b62e27d8a83c40327a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"07aa52c9f0dcefe3ef92b28ec359b261f3bacab706f8dae8651a7d4b9b797f59","time_unix":1790661489.2167397} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":3,"time_unix":1790661489.2174702} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-badtoken/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-badtoken/island-0/events.jsonl deleted file mode 100644 index 202d145a2..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-badtoken/island-0/events.jsonl +++ /dev/null @@ -1,14 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790650005.845268} -{"event":"rl_driver_start","execution_mode":"serial-colocated","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:1042b4ba58550335e6df64ffcb0f60adf83c9fddf45dfc195c1dfa4458fa579a","time_unix":1790650170.6290338} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790650173.762518} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790650272.906748} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790650272.907182} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790650272.9130323} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790650285.6347823} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790650286.6377661} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790650308.2272987} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6349465847015381,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rollout_seconds":12.727607785999908,"time_unix":1790650308.8301606,"tool_wait_seconds":0.0,"train_seconds":21.589066717999913,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790650308.9050484} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"9b28cafc782f0959717e92aef89a30a92c3384a708cb818791ccb23937a34b9c","time_unix":1790650310.3888452} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790650310.389324} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790650312.075056} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-normal/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-normal/island-0/events.jsonl deleted file mode 100644 index 490d53326..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-normal/island-0/events.jsonl +++ /dev/null @@ -1,29 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790649188.1251118} -{"event":"rl_driver_start","execution_mode":"serial-colocated","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:cb4f9a6bf0eb3e60868658a401014d18244d10db310d461d857109a74c90f510","time_unix":1790649320.7835104} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790649324.052898} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790649479.107763} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790649479.108097} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790649479.1148367} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790649496.1278296} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790649497.090027} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790649532.4556794} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6349465847015381,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rollout_seconds":17.019723989,"time_unix":1790649533.0543826,"tool_wait_seconds":0.0,"train_seconds":35.365249602999995,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790649533.1293464} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:f9d7b463b3c156eb2429eddbe4db6dfd6fb8d2cfe31a9bf88836d952b2de397a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"9b28cafc782f0959717e92aef89a30a92c3384a708cb818791ccb23937a34b9c","time_unix":1790649534.5730321} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790649538.4604275} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790649539.9014623} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1790649547.3866007} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790649548.6753159} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790649564.6018178} -{"action_tokens":6022,"active_groups":4,"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.4057203531265259,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.375,"reward_std":0.4841229182759271,"rollout_seconds":8.926189775000012,"time_unix":1790649565.076761,"tool_wait_seconds":0.0,"train_seconds":15.925966893000009,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790649565.1494906} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:372a77f480c439518a3226df373c0aa5a4639a8e5e7a85d6a6ca7a51fcb1c212","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"8d4d0c338efef8b48de7e7cb6e84ceb058119d1868eec507393116e8307ace3b","time_unix":1790649566.409647} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790649566.4106238} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790649567.873108} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790649575.3137164} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790649576.5078714} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790649593.1651673} -{"action_tokens":9426,"active_groups":4,"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.26242998242378235,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.125,"reward_std":0.3307189138830738,"rollout_seconds":8.903192531999991,"time_unix":1790649593.6460762,"tool_wait_seconds":0.0,"train_seconds":16.656706792999955,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790649593.7202861} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:635ac0788eff8f0da32c7712fe7a03c22859322a8f2f8dc7e8521b894ff02c50","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"3dd463fca284b832fc751fd3bdedd72a8a057ef94c81117441180036bcc4f4ac","time_unix":1790649594.9466057} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":3,"time_unix":1790649594.9474022} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-pubfail/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-pubfail/island-0/events.jsonl deleted file mode 100644 index 014b66326..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-pubfail/island-0/events.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790654833.6955497} -{"event":"rl_driver_start","execution_mode":"serial-colocated","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:02bbb4dd2e1fd21d04686c4aeb3895395d59bb4c014856d799d415bf7240b8ff","time_unix":1790654969.5356412} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790654972.9552376} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790655072.148945} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790655072.1492546} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790655072.1550176} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790655084.9311748} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790655085.66401} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790655107.9185562} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6349465847015381,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rollout_seconds":12.781890497999484,"time_unix":1790655108.4934247,"tool_wait_seconds":0.0,"train_seconds":22.254192657999738,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790655108.5601099} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-zerograd/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-zerograd/island-0/events.jsonl deleted file mode 100644 index 8b2ba4b73..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-smoke-zerograd/island-0/events.jsonl +++ /dev/null @@ -1,9 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790649664.0843573} -{"event":"rl_driver_start","execution_mode":"serial-colocated","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:5d88d5d2f64a1cbbe5fb984d64b10f77d5cae9e3250a4dcd5cc3cc21cee37d8a","time_unix":1790649794.1197073} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790649797.5746028} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790649901.1004915} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790649901.1007314} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790649901.1073651} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790649914.8048875} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790649915.666912} -{"error":"StrictRlInvariantError: rollout 0: non-zero advantages produced grad_norm 0; adapter gradients are not flowing","event":"rl_strict_failure","island_id":0,"metric":"zero_grad_norm","time_unix":1790649937.8826425,"value":1} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/benchmark.log b/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/benchmark.log deleted file mode 100644 index c40e603de..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/benchmark.log +++ /dev/null @@ -1,103 +0,0 @@ -warning: method `start_pool` is never used - --> src/iso_worker.rs:116:12 - | -108 | impl IsoBackendConfig { - | --------------------- method in this implementation -... -116 | pub fn start_pool(&self, devices: Vec, queue_capacity: usize) -> Result { - | ^^^^^^^^^^ - | - = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default - -warning: associated items `start_pool` and `flatten` are never used - --> src/iso_worker.rs:158:12 - | -134 | impl IsoBackend { - | --------------- associated items in this implementation -... -158 | pub fn start_pool( - | ^^^^^^^^^^ -... -182 | pub fn flatten(&mut self, matrix: &mut [f32], rows: usize, cols: usize) -> Result<()> { - | ^^^^^^^ - -warning: associated function `start` is never used - --> src/iso_worker.rs:518:8 - | -517 | impl TorchIsoPool { - | ----------------- associated function in this implementation -518 | fn start(python: PathBuf, devices: Vec, queue_capacity: usize) -> Result { - | ^^^^^ - -warning: function `merge_iso` is never used - --> src/merge.rs:105:8 - | -105 | pub fn merge_iso(deltas: &[&[f32]], weights: &[f64], rows: usize, cols: usize, out: &mut [f32]) { - | ^^^^^^^^^ - -warning: function `next_committable_round` is never used - --> src/server.rs:2768:4 - | -2768 | fn next_committable_round(rounds: &[Round], now: Instant) -> Option { - | ^^^^^^^^^^^^^^^^^^^^^^ - -warning: function `fragment_available` is never used - --> src/server.rs:2776:4 - | -2776 | fn fragment_available(rounds: &[Round], fragment_id: usize) -> bool { - | ^^^^^^^^^^^^^^^^^^ - -warning: function `broadcast_updated_fragment` is never used - --> src/server.rs:3463:10 - | -3463 | async fn broadcast_updated_fragment(st: &GlobalState, registry: &Registry, p: usize) -> Result<()> { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: methods `fid`, `base_version`, and `finish_inline` are never used - --> src/state.rs:190:12 - | -189 | impl PreparedMerge { - | ------------------ methods in this implementation -190 | pub fn fid(&self) -> usize { - | ^^^ -... -194 | pub fn base_version(&self) -> u64 { - | ^^^^^^^^^^^^ -... -241 | fn finish_inline(self) -> Result { - | ^^^^^^^^^^^^^ - -warning: associated items `new_with_layout_fingerprint` and `merge_and_step` are never used - --> src/state.rs:259:12 - | -253 | impl GlobalState { - | ---------------- associated items in this implementation -... -259 | pub fn new_with_layout_fingerprint( - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -... -589 | pub fn merge_and_step( - | ^^^^^^^^^^^^^^ - -yeto-federated-m2: 2 island(s) x 1 GPU, 4 groups/round -PLAN_JSON {"arms": [{"benchmark_islands": 2, "fragments": 1, "gpus_per_island": 1, "groups_per_gpu_per_round": 4.0, "groups_per_round": 4, "islands": 2, "kind": "federated", "local_horizon": 1, "name": "yeto-federated-m2", "pipeline": 1, "prompt_groups": 24, "total_gpus": 2, "trajectories": 192}], "expert_parallel": 1, "fairness": {"paired_prompt_streams": true, "same_expert_parallel": true, "same_per_rank_batch": true, "same_pipeline_parallel": true, "same_prompt_groups": true, "same_total_gpus": true, "same_trajectories": true}, "model": "Qwen/Qwen3-0.6B", "pipeline_parallel": 1, "seeds": [17]} -RL_ENGINE ports - Fetching 10 files: 0%| | 0/10 [00:00 failed for pad_packed_seq_alignment: typing.Union[int, typing.Literal['max'], NoneType]. -Deferring to attribute metadata. If the metadata is incomplete, 'parser.add_argument()' may fail. -Original failure: Unions not supported by argparse: typing.Union[int, typing.Literal['max'], NoneType] -using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 -Number of virtual stages per pipeline stage: None -using torch.bfloat16 for parameters ... ------------------------- arguments ------------------------ - _is_global_batch_size_explicitly_specified ...... True - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - activation_func_clamp_shared_expert ............. True - activation_func_clamp_value ..................... None - activation_offload_fraction ..................... 1.0 - actor_num_gpus_per_node ......................... 1 - actor_num_nodes ................................. 1 - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.999 - adam_eps ........................................ 1e-08 - add_bias_linear ................................. False - add_position_embedding .......................... True - add_qkv_bias .................................... False - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - advantage_estimator ............................. grpo - align_grad_reduce ............................... True - align_param_gather .............................. False - allgather_cp .................................... False - allow_ambiguous_pad_tokens ...................... False - allow_partial_train_step ........................ False - api_server_host ................................. 127.0.0.1 - api_server_port ................................. 0 - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_chat_template ............................. True - apply_chat_template_kwargs ...................... {'enable_thinking': False} - apply_dsa_kernel_fusion ......................... None - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - apply_wd_to_qk_layernorm ........................ False - async_ckpt_cpu_priority ......................... 10 - async_ckpt_io_priority .......................... 3 - async_ckpt_use_cpu_shm .......................... False - async_data_buffer_capacity_factor ............... 2.0 - async_max_concurrent_samples .................... None - async_save ...................................... False - async_strategy .................................. mcore - async_unused_samples_handler .................... drop - attention_backend ............................... AttnBackend.unfused - attention_dropout ............................... 0.1 - attention_output_gate ........................... False - attention_softmax_in_fp32 ....................... True - auto_detect_ckpt_format ......................... False - balance_by_flops ................................ False - balance_data .................................... True - barrier_with_L1_time ............................ True - batch_invariant_mode ............................ False - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - beta_fast ....................................... 32 - beta_slow ....................................... 1 - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - buffer_filter_path .............................. yeto.rl.engine.miles_adapter.rollout_meta_hook.policy_buffer_filter - cache_mla_latents ............................... False - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - chat_template ................................... None - chat_template_path .............................. None - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_lora_weight_equal ......................... False - check_rematerialize_param_from_master_weight .... False - check_weight_hash_across_dp_replicas_interval ... None - check_weight_update_allow_quant_error ........... False - check_weight_update_equal ....................... False - check_weight_update_selector .................... all - check_weight_update_skip_list ................... None - chunked_optimizer_state_offload ................. False - ci_check_model_hash ............................. False - ci_disable_kl_checker ........................... False - ci_disable_logprobs_checker ..................... False - ci_disable_weight_update_checker ................ False - ci_enable_metrics_capture ....................... False - ci_ft_test_actions .............................. None - ci_ft_test_actions_path ......................... None - ci_inject_rollout_data_min_match_ratio .......... 0.9 - ci_inject_rollout_data_path ..................... None - ci_inject_rollout_data_start_rollout_id ......... None - ci_load_grad_norm ............................... None - ci_metric_checker_expect_num .................... None - ci_metric_checker_key ........................... None - ci_metric_checker_threshold ..................... None - ci_save_grad_norm ............................... None - ci_save_model_hash .............................. False - ci_test ......................................... False - ci_tito_special_token_count_threshold ........... 0.0 - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch_dist - ckpt_fully_parallel_load ........................ False - ckpt_fully_parallel_load_exchange_algo .......... broadcast - ckpt_fully_parallel_load_process_group .......... dp - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_fully_parallel_save_process_group .......... dp - ckpt_load_validate_sharding_integrity ........... True - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clear_quantized_weight_workspaces_on_offload .... True - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - cluster_backend ................................. ray - colocate ........................................ True - colocate_memory_peak_device ..................... cpu - compress_ratios ................................. None - compute_advantages_and_returns .................. True - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 -... [truncated: 2442 of 2842 lines removed; full evidence archived outside repo] ... -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:29.147 actor_cell00000_rank00000] timer.py:42 - Timer finalize_and_resume_engines end (elapsed: 0.1s) -(CommandActor pid=15596) [2026-09-29 03:16:29] [continue_generation] torch.cuda.empty_cache() called: reserved 32306.0 MB -> 32266.0 MB (freed 40.0 MB) -(InferenceController pid=12655) [2026-09-29 03:16:29.265 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.0.14:21102/resume_memory_occupation "HTTP/1.1 200 OK" -(TrainerController pid=12659) [2026-09-29 03:16:29.199 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=update_weights ok=true elapsed_s=0.2 -(MegatronTrainRayActor pid=15836) [miles-colocate] Update weights: 1it [00:00, 4.41it/s] -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:29.195 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.96, 'used_GB': 7.22, 'allocated_GB': 1.38, 'reserved_GB': 1.51} -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:29.196 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 0.2s) -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:29.197 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.2 -(InferenceController pid=12655) [2026-09-29 03:16:29.314 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.0.14:21102/weights_checker "HTTP/1.1 200 OK" -(TrainerController pid=12659) [2026-09-29 03:16:29.324 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=sleep n_actors=1 -(CommandActor pid=15596) [2026-09-29 03:16:29] Weight version changed. old_version='3' new_version='yeto:2:63b0317bb2ce36b7906fce2e3e8ceb45e4d87a67e8617a607fd5f6d5c886f3d5' num_recorded=0 -(CommandActor pid=15596) [2026-09-29 03:16:29] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(CommandActor pid=15596) [2026-09-29 03:16:29] [WeightChecker] checksum computed for 226 tensors in 0.021s -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:29.326 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:29.326 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:30.167 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 41.93, 'used_GB': 37.25, 'allocated_GB': 1.38, 'reserved_GB': 1.49} -(RolloutExecutor pid=12653) Rollout generation: 0%| | 0/32 [00:00user\nIt takes Roque two hours to walk to work and one hour to ride his bike to work. Roque walks to and from work three times a week and rides his bike to and from work twice a week. How many hours in total does he take to get to and from work a week with walking and biking?\nPut the final numeric answer in \boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nWe are told the following information:\n\n- **Walking to work and back**: 3 times a week \n- **Time per round trip**: 2 ho..., label=16, reward_summary=0.0 -(RolloutExecutor pid=12653) Rollout generation: 25%|██▌ | 8/32 [00:04<00:12, 1.94it/s] -(CommandActor pid=15596) [2026-09-29 03:16:34] Decode batch, #running-req: 10, #token: 3467, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2210.22, #queue-req: 0 -(CommandActor pid=15596) [2026-09-29 03:16:35] Decode batch, #running-req: 4, #token: 1701, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1434.74, #queue-req: 0 -(RolloutExecutor pid=12653) Rollout generation: 50%|█████ | 16/32 [00:04<00:03, 4.10it/s] Rollout generation: 100%|██████████| 32/32 [00:04<00:00, 7.02it/s] -(RolloutExecutor pid=12653) [2026-09-29 03:16:35.301 rollout_executor] inference_rollout_train.py:161 - Finish rollout: text_preview=<|im_start|>user\nTim rides his bike back and forth to work for each of his 5 workdays. His work is 20 miles away. He also goes for a weekend bike ride of 200 miles. If he can bike at 25 mph how much time does he spend biking a week?\nPut the final numeric answer in \boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nWe are told the following information:\n\n- Tim rides his bike **back and forth** to work **for 5 workdays**.\n- His job is **20 miles away**, and he goes to work **a weeken..., label=16, reward_summary=1.0 -(RolloutExecutor pid=12653) [2026-09-29 03:16:35.303 rollout_executor] inference_rollout_train.py:33 - Abort request for ['http://172.20.0.14:21102'] -(RolloutExecutor pid=12653) [2026-09-29 03:16:35.310 rollout_executor] rollout_data_conversion.py:39 - Final collected 32 samples from rollout to train -(RolloutExecutor pid=12653) [2026-09-29 03:16:35.311 rollout_executor] debug_data.py:124 - Save debug rollout data to /work/out/strict2-federated/work/seed-17/yeto-federated-m2/rollouts/island-0/2.pt -(RolloutExecutor pid=12653) [2026-09-29 03:16:35.322 rollout_executor] metrics.py:93 - perf 2: {'rollout/raw_reward_unfiltered': 0.28125, 'rollout/num_training_samples': 32, 'rollout/episode_raw_reward': 0.28125, 'rollout/episode_response_length/mean': 273.0, 'rollout/episode_response_length/median': 272.0, 'rollout/episode_response_length/max': 384, 'rollout/episode_response_length/min': 148, 'rollout/episode_total_response_length/mean': 273.0, 'rollout/response_len/mean': 273.0, 'rollout/response_len/median': 272.0, 'rollout/response_len/max': 384, 'rollout/response_len/min': 148, 'rollout/zero_std/count_0.0': 1, 'rollout/zero_std/all_zero_percentage': 0.25, 'rollout/zero_std/all_one_percentage': 0.0, 'rollout/prefix_cache_hit_rate': 0.0, 'rollout/avg_cached_tokens_per_sample': 0.0, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.0625, 'perf/rollout_time': 4.581277847290039, 'perf/tokens_per_gpu_per_sec': 1906.8915466822432, 'perf/longest_sample_tokens_per_sec': 83.81940864537333, 'perf/effective_tokens_per_gpu_per_sec': 1906.8915466822432, 'perf/longest_effective_sample_tokens_per_sec': 83.81940864537333} -(RolloutExecutor pid=12653) [2026-09-29 03:16:35.323 rollout_executor] train_data_conversion.py:334 - Rollout-side DP schedule: num_samples=32, num_rollouts=[32], num_microbatches=[32] -(InferenceController pid=12655) [2026-09-29 03:16:35.346 inference_controller] _client.py:1740 - HTTP Request: GET http://172.20.0.14:21102/flush_cache "HTTP/1.1 200 OK" -(CommandActor pid=15596) [2026-09-29 03:16:35] Cache flushed successfully! -(CommandActor pid=15596) [2026-09-29 03:16:35] Cache flushed successfully! -(InferenceController pid=12655) [2026-09-29 03:16:35.841 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.0.14:21102/release_memory_occupation "HTTP/1.1 200 OK" -(TrainerController pid=12659) [2026-09-29 03:16:35.848 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=wake_up n_actors=1 -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:35.849 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:35.850 actor_cell00000_rank00000] timer.py:31 - Timer wake_up start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:35.850 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 74.61, 'used_GB': 4.57, 'allocated_GB': 1.38, 'reserved_GB': 1.49} -(TrainerController pid=12659) [2026-09-29 03:16:36.756 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=wake_up ok=true elapsed_s=0.9 -(TrainerController pid=12659) [2026-09-29 03:16:36.760 trainer_controller_actor] group.py:190 - ft op=train phase=start rollout=2 attempt=0 -(TrainerController pid=12659) [2026-09-29 03:16:36.760 trainer_controller_actor] group.py:534 - ft op=refresh phase=start rollout=2 alive=0 healing= all_states="(0, 'StateAllocatedAlive')" quorum=0 -(TrainerController pid=12659) [2026-09-29 03:16:36.761 trainer_controller_actor] group.py:557 - ft op=refresh phase=decision rollout=2 needs_reconfigure=false reason=alive_config_unchanged,no_healing quorum=0 -(TrainerController pid=12659) [2026-09-29 03:16:36.761 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=train n_actors=1 -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:36.753 actor_cell00000_rank00000] reloadable_process_group.py:151 - Reloading 0 process groups in pid 15836 -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:36.754 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 73.14, 'used_GB': 6.04, 'allocated_GB': 1.38, 'reserved_GB': 1.49} -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:36.754 actor_cell00000_rank00000] timer.py:42 - Timer wake_up end (elapsed: 0.9s) -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:36.755 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=0.9 -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:36.762 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:36.762 actor_cell00000_rank00000] timer.py:31 - Timer data_preprocess start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:36.768 actor_cell00000_rank00000] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:36.768 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:36.768 actor_cell00000_rank00000] timer.py:42 - Timer train_wait end (elapsed: 63.7s) -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:36.768 actor_cell00000_rank00000] timer.py:31 - Timer train start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:36.768 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:36.769 actor_cell00000_rank00000] timer.py:31 - Timer log_probs start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:42.851 actor_cell00000_rank00000] timer.py:42 - Timer log_probs end (elapsed: 6.1s) -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:42.851 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=6.1 -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:42.858 actor_cell00000_rank00000] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=0 -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:42.859 actor_cell00000_rank00000] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:42.859 actor_cell00000_rank00000] log_utils.py:126 - rollout 2: {'rollout/response_lengths': 273.0, 'rollout/rewards': -3.725290298461914e-09, 'rollout/truncated': 0.0625, 'rollout/rollout_log_probs': -0.36793065071105957, 'rollout/raw_reward': 0.28125, 'rollout/total_lengths': 360.0, 'rollout/log_probs': -0.36848196387290955, 'rollout/advantages': 1.1175870895385742e-08, 'rollout/returns': 1.1175870895385742e-08} -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:42.859 actor_cell00000_rank00000] timer.py:31 - Timer actor_train start -(TrainerController pid=12659) [2026-09-29 03:16:54.081 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=train ok=true elapsed_s=17.3 -(TrainerController pid=12659) [2026-09-29 03:16:54.081 trainer_controller_actor] group.py:290 - ft op=check errored= discarded= normal=0 decision=no_retry reason="survivors normal, gradients valid" -(TrainerController pid=12659) [2026-09-29 03:16:54.085 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=12659) [2026-09-29 03:16:54.088 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(TrainerController pid=12659) [2026-09-29 03:16:54.091 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:54.076 actor_cell00000_rank00000] model.py:637 - train op=train_step rollout=2 step=0 attempt=0 outcome=NORMAL valid_step=true -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:54.077 actor_cell00000_rank00000] log_utils.py:554 - step 2: {'train/loss': -1.1175870895385742e-08, 'train/pg_loss': -1.1175870895385742e-08, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.014273852109909058, 'train/train_rollout_kl': 0.0007000528858043253, 'train/grad_norm': 0.34883660078048706, 'train/lr-pg_0': 0.0, 'train/step': 2} -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:54.077 actor_cell00000_rank00000] model.py:847 - step 2: {'train/loss': -1.1175870895385742e-08, 'train/pg_loss': -1.1175870895385742e-08, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.014273852109909058, 'train/train_rollout_kl': 0.0007000528858043253, 'train/grad_norm': 0.34883660078048706, 'train/lr-pg_0': 0.0, 'train/step': 2} -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:54.077 actor_cell00000_rank00000] timer.py:42 - Timer actor_train end (elapsed: 11.2s) -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:54.077 actor_cell00000_rank00000] timer.py:42 - Timer train end (elapsed: 17.3s) -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:54.077 actor_cell00000_rank00000] timer.py:31 - Timer train_wait start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:54.078 actor_cell00000_rank00000] train_metric_utils.py:56 - perf 2: {'perf/update_weights_implementation_time': 0.1061704158782959, 'perf/finalize_and_resume_engines_time': 0.07411813735961914, 'perf/update_weights_time': 0.2443220615386963, 'perf/sleep_time': 1.4020717144012451, 'perf/wake_up_time': 0.9047815799713135, 'perf/data_preprocess_time': 0.0056591033935546875, 'perf/train_wait_time': 63.682080030441284, 'perf/log_probs_time': 6.082097291946411, 'perf/actor_train_time': 11.218274354934692, 'perf/train_time': 17.30893349647522, 'perf/log_probs_tflops': 2.049669530388341, 'perf/actor_train_tflops': 3.3337452193820694, 'perf/actor_train_tok_per_s': 1026.8959053343694, 'perf/mfu_peak_tflops': 989.0, 'perf/actor_train_mfu': 0.0033708242865339425, 'perf/step_time': 80.9910135269165, 'perf/wait_time_ratio': 0.7862857526690565} -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:54.078 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=17.3 -(MegatronTrainRayActor pid=15836) [2026-09-29 03:16:54.079 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=17.3 -(TrainerController pid=12659) [2026-09-29 03:16:54.378 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.3 -(TrainerController pid=12659) [2026-09-29 03:17:23.262 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=12659) [2026-09-29 03:17:49.424 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=26.2 -(TrainerController pid=12659) [2026-09-29 03:17:49.520 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=12659) [2026-09-29 03:17:49.844 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.3 -(TrainerController pid=12659) [2026-09-29 03:17:50.378 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=12659) [2026-09-29 03:17:50.756 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.4 -(InferenceController pid=12655) [2026-09-29 03:17:51.163 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.0.14:21102/resume_memory_occupation "HTTP/1.1 200 OK" -(TrainerController pid=12659) [2026-09-29 03:17:51.171 trainer_controller_actor] group.py:392 - ft op=update_weights phase=start rollout= -(TrainerController pid=12659) [2026-09-29 03:17:51.172 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=update_weights n_actors=1 -(CommandActor pid=15596) [2026-09-29 03:17:51] Cache flush skipped: KV cache memory is released. -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.174 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.174 actor_cell00000_rank00000] timer.py:31 - Timer update_weights start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.175 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 69.36, 'used_GB': 9.82, 'allocated_GB': 1.38, 'reserved_GB': 4.09} -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.179 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:21102/pause_generation "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.183 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: GET http://172.20.0.14:21102/flush_cache "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.187 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:21102/begin_weight_update "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.188 actor_cell00000_rank00000] timer.py:31 - Timer update_weights_implementation start -(MegatronTrainRayActor pid=15836) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(CommandActor pid=15596) [2026-09-29 03:17:51] LoRA adapter loading from tensors starts: LoRARef(lora_id=57047798205c462da2967b12f5666406, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(CommandActor pid=15596) [2026-09-29 03:17:51] LoRA adapter '57047798205c462da2967b12f5666406': loaded weights for target modules ['down_proj', 'gate_up_proj', 'o_proj', 'qkv_proj']. -(CommandActor pid=15596) [2026-09-29 03:17:51] LoRA adapter loading from tensors completes: LoRARef(lora_id=57047798205c462da2967b12f5666406, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(CommandActor pid=15596) [2026-09-29 03:17:51] Weight version changed. old_version='yeto:2:63b0317bb2ce36b7906fce2e3e8ceb45e4d87a67e8617a607fd5f6d5c886f3d5' new_version='4' num_recorded=0 -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.267 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:21102/update_weights_from_tensor "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.268 actor_cell00000_rank00000] timer.py:42 - Timer update_weights_implementation end (elapsed: 0.1s) -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.268 actor_cell00000_rank00000] timer.py:31 - Timer finalize_and_resume_engines start -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.314 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:21102/end_weight_update "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.318 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:21102/update_weight_version "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.321 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:21102/continue_generation "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.321 actor_cell00000_rank00000] timer.py:42 - Timer finalize_and_resume_engines end (elapsed: 0.1s) -(CommandActor pid=15596) [2026-09-29 03:17:51] [continue_generation] torch.cuda.empty_cache() called: reserved 32306.0 MB -> 32266.0 MB (freed 40.0 MB) -(TrainerController pid=12659) [2026-09-29 03:17:51.606 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=update_weights ok=true elapsed_s=0.4 -(MegatronTrainRayActor pid=15836) [miles-colocate] Update weights: 1it [00:00, 2.42it/s] -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.603 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.96, 'used_GB': 7.22, 'allocated_GB': 1.38, 'reserved_GB': 1.51} -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.604 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 0.4s) -(MegatronTrainRayActor pid=15836) [2026-09-29 03:17:51.605 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.4 -(InferenceController pid=12655) [2026-09-29 03:17:51.660 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.0.14:21102/resume_memory_occupation "HTTP/1.1 200 OK" -(InferenceController pid=12655) [2026-09-29 03:17:51.705 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.0.14:21102/weights_checker "HTTP/1.1 200 OK" -(CommandActor pid=15596) [2026-09-29 03:17:51] Weight version changed. old_version='4' new_version='yeto:3:2d30c85bbf38b14b9a93fa5a5f4ddb04a896235c2c81b9b62e27d8a83c40327a' num_recorded=0 -(CommandActor pid=15596) [2026-09-29 03:17:51] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(CommandActor pid=15596) [2026-09-29 03:17:51] [WeightChecker] checksum computed for 226 tensors in 0.021s -(InferenceController pid=12655) [2026-09-29 03:17:51.865 inference_controller] rollout_server.py:132 - Killing server cell_id='inference-engine-all-0-0-00000'... -(InferenceController pid=12655) [2026-09-29 03:17:51.868 inference_controller] _client.py:1740 - HTTP Request: GET http://172.20.0.14:21000/workers "HTTP/1.1 200 OK" -(InferenceController pid=12655) [2026-09-29 03:17:51.869 inference_controller] _client.py:1740 - HTTP Request: DELETE http://172.20.0.14:21000/workers/af77e186-53f3-41bb-8f64-3d62c1236455 "HTTP/1.1 202 Accepted" -(InferenceController pid=12655) [2026-09-29 03:17:51.870 inference_controller] server_cell.py:268 - Cell inference-engine-all-0-0-00000 dispose start old=addr_info=CellAddrInfo(server_url='http://172.20.0.14:21102', bootstrap_port=None, gate_url='http://172.20.0.14:21136') -(InferenceController pid=12655) [2026-09-29 03:17:51.870 inference_controller] server_cell.py:271 - Cell inference-engine-all-0-0-00000 dispose end new= -(RayWorkerManager pid=12651) [2026-09-29 03:17:51.874 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='rollout-executor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b59889d23e0>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.rollout_executor.RolloutExecutor', ctor_kwargs=. at 0x2b59889d3600>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=ServeWorkerSpec(name='rollout-executor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b59889d23e0>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.rollout_executor.RolloutExecutor', ctor_kwargs=. at 0x2b59889d3600>, concurrency_groups=None), actor_handle=Actor(RolloutExecutor, c7cddfd8fdbe69f3704eb82f02000000), self_addrs={'rpc': HostAndPort(host='172.20.0.14', port=21137, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(RayWorkerManager pid=12651) [2026-09-29 03:17:51.874 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='inference-controller', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b59889d3740>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.inference_controller.InferenceController', ctor_kwargs=. at 0x2b59a644bba0>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=ServeWorkerSpec(name='inference-controller', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b59889d3740>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.inference_controller.InferenceController', ctor_kwargs=. at 0x2b59a644bba0>, concurrency_groups=None), actor_handle=Actor(InferenceController, bbd0db2679aea8b58219675d02000000), self_addrs={'rpc': HostAndPort(host='172.20.0.14', port=21100, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(RayWorkerManager pid=12651) [2026-09-29 03:17:51.877 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-controller-actor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b59a6470220>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=False), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.train.group.TrainerController', ctor_kwargs=. at 0x2b59a6470360>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-controller-actor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b59a6470220>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=False), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.train.group.TrainerController', ctor_kwargs=. at 0x2b59a6470360>, concurrency_groups=None), actor_handle=Actor(TrainerController, 5bb45a25e0f4f56135e2dab202000000), self_addrs={'rpc': HostAndPort(host='172.20.0.14', port=21140, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(RayWorkerManager pid=12651) [2026-09-29 03:17:51.877 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b59a6470400>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=1, pg_name='actor', pg_slot_offset=0, pin_to_head=False), meta=. at 0x2b59a64718a0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x2b59a6472480>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b59a6470400>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=1, pg_name='actor', pg_slot_offset=0, pin_to_head=False), meta=. at 0x2b59a64718a0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x2b59a6472480>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, ad5af937c3c5a415543bc4d002000000), self_addrs={'master': HostAndPort(host='172.20.0.14', port=21138, external_host=None), 'rpc': HostAndPort(host='172.20.0.14', port=21139, external_host=None)}, local_gpu_ids=[0], gpu_slot_index=0) -(CommandActor pid=15596) [2026-09-29 03:17:51] SIGTERM received. signum=None frame=None. Draining requests and shutting down... -(RayWorkerManager pid=12651) [2026-09-29 03:17:51.993 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_CommandActorManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=CommandWorkerSpec(name='inference-router-0', category=None, port_infos=[PortInfo(name='primary', static_port=21000, mode='per_worker', allow_dynamic=False, num_consecutive=1, offset_by_cell=False), PortInfo(name='prometheus', static_port=9000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b59a644bc40>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x2b59a644bce0>), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=CommandWorkerSpec(name='inference-router-0', category=None, port_infos=[PortInfo(name='primary', static_port=21000, mode='per_worker', allow_dynamic=False, num_consecutive=1, offset_by_cell=False), PortInfo(name='prometheus', static_port=9000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b59a644bc40>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x2b59a644bce0>), actor_handle=Actor(CommandActor, 8508d8a663facc146d080ce202000000), self_addrs={'primary': HostAndPort(host='172.20.0.14', port=21000, external_host=None), 'prometheus': HostAndPort(host='172.20.0.14', port=21101, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(CommandActor pid=15596) [2026-09-29 03:17:52] Subprocess detokenizer (pid=18575) crashed with exit code -15. Triggering SIGQUIT for cleanup... -(CommandActor pid=15596) [2026-09-29 03:17:52] SIGQUIT received. signum=None, frame=None. It usually means one child failed. -(CommandActor pid=15596) [2026-09-29 03:17:52] Sleeping 5 seconds before crash diagnostics to let GPU activity settle. -(CommandActor pid=15596) WARNING:miles.utils.workers.process_utils:Process 17640 did not exit within 5.0s after SIGTERM; escalating to SIGKILL -(RayWorkerManager pid=12651) [2026-09-29 03:17:56.955 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_CommandActorManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=CommandWorkerSpec(name='inference-engine-all-0-0', category='inference_engine', port_infos=[PortInfo(name='primary', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='dist_init', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=31, offset_by_cell=False), PortInfo(name='nccl', static_port=10000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='engine_info_bootstrap', static_port=12000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='gate', static_port=13000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b59a644bec0>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.2, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=1, num_gpus_per_node=1, pg_name='rollout', pg_slot_offset=0, pin_to_head=False), meta=. at 0x2b59a6470040>, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x2b59a6470180>), actors=[...], generation=1, liveness_scan_task= wait_for=<_GatheringFuture pending cb=[Task.task_wakeup()]>>), worker_in_cell_index=0, spec=CommandWorkerSpec(name='inference-engine-all-0-0', category='inference_engine', port_infos=[PortInfo(name='primary', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='dist_init', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=31, offset_by_cell=False), PortInfo(name='nccl', static_port=10000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='engine_info_bootstrap', static_port=12000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='gate', static_port=13000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b59a644bec0>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.2, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=1, num_gpus_per_node=1, pg_name='rollout', pg_slot_offset=0, pin_to_head=False), meta=. at 0x2b59a6470040>, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x2b59a6470180>), actor_handle=Actor(CommandActor, b9a886e549015aad89c2194002000000), self_addrs={'primary': HostAndPort(host='172.20.0.14', port=21102, external_host=None), 'dist_init': HostAndPort(host='172.20.0.14', port=21103, external_host=None), 'nccl': HostAndPort(host='172.20.0.14', port=21134, external_host=None), 'engine_info_bootstrap': HostAndPort(host='172.20.0.14', port=21135, external_host=None), 'gate': HostAndPort(host='172.20.0.14', port=21136, external_host=None)}, local_gpu_ids=[0], gpu_slot_index=0) - -(CommandActor pid=15596) [2026-09-29 03:16:34] INFO: 172.20.0.14:19660 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:34] INFO: 172.20.0.14:46059 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:34] INFO: 172.20.0.14:21225 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:34] INFO: 172.20.0.14:52070 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:34] INFO: 172.20.0.14:25455 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:34] INFO: 172.20.0.14:29774 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:34] INFO: 172.20.0.14:45662 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:34] INFO: 172.20.0.14:23242 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:34] INFO: 172.20.0.14:25935 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:34] INFO: 172.20.0.14:56246 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:35] INFO: 172.20.0.14:38160 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:35] INFO: 172.20.0.14:54350 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:35] INFO: 172.20.0.14:40287 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:35] INFO: 172.20.0.14:22524 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:35] INFO: 172.20.0.14:34987 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:35] INFO: 172.20.0.14:44346 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:35] INFO: 172.20.0.14:44981 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:35] INFO: 172.20.0.14:40025 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:35] INFO: 172.20.0.14:57536 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:35] INFO: 172.20.0.14:21598 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:35] INFO: 172.20.0.14:44969 - "POST /abort_request HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:35] INFO: 172.20.0.14:62566 - "GET /flush_cache HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:16:35] INFO: 172.20.0.14:62566 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:17:27] INFO: 172.20.0.14:47909 - "GET /health HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:17:51] INFO: 172.20.0.14:16868 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:17:51] INFO: 172.20.0.14:46924 - "POST /pause_generation HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:17:51] INFO: 172.20.0.14:46924 - "GET /flush_cache HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:17:51] INFO: 172.20.0.14:46924 - "POST /begin_weight_update HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:17:51] INFO: 172.20.0.14:46924 - "POST /update_weights_from_tensor HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:17:51] INFO: 172.20.0.14:46924 - "POST /end_weight_update HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:17:51] INFO: 172.20.0.14:46924 - "POST /update_weight_version HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:17:51] INFO: 172.20.0.14:46924 - "POST /continue_generation HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:17:51] INFO: 172.20.0.14:16868 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:17:51] INFO: 172.20.0.14:44880 - "POST /update_weight_version HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:17:51] INFO: 172.20.0.14:44880 - "GET /model_info HTTP/1.1" 200 OK -(CommandActor pid=15596) [2026-09-29 03:17:51] INFO: 172.20.0.14:16868 - "POST /weights_checker HTTP/1.1" 200 OK -[rl] learner 0 finalized (rl_engine=ports) diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-1/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-1/events.jsonl deleted file mode 100644 index a2b6c76bd..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-1/events.jsonl +++ /dev/null @@ -1,43 +0,0 @@ -{"event":"rl_engine_selected","island_id":1,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790651090.8085089} -{"event":"rl_driver_start","execution_mode":"serial-colocated","island_id":1,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:4b34f2e86950fe2b21f9661368934884cd7fbe163d6591a4c7ee8a8568451c07","time_unix":1790651264.7407322} -{"event":"rl_driver_phase","island_id":1,"optimizer":"reset","phase":"apply","policy_version":0,"time_unix":1790651268.5215974} -{"event":"rl_policy_apply","island_id":1,"local_step":0,"optimizer":"reset","partial_fragment_apply":false,"policy_version":0,"sync/apply_seconds":142.19399161400008,"sync/global_policy_hash":"a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","time_unix":1790651410.7164354} -{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":0,"time_unix":1790651410.80457} -{"event":"rl_publication","island_id":1,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790651535.6116502} -{"event":"rl_driver_phase","island_id":1,"phase":"offload","rollout_id":0,"time_unix":1790651535.611879} -{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790651535.6198583} -{"event":"rl_driver_phase","island_id":1,"phase":"onload","rollout_id":0,"time_unix":1790651555.0879707} -{"event":"rl_driver_phase","island_id":1,"phase":"train","rollout_id":0,"time_unix":1790651555.8707862} -{"event":"rl_driver_phase","island_id":1,"phase":"sync","rollout_id":0,"time_unix":1790651598.8130503} -{"event":"rl_driver_phase","island_id":1,"phase":"export_push","rollout_id":0,"time_unix":1790651598.8133266} -{"action_tokens":6145,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.23339484632015228,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.40183985233306885,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":1,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.4375,"reward_std":0.49607837082461076,"rl/action_tokens":6145,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":0,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.23339484632015228,"rl/mixed_version_group_count":0,"rl/reward_mean":0.4375,"rl/reward_std":0.49607837082461076,"rl/rollout_policy_version":0,"rl/rollout_seconds":19.47517814799994,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":19.47517814799994,"sync/bytes_sent":40370224,"time_unix":1790651599.876816,"tool_wait_seconds":0.0,"train_seconds":42.94183087900001,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":1,"phase":"wait_global","policy_version":1,"time_unix":1790651599.9232972} -{"event":"rl_driver_phase","island_id":1,"optimizer":"reset","phase":"apply","policy_version":1,"time_unix":1790651600.3867714} -{"event":"rl_policy_apply","island_id":1,"local_step":1,"optimizer":"reset","partial_fragment_apply":false,"policy_version":1,"sync/apply_seconds":101.067884069,"sync/global_policy_hash":"42a3838531e620dc7029bf0a51c979f4e65ea8a052cc700e4552485c95a0cae3","time_unix":1790651701.4552052} -{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":1,"time_unix":1790651701.5283265} -{"event":"rl_publication","island_id":1,"policy_version":1,"rl/policy_token":"yeto:1:42a3838531e620dc7029bf0a51c979f4e65ea8a052cc700e4552485c95a0cae3","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"b7468b340cbd6e544958f986bf3af2f0313b1962ea08bd15dd798f1ddfe594a8","time_unix":1790651702.8013208} -{"event":"rl_driver_phase","island_id":1,"phase":"offload","rollout_id":1,"time_unix":1790651702.8016245} -{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790651704.1396508} -{"event":"rl_driver_phase","island_id":1,"phase":"onload","rollout_id":1,"time_unix":1790651714.4040644} -{"event":"rl_driver_phase","island_id":1,"phase":"train","rollout_id":1,"time_unix":1790651715.334272} -{"event":"rl_driver_phase","island_id":1,"phase":"sync","rollout_id":1,"time_unix":1790651731.5168536} -{"event":"rl_driver_phase","island_id":1,"phase":"export_push","rollout_id":1,"time_unix":1790651731.5173311} -{"action_tokens":8050,"active_groups":4,"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.20019644498825073,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.3091374635696411,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":1,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.5625,"reward_std":0.49607837082461076,"rl/action_tokens":8050,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":1,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.20019644498825073,"rl/mixed_version_group_count":0,"rl/reward_mean":0.5625,"rl/reward_std":0.49607837082461076,"rl/rollout_policy_version":1,"rl/rollout_seconds":11.600881930000014,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.5,"rollout_seconds":11.600881930000014,"sync/bytes_sent":40370224,"time_unix":1790651732.4281118,"tool_wait_seconds":0.0,"train_seconds":16.1822536090001,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":1,"phase":"wait_global","policy_version":2,"time_unix":1790651732.4722278} -{"event":"rl_driver_phase","island_id":1,"optimizer":"reset","phase":"apply","policy_version":2,"time_unix":1790651734.5387485} -{"event":"rl_policy_apply","island_id":1,"local_step":2,"optimizer":"reset","partial_fragment_apply":false,"policy_version":2,"sync/apply_seconds":55.4298694470001,"sync/global_policy_hash":"63b0317bb2ce36b7906fce2e3e8ceb45e4d87a67e8617a607fd5f6d5c886f3d5","time_unix":1790651789.9690254} -{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":2,"time_unix":1790651790.0431926} -{"event":"rl_publication","island_id":1,"policy_version":2,"rl/policy_token":"yeto:2:63b0317bb2ce36b7906fce2e3e8ceb45e4d87a67e8617a607fd5f6d5c886f3d5","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"2b25fc9ebfc90f83413464c9bba8abb9676b031bda2493d9c1001ada30ad5991","time_unix":1790651791.2741423} -{"event":"rl_driver_phase","island_id":1,"phase":"offload","rollout_id":2,"time_unix":1790651791.2749147} -{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790651792.7210584} -{"event":"rl_driver_phase","island_id":1,"phase":"onload","rollout_id":2,"time_unix":1790651797.79381} -{"event":"rl_driver_phase","island_id":1,"phase":"train","rollout_id":2,"time_unix":1790651798.7350502} -{"event":"rl_driver_phase","island_id":1,"phase":"sync","rollout_id":2,"time_unix":1790651815.983548} -{"event":"rl_driver_phase","island_id":1,"phase":"export_push","rollout_id":2,"time_unix":1790651815.983732} -{"action_tokens":8451,"active_groups":4,"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.09777509421110153,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.16807568073272705,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":1,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.625,"reward_std":0.4841229182759271,"rl/action_tokens":8451,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":2,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.09777509421110153,"rl/mixed_version_group_count":0,"rl/reward_mean":0.625,"rl/reward_std":0.4841229182759271,"rl/rollout_policy_version":2,"rl/rollout_seconds":6.51709208300008,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.75,"rollout_seconds":6.51709208300008,"sync/bytes_sent":40370224,"time_unix":1790651816.960276,"tool_wait_seconds":0.0,"train_seconds":17.24811845499994,"train_step":null,"zero_variance_group_ratio":0.75} -{"event":"rl_driver_phase","island_id":1,"phase":"wait_global","policy_version":3,"time_unix":1790651817.0020685} -{"event":"rl_driver_phase","island_id":1,"optimizer":"reset","phase":"apply","policy_version":3,"time_unix":1790651817.4624383} -{"event":"rl_policy_apply","island_id":1,"local_step":3,"optimizer":"reset","partial_fragment_apply":false,"policy_version":3,"sync/apply_seconds":52.31304204100002,"sync/global_policy_hash":"2d30c85bbf38b14b9a93fa5a5f4ddb04a896235c2c81b9b62e27d8a83c40327a","time_unix":1790651869.7760184} -{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":3,"time_unix":1790651869.8479905} -{"event":"rl_publication","island_id":1,"policy_version":3,"rl/policy_token":"yeto:3:2d30c85bbf38b14b9a93fa5a5f4ddb04a896235c2c81b9b62e27d8a83c40327a","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"07aa52c9f0dcefe3ef92b28ec359b261f3bacab706f8dae8651a7d4b9b797f59","time_unix":1790651871.0108688} -{"event":"rl_driver_phase","island_id":1,"phase":"finish","rollout_id":3,"time_unix":1790651871.011526} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-1/miles.log b/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-1/miles.log deleted file mode 100644 index 26aecbd8d..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-1/miles.log +++ /dev/null @@ -1,401 +0,0 @@ -/opt/sglang/lib/python3.12/site-packages/transformer_engine/__init__.py:103: RuntimeWarning: Detected a Jax installation but could not find the shared object file for the Transformer Engine Jax extension library. If this is not intentional, please reinstall Transformer Engine with `pip install transformer_engine[jax]` or build from source with `NVTE_FRAMEWORK=jax`. - warnings.warn( -Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -WARNING:megatron.bridge.models.conversion.auto_bridge:Loading a model with trust_remote_code=True allows arbitrary code execution from the model repository. Only use this with models you trust. -/opt/sglang/lib/python3.12/site-packages/megatron/bridge/models/qwen/qwen3_bridge.py:45: FutureWarning: Qwen3Bridge.provider_bridge() and AutoBridge.to_megatron_provider() are deprecated for builder-backed models. Use AutoBridge.get_model_config() for training recipes or AutoBridge.get_model() to construct a model. Legacy provider support will be removed in a future release. - provider = super().provider_bridge(hf_pretrained) -[transformers] `torch_dtype` is deprecated! Use `dtype` instead! -/root/Megatron-LM/megatron/core/transformer/transformer_config.py:3249: UserWarning: full scope is deprecated. Use empty cuda_graph_modules to capture the whole layer. - warnings.warn( -[2026-09-29 03:04:50.597 main] arguments.py:49 - --tokenizer-model not set, use --hf-checkpoint as tokenizer model. -[2026-09-29 03:04:50.646 main] arguments.py:152 - MoE-expert LoRA layout: per-expert (--experts-shared-outer-loras). -[2026-09-29 03:04:50.647 main] arguments.py:3616 - Colocate mode: defaulting --sglang-cuda-graph-backend-prefill=disabled to avoid NVLS OOM. Set --sglang-cuda-graph-backend-prefill explicitly to override. -[2026-09-29 03:04:50.647 main] arguments.py:3626 - rollout_num_gpus None != actor_num_gpus_per_node 1 * actor_num_nodes 1, overriding rollout_num_gpus to match actor_num_gpus_per_node * actor_num_nodes. -[2026-09-29 03:04:50.648 main] arguments.py:3858 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 1024 as default value. -[2026-09-29 03:04:50.648 main] arguments.py:3866 - args.rollout_max_prompt_len is not set. Use args.rollout_max_context_len - 1 (1024 - 1) as default value so that there is at least one generated token to compute loss. -/root/Megatron-LM/megatron/core/_rank_utils.py:39: UserWarning: Could not determine rank from torch.distributed, RANK, or SLURM_PROCID. Defaulting to rank 0. - warnings.warn( -WARNING: Inferring the appropriate argparse argument type from failed for pad_packed_seq_alignment: typing.Union[int, typing.Literal['max'], NoneType]. -Deferring to attribute metadata. If the metadata is incomplete, 'parser.add_argument()' may fail. -Original failure: Unions not supported by argparse: typing.Union[int, typing.Literal['max'], NoneType] -using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 -Number of virtual stages per pipeline stage: None -using torch.bfloat16 for parameters ... ------------------------- arguments ------------------------ - _is_global_batch_size_explicitly_specified ...... True - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - activation_func_clamp_shared_expert ............. True - activation_func_clamp_value ..................... None - activation_offload_fraction ..................... 1.0 - actor_num_gpus_per_node ......................... 1 - actor_num_nodes ................................. 1 - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.999 - adam_eps ........................................ 1e-08 - add_bias_linear ................................. False - add_position_embedding .......................... True - add_qkv_bias .................................... False - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - advantage_estimator ............................. grpo - align_grad_reduce ............................... True - align_param_gather .............................. False - allgather_cp .................................... False - allow_ambiguous_pad_tokens ...................... False - allow_partial_train_step ........................ False - api_server_host ................................. 127.0.0.1 - api_server_port ................................. 0 - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_chat_template ............................. True - apply_chat_template_kwargs ...................... {'enable_thinking': False} - apply_dsa_kernel_fusion ......................... None - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - apply_wd_to_qk_layernorm ........................ False - async_ckpt_cpu_priority ......................... 10 - async_ckpt_io_priority .......................... 3 - async_ckpt_use_cpu_shm .......................... False - async_data_buffer_capacity_factor ............... 2.0 - async_max_concurrent_samples .................... None - async_save ...................................... False - async_strategy .................................. mcore - async_unused_samples_handler .................... drop - attention_backend ............................... AttnBackend.unfused - attention_dropout ............................... 0.1 - attention_output_gate ........................... False - attention_softmax_in_fp32 ....................... True - auto_detect_ckpt_format ......................... False - balance_by_flops ................................ False - balance_data .................................... True - barrier_with_L1_time ............................ True - batch_invariant_mode ............................ False - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - beta_fast ....................................... 32 - beta_slow ....................................... 1 - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - buffer_filter_path .............................. yeto.rl.engine.miles_adapter.rollout_meta_hook.policy_buffer_filter - cache_mla_latents ............................... False - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - chat_template ................................... None - chat_template_path .............................. None - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_lora_weight_equal ......................... False - check_rematerialize_param_from_master_weight .... False - check_weight_hash_across_dp_replicas_interval ... None - check_weight_update_allow_quant_error ........... False - check_weight_update_equal ....................... False - check_weight_update_selector .................... all - check_weight_update_skip_list ................... None - chunked_optimizer_state_offload ................. False - ci_check_model_hash ............................. False - ci_disable_kl_checker ........................... False - ci_disable_logprobs_checker ..................... False - ci_disable_weight_update_checker ................ False - ci_enable_metrics_capture ....................... False - ci_ft_test_actions .............................. None - ci_ft_test_actions_path ......................... None - ci_inject_rollout_data_min_match_ratio .......... 0.9 - ci_inject_rollout_data_path ..................... None - ci_inject_rollout_data_start_rollout_id ......... None - ci_load_grad_norm ............................... None - ci_metric_checker_expect_num .................... None - ci_metric_checker_key ........................... None - ci_metric_checker_threshold ..................... None - ci_save_grad_norm ............................... None - ci_save_model_hash .............................. False - ci_test ......................................... False - ci_tito_special_token_count_threshold ........... 0.0 - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch_dist - ckpt_fully_parallel_load ........................ False - ckpt_fully_parallel_load_exchange_algo .......... broadcast - ckpt_fully_parallel_load_process_group .......... dp - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_fully_parallel_save_process_group .......... dp - ckpt_load_validate_sharding_integrity ........... True - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clear_quantized_weight_workspaces_on_offload .... True - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - cluster_backend ................................. ray - colocate ........................................ True - colocate_memory_peak_device ..................... cpu - compress_ratios ................................. None - compute_advantages_and_returns .................. True - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 -... [truncated: 2448 of 2848 lines removed; full evidence archived outside repo] ... -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:31.131 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/update_weight_version "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:31.134 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/continue_generation "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:31.135 actor_cell00000_rank00000] timer.py:42 - Timer finalize_and_resume_engines end (elapsed: 0.1s) -(MegatronTrainRayActor pid=15843) [miles-colocate] Update weights: 1it [00:00, 5.56it/s] -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:31.167 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 71.89, 'used_GB': 7.29, 'allocated_GB': 1.38, 'reserved_GB': 1.58} -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:31.168 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 0.2s) -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:31.168 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.2 -(InferenceController pid=12656) [2026-09-29 03:16:31.215 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/resume_memory_occupation "HTTP/1.1 200 OK" -(InferenceController pid=12656) [2026-09-29 03:16:31.268 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/weights_checker "HTTP/1.1 200 OK" -(TrainerController pid=12660) [2026-09-29 03:16:31.276 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=sleep n_actors=1 -(CommandActor pid=15633) [2026-09-29 03:16:31] Weight version changed. old_version='3' new_version='yeto:2:63b0317bb2ce36b7906fce2e3e8ceb45e4d87a67e8617a607fd5f6d5c886f3d5' num_recorded=0 -(CommandActor pid=15633) [2026-09-29 03:16:31] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(CommandActor pid=15633) [2026-09-29 03:16:31] [WeightChecker] checksum computed for 226 tensors in 0.030s -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:31.277 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:31.278 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:32.142 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 41.86, 'used_GB': 37.32, 'allocated_GB': 1.38, 'reserved_GB': 1.56} -(RolloutExecutor pid=12654) Rollout generation: 0%| | 0/32 [00:00user\nBella bought stamps at the post office. Some of the stamps had a snowflake design, some had a truck design, and some had a rose design. Bella bought 11 snowflake stamps. She bought 9 more truck stamps than snowflake stamps, and 13 fewer rose stamps than truck stamps. How many stamps did Bella buy in all?\nPut the final numeric answer in \boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nWe are given the following information:\n\n- Bella bought **11 snowflake stamps**.\n- Sh..., label=38, reward_summary=1.0 -(RolloutExecutor pid=12654) Rollout generation: 25%|██▌ | 8/32 [00:03<00:11, 2.05it/s] -(CommandActor pid=15633) [2026-09-29 03:16:36] Decode batch, #running-req: 11, #token: 3240, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2945.55, #queue-req: 0 -(CommandActor pid=15633) [2026-09-29 03:16:36] Decode batch, #running-req: 7, #token: 2396, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1620.96, #queue-req: 0 -(CommandActor pid=15633) [2026-09-29 03:16:37] Decode batch, #running-req: 3, #token: 1232, token usage: 0.00, cuda graph: True, gen throughput (token/s): 995.18, #queue-req: 0 -(RolloutExecutor pid=12654) Rollout generation: 75%|███████▌ | 24/32 [00:04<00:01, 6.47it/s] Rollout generation: 100%|██████████| 32/32 [00:04<00:00, 7.10it/s] -(RolloutExecutor pid=12654) [2026-09-29 03:16:37.235 rollout_executor] inference_rollout_train.py:161 - Finish rollout: text_preview=<|im_start|>user\nSamantha’s last name has three fewer letters than Bobbie’s last name. If Bobbie took two letters off her last name, she would have a last name twice the length of Jamie’s. Jamie’s full name is Jamie Grey. How many letters are in Samantha’s last name?\nPut the final numeric answer in \boxed{}.<|im_end|>\n<|im_start|>assistant\n\n\n\n\nWe are given the following information:\n\n1. **Bbie’s last name has three fewer letters than Saman’s.**\n - So, if Bobbie’s last name has $ ..., label=7, reward_summary=0.0 -(RolloutExecutor pid=12654) [2026-09-29 03:16:37.237 rollout_executor] inference_rollout_train.py:33 - Abort request for ['http://172.20.0.14:22102'] -(RolloutExecutor pid=12654) [2026-09-29 03:16:37.244 rollout_executor] rollout_data_conversion.py:39 - Final collected 32 samples from rollout to train -(RolloutExecutor pid=12654) [2026-09-29 03:16:37.245 rollout_executor] debug_data.py:124 - Save debug rollout data to /work/out/strict2-federated/work/seed-17/yeto-federated-m2/rollouts/island-1/2.pt -(RolloutExecutor pid=12654) [2026-09-29 03:16:37.256 rollout_executor] metrics.py:93 - perf 2: {'rollout/raw_reward_unfiltered': 0.625, 'rollout/num_training_samples': 32, 'rollout/episode_raw_reward': 0.625, 'rollout/episode_response_length/mean': 264.09375, 'rollout/episode_response_length/median': 260.0, 'rollout/episode_response_length/max': 384, 'rollout/episode_response_length/min': 161, 'rollout/episode_total_response_length/mean': 264.09375, 'rollout/response_len/mean': 264.09375, 'rollout/response_len/median': 260.0, 'rollout/response_len/max': 384, 'rollout/response_len/min': 161, 'rollout/zero_std/count_1.0': 2, 'rollout/zero_std/count_0.0': 1, 'rollout/zero_std/all_zero_percentage': 0.25, 'rollout/zero_std/all_one_percentage': 0.5, 'rollout/prefix_cache_hit_rate': 0.0, 'rollout/avg_cached_tokens_per_sample': 0.0, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.09375, 'perf/rollout_time': 4.528940677642822, 'perf/tokens_per_gpu_per_sec': 1865.999270363261, 'perf/longest_sample_tokens_per_sec': 84.78803926393235, 'perf/effective_tokens_per_gpu_per_sec': 1865.999270363261, 'perf/longest_effective_sample_tokens_per_sec': 84.78803926393235} -(RolloutExecutor pid=12654) [2026-09-29 03:16:37.258 rollout_executor] train_data_conversion.py:334 - Rollout-side DP schedule: num_samples=32, num_rollouts=[32], num_microbatches=[32] -(InferenceController pid=12656) [2026-09-29 03:16:37.271 inference_controller] _client.py:1740 - HTTP Request: GET http://172.20.0.14:22102/flush_cache "HTTP/1.1 200 OK" -(CommandActor pid=15633) [2026-09-29 03:16:37] Cache flushed successfully! -(CommandActor pid=15633) [2026-09-29 03:16:37] Cache flushed successfully! -(InferenceController pid=12656) [2026-09-29 03:16:37.787 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/release_memory_occupation "HTTP/1.1 200 OK" -(TrainerController pid=12660) [2026-09-29 03:16:37.795 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=wake_up n_actors=1 -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:37.796 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:37.796 actor_cell00000_rank00000] timer.py:31 - Timer wake_up start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:37.797 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 74.61, 'used_GB': 4.57, 'allocated_GB': 1.38, 'reserved_GB': 1.56} -(TrainerController pid=12660) [2026-09-29 03:16:38.732 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=wake_up ok=true elapsed_s=0.9 -(TrainerController pid=12660) [2026-09-29 03:16:38.737 trainer_controller_actor] group.py:190 - ft op=train phase=start rollout=2 attempt=0 -(TrainerController pid=12660) [2026-09-29 03:16:38.737 trainer_controller_actor] group.py:534 - ft op=refresh phase=start rollout=2 alive=0 healing= all_states="(0, 'StateAllocatedAlive')" quorum=0 -(TrainerController pid=12660) [2026-09-29 03:16:38.737 trainer_controller_actor] group.py:557 - ft op=refresh phase=decision rollout=2 needs_reconfigure=false reason=alive_config_unchanged,no_healing quorum=0 -(TrainerController pid=12660) [2026-09-29 03:16:38.737 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=train n_actors=1 -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:38.729 actor_cell00000_rank00000] reloadable_process_group.py:151 - Reloading 0 process groups in pid 15843 -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:38.731 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 73.07, 'used_GB': 6.11, 'allocated_GB': 1.38, 'reserved_GB': 1.56} -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:38.731 actor_cell00000_rank00000] timer.py:42 - Timer wake_up end (elapsed: 0.9s) -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:38.731 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=0.9 -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:38.739 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:38.739 actor_cell00000_rank00000] timer.py:31 - Timer data_preprocess start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:38.745 actor_cell00000_rank00000] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:38.745 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:38.745 actor_cell00000_rank00000] timer.py:42 - Timer train_wait end (elapsed: 67.2s) -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:38.745 actor_cell00000_rank00000] timer.py:31 - Timer train start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:38.745 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:38.745 actor_cell00000_rank00000] timer.py:31 - Timer log_probs start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:44.772 actor_cell00000_rank00000] timer.py:42 - Timer log_probs end (elapsed: 6.0s) -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:44.772 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=6.0 -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:44.782 actor_cell00000_rank00000] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=0 -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:44.783 actor_cell00000_rank00000] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:44.784 actor_cell00000_rank00000] log_utils.py:126 - rollout 2: {'rollout/response_lengths': 264.09375, 'rollout/rewards': 0.0, 'rollout/truncated': 0.09375, 'rollout/rollout_log_probs': -0.35195159912109375, 'rollout/raw_reward': 0.625, 'rollout/total_lengths': 345.09375, 'rollout/log_probs': -0.3519270420074463, 'rollout/advantages': 3.725290298461914e-09, 'rollout/returns': 3.725290298461914e-09} -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:44.784 actor_cell00000_rank00000] timer.py:31 - Timer actor_train start -(TrainerController pid=12660) [2026-09-29 03:16:55.975 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=train ok=true elapsed_s=17.2 -(TrainerController pid=12660) [2026-09-29 03:16:55.976 trainer_controller_actor] group.py:290 - ft op=check errored= discarded= normal=0 decision=no_retry reason="survivors normal, gradients valid" -(TrainerController pid=12660) [2026-09-29 03:16:55.979 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=12660) [2026-09-29 03:16:55.982 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(TrainerController pid=12660) [2026-09-29 03:16:55.985 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:55.970 actor_cell00000_rank00000] model.py:637 - train op=train_step rollout=2 step=0 attempt=0 outcome=NORMAL valid_step=true -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:55.972 actor_cell00000_rank00000] log_utils.py:554 - step 2: {'train/loss': -3.725290298461914e-09, 'train/pg_loss': -3.725290298461914e-09, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.011719481088221073, 'train/train_rollout_kl': 0.0005182739114388824, 'train/grad_norm': 0.16807568073272705, 'train/lr-pg_0': 0.0, 'train/step': 2} -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:55.972 actor_cell00000_rank00000] model.py:847 - step 2: {'train/loss': -3.725290298461914e-09, 'train/pg_loss': -3.725290298461914e-09, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.011719481088221073, 'train/train_rollout_kl': 0.0005182739114388824, 'train/grad_norm': 0.16807568073272705, 'train/lr-pg_0': 0.0, 'train/step': 2} -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:55.972 actor_cell00000_rank00000] timer.py:42 - Timer actor_train end (elapsed: 11.2s) -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:55.972 actor_cell00000_rank00000] timer.py:42 - Timer train end (elapsed: 17.2s) -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:55.972 actor_cell00000_rank00000] timer.py:31 - Timer train_wait start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:55.972 actor_cell00000_rank00000] train_metric_utils.py:56 - perf 2: {'perf/update_weights_implementation_time': 0.08281254768371582, 'perf/finalize_and_resume_engines_time': 0.06633853912353516, 'perf/update_weights_time': 0.2015678882598877, 'perf/sleep_time': 1.4384593963623047, 'perf/wake_up_time': 0.9345066547393799, 'perf/data_preprocess_time': 0.005620479583740234, 'perf/train_wait_time': 67.2408857345581, 'perf/log_probs_time': 6.026472568511963, 'perf/actor_train_time': 11.187910795211792, 'perf/train_time': 17.226786375045776, 'perf/log_probs_tflops': 1.9780997910874896, 'perf/actor_train_tflops': 3.196565743231509, 'perf/actor_train_tok_per_s': 987.04755536004, 'perf/mfu_peak_tflops': 989.0, 'perf/actor_train_mfu': 0.0032321190528124457, 'perf/step_time': 84.46767210960388, 'perf/wait_time_ratio': 0.7960546805090997} -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:55.972 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=17.2 -(MegatronTrainRayActor pid=15843) [2026-09-29 03:16:55.973 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=17.2 -(TrainerController pid=12660) [2026-09-29 03:16:56.272 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.3 -(TrainerController pid=12660) [2026-09-29 03:17:23.330 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=12660) [2026-09-29 03:17:48.977 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=25.6 -(TrainerController pid=12660) [2026-09-29 03:17:49.080 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=12660) [2026-09-29 03:17:49.445 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.4 -(TrainerController pid=12660) [2026-09-29 03:17:49.917 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=1 -(TrainerController pid=12660) [2026-09-29 03:17:50.278 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.4 -(InferenceController pid=12656) [2026-09-29 03:17:50.695 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/resume_memory_occupation "HTTP/1.1 200 OK" -(TrainerController pid=12660) [2026-09-29 03:17:50.702 trainer_controller_actor] group.py:392 - ft op=update_weights phase=start rollout= -(TrainerController pid=12660) [2026-09-29 03:17:50.702 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=update_weights n_actors=1 -(CommandActor pid=15633) [2026-09-29 03:17:50] Cache flush skipped: KV cache memory is released. -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.705 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.705 actor_cell00000_rank00000] timer.py:31 - Timer update_weights start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.706 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 69.61, 'used_GB': 9.56, 'allocated_GB': 1.38, 'reserved_GB': 3.83} -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.711 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/pause_generation "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.714 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: GET http://172.20.0.14:22102/flush_cache "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.718 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/begin_weight_update "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.719 actor_cell00000_rank00000] timer.py:31 - Timer update_weights_implementation start -(MegatronTrainRayActor pid=15843) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(CommandActor pid=15633) [2026-09-29 03:17:50] LoRA adapter loading from tensors starts: LoRARef(lora_id=681869fce0214fa2ae8b791b1d9aabbd, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(CommandActor pid=15633) [2026-09-29 03:17:50] LoRA adapter '681869fce0214fa2ae8b791b1d9aabbd': loaded weights for target modules ['down_proj', 'gate_up_proj', 'o_proj', 'qkv_proj']. -(CommandActor pid=15633) [2026-09-29 03:17:50] LoRA adapter loading from tensors completes: LoRARef(lora_id=681869fce0214fa2ae8b791b1d9aabbd, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(CommandActor pid=15633) [2026-09-29 03:17:50] Weight version changed. old_version='yeto:2:63b0317bb2ce36b7906fce2e3e8ceb45e4d87a67e8617a607fd5f6d5c886f3d5' new_version='4' num_recorded=0 -(CommandActor pid=15633) [2026-09-29 03:17:50] [continue_generation] torch.cuda.empty_cache() called: reserved 32306.0 MB -> 32266.0 MB (freed 40.0 MB) -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.827 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/update_weights_from_tensor "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15843) [miles-colocate] Update weights: 1it [00:00, 9.22it/s][2026-09-29 03:17:50.828 actor_cell00000_rank00000] timer.py:42 - Timer update_weights_implementation end (elapsed: 0.1s) -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.828 actor_cell00000_rank00000] timer.py:31 - Timer finalize_and_resume_engines start -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.872 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/end_weight_update "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.876 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/update_weight_version "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.879 actor_cell00000_rank00000] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/continue_generation "HTTP/1.1 200 OK" -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.880 actor_cell00000_rank00000] timer.py:42 - Timer finalize_and_resume_engines end (elapsed: 0.1s) -(InferenceController pid=12656) [2026-09-29 03:17:50.959 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/resume_memory_occupation "HTTP/1.1 200 OK" -(TrainerController pid=12660) [2026-09-29 03:17:50.917 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=update_weights ok=true elapsed_s=0.2 -(CommandActor pid=15633) [2026-09-29 03:17:50] Weight version changed. old_version='4' new_version='yeto:3:2d30c85bbf38b14b9a93fa5a5f4ddb04a896235c2c81b9b62e27d8a83c40327a' num_recorded=0 -(CommandActor pid=15633) [2026-09-29 03:17:50] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(CommandActor pid=15633) [2026-09-29 03:17:51] [WeightChecker] checksum computed for 226 tensors in 0.021s -(MegatronTrainRayActor pid=15843) [miles-colocate] Update weights: 1it [00:00, 5.16it/s] -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.914 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 71.88, 'used_GB': 7.29, 'allocated_GB': 1.38, 'reserved_GB': 1.58} -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.915 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 0.2s) -(MegatronTrainRayActor pid=15843) [2026-09-29 03:17:50.915 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=0.2 -(InferenceController pid=12656) [2026-09-29 03:17:51.004 inference_controller] _client.py:1740 - HTTP Request: POST http://172.20.0.14:22102/weights_checker "HTTP/1.1 200 OK" -(RayWorkerManager pid=12652) [2026-09-29 03:17:51.183 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='rollout-executor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b7612932340>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.rollout_executor.RolloutExecutor', ctor_kwargs=. at 0x2b7612933560>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=ServeWorkerSpec(name='rollout-executor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b7612932340>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.rollout_executor.RolloutExecutor', ctor_kwargs=. at 0x2b7612933560>, concurrency_groups=None), actor_handle=Actor(RolloutExecutor, 78288e2e9c19003caa479a6b03000000), self_addrs={'rpc': HostAndPort(host='172.20.0.14', port=22137, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(RayWorkerManager pid=12652) [2026-09-29 03:17:51.184 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='inference-controller', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b76129336a0>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.inference_controller.InferenceController', ctor_kwargs=. at 0x2b7644483b00>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=ServeWorkerSpec(name='inference-controller', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b76129336a0>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.inference_controller.InferenceController', ctor_kwargs=. at 0x2b7644483b00>, concurrency_groups=None), actor_handle=Actor(InferenceController, 5326985c32cc62798b394a1c03000000), self_addrs={'rpc': HostAndPort(host='172.20.0.14', port=22100, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(RayWorkerManager pid=12652) [2026-09-29 03:17:51.186 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-controller-actor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b76444ac180>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=False), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.train.group.TrainerController', ctor_kwargs=. at 0x2b76444ac2c0>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-controller-actor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b76444ac180>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=False), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.train.group.TrainerController', ctor_kwargs=. at 0x2b76444ac2c0>, concurrency_groups=None), actor_handle=Actor(TrainerController, 2e127d6b66b7c4f5519b013e03000000), self_addrs={'rpc': HostAndPort(host='172.20.0.14', port=22140, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(RayWorkerManager pid=12652) [2026-09-29 03:17:51.187 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b76444ac360>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=1, pg_name='actor', pg_slot_offset=0, pin_to_head=False), meta=. at 0x2b76444ad800>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x2b76444ae3e0>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b76444ac360>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=1, pg_name='actor', pg_slot_offset=0, pin_to_head=False), meta=. at 0x2b76444ad800>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x2b76444ae3e0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 810111257d27f5f66531495e03000000), self_addrs={'master': HostAndPort(host='172.20.0.14', port=22138, external_host=None), 'rpc': HostAndPort(host='172.20.0.14', port=22139, external_host=None)}, local_gpu_ids=[1], gpu_slot_index=0) -(InferenceController pid=12656) [2026-09-29 03:17:51.176 inference_controller] rollout_server.py:132 - Killing server cell_id='inference-engine-all-0-0-00000'... -(InferenceController pid=12656) [2026-09-29 03:17:51.178 inference_controller] _client.py:1740 - HTTP Request: GET http://172.20.0.14:22000/workers "HTTP/1.1 200 OK" -(InferenceController pid=12656) [2026-09-29 03:17:51.180 inference_controller] _client.py:1740 - HTTP Request: DELETE http://172.20.0.14:22000/workers/71d480a9-d5a8-40cc-a8ee-3318feaf365a "HTTP/1.1 202 Accepted" -(InferenceController pid=12656) [2026-09-29 03:17:51.180 inference_controller] server_cell.py:268 - Cell inference-engine-all-0-0-00000 dispose start old=addr_info=CellAddrInfo(server_url='http://172.20.0.14:22102', bootstrap_port=None, gate_url='http://172.20.0.14:22136') -(InferenceController pid=12656) [2026-09-29 03:17:51.180 inference_controller] server_cell.py:271 - Cell inference-engine-all-0-0-00000 dispose end new= -(CommandActor pid=15633) [2026-09-29 03:17:51] SIGTERM received. signum=None frame=None. Draining requests and shutting down... -(RayWorkerManager pid=12652) [2026-09-29 03:17:51.302 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_CommandActorManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=CommandWorkerSpec(name='inference-router-0', category=None, port_infos=[PortInfo(name='primary', static_port=22000, mode='per_worker', allow_dynamic=False, num_consecutive=1, offset_by_cell=False), PortInfo(name='prometheus', static_port=9000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b7644483ba0>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x2b7644483c40>), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=CommandWorkerSpec(name='inference-router-0', category=None, port_infos=[PortInfo(name='primary', static_port=22000, mode='per_worker', allow_dynamic=False, num_consecutive=1, offset_by_cell=False), PortInfo(name='prometheus', static_port=9000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b7644483ba0>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True), meta=None, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x2b7644483c40>), actor_handle=Actor(CommandActor, 283750e48db22d30d6b7cd1603000000), self_addrs={'primary': HostAndPort(host='172.20.0.14', port=22000, external_host=None), 'prometheus': HostAndPort(host='172.20.0.14', port=22101, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(CommandActor pid=15633) [2026-09-29 03:17:51] Subprocess scheduler_0 (pid=18574) crashed with exit code -15. Triggering SIGQUIT for cleanup... -(CommandActor pid=15633) [2026-09-29 03:17:51] SIGQUIT received. signum=None, frame=None. It usually means one child failed. -(CommandActor pid=15633) [2026-09-29 03:17:51] Sleeping 5 seconds before crash diagnostics to let GPU activity settle. -(CommandActor pid=15633) WARNING:miles.utils.workers.process_utils:Process 17641 did not exit within 5.0s after SIGTERM; escalating to SIGKILL - -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:33305 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:58467 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:33338 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:17740 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:58685 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:23984 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:25894 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:26195 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:58962 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:43908 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:28108 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:21352 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:24297 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:30754 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:36] INFO: 172.20.0.14:52053 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:37] INFO: 172.20.0.14:60134 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:37] INFO: 172.20.0.14:19363 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:37] INFO: 172.20.0.14:61280 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:37] INFO: 172.20.0.14:34523 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:37] INFO: 172.20.0.14:39854 - "POST /generate HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:37] INFO: 172.20.0.14:50090 - "POST /abort_request HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:37] INFO: 172.20.0.14:46446 - "GET /flush_cache HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:16:37] INFO: 172.20.0.14:46446 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:17:27] INFO: 172.20.0.14:36462 - "GET /health HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:17:50] INFO: 172.20.0.14:50790 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:17:50] INFO: 172.20.0.14:27452 - "POST /pause_generation HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:17:50] INFO: 172.20.0.14:27452 - "GET /flush_cache HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:17:50] INFO: 172.20.0.14:27452 - "POST /begin_weight_update HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:17:50] INFO: 172.20.0.14:27452 - "POST /update_weights_from_tensor HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:17:50] INFO: 172.20.0.14:27452 - "POST /end_weight_update HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:17:50] INFO: 172.20.0.14:27452 - "POST /update_weight_version HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:17:50] INFO: 172.20.0.14:27452 - "POST /continue_generation HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:17:50] INFO: 172.20.0.14:50790 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:17:50] INFO: 172.20.0.14:45621 - "POST /update_weight_version HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:17:50] INFO: 172.20.0.14:45621 - "GET /model_info HTTP/1.1" 200 OK -(CommandActor pid=15633) [2026-09-29 03:17:51] INFO: 172.20.0.14:50790 - "POST /weights_checker HTTP/1.1" 200 OK -[rl] learner 1 finalized (rl_engine=ports) -(RayWorkerManager pid=12652) [2026-09-29 03:17:56.281 worker_manager] ray_worker_manager.py:409 - Killed actor at self=_CommandActorManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=CommandWorkerSpec(name='inference-engine-all-0-0', category='inference_engine', port_infos=[PortInfo(name='primary', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='dist_init', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=31, offset_by_cell=False), PortInfo(name='nccl', static_port=10000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='engine_info_bootstrap', static_port=12000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='gate', static_port=13000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b7644483e20>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.2, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=1, num_gpus_per_node=1, pg_name='rollout', pg_slot_offset=0, pin_to_head=False), meta=. at 0x2b7644483f60>, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x2b76444ac0e0>), actors=[...], generation=1, liveness_scan_task= wait_for=>), worker_in_cell_index=0, spec=CommandWorkerSpec(name='inference-engine-all-0-0', category='inference_engine', port_infos=[PortInfo(name='primary', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='dist_init', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=31, offset_by_cell=False), PortInfo(name='nccl', static_port=10000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='engine_info_bootstrap', static_port=12000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='gate', static_port=13000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x2b7644483e20>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.2, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=1, num_gpus_per_node=1, pg_name='rollout', pg_slot_offset=0, pin_to_head=False), meta=. at 0x2b7644483f60>, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x2b76444ac0e0>), actor_handle=Actor(CommandActor, 9721943b83bb109e2189aaa903000000), self_addrs={'primary': HostAndPort(host='172.20.0.14', port=22102, external_host=None), 'dist_init': HostAndPort(host='172.20.0.14', port=22103, external_host=None), 'nccl': HostAndPort(host='172.20.0.14', port=22134, external_host=None), 'engine_info_bootstrap': HostAndPort(host='172.20.0.14', port=22135, external_host=None), 'gate': HostAndPort(host='172.20.0.14', port=22136, external_host=None)}, local_gpu_ids=[1], gpu_slot_index=0) diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/syncer.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/syncer.jsonl deleted file mode 100644 index 2a39bb495..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/syncer.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","sync/base_version":0,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.337,"sync/global_delta_norm":0.16523045229077404,"step":1,"fragment":0,"launch_base_version":0,"attempt":1,"gnorm":0.16523045229077404,"ms":331887,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":8207408449347384639},{"id":1,"generation":8539465229979669959}],"responded":[0,1],"responded_members":[{"id":0,"generation":8207408449347384639},{"id":1,"generation":8539465229979669959}],"missed_grace":[],"missed_members":[],"quorum_ms":331526,"grace_ms":0,"sync_ms":337,"responders":[{"id":0,"generation":8207408449347384639,"base_version":0,"staleness":0,"c_steps":1,"c_tokens":1,"weight":1,"contribution":0.5},{"id":1,"generation":8539465229979669959,"base_version":0,"staleness":0,"c_steps":1,"c_tokens":1,"weight":1,"contribution":0.5}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","sync/base_version":1,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.311,"sync/global_delta_norm":0.1429024636732199,"step":2,"fragment":0,"launch_base_version":1,"attempt":1,"gnorm":0.1429024636732199,"ms":134144,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":8207408449347384639},{"id":1,"generation":8539465229979669959}],"responded":[0,1],"responded_members":[{"id":0,"generation":8207408449347384639},{"id":1,"generation":8539465229979669959}],"missed_grace":[],"missed_members":[],"quorum_ms":133811,"grace_ms":0,"sync_ms":311,"responders":[{"id":0,"generation":8207408449347384639,"base_version":1,"staleness":0,"c_steps":1,"c_tokens":1,"weight":1,"contribution":0.5},{"id":1,"generation":8539465229979669959,"base_version":1,"staleness":0,"c_steps":1,"c_tokens":1,"weight":1,"contribution":0.5}]} -{"protocol_version":4,"delta_semantics":"local_minus_raw_anchor","sync/layout_hash":"f1c29481efeffd57ff1451a1e510ab088b2fdb082e01341d6313140ccf03b8b3","sync/base_version":2,"sync/responders":2,"sync/quorum":2,"sync/rejected_stale_updates":0,"sync/merge_seconds":0.242,"sync/global_delta_norm":0.07067725849102306,"step":3,"fragment":0,"launch_base_version":2,"attempt":1,"gnorm":0.07067725849102306,"ms":82850,"quorum":2,"expected":[0,1],"expected_members":[{"id":0,"generation":8207408449347384639},{"id":1,"generation":8539465229979669959}],"responded":[0,1],"responded_members":[{"id":0,"generation":8207408449347384639},{"id":1,"generation":8539465229979669959}],"missed_grace":[],"missed_members":[],"quorum_ms":82583,"grace_ms":0,"sync_ms":242,"responders":[{"id":0,"generation":8207408449347384639,"base_version":2,"staleness":0,"c_steps":1,"c_tokens":1,"weight":1,"contribution":0.5},{"id":1,"generation":8539465229979669959,"base_version":2,"staleness":0,"c_steps":1,"c_tokens":1,"weight":1,"contribution":0.5}]} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/syncer.log b/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/syncer.log deleted file mode 100644 index 8f8c39a6f..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/syncer.log +++ /dev/null @@ -1,29 +0,0 @@ -2026-09-29T03:04:13.570865Z INFO yeto_syncer::server: syncer listening port=35871 semantic_profile_sha256="1e3a092fbeb8c06c79dcc3d771088ab76debcdc692d0507904484411c0864cd6" require_profile_binding=false -2026-09-29T03:04:13.570992Z INFO yeto_syncer::server: waiting for learners and INIT_PARAMS expected=2 -2026-09-29T03:04:13.651988Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:58402 -2026-09-29T03:07:47.778729Z INFO yeto_syncer::server: learner connected (layout: 1 fragments) learner_id=0 generation=8207408449347384639 num_streams=4 -2026-09-29T03:07:47.778875Z INFO yeto_syncer::server: initialized global state iso_backend=scalar -2026-09-29T03:07:47.875950Z INFO yeto_syncer::server: learner connected (layout: 1 fragments) learner_id=1 generation=8539465229979669959 num_streams=4 -2026-09-29T03:07:47.877766Z INFO yeto_syncer::server: global parameters initialized -2026-09-29T03:07:48.352465Z INFO yeto_syncer::server: checkpoint committed step=0 path=/work/out/strict2-federated/work/seed-17/yeto-federated-m2/state.ckpt -2026-09-29T03:13:20.249377Z INFO yeto_syncer::server: checkpoint committed step=1 path=/work/out/strict2-federated/work/seed-17/yeto-federated-m2/state.ckpt -2026-09-29T03:13:20.328671Z INFO yeto_syncer::server: outer step step=1 fragment=0 responders=[Member { learner_id: 0, generation: 8207408449347384639 }, Member { learner_id: 1, generation: 8539465229979669959 }] gnorm="0.1652" ms=331887 -2026-09-29T03:15:34.385015Z INFO yeto_syncer::server: checkpoint committed step=2 path=/work/out/strict2-federated/work/seed-17/yeto-federated-m2/state.ckpt -2026-09-29T03:15:34.473504Z INFO yeto_syncer::server: outer step step=2 fragment=0 responders=[Member { learner_id: 0, generation: 8207408449347384639 }, Member { learner_id: 1, generation: 8539465229979669959 }] gnorm="0.1429" ms=134144 -2026-09-29T03:16:57.318193Z INFO yeto_syncer::server: checkpoint committed step=3 path=/work/out/strict2-federated/work/seed-17/yeto-federated-m2/state.ckpt -2026-09-29T03:16:57.324233Z INFO yeto_syncer::server: outer step step=3 fragment=0 responders=[Member { learner_id: 0, generation: 8207408449347384639 }, Member { learner_id: 1, generation: 8539465229979669959 }] gnorm="0.0707" ms=82850 -2026-09-29T03:17:51.012133Z INFO yeto_syncer::server: learner finalized authoritative parameters learner_id=1 generation=8539465229979669959 step=3 remaining=1 -2026-09-29T03:17:51.165942Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:38080 -2026-09-29T03:17:51.165969Z WARN yeto_syncer::server: learner disconnected during finalization; waiting for reconnect learner_id=1 generation=8539465229979669959 -2026-09-29T03:17:51.166938Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:61421 -2026-09-29T03:17:51.167219Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:38116 -2026-09-29T03:17:51.167543Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:53124 -2026-09-29T03:17:51.167763Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:63703 -2026-09-29T03:17:51.714806Z INFO yeto_syncer::server: learner finalized authoritative parameters learner_id=0 generation=8207408449347384639 step=3 remaining=0 -2026-09-29T03:17:51.714858Z INFO yeto_syncer::server: all learners acknowledged final cut learners=2 -2026-09-29T03:17:51.714890Z INFO yeto_syncer::server: training complete after 3 outer steps -2026-09-29T03:17:51.853582Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:21335 -2026-09-29T03:17:51.855851Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:26703 -2026-09-29T03:17:51.855923Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:37250 -2026-09-29T03:17:51.856161Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:57422 -2026-09-29T03:17:51.856401Z WARN yeto_syncer::server: connection ended: unexpected end of file peer=127.0.0.1:43222 diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-a-failed-race/work/seed-17/yeto-federated-m2/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-a-failed-race/work/seed-17/yeto-federated-m2/island-0/events.jsonl deleted file mode 100644 index 1da44d1f1..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-a-failed-race/work/seed-17/yeto-federated-m2/island-0/events.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":0,"time_unix":1790675624.3365195} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":0,"time_unix":1790675624.6095378} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":0,"time_unix":1790675624.6100786} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":0,"time_unix":1790675625.0213344} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":1,"policy_version":0,"reset_parameter_count":392,"rl/global_policy_version":0,"rl/optimizer_reset_count":1,"sync/global_policy_hash":"62f3b8f69c1cb9f2db1ab500f19da0dd76a5323536e6f38b0ee81bf1a486eaa6","time_unix":1790675625.0419207} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-a/work/seed-17/yeto-federated-m2/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-a/work/seed-17/yeto-federated-m2/island-0/events.jsonl deleted file mode 100644 index 50c20622f..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-a/work/seed-17/yeto-federated-m2/island-0/events.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":0,"time_unix":1790676205.1255362} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":0,"time_unix":1790676205.5254612} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":0,"time_unix":1790676205.5257957} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":0,"time_unix":1790676205.918825} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":1,"policy_version":0,"reset_parameter_count":392,"rl/global_policy_version":0,"rl/optimizer_reset_count":1,"sync/global_policy_hash":"62f3b8f69c1cb9f2db1ab500f19da0dd76a5323536e6f38b0ee81bf1a486eaa6","time_unix":1790676205.943197} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.23342479765415192,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":0.06565981299996793,"group_p95_seconds":0.06567054000002485,"group_p99_seconds":0.06567139200003226,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":0.0005554890958592296,"nonzero_advantage_count":24,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5227,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0005554890958592296,"rl/ess_ratio":1.0,"rl/global_policy_version":0,"rl/group_p50_seconds":0.06565981299996793,"rl/group_p95_seconds":0.06567054000002485,"rl/group_p99_seconds":0.06567139200003226,"rl/local_delta_norm":0.23342479765415192,"rl/mixed_version_group_count":0,"rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_policy_version":0,"rl/rollout_seconds":0.502177455999913,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":0.502177455999913,"sync/bytes_sent":40370224,"time_unix":1790676246.569311,"tool_wait_seconds":0.0,"train_seconds":20.580076194999947,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":1,"time_unix":1790676247.8050554} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":1,"time_unix":1790676248.127343} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":1,"time_unix":1790676248.128019} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":1,"time_unix":1790676248.6024125} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":2,"policy_version":1,"reset_parameter_count":392,"rl/global_policy_version":1,"rl/optimizer_reset_count":2,"sync/global_policy_hash":"a4bac44b623f4e11c79353a73fbc1800a1728371053c77ba6b45885e1f5ac0f0","time_unix":1790676248.6145828} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-b/work/seed-17/yeto-federated-m2/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-b/work/seed-17/yeto-federated-m2/island-0/events.jsonl deleted file mode 100644 index fe97691aa..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfl-b/work/seed-17/yeto-federated-m2/island-0/events.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":0,"time_unix":1790676695.7680018} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":0,"time_unix":1790676696.139278} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":0,"time_unix":1790676696.1397235} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":0,"time_unix":1790676696.6018827} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":1,"policy_version":0,"reset_parameter_count":392,"rl/global_policy_version":0,"rl/optimizer_reset_count":1,"sync/global_policy_hash":"62f3b8f69c1cb9f2db1ab500f19da0dd76a5323536e6f38b0ee81bf1a486eaa6","time_unix":1790676696.6260607} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":0.0,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.23342479765415192,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":4,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":null,"group_p50_seconds":0.06215635149993659,"group_p95_seconds":0.062168640949983,"group_p99_seconds":0.06216962818999264,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":0.0005554890958592296,"nonzero_advantage_count":24,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5227,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":0.0,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":0.0005554890958592296,"rl/ess_ratio":1.0,"rl/global_policy_version":0,"rl/group_p50_seconds":0.06215635149993659,"rl/group_p95_seconds":0.062168640949983,"rl/group_p99_seconds":0.06216962818999264,"rl/local_delta_norm":0.23342479765415192,"rl/mixed_version_group_count":0,"rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_policy_version":0,"rl/rollout_seconds":0.5607653140000366,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":0.5607653140000366,"sync/bytes_sent":40370224,"time_unix":1790676727.4968479,"tool_wait_seconds":0.0,"train_seconds":15.286655385999893,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"begin","policy_version":1,"time_unix":1790676728.3704665} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"apply_finished","policy_version":1,"time_unix":1790676728.684817} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_begin","policy_version":1,"time_unix":1790676728.6853132} -{"event":"rl_policy_apply_progress","island_id":0,"phase":"hash_finished","policy_version":1,"time_unix":1790676729.268348} -{"event":"rl_policy_apply","island_id":0,"optimizer_reset_count":2,"policy_version":1,"reset_parameter_count":392,"rl/global_policy_version":1,"rl/optimizer_reset_count":2,"sync/global_policy_hash":"a4bac44b623f4e11c79353a73fbc1800a1728371053c77ba6b45885e1f5ac0f0","time_unix":1790676729.2805007} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfp-a/work/seed-17/yeto-federated-m2/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfp-a/work/seed-17/yeto-federated-m2/island-0/events.jsonl deleted file mode 100644 index 952fc8258..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfp-a/work/seed-17/yeto-federated-m2/island-0/events.jsonl +++ /dev/null @@ -1,19 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790675388.6220732} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:19a443471da34825cd6ad8f40fb2ea77df0b519b68a9761822c6b75f10915a2a","time_unix":1790675540.2210076} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":0,"time_unix":1790675544.1651087} -{"event":"rl_policy_apply","island_id":0,"local_step":0,"optimizer":"reset","partial_fragment_apply":false,"policy_version":0,"sync/apply_seconds":133.97908590999998,"sync/global_policy_hash":"a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","time_unix":1790675678.1445515} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790675678.2288167} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790675786.8410404} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790675786.8412669} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790675786.8477929} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790675788.857888} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790675789.743749} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790675823.7421436} -{"event":"rl_driver_phase","island_id":0,"phase":"export_push","rollout_id":0,"time_unix":1790675823.742306} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.2334245890378952,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6349465847015381,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5227,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":0,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.2334245890378952,"rl/mixed_version_group_count":0,"rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_policy_version":0,"rl/rollout_seconds":2.015859567000007,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":2.015859567000007,"sync/bytes_sent":40370224,"time_unix":1790675824.7431793,"tool_wait_seconds":0.0,"train_seconds":33.99795263699991,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"wait_global","policy_version":1,"time_unix":1790675824.793935} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":1,"time_unix":1790675825.3179612} -{"event":"rl_policy_apply","island_id":0,"local_step":1,"optimizer":"reset","partial_fragment_apply":false,"policy_version":1,"sync/apply_seconds":104.34266516399998,"sync/global_policy_hash":"f1988b39018907f8f2a294b1b78607356f9a20ae443f32e19f11728d6f3cc462","time_unix":1790675929.661133} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790675929.7360785} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:f1988b39018907f8f2a294b1b78607356f9a20ae443f32e19f11728d6f3cc462","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"0f32b878c1abc1596dd3be5e292389c10f1540c438b1517f5f7f6cbf24c5beea","time_unix":1790675931.1261024} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":1,"time_unix":1790675931.126497} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfp-b/work/seed-17/yeto-federated-m2/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfp-b/work/seed-17/yeto-federated-m2/island-0/events.jsonl deleted file mode 100644 index 22285c29a..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-tf-diagnosis/tfp-b/work/seed-17/yeto-federated-m2/island-0/events.jsonl +++ /dev/null @@ -1,19 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790676106.4100897} -{"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:50f7cc565703734647f92f1647dbe32d661bed8821f74ebc3e48d33623ec7eaf","time_unix":1790676265.2531772} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":0,"time_unix":1790676268.7526982} -{"event":"rl_policy_apply","island_id":0,"local_step":0,"optimizer":"reset","partial_fragment_apply":false,"policy_version":0,"sync/apply_seconds":126.27690912000003,"sync/global_policy_hash":"a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","time_unix":1790676395.03036} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790676395.1245227} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f","time_unix":1790676461.5998445} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790676461.6007543} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790676461.6068861} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790676464.1824112} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790676465.0022233} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790676488.0130813} -{"event":"rl_driver_phase","island_id":0,"phase":"export_push","rollout_id":0,"time_unix":1790676488.0132725} -{"action_tokens":5227,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.23342415690422058,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.6349620223045349,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.8125,"reward_std":0.3903123748998999,"rl/action_tokens":5227,"rl/active_groups":4,"rl/cancelled_groups":0,"rl/clip_fraction":null,"rl/completed_groups":4,"rl/completed_trajectories":32,"rl/current_vs_rollout_kl":null,"rl/ess_ratio":null,"rl/global_policy_version":0,"rl/group_p50_seconds":0.0,"rl/group_p95_seconds":0.0,"rl/group_p99_seconds":0.0,"rl/local_delta_norm":0.23342415690422058,"rl/mixed_version_group_count":0,"rl/reward_mean":0.8125,"rl/reward_std":0.3903123748998999,"rl/rollout_policy_version":0,"rl/rollout_seconds":2.5799611909999385,"rl/tool_wait_seconds":0.0,"rl/zero_variance_group_ratio":0.25,"rollout_seconds":2.5799611909999385,"sync/bytes_sent":40370224,"time_unix":1790676489.0009284,"tool_wait_seconds":0.0,"train_seconds":23.010525443999995,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"wait_global","policy_version":1,"time_unix":1790676489.0535777} -{"event":"rl_driver_phase","island_id":0,"optimizer":"reset","phase":"apply","policy_version":1,"time_unix":1790676489.5955925} -{"event":"rl_policy_apply","island_id":0,"local_step":1,"optimizer":"reset","partial_fragment_apply":false,"policy_version":1,"sync/apply_seconds":57.84778806300005,"sync/global_policy_hash":"d039b1386225ff7e9e0e6685cec86018dead372a028f0a536838e7349e89211e","time_unix":1790676547.4439137} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790676547.5358477} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:d039b1386225ff7e9e0e6685cec86018dead372a028f0a536838e7349e89211e","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"26c8204bc8d51c29f5333068551950d2aea004a2f4aa77c5337964234bce1981","time_unix":1790676548.793009} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":1,"time_unix":1790676548.7934983} diff --git a/openspec/changes/rl-engine-ports/evidence/2026-09-29-verda-smoke/island-0/events.jsonl b/openspec/changes/rl-engine-ports/evidence/2026-09-29-verda-smoke/island-0/events.jsonl deleted file mode 100644 index bdc76c22a..000000000 --- a/openspec/changes/rl-engine-ports/evidence/2026-09-29-verda-smoke/island-0/events.jsonl +++ /dev/null @@ -1,29 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"0394715083c91182b5eb0c526eeee4196ac694b9","placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl_engine":"ports","time_unix":1790655618.9032362} -{"event":"rl_driver_start","execution_mode":"serial-colocated","island_id":0,"placement":"colocated","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:a206596e07bd148fc0ac01475d5019ed43a95653521be57288ef14d46a26ef2d","time_unix":1790655678.4944916} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790655683.2633674} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:1188152a0b19bf77f2a7e9e6ecf2a2745516ed2e56fdde51dc86c330b4a2c6ad","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"d486f19ac85362c5bd7264dabfab1770988a29ea807e01f5dd7f1fb015b8ef27","time_unix":1790655760.948427} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1790655760.9485743} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790655760.9523327} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1790655770.582674} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790655771.4928367} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790655805.021604} -{"action_tokens":5222,"active_groups":4,"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.5958814024925232,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":null,"lr":null,"mean_kl":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.90625,"reward_std":0.2914805954090255,"rollout_seconds":9.634057805999873,"time_unix":1790655805.557966,"tool_wait_seconds":0.0,"train_seconds":33.528509833000044,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790655805.6281712} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:c93f64138c688683cbc1fc954e3345aa2df2f7cc61a90fb7aa17a8a396d96e26","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"7f78e6b3abf52c4ae0f0e9a50e82742116a78af8930347d5deb6f2d28f22f7ce","time_unix":1790655807.0543776} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1790655811.1506631} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790655813.0252967} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1790655818.7477536} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790655819.7912161} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790655836.8179746} -{"action_tokens":5619,"active_groups":4,"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.4945510923862457,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":null,"lr":null,"mean_kl":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.4375,"reward_std":0.49607837082461076,"rollout_seconds":7.597040026000059,"time_unix":1790655837.2693474,"tool_wait_seconds":0.0,"train_seconds":17.02650296999991,"train_step":null,"zero_variance_group_ratio":0.25} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790655837.3394194} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:49df7f1c9ed6b85a7e9b635cc000339b9b05bfb0497f55404fac71a176cd1d4b","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"04a4f656495feefd0a4e1209701e35c685749a9711bc4cdb82cfb95876d74afc","time_unix":1790655838.664413} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1790655838.6648507} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790655840.3034742} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1790655846.0136938} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790655847.0411458} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790655865.1646872} -{"action_tokens":9485,"active_groups":4,"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":4,"completed_trajectories":32,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":0,"dynamic_filter_replacement_attempts":0,"ess_ratio":null,"event":"rl_local_round","grad_norm":0.2542969882488251,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":null,"lr":null,"mean_kl":null,"pass_rate":null,"pg_loss":null,"reward_mean":0.09375,"reward_std":0.2914805954090255,"rollout_seconds":7.348813269999937,"time_unix":1790655865.6142504,"tool_wait_seconds":0.0,"train_seconds":18.12329972700013,"train_step":null,"zero_variance_group_ratio":0.5} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790655865.6841695} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:23edd7555b184d73dbf49d3e7929bc90741faf9b873529c03cdc2d999a0d63f2","sync/publication_members":["engine:inference-engine-all-0-0-00000"],"sync/publication_payload_bytes":40370176,"sync/publication_payload_hash":"9988db222b659c244d64271d8a713ed23935345a8b82876ed2ddf6585862876f","time_unix":1790655867.0038555} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":3,"time_unix":1790655867.00463} diff --git a/openspec/changes/rl-engine-ports/evidence/EVIDENCE.md b/openspec/changes/rl-engine-ports/evidence/EVIDENCE.md new file mode 100644 index 000000000..969e73e5c --- /dev/null +++ b/openspec/changes/rl-engine-ports/evidence/EVIDENCE.md @@ -0,0 +1,106 @@ +# Evidence moved out of the repository + +Run logs that used to live under this directory were removed from git by change `rl-code-quality-cleanup` (2026-10-12). +Copies are kept outside the repository at: + + /home/michael/work/infra-drafts/evidence/rl-engine-ports/evidence/ + +The relative path under that directory is the same as the old path under this directory. +Moved files: + +- 2026-09-29-decoupled2/verify_peft.log +- 2026-09-29-decoupled2/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-default-params-modal/export.log +- 2026-09-29-default-params-modal/pulled/rl-island-0.jsonl +- 2026-09-29-eq62-v2/legacy-s17/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62-v2/ports-s17/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62-v2/tf-legacy/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62-v2/tf-ports/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62/legacy-s17/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62/legacy-s18/work/seed-18/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62/legacy-s19/work/seed-19/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62/legacy-s20/work/seed-20/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62/legacy-s21/work/seed-21/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62/ports-s17/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62/ports-s18/work/seed-18/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62/ports-s19/work/seed-19/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62/ports-s20/work/seed-20/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62/ports-s21/work/seed-21/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62/tf-legacy/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq62/tf-ports/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-eq63/legacy-s17/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-eq63/legacy-s18/work/seed-18/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-eq63/legacy-s19/work/seed-19/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-eq63/legacy-s20/work/seed-20/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-eq63/legacy-s21/work/seed-21/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-eq63/ports-s17/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-eq63/ports-s18/work/seed-18/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-eq63/ports-s19/work/seed-19/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-eq63/ports-s20/work/seed-20/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-eq63/ports-s21/work/seed-21/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-eq63/tf-legacy/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-eq63/tf-ports/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-kill44-legacy-vs-ports/drive-legacy.log +- 2026-09-29-kill44-legacy-vs-ports/drive-ports.log +- 2026-09-29-kill44-legacy-vs-ports/kill44-legacy/island-0/events.jsonl +- 2026-09-29-kill44-legacy-vs-ports/kill44-legacy/miles-first.log +- 2026-09-29-kill44-legacy-vs-ports/kill44-legacy/miles-restart.log +- 2026-09-29-kill44-legacy-vs-ports/kill44-legacy/run.log +- 2026-09-29-kill44-legacy-vs-ports/kill44-legacy/syncer.log +- 2026-09-29-kill44-legacy-vs-ports/kill44-ports/island-0/events.jsonl +- 2026-09-29-kill44-legacy-vs-ports/kill44-ports/miles-first.log +- 2026-09-29-kill44-legacy-vs-ports/kill44-ports/miles-restart.log +- 2026-09-29-kill44-legacy-vs-ports/kill44-ports/run.log +- 2026-09-29-kill44-legacy-vs-ports/kill44-ports/syncer.log +- 2026-09-29-kill44-strict1/island-0/events.jsonl +- 2026-09-29-legacy-baseline-v2/legacy-0/benchmark.log +- 2026-09-29-legacy-baseline-v2/legacy-0/report/results.jsonl +- 2026-09-29-legacy-baseline-v2/legacy-0/work/data/m2/combined.jsonl +- 2026-09-29-legacy-baseline-v2/legacy-0/work/data/m2/eval.jsonl +- 2026-09-29-legacy-baseline-v2/legacy-0/work/data/m2/island-0.jsonl +- 2026-09-29-legacy-baseline-v2/legacy-0/work/data/m2/island-1.jsonl +- 2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/eval-samples.jsonl +- 2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/eval.log +- 2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-0/miles.log +- 2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-1/events.jsonl +- 2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/island-1/miles.log +- 2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/syncer.jsonl +- 2026-09-29-legacy-baseline-v2/legacy-0/work/seed-17/yeto-federated-m2/syncer.log +- 2026-09-29-legacy-baseline-v2/legacy-1/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-legacy-baseline-v2/legacy-2/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-legacy-baseline/legacy-0/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-legacy-baseline/legacy-1/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-legacy-baseline/legacy-2/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-rerun-decoupled/work/seed-17/yeto-decoupled-m2/island-0/events.jsonl +- 2026-09-29-rerun-kill44/island-0/events.jsonl +- 2026-09-29-rerun-smoke-badtoken/island-0/events.jsonl +- 2026-09-29-rerun-smoke-normal/island-0/events.jsonl +- 2026-09-29-rerun-smoke-normal/probe.jsonl +- 2026-09-29-rerun-smoke-pubfail/island-0/events.jsonl +- 2026-09-29-rerun-smoke-zerograd/island-0/events.jsonl +- 2026-09-29-rerun-strict2/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-smoke-badtoken/island-0/events.jsonl +- 2026-09-29-smoke-normal/island-0/events.jsonl +- 2026-09-29-smoke-pubfail/island-0/events.jsonl +- 2026-09-29-smoke-zerograd/island-0/events.jsonl +- 2026-09-29-strict2-federated/benchmark.log +- 2026-09-29-strict2-federated/report/results.jsonl +- 2026-09-29-strict2-federated/work/data/m2/combined.jsonl +- 2026-09-29-strict2-federated/work/data/m2/eval.jsonl +- 2026-09-29-strict2-federated/work/data/m2/island-0.jsonl +- 2026-09-29-strict2-federated/work/data/m2/island-1.jsonl +- 2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/eval-samples.jsonl +- 2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/eval.log +- 2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-0/miles.log +- 2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-1/events.jsonl +- 2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/island-1/miles.log +- 2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/syncer.jsonl +- 2026-09-29-strict2-federated/work/seed-17/yeto-federated-m2/syncer.log +- 2026-09-29-tf-diagnosis/tfl-a-failed-race/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-tf-diagnosis/tfl-a/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-tf-diagnosis/tfl-b/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-tf-diagnosis/tfp-a/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-tf-diagnosis/tfp-b/work/seed-17/yeto-federated-m2/island-0/events.jsonl +- 2026-09-29-verda-smoke/island-0/events.jsonl diff --git a/openspec/changes/rl-infra-spec/evidence/2026-09-30-img-5c1b49e/check_flags.log b/openspec/changes/rl-infra-spec/evidence/2026-09-30-img-5c1b49e/check_flags.log deleted file mode 100644 index 2399f290b..000000000 --- a/openspec/changes/rl-infra-spec/evidence/2026-09-30-img-5c1b49e/check_flags.log +++ /dev/null @@ -1,9 +0,0 @@ -miles from /tmp/img-verify-5c1b/root/miles/miles/utils/arguments.py -git HEAD 5c1b49ebccbc7508c1d9ef89eacc2db3e448b6ba -defaults policy_loss 1.0 1.05 0.4 0.4 -OK ['--policy-loss-variant', 'cispo'] cispo 1.0 1.05 0.4 0.4 -OK ['--policy-loss-variant', 'sapo', '--sapo-tau-pos', '0.9', '--sapo-tau-neg', '1.2'] sapo 0.9 1.2 0.4 0.4 -OK ['--policy-loss-variant', 'gmpo', '--gmpo-log-clip-low', '0.3', '--gmpo-log-clip-high', '0.5'] gmpo 1.0 1.05 0.3 0.5 -rejected as expected ['--policy-loss-variant', 'gmpo', '--calculate-per-token-loss'] -> --policy-loss-variant gmpo cannot be combined with --calculate-per-token-loss -rejected as expected ['--policy-loss-variant', 'sapo', '--sapo-tau-pos', '0'] -> --sapo-tau-pos must be finite and > 0, got 0.0 -argparse rejects bogus variant diff --git a/openspec/changes/rl-infra-spec/evidence/EVIDENCE.md b/openspec/changes/rl-infra-spec/evidence/EVIDENCE.md new file mode 100644 index 000000000..ddacfb768 --- /dev/null +++ b/openspec/changes/rl-infra-spec/evidence/EVIDENCE.md @@ -0,0 +1,54 @@ +# Evidence moved out of the repository + +Run logs that used to live under this directory were removed from git by change `rl-code-quality-cleanup` (2026-10-12). +Copies are kept outside the repository at: + + /home/michael/work/infra-drafts/evidence/rl-infra-spec/evidence/ + +The relative path under that directory is the same as the old path under this directory. +Moved files: + +- 2026-09-30-img-5c1b49e/check_flags.log +- fn/try27-s12-h100-20261006a/fna/pulled/rl-island-0.jsonl +- fn/try27-s12-h100-20261006a/fna/pulled/run.log +- fn/try27-s12-h100-20261006a/fnprep/fnprep-remote.log +- image/attempt2/imports_cpu.log +- image/attempt2/ns_probe.log +- infra-a/2.2-2.3/round2-e0a/tape.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-b1/state/elastic-state/ledger/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-b1/state/elastic-state/reconfig/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-f1/state/elastic-state/ledger/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-f1/state/elastic-state/reconfig/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-f2/state/elastic-state/ledger/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-f2/state/elastic-state/reconfig/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-f3/state/elastic-state/ledger/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-f3/state/elastic-state/reconfig/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-f5-e2r/state/elastic-state/ledger/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-f5-e2r/state/elastic-state/reconfig/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-f5/state/elastic-state/ledger/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-f5/state/elastic-state/reconfig/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-f6a/state/elastic-state/ledger/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-f6a/state/elastic-state/reconfig/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-rb/state/elastic-state/ledger/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-rb/state/elastic-state/reconfig/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-rbold/state/elastic-state/ledger/journal.jsonl +- infra-e2/4.2-4.5/gpu-v6-c3-rbold/state/elastic-state/reconfig/journal.jsonl +- infra-v2-b1/a4/base/elastic-state/ledger/journal.jsonl +- infra-v2-b1/a4/base/elastic-state/reconfig/journal.jsonl +- infra-v2-b1/a4/e1b/elastic-state/ledger/journal.jsonl +- infra-v2-b1/a4/e1b/elastic-state/reconfig/journal.jsonl +- infra-v2-b1/a4/x2/elastic-state/ledger/journal.jsonl +- infra-v2-b1/a4/x2/elastic-state/reconfig/journal.jsonl +- infra-v2-b1/a4s3/e1b/elastic-state/ledger/journal.jsonl +- infra-v2-b1/a4s3/e1b/elastic-state/reconfig/journal.jsonl +- infra-v2-b1/a4s3/watchdog/elastic-state/ledger/journal.jsonl +- infra-v2-b1/a4s3/watchdog/elastic-state/reconfig/journal.jsonl +- infra-v2-b1/a4s3/watchdog/watchdog.out +- infra-v2-b1/fe1/elastic-state/ledger/journal.jsonl +- infra-v2-b1/fe1/elastic-state/reconfig/journal.jsonl +- infra-v2-b1/fe1r/fe1r/elastic-state/ledger/journal.jsonl +- infra-v2-b1/fe1r/fe1r/elastic-state/reconfig/journal.jsonl +- infra-v2-b1/fe1r4/fe1r5/elastic-state/ledger/journal.jsonl +- infra-v2-b1/fe1r4/fe1r5/elastic-state/reconfig/journal.jsonl +- infra-v2-b1/nsmoke/launcher.log +- ports-image/2026-10-02-m3a27/check-old.log diff --git a/openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fna/pulled/rl-island-0.jsonl b/openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fna/pulled/rl-island-0.jsonl deleted file mode 100644 index 777499b6f..000000000 --- a/openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fna/pulled/rl-island-0.jsonl +++ /dev/null @@ -1,128 +0,0 @@ -{"event":"rl_engine_selected","island_id":0,"miles_commit":"c35702eefcf2862cee155e46870e6ad30568d2c6","placement":"colocated","rl/algorithm_absorbed_flags":{},"rl/algorithm_spec":"{\"advantage_estimator\":\"grpo\",\"dynamic_sampling_filter\":null,\"dynamic_sampling_max_replacements\":null,\"kl_coef\":null,\"loss\":\"policy_loss\",\"schema\":\"yeto-rl-algorithm-spec-v1\"}","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl/outer_sync":false,"rl_engine":"ports","time_unix":1791285031.9600205} -{"config_epoch":0,"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:f70d5baffa871590366c1eb4164ea682b9947a050d79a6c2f9fc306dd61517e3","time_unix":1791285122.8190894} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1791285136.3302248} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"publish","pid":18336,"policy_version":null,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":null,"rounds_completed":0,"seq":1,"time_unix":1791285152.7571907,"trained_version":null,"uptime_s":30.001,"weight_transport":"cuda-ipc"} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"publish","pid":18336,"policy_version":null,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":null,"rounds_completed":0,"seq":2,"time_unix":1791285182.7578611,"trained_version":null,"uptime_s":60.001,"weight_transport":"cuda-ipc"} -{"available":true,"config_epoch":0,"cpu_rss_bytes":3774410752,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":67098.625,"power_w":122.102,"util_pct":0,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":67242.625,"power_w":131.11,"util_pct":0,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":67242.625,"power_w":129.555,"util_pct":0,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":66762.625,"power_w":123.547,"util_pct":0,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":67146.625,"power_w":126.839,"util_pct":0,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":67242.625,"power_w":123.279,"util_pct":0,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":67242.625,"power_w":126.596,"util_pct":0,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":66522.625,"power_w":125.32,"util_pct":0,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1025.398844937,"time_unix":1791285182.910941,"weight_transport":"cuda-ipc"} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"publish","pid":18336,"policy_version":null,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":null,"rounds_completed":0,"seq":3,"time_unix":1791285212.7584136,"trained_version":null,"uptime_s":90.002,"weight_transport":"cuda-ipc"} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"publish","pid":18336,"policy_version":null,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":null,"rounds_completed":0,"seq":4,"time_unix":1791285242.7589486,"trained_version":null,"uptime_s":120.002,"weight_transport":"cuda-ipc"} -{"available":true,"config_epoch":0,"cpu_rss_bytes":3774410752,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":69016.625,"power_w":119.589,"util_pct":0,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":69160.625,"power_w":127.132,"util_pct":0,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":69160.625,"power_w":125.889,"util_pct":0,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":68680.625,"power_w":121.374,"util_pct":0,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":69064.625,"power_w":122.916,"util_pct":0,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":69160.625,"power_w":119.077,"util_pct":0,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":69160.625,"power_w":122.757,"util_pct":0,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":68440.625,"power_w":121.62,"util_pct":0,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1085.399712235,"time_unix":1791285242.9117813,"weight_transport":"cuda-ipc"} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"publish","pid":18336,"policy_version":null,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":null,"rounds_completed":0,"seq":5,"time_unix":1791285272.7594874,"trained_version":null,"uptime_s":150.003,"weight_transport":"cuda-ipc"} -{"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285279.1716428} -{"end":1121.659985907,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":978.818225393,"task":"publish","time_unix":1791285279.1720073,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1791285279.1721241} -{"end":1121.665593291,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":0,"start":1121.660211319,"task":"offload","time_unix":1791285279.1776223,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1791285279.17777} -{"end":1140.45609471,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"rollout","rollout_id":0,"start":1121.665857615,"task":"generate","time_unix":1791285297.9681737,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1791285297.9683726} -{"end":1141.889917275,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":0,"start":1140.456458337,"task":"onload","time_unix":1791285299.4020224,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1791285299.402256} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":0,"rounds_completed":0,"seq":6,"time_unix":1791285302.7600424,"trained_version":null,"uptime_s":180.003,"weight_transport":"cuda-ipc"} -{"available":true,"config_epoch":0,"cpu_rss_bytes":3355287552,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":13592.625,"power_w":220.123,"util_pct":83,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":13738.625,"power_w":239.64,"util_pct":70,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":13736.625,"power_w":239.441,"util_pct":82,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":13256.625,"power_w":230.92,"util_pct":39,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":12628.625,"power_w":123.401,"util_pct":0,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":12724.625,"power_w":119.548,"util_pct":0,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":12724.625,"power_w":123.401,"util_pct":0,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":12006.625,"power_w":119.573,"util_pct":0,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1145.400624592,"time_unix":1791285302.912751,"weight_transport":"cuda-ipc"} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":0,"rounds_completed":0,"seq":7,"time_unix":1791285332.7605896,"trained_version":null,"uptime_s":210.004,"weight_transport":"cuda-ipc"} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":0,"rounds_completed":0,"seq":8,"time_unix":1791285362.7611294,"trained_version":null,"uptime_s":240.004,"weight_transport":"cuda-ipc"} -{"available":true,"config_epoch":0,"cpu_rss_bytes":3337641984,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":18152.625,"power_w":162.666,"util_pct":0,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":18394.625,"power_w":173.579,"util_pct":0,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":18390.625,"power_w":170.784,"util_pct":0,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":17430.625,"power_w":165.541,"util_pct":0,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":19970.625,"power_w":202.606,"util_pct":0,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":19904.625,"power_w":191.969,"util_pct":0,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":20162.625,"power_w":203.501,"util_pct":0,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":18964.625,"power_w":200.51,"util_pct":2,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1205.401699242,"time_unix":1791285362.9137716,"weight_transport":"cuda-ipc"} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":0,"rounds_completed":0,"seq":9,"time_unix":1791285392.761656,"trained_version":null,"uptime_s":270.005,"weight_transport":"cuda-ipc"} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":0,"rounds_completed":0,"seq":10,"time_unix":1791285422.7621913,"trained_version":null,"uptime_s":300.006,"weight_transport":"cuda-ipc"} -{"available":true,"config_epoch":0,"cpu_rss_bytes":3337641984,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":18152.625,"power_w":119.296,"util_pct":0,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":18394.625,"power_w":128.382,"util_pct":0,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":18390.625,"power_w":125.811,"util_pct":0,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":17430.625,"power_w":120.564,"util_pct":0,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":20042.625,"power_w":187.439,"util_pct":0,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":19978.625,"power_w":189.163,"util_pct":31,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":20234.625,"power_w":188.889,"util_pct":83,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":19036.625,"power_w":184.099,"util_pct":0,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1265.402538667,"time_unix":1791285422.9146152,"weight_transport":"cuda-ipc"} -{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[1e-05],"clip_fraction":null,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"masked_fraction":null,"nonzero_advantages":null,"resp_len_mean":1000.828125,"resp_len_p95":1024.0,"reward_p10":0.0,"reward_p50":0.0,"reward_p90":0.0,"rollout_id":0,"step_seconds":150.3467451429999,"submitted_groups":null,"time_unix":1791285449.74938,"tok_per_s":426.0351625110142,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0,"pg_clipfrac":0.0,"pg_loss":0.0,"ppo_kl":-1.000444171950221e-09,"train_rollout_kl":0.0018599902978166938,"train_rollout_logprob_abs_diff":0.03606637567281723},"train_step":1,"trained_groups":8,"trained_sample_ids_sha256":"d82d6a02dabaac03a6d9f9876c2587054e271f74ce351cdcafbd26939ae9af43","trained_samples":64,"truncated_frac":0.953125} -{"config_epoch":0,"event":"rl_round_labels","execution_mode":"colocated-serial","island_id":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/aborted_groups":0,"rl/carried_over_groups":null,"rl/clip_fraction":null,"rl/ess_ratio":1.0,"rl/filtered_groups":0,"rl/groups":8,"rl/masked_fraction":null,"rl/mean_kl":-1.000444171950221e-09,"rollout_id":0,"t":1292.237757907,"time_unix":1791285449.7497633,"weight_transport":"cuda-ipc"} -{"end":1292.237971355,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":0,"start":1141.890336426,"task":"train","time_unix":1791285449.749942,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1791285449.7500346} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"sync","pid":18336,"policy_version":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":0,"rounds_completed":0,"seq":11,"time_unix":1791285452.811284,"trained_version":1,"uptime_s":330.055,"weight_transport":"cuda-ipc"} -{"action_tokens":64053,"active_groups":8,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":8,"completed_trajectories":64,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":8,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":0.0,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":0.0,"lr":1e-05,"mean_kl":-1.000444171950221e-09,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":0.0,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":18.796108502000152,"time_unix":1791285454.5753045,"tool_wait_seconds":0.0,"train_seconds":150.3467451429999,"train_step":1,"zero_variance_group_ratio":1.0} -{"end":1297.9451991,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":0,"start":1292.238123853,"task":"outer_sync","time_unix":1791285455.4573538,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1791285455.4575727} -{"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285465.785754} -{"end":1308.274093762,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":1297.945657347,"task":"publish","time_unix":1791285465.7861466,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1791285465.7863526} -{"end":1313.499608623,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":1,"start":1308.274435918,"task":"offload","time_unix":1791285471.0116932,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1791285471.01188} -{"end":1318.698010377,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"rollout","rollout_id":1,"start":1313.499965289,"task":"generate","time_unix":1791285476.210133,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1791285476.2103372} -{"end":1320.136628849,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":1,"start":1318.698422893,"task":"onload","time_unix":1791285477.6487236,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1791285477.6489222} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":1,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":1,"rounds_completed":1,"seq":12,"time_unix":1791285482.8119621,"trained_version":1,"uptime_s":360.055,"weight_transport":"cuda-ipc"} -{"available":true,"config_epoch":0,"cpu_rss_bytes":3804639232,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":18556.625,"power_w":207.945,"util_pct":54,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":18804.625,"power_w":229.677,"util_pct":37,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":18726.625,"power_w":213.555,"util_pct":66,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":17768.625,"power_w":204.834,"util_pct":37,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":20480.625,"power_w":256.025,"util_pct":53,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":20658.625,"power_w":242.575,"util_pct":36,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":20154.625,"power_w":252.484,"util_pct":46,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":18964.625,"power_w":253.466,"util_pct":53,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1325.403583624,"time_unix":1791285482.915712,"weight_transport":"cuda-ipc"} -{"aborted_in_flight_groups":0,"adv_mean":null,"adv_std":null,"applied_lrs":[8.333333333333334e-06],"clip_fraction":null,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"masked_fraction":null,"nonzero_advantages":null,"resp_len_mean":1013.78125,"resp_len_p95":1024.0,"reward_p10":0.0,"reward_p50":0.0,"reward_p90":0.0,"rollout_id":1,"step_seconds":10.55372185200008,"submitted_groups":8,"time_unix":1791285488.2030578,"tok_per_s":6147.783778071045,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0,"pg_clipfrac":0.0,"pg_loss":0.0,"ppo_kl":0.0,"train_rollout_kl":0.0019241459667682648,"train_rollout_logprob_abs_diff":0.03648190200328827},"train_step":2,"trained_groups":8,"trained_sample_ids_sha256":"5883e6fff48f762828700c935c3f3dddb9079f20a565f30b5aa34ce943e82277","trained_samples":64,"truncated_frac":0.96875} -{"config_epoch":0,"event":"rl_round_labels","execution_mode":"colocated-serial","island_id":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/aborted_groups":0,"rl/carried_over_groups":null,"rl/clip_fraction":null,"rl/ess_ratio":1.0,"rl/filtered_groups":0,"rl/groups":8,"rl/masked_fraction":null,"rl/mean_kl":0.0,"rollout_id":1,"t":1330.691430879,"time_unix":1791285488.2034228,"weight_transport":"cuda-ipc"} -{"end":1330.691631206,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":1,"start":1320.136991544,"task":"train","time_unix":1791285488.20359,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1791285488.2036805} -{"action_tokens":64882,"active_groups":8,"applied_lr":8.333333333333334e-06,"applied_lrs":[8.333333333333334e-06],"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":8,"completed_trajectories":64,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":8,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":0.0,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":0.0,"lr":8.333333333333334e-06,"mean_kl":0.0,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":0.0,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":10.4235576760002,"time_unix":1791285493.0537043,"tool_wait_seconds":0.0,"train_seconds":10.55372185200008,"train_step":2,"zero_variance_group_ratio":1.0} -{"end":1336.426499783,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":1,"start":1330.691771297,"task":"outer_sync","time_unix":1791285493.938855,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1791285493.939187} -{"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285503.703498} -{"end":1346.19180621,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":1336.427271178,"task":"publish","time_unix":1791285503.7038512,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1791285503.718487} -{"end":1351.976057432,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":2,"start":1346.206538009,"task":"offload","time_unix":1791285509.4881387,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1791285509.48834} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"generate","pid":18336,"policy_version":2,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":2,"rounds_completed":2,"seq":13,"time_unix":1791285512.812508,"trained_version":2,"uptime_s":390.056,"weight_transport":"cuda-ipc"} -{"end":1356.658371898,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"rollout","rollout_id":2,"start":1351.976419693,"task":"generate","time_unix":1791285514.1704533,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1791285514.1706393} -{"end":1358.050608167,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":2,"start":1356.658724345,"task":"onload","time_unix":1791285515.5626972,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1791285515.5629075} -{"aborted_in_flight_groups":0,"adv_mean":null,"adv_std":null,"applied_lrs":[6.666666666666668e-06],"clip_fraction":null,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"masked_fraction":null,"nonzero_advantages":null,"resp_len_mean":1024.0,"resp_len_p95":1024.0,"reward_p10":0.0,"reward_p50":0.0,"reward_p90":0.0,"rollout_id":2,"step_seconds":10.461073258999932,"submitted_groups":8,"time_unix":1791285526.0243704,"tok_per_s":6264.749168410391,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0,"pg_clipfrac":0.0,"pg_loss":0.0,"ppo_kl":0.0,"train_rollout_kl":0.0018938753055408597,"train_rollout_logprob_abs_diff":0.036156367510557175},"train_step":3,"trained_groups":8,"trained_sample_ids_sha256":"7cb590236a092586fda2fe77d6388783755fbee91cab2e6056c152578caf1fa5","trained_samples":64,"truncated_frac":1.0} -{"config_epoch":0,"event":"rl_round_labels","execution_mode":"colocated-serial","island_id":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/aborted_groups":0,"rl/carried_over_groups":null,"rl/clip_fraction":null,"rl/ess_ratio":1.0,"rl/filtered_groups":0,"rl/groups":8,"rl/masked_fraction":null,"rl/mean_kl":0.0,"rollout_id":2,"t":1368.512745413,"time_unix":1791285526.0247386,"weight_transport":"cuda-ipc"} -{"end":1368.512946339,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":2,"start":1358.050978623,"task":"train","time_unix":1791285526.0249188,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1791285526.0250077} -{"action_tokens":65536,"active_groups":8,"applied_lr":6.666666666666668e-06,"applied_lrs":[6.666666666666668e-06],"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":8,"completed_trajectories":64,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":8,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":0.0,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":0.0,"lr":6.666666666666668e-06,"mean_kl":0.0,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":0.0,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":10.451829019000115,"time_unix":1791285530.7120347,"tool_wait_seconds":0.0,"train_seconds":10.461073258999932,"train_step":3,"zero_variance_group_ratio":1.0} -{"end":1374.075931474,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":2,"start":1368.513098128,"task":"outer_sync","time_unix":1791285531.588047,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1791285531.5882518} -{"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285541.456038} -{"end":1383.944375504,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":1374.076338792,"task":"publish","time_unix":1791285541.4564235,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":3,"time_unix":1791285541.4700155} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"offload","pid":18336,"policy_version":3,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":3,"rounds_completed":3,"seq":14,"time_unix":1791285542.813056,"trained_version":3,"uptime_s":420.056,"weight_transport":"cuda-ipc"} -{"available":true,"config_epoch":0,"cpu_rss_bytes":3852828672,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":79252.625,"power_w":120.782,"util_pct":1,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":79580.625,"power_w":128.019,"util_pct":0,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":79522.625,"power_w":125.857,"util_pct":0,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":78302.625,"power_w":120.933,"util_pct":0,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":79278.625,"power_w":124.591,"util_pct":1,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":79452.625,"power_w":121.184,"util_pct":0,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":79766.625,"power_w":124.212,"util_pct":1,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":77692.625,"power_w":121.471,"util_pct":0,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1385.413323022,"time_unix":1791285542.9253833,"weight_transport":"cuda-ipc"} -{"end":1389.506782892,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":3,"start":1383.958063977,"task":"offload","time_unix":1791285547.0188613,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":3,"rollout_id":3,"time_unix":1791285547.0190797} -{"end":1394.435067201,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"rollout","rollout_id":3,"start":1389.507162434,"task":"generate","time_unix":1791285551.9471428,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":3,"time_unix":1791285551.9473333} -{"end":1395.973435714,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":3,"start":1394.435419066,"task":"onload","time_unix":1791285553.485534,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":3,"time_unix":1791285553.4857311} -{"aborted_in_flight_groups":0,"adv_mean":null,"adv_std":null,"applied_lrs":[5e-06],"clip_fraction":null,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"masked_fraction":null,"nonzero_advantages":null,"resp_len_mean":1020.15625,"resp_len_p95":1024.0,"reward_p10":0.0,"reward_p50":0.0,"reward_p90":0.0,"rollout_id":3,"step_seconds":10.04114407599991,"submitted_groups":8,"time_unix":1791285563.5272672,"tok_per_s":6502.247105093783,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0,"pg_clipfrac":0.0,"pg_loss":0.0,"ppo_kl":0.0,"train_rollout_kl":0.0019020842155441642,"train_rollout_logprob_abs_diff":0.03649664670228958},"train_step":4,"trained_groups":8,"trained_sample_ids_sha256":"7db6f0b987b89946ae44941cfccabab48ebe099810c620f74884bca9ea10ed96","trained_samples":64,"truncated_frac":0.984375} -{"config_epoch":0,"event":"rl_round_labels","execution_mode":"colocated-serial","island_id":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/aborted_groups":0,"rl/carried_over_groups":null,"rl/clip_fraction":null,"rl/ess_ratio":1.0,"rl/filtered_groups":0,"rl/groups":8,"rl/masked_fraction":null,"rl/mean_kl":0.0,"rollout_id":3,"t":1406.015629819,"time_unix":1791285563.5276225,"weight_transport":"cuda-ipc"} -{"end":1406.015830443,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":3,"start":1395.973817531,"task":"train","time_unix":1791285563.527785,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":3,"time_unix":1791285563.5278757} -{"action_tokens":65290,"active_groups":8,"applied_lr":5e-06,"applied_lrs":[5e-06],"base_policy_version":3,"cancelled_groups":0,"clip_fraction":null,"completed_groups":8,"completed_trajectories":64,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":8,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":0.0,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":4,"loss":0.0,"lr":5e-06,"mean_kl":0.0,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":0.0,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":10.477000067000063,"time_unix":1791285568.2051337,"tool_wait_seconds":0.0,"train_seconds":10.04114407599991,"train_step":4,"zero_variance_group_ratio":1.0} -{"end":1411.576559265,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":3,"start":1406.01596646,"task":"outer_sync","time_unix":1791285569.088938,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":4,"time_unix":1791285569.089251} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"publish","pid":18336,"policy_version":3,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":null,"rounds_completed":3,"seq":15,"time_unix":1791285572.8138266,"trained_version":4,"uptime_s":450.057,"weight_transport":"cuda-ipc"} -{"event":"rl_publication","island_id":0,"policy_version":4,"rl/policy_token":"yeto:4:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285578.7419362} -{"end":1421.230272057,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":1411.577336723,"task":"publish","time_unix":1791285578.742331,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":4,"time_unix":1791285578.7562606} -{"end":1427.538119505,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":4,"start":1421.24431026,"task":"offload","time_unix":1791285585.0502071,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":4,"rollout_id":4,"time_unix":1791285585.0503945} -{"end":1432.560233736,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"rollout","rollout_id":4,"start":1427.538480802,"task":"generate","time_unix":1791285590.0723002,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":4,"time_unix":1791285590.072497} -{"end":1433.980336106,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":4,"start":1432.56058292,"task":"onload","time_unix":1791285591.4924161,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":4,"time_unix":1791285591.4926155} -{"aborted_in_flight_groups":0,"adv_mean":null,"adv_std":null,"applied_lrs":[3.333333333333334e-06],"clip_fraction":null,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"masked_fraction":null,"nonzero_advantages":null,"resp_len_mean":999.953125,"resp_len_p95":1024.0,"reward_p10":0.0,"reward_p50":0.0,"reward_p90":0.0,"rollout_id":4,"step_seconds":9.901529057999824,"submitted_groups":8,"time_unix":1791285601.3945644,"tok_per_s":6463.345168723651,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0,"pg_clipfrac":0.0,"pg_loss":0.0,"ppo_kl":0.0,"train_rollout_kl":0.0019019710598513484,"train_rollout_logprob_abs_diff":0.036261189728975296},"train_step":5,"trained_groups":8,"trained_sample_ids_sha256":"5662988e66c78ab4f31c26050379f5f0c1c6341ce00255741b05527f544c6708","trained_samples":64,"truncated_frac":0.96875} -{"config_epoch":0,"event":"rl_round_labels","execution_mode":"colocated-serial","island_id":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/aborted_groups":0,"rl/carried_over_groups":null,"rl/clip_fraction":null,"rl/ess_ratio":1.0,"rl/filtered_groups":0,"rl/groups":8,"rl/masked_fraction":null,"rl/mean_kl":0.0,"rollout_id":4,"t":1443.882962364,"time_unix":1791285601.3949869,"weight_transport":"cuda-ipc"} -{"end":1443.883215266,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":4,"start":1433.980700018,"task":"train","time_unix":1791285601.3951726,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":4,"time_unix":1791285601.3952582} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"sync","pid":18336,"policy_version":4,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":4,"rounds_completed":4,"seq":16,"time_unix":1791285602.814457,"trained_version":5,"uptime_s":480.058,"weight_transport":"cuda-ipc"} -{"available":true,"config_epoch":0,"cpu_rss_bytes":3852902400,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":21032.625,"power_w":120.757,"util_pct":0,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":21250.625,"power_w":128.811,"util_pct":0,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":21242.625,"power_w":126.728,"util_pct":0,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":20406.625,"power_w":121.819,"util_pct":0,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":22374.625,"power_w":125.05,"util_pct":0,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":22404.625,"power_w":123.251,"util_pct":0,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":22082.625,"power_w":124.423,"util_pct":0,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":20568.625,"power_w":121.369,"util_pct":0,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1445.414239647,"time_unix":1791285602.9263175,"weight_transport":"cuda-ipc"} -{"action_tokens":63997,"active_groups":8,"applied_lr":3.333333333333334e-06,"applied_lrs":[3.333333333333334e-06],"base_policy_version":4,"cancelled_groups":0,"clip_fraction":null,"completed_groups":8,"completed_trajectories":64,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":8,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":0.0,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":5,"loss":0.0,"lr":3.333333333333334e-06,"mean_kl":0.0,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":0.0,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":11.315920646999984,"time_unix":1791285606.0933363,"tool_wait_seconds":0.0,"train_seconds":9.901529057999824,"train_step":5,"zero_variance_group_ratio":1.0} -{"end":1449.462540842,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":4,"start":1443.883346481,"task":"outer_sync","time_unix":1791285606.9749565,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":5,"time_unix":1791285606.9752853} -{"event":"rl_publication","island_id":0,"policy_version":5,"rl/policy_token":"yeto:5:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285616.3555238} -{"end":1458.843831943,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":1449.463370069,"task":"publish","time_unix":1791285616.3558664,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":5,"time_unix":1791285616.3702774} -{"end":1464.16865433,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":5,"start":1458.858330573,"task":"offload","time_unix":1791285621.6807451,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":5,"rollout_id":5,"time_unix":1791285621.680951} -{"end":1468.909227733,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"rollout","rollout_id":5,"start":1464.169036524,"task":"generate","time_unix":1791285626.4212992,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":5,"time_unix":1791285626.4214826} -{"end":1470.331685289,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":5,"start":1468.909568983,"task":"onload","time_unix":1791285627.8437645,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":5,"time_unix":1791285627.8439708} -{"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":5,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":5,"rounds_completed":5,"seq":17,"time_unix":1791285632.8150492,"trained_version":5,"uptime_s":510.058,"weight_transport":"cuda-ipc"} -{"aborted_in_flight_groups":0,"adv_mean":null,"adv_std":null,"applied_lrs":[1.6666666666666665e-06],"clip_fraction":null,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"masked_fraction":null,"nonzero_advantages":null,"resp_len_mean":999.03125,"resp_len_p95":1024.0,"reward_p10":0.0,"reward_p50":0.0,"reward_p90":0.0,"rollout_id":5,"step_seconds":10.804810546999988,"submitted_groups":8,"time_unix":1791285638.6492066,"tok_per_s":5917.549384311298,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0,"pg_clipfrac":0.0,"pg_loss":0.0,"ppo_kl":0.0,"train_rollout_kl":0.0018683435628190637,"train_rollout_logprob_abs_diff":0.0359635204076767},"train_step":6,"trained_groups":8,"trained_sample_ids_sha256":"4f9af3ed01dbaa4e2e7788efbcbea5c6ce3ec4901d040d08a2037abcd607687d","trained_samples":64,"truncated_frac":0.96875} -{"config_epoch":0,"event":"rl_round_labels","execution_mode":"colocated-serial","island_id":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/aborted_groups":0,"rl/carried_over_groups":null,"rl/clip_fraction":null,"rl/ess_ratio":1.0,"rl/filtered_groups":0,"rl/groups":8,"rl/masked_fraction":null,"rl/mean_kl":0.0,"rollout_id":5,"t":1481.137571521,"time_unix":1791285638.6495662,"weight_transport":"cuda-ipc"} -{"end":1481.137779501,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":5,"start":1470.332056308,"task":"train","time_unix":1791285638.6497386,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":5,"time_unix":1791285638.6498363} -{"action_tokens":63938,"active_groups":8,"applied_lr":1.6666666666666665e-06,"applied_lrs":[1.6666666666666665e-06],"base_policy_version":5,"cancelled_groups":0,"clip_fraction":null,"completed_groups":8,"completed_trajectories":64,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":8,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":0.0,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":6,"loss":0.0,"lr":1.6666666666666665e-06,"mean_kl":0.0,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":0.0,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":10.050893723999934,"time_unix":1791285643.5051532,"tool_wait_seconds":0.0,"train_seconds":10.804810546999988,"train_step":6,"zero_variance_group_ratio":1.0} -{"end":1486.881580093,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":5,"start":1481.137926151,"task":"outer_sync","time_unix":1791285644.3938332,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":6,"time_unix":1791285644.3941019} -{"event":"rl_publication","island_id":0,"policy_version":6,"rl/policy_token":"yeto:6:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285653.8974712} -{"end":1496.385882499,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":1486.882185187,"task":"publish","time_unix":1791285653.8980012,"weight_transport":"cuda-ipc"} -{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":6,"time_unix":1791285653.9136086} -{"event":"rl_learner_finalized","island_id":0,"time_unix":1791285659.0665615} diff --git a/openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fna/pulled/run.log b/openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fna/pulled/run.log deleted file mode 100644 index ee9bec224..000000000 --- a/openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fna/pulled/run.log +++ /dev/null @@ -1,9075 +0,0 @@ -==> /root/sky_logs/1-yeto-rl-island-0/run.log <== -2026-10-06 11:06:03,545 INFO worker.py:1544 -- Connecting to existing Ray cluster at address: 10.0.0.8:6380... -2026-10-06 11:06:03,553 INFO worker.py:1719 -- Connected to Ray cluster. View the dashboard at 127.0.0.1:8266  -├── Waiting for task resources on 1 node. -└── Job started. Streaming logs... (Ctrl-C to exit log streaming; job will not be killed) -(setup pid=3393) [yeto-setup] image provides miles c35702eefcf2862cee155e46870e6ad30568d2c6 (/opt/yeto/image-manifest.json); editable install kept -(setup pid=3393) ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. -(setup pid=3393) megatron-bridge 0.7.0+8cd3466d requires comet-ml>=3.50.0, which is not installed. -(setup pid=3393) megatron-bridge 0.7.0+8cd3466d requires hydra-core<1.4,>=1.3.4, which is not installed. -(setup pid=3393) megatron-bridge 0.7.0+8cd3466d requires mlflow>=3.16.0, which is not installed. -(setup pid=3393) megatron-bridge 0.7.0+8cd3466d requires open-clip-torch>=3.2.0, which is not installed. -(setup pid=3393) cosmos-guardrail 0.3.1 requires opencv-python>=4.10.0.84, which is not installed. -(setup pid=3393) megatron-bridge 0.7.0+8cd3466d requires flashinfer-cubin==0.6.18.post1, but you have flashinfer-cubin 0.6.18 which is incompatible. -(setup pid=3393) megatron-bridge 0.7.0+8cd3466d requires flashinfer-python==0.6.18.post1, but you have flashinfer-python 0.6.18 which is incompatible. -(setup pid=3393) megatron-bridge 0.7.0+8cd3466d requires pyyaml>=6.0.2, but you have pyyaml 6.0.1 which is incompatible. -(setup pid=3393) cosmos-guardrail 0.3.1 requires imageio>=2.37.0, but you have imageio 2.36.0 which is incompatible. -(setup pid=3393) [yeto-setup] image provides sglang 4e4148f1b4fe9f05973da0d1e5cfe237512d5155 -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:37,319 INFO usage_lib.py:448 -- Usage stats collection is disabled. -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:37,319 INFO scripts.py:1042 -- Local node IP: 10.0.0.8 -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 SUCC scripts.py:1081 -- -------------------- -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 SUCC scripts.py:1082 -- Ray runtime started. -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 SUCC scripts.py:1083 -- -------------------- -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1085 -- Next steps -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO cli_output_helpers.py:16 -- Note: The following commands are intended for use on -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO cli_output_helpers.py:17 -- the head node or within the cluster network. -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1089 -- To add another node to this Ray cluster, run -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1092 --  ray start --address='10.0.0.8:6379' -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1103 -- To connect to this Ray cluster: -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1105 -- import ray -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1106 -- ray.init(_node_ip_address='10.0.0.8') -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1120 -- To submit a Ray job using the Ray Jobs CLI: -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1121 --  RAY_API_SERVER_ADDRESS='http://127.0.0.1:8265' ray job submit --working-dir . -- python my_script.py -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1130 -- See https://docs.ray.io/en/latest/cluster/running-applications/job-submission/index.html -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1134 -- for more information on submitting Ray jobs to the Ray cluster. -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1139 -- To terminate the Ray runtime, run -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1140 --  ray stop -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1143 -- To view the status of the cluster, use -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1144 -- ray status -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1148 -- To monitor and debug Ray, view the dashboard at -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1149 -- 127.0.0.1:8265 -(yeto-rl-island-0, pid=3393) 2026-10-06 11:06:46,239 INFO scripts.py:1156 -- If connection to the dashboard fails, check your firewall settings and network configuration. -(yeto-rl-island-0, pid=3393) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=3393) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=3393) LANG_RL Log directory: None -(yeto-rl-island-0, pid=3393) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=3393) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=3393) (yeto-rl-island-0, pid=3393) Fetching 22 files: 0%| | 0/22 [00:00 failed for pad_packed_seq_alignment: typing.Union[int, typing.Literal['max'], NoneType]. -(yeto-rl-island-0, pid=3393) Deferring to attribute metadata. If the metadata is incomplete, 'parser.add_argument()' may fail. -(yeto-rl-island-0, pid=3393) Original failure: Unions not supported by argparse: typing.Union[int, typing.Literal['max'], NoneType] -(yeto-rl-island-0, pid=3393) [2026-10-06 11:07:22.914 main] arguments.py:49 - --tokenizer-model not set, use --hf-checkpoint as tokenizer model. -(yeto-rl-island-0, pid=3393) [2026-10-06 11:07:22.927 main] arguments.py:239 - MoE-expert LoRA layout: per-expert (--experts-shared-outer-loras). -(yeto-rl-island-0, pid=3393) [2026-10-06 11:07:22.927 main] arguments.py:3688 - Colocate mode: defaulting --sglang-cuda-graph-backend-prefill=disabled to avoid NVLS OOM. Set --sglang-cuda-graph-backend-prefill explicitly to override. -(yeto-rl-island-0, pid=3393) [2026-10-06 11:07:22.927 main] arguments.py:3698 - rollout_num_gpus None != actor_num_gpus_per_node 8 * actor_num_nodes 1, overriding rollout_num_gpus to match actor_num_gpus_per_node * actor_num_nodes. -(yeto-rl-island-0, pid=3393) [2026-10-06 11:07:22.927 main] arguments.py:3930 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 2048 as default value. -(yeto-rl-island-0, pid=3393) [2026-10-06 11:07:22.927 main] arguments.py:3938 - args.rollout_max_prompt_len is not set. Use args.rollout_max_context_len - 1 (2048 - 1) as default value so that there is at least one generated token to compute loss. -(yeto-rl-island-0, pid=3393) using world size: 8, data-parallel size: 2, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 2, pipeline-model-parallel size: 2 -(yeto-rl-island-0, pid=3393) WARNING: Setting args.overlap_p2p_comm and args.align_param_gather to False since non-interleaved schedule does not support overlapping p2p communication and aligned param AG -(yeto-rl-island-0, pid=3393) Number of virtual stages per pipeline stage: None -(yeto-rl-island-0, pid=3393) using torch.bfloat16 for parameters ... -(yeto-rl-island-0, pid=3393) ------------------------ arguments ------------------------ -(yeto-rl-island-0, pid=3393) _is_global_batch_size_explicitly_specified ...... True -(yeto-rl-island-0, pid=3393) account_for_embedding_in_pipeline_split ......... False -(yeto-rl-island-0, pid=3393) account_for_loss_in_pipeline_split .............. False -(yeto-rl-island-0, pid=3393) accumulate_allreduce_grads_in_fp32 .............. True -(yeto-rl-island-0, pid=3393) activation_func_clamp_shared_expert ............. True -(yeto-rl-island-0, pid=3393) activation_func_clamp_value ..................... None -(yeto-rl-island-0, pid=3393) activation_offload_fraction ..................... 1.0 -(yeto-rl-island-0, pid=3393) actor_num_gpus_per_node ......................... 8 -(yeto-rl-island-0, pid=3393) actor_num_nodes ................................. 1 -(yeto-rl-island-0, pid=3393) adam_beta1 ...................................... 0.9 -(yeto-rl-island-0, pid=3393) adam_beta2 ...................................... 0.999 -(yeto-rl-island-0, pid=3393) adam_eps ........................................ 1e-08 -(yeto-rl-island-0, pid=3393) add_bias_linear ................................. False -(yeto-rl-island-0, pid=3393) add_position_embedding .......................... True -(yeto-rl-island-0, pid=3393) add_qkv_bias .................................... False -(yeto-rl-island-0, pid=3393) adlr_autoresume ................................. False -(yeto-rl-island-0, pid=3393) adlr_autoresume_interval ........................ 1000 -(yeto-rl-island-0, pid=3393) advantage_estimator ............................. grpo -(yeto-rl-island-0, pid=3393) align_grad_reduce ............................... True -(yeto-rl-island-0, pid=3393) align_param_gather .............................. False -(yeto-rl-island-0, pid=3393) allgather_cp .................................... False -(yeto-rl-island-0, pid=3393) allow_ambiguous_pad_tokens ...................... False -(yeto-rl-island-0, pid=3393) allow_partial_train_step ........................ False -(yeto-rl-island-0, pid=3393) api_server_host ................................. 127.0.0.1 -(yeto-rl-island-0, pid=3393) api_server_port ................................. 0 -(yeto-rl-island-0, pid=3393) app_tag_run_name ................................ None -(yeto-rl-island-0, pid=3393) app_tag_run_version ............................. 0.0.0 -(yeto-rl-island-0, pid=3393) apply_chat_template ............................. True -(yeto-rl-island-0, pid=3393) apply_chat_template_kwargs ...................... {} -(yeto-rl-island-0, pid=3393) apply_dsa_kernel_fusion ......................... None -(yeto-rl-island-0, pid=3393) apply_query_key_layer_scaling ................... False -(yeto-rl-island-0, pid=3393) apply_residual_connection_post_layernorm ........ False -(yeto-rl-island-0, pid=3393) apply_rope_fusion ............................... True -(yeto-rl-island-0, pid=3393) apply_wd_to_qk_layernorm ........................ False -(yeto-rl-island-0, pid=3393) async_ckpt_cpu_priority ......................... 10 -(yeto-rl-island-0, pid=3393) async_ckpt_io_priority .......................... 3 -(yeto-rl-island-0, pid=3393) async_ckpt_use_cpu_shm .......................... False -(yeto-rl-island-0, pid=3393) async_data_buffer_capacity_factor ............... 2.0 -(yeto-rl-island-0, pid=3393) async_max_concurrent_samples .................... None -(yeto-rl-island-0, pid=3393) async_save ...................................... False -(yeto-rl-island-0, pid=3393) async_strategy .................................. mcore -(yeto-rl-island-0, pid=3393) async_unused_samples_handler .................... drop -(yeto-rl-island-0, pid=3393) attention_backend ............................... AttnBackend.auto -(yeto-rl-island-0, pid=3393) attention_dropout ............................... 0.0 -(yeto-rl-island-0, pid=3393) attention_output_gate ........................... True -(yeto-rl-island-0, pid=3393) attention_softmax_in_fp32 ....................... True -(yeto-rl-island-0, pid=3393) auto_detect_ckpt_format ......................... False -(yeto-rl-island-0, pid=3393) balance_by_flops ................................ False -(yeto-rl-island-0, pid=3393) balance_data .................................... True -(yeto-rl-island-0, pid=3393) barrier_with_L1_time ............................ True -(yeto-rl-island-0, pid=3393) batch_invariant_mode ............................ False -(yeto-rl-island-0, pid=3393) bert_binary_head ................................ True -(yeto-rl-island-0, pid=3393) bert_embedder_type .............................. megatron -(yeto-rl-island-0, pid=3393) bert_load ....................................... None -(yeto-rl-island-0, pid=3393) beta_fast ....................................... 32 -(yeto-rl-island-0, pid=3393) beta_slow ....................................... 1 -(yeto-rl-island-0, pid=3393) bf16 ............................................ True -(yeto-rl-island-0, pid=3393) bias_dropout_fusion ............................. True -(yeto-rl-island-0, pid=3393) bias_gelu_fusion ................................ False -(yeto-rl-island-0, pid=3393) bias_swiglu_fusion .............................. True -(yeto-rl-island-0, pid=3393) biencoder_projection_dim ........................ 0 -(yeto-rl-island-0, pid=3393) biencoder_shared_query_context_model ............ False -(yeto-rl-island-0, pid=3393) block_data_path ................................. None -(yeto-rl-island-0, pid=3393) buffer_filter_path .............................. yeto.rl.engine.miles_adapter.rollout_meta_hook.policy_buffer_filter -(yeto-rl-island-0, pid=3393) cache_mla_latents ............................... False -(yeto-rl-island-0, pid=3393) calc_ft_timeouts ................................ False -(yeto-rl-island-0, pid=3393) calculate_per_token_loss ........................ False -(yeto-rl-island-0, pid=3393) chat_template ................................... None -(yeto-rl-island-0, pid=3393) chat_template_path .............................. None -(yeto-rl-island-0, pid=3393) check_for_large_grads ........................... False -(yeto-rl-island-0, pid=3393) check_for_nan_in_loss_and_grad .................. True -(yeto-rl-island-0, pid=3393) check_for_spiky_loss ............................ False -(yeto-rl-island-0, pid=3393) check_lora_weight_equal ......................... True -(yeto-rl-island-0, pid=3393) check_rematerialize_param_from_master_weight .... False -(yeto-rl-island-0, pid=3393) check_weight_hash_across_dp_replicas_interval ... None -(yeto-rl-island-0, pid=3393) check_weight_update_allow_quant_error ........... False -(yeto-rl-island-0, pid=3393) check_weight_update_equal ....................... False -(yeto-rl-island-0, pid=3393) check_weight_update_selector .................... all -(yeto-rl-island-0, pid=3393) check_weight_update_skip_list ................... None -(yeto-rl-island-0, pid=3393) chunked_optimizer_state_offload ................. False -(yeto-rl-island-0, pid=3393) ci_check_model_hash ............................. False -(yeto-rl-island-0, pid=3393) ci_disable_kl_checker ........................... False -(yeto-rl-island-0, pid=3393) ci_disable_logprobs_checker ..................... False -(yeto-rl-island-0, pid=3393) ci_disable_weight_update_checker ................ False -(yeto-rl-island-0, pid=3393) ci_enable_metrics_capture ....................... False -(yeto-rl-island-0, pid=3393) ci_ft_test_actions .............................. None -(yeto-rl-island-0, pid=3393) ci_ft_test_actions_path ......................... None -(yeto-rl-island-0, pid=3393) ci_inject_rollout_data_min_match_ratio .......... 0.9 -(yeto-rl-island-0, pid=3393) ci_inject_rollout_data_path ..................... None -(yeto-rl-island-0, pid=3393) ci_inject_rollout_data_start_rollout_id ......... None -(yeto-rl-island-0, pid=3393) ci_load_grad_norm ............................... None -(yeto-rl-island-0, pid=3393) ci_metric_checker_expect_num .................... None -(yeto-rl-island-0, pid=3393) ci_metric_checker_key ........................... None -(yeto-rl-island-0, pid=3393) ci_metric_checker_threshold ..................... None -(yeto-rl-island-0, pid=3393) ci_save_grad_norm ............................... None -(yeto-rl-island-0, pid=3393) ci_save_model_hash .............................. False -(yeto-rl-island-0, pid=3393) ci_test ......................................... False -(yeto-rl-island-0, pid=3393) ci_tito_special_token_count_threshold ........... 0.0 -(yeto-rl-island-0, pid=3393) ckpt_assume_constant_structure .................. False -(yeto-rl-island-0, pid=3393) ckpt_convert_format ............................. None -(yeto-rl-island-0, pid=3393) ckpt_convert_save ............................... None -(yeto-rl-island-0, pid=3393) ckpt_convert_update_legacy_dist_opt_format ...... False -(yeto-rl-island-0, pid=3393) ckpt_format ..................................... torch_dist -(yeto-rl-island-0, pid=3393) ckpt_fully_parallel_load ........................ False -(yeto-rl-island-0, pid=3393) ckpt_fully_parallel_load_exchange_algo .......... broadcast -(yeto-rl-island-0, pid=3393) ckpt_fully_parallel_load_process_group .......... dp -(yeto-rl-island-0, pid=3393) ckpt_fully_parallel_save ........................ True -(yeto-rl-island-0, pid=3393) ckpt_fully_parallel_save_deprecated ............. False -(yeto-rl-island-0, pid=3393) ckpt_fully_parallel_save_process_group .......... dp -(yeto-rl-island-0, pid=3393) ckpt_load_validate_sharding_integrity ........... True -(yeto-rl-island-0, pid=3393) ckpt_step ....................................... None -(yeto-rl-island-0, pid=3393) classes_fraction ................................ 1.0 -(yeto-rl-island-0, pid=3393) clear_quantized_weight_workspaces_on_offload .... True -(yeto-rl-island-0, pid=3393) clip_grad ....................................... 1.0 -(yeto-rl-island-0, pid=3393) clone_scatter_output_in_embedding ............... True -(yeto-rl-island-0, pid=3393) cluster_backend ................................. ray -(yeto-rl-island-0, pid=3393) colocate ........................................ True -(yeto-rl-island-0, pid=3393) colocate_memory_peak_device ..................... cpu -(yeto-rl-island-0, pid=3393) compress_ratios ................................. None -(yeto-rl-island-0, pid=3393) compute_advantages_and_returns .................. True -(yeto-rl-island-0, pid=3393) config_logger_dir ............................... -(yeto-rl-island-0, pid=3393) consumed_train_samples .......................... 0 -(yeto-rl-island-0, pid=3393) consumed_valid_samples .......................... 0 -(yeto-rl-island-0, pid=3393) context_parallel_size ........................... 1 -(yeto-rl-island-0, pid=3393) cp_comm_type .................................... ['p2p'] -(yeto-rl-island-0, pid=3393) cp_partition_mode ............................... zigzag -(yeto-rl-island-0, pid=3393) cpu_offloading_num_layers ....................... 0 -(yeto-rl-island-0, pid=3393) cpu_offloading_retain_pinned_cpu_buffers ........ False -(yeto-rl-island-0, pid=3393) create_all_gather_group ......................... False -(yeto-rl-island-0, pid=3393) create_attention_mask_in_dataloader ............. True -(yeto-rl-island-0, pid=3393) critic_load ..................................... /mnt/yeto-models/torch_dist/qwen3.8-flash-next-4layer_torch_dist -(yeto-rl-island-0, pid=3393) critic_lr ....................................... 1e-05 -(yeto-rl-island-0, pid=3393) critic_lr_warmup_iters .......................... 0 -(yeto-rl-island-0, pid=3393) critic_num_gpus_per_node ........................ None -(yeto-rl-island-0, pid=3393) critic_num_nodes ................................ None -(yeto-rl-island-0, pid=3393) critic_save ..................................... None -(yeto-rl-island-0, pid=3393) cross_entropy_fusion_impl ....................... native -(yeto-rl-island-0, pid=3393) cross_entropy_loss_fusion ....................... False -(yeto-rl-island-0, pid=3393) csa_compress_ratios ............................. None -(yeto-rl-island-0, pid=3393) csa_compress_rotary_base ........................ 40000.0 -(yeto-rl-island-0, pid=3393) csa_dense_mode .................................. False -(yeto-rl-island-0, pid=3393) csa_window_size ................................. 128 -(yeto-rl-island-0, pid=3393) cuda_graph_dynamic_microbatches ................. False -(yeto-rl-island-0, pid=3393) cuda_graph_impl ................................. none -(yeto-rl-island-0, pid=3393) cuda_graph_modules .............................. [] -(yeto-rl-island-0, pid=3393) cuda_graph_warmup_steps ......................... 3 -(yeto-rl-island-0, pid=3393) custom_async_data_buffer_path ................... None -(yeto-rl-island-0, pid=3393) custom_config_path .............................. None -(yeto-rl-island-0, pid=3393) custom_convert_samples_to_train_data_path ....... None -(yeto-rl-island-0, pid=3393) custom_eval_rollout_log_function_path ........... None -(yeto-rl-island-0, pid=3393) custom_generate_function_path ................... None -(yeto-rl-island-0, pid=3393) custom_inference_engine_provider_path ........... miles.ray.specs.inference.backend_inference_engine_provider -(yeto-rl-island-0, pid=3393) custom_loss_function_path ....................... None -(yeto-rl-island-0, pid=3393) custom_megatron_before_log_prob_hook_path ....... None -(yeto-rl-island-0, pid=3393) custom_megatron_before_train_step_hook_path ..... None -(yeto-rl-island-0, pid=3393) custom_megatron_init_path ....................... None -(yeto-rl-island-0, pid=3393) custom_megatron_post_save_hook_path ............. None -(yeto-rl-island-0, pid=3393) custom_model_provider_path ...................... miles_plugins.models.qwen3_8_next.model_provider.get_qwen3_8_next_model_provider -(yeto-rl-island-0, pid=3393) custom_pg_loss_reducer_function_path ............ None -(yeto-rl-island-0, pid=3393) custom_reward_post_process_path ................. None -(yeto-rl-island-0, pid=3393) custom_rm_path .................................. gsm8k_reward.score -(yeto-rl-island-0, pid=3393) custom_rollout_log_function_path ................ None -(yeto-rl-island-0, pid=3393) custom_tis_function_path ........................ None -(yeto-rl-island-0, pid=3393) custom_update_weight_post_write_path ............ None -(yeto-rl-island-0, pid=3393) dashboard_flush_interval ........................ 5.0 -(yeto-rl-island-0, pid=3393) dashboard_forward_prometheus .................... False -(yeto-rl-island-0, pid=3393) dashboard_gpu_sample_interval ................... 1.0 -(yeto-rl-island-0, pid=3393) dashboard_sglang_metrics ........................ None -(yeto-rl-island-0, pid=3393) dashboard_sglang_scrape_interval ................ 2.0 -(yeto-rl-island-0, pid=3393) dashboard_sglang_scrape_mode .................... auto -(yeto-rl-island-0, pid=3393) data_args_path .................................. None -(yeto-rl-island-0, pid=3393) data_cache_path ................................. None -(yeto-rl-island-0, pid=3393) data_pad_size_multiplier ........................ 128 -(yeto-rl-island-0, pid=3393) data_parallel_random_init ....................... False -(yeto-rl-island-0, pid=3393) data_parallel_sharding_strategy ................. optim_grads_params -(yeto-rl-island-0, pid=3393) data_parallel_size .............................. 2 -(yeto-rl-island-0, pid=3393) data_path ....................................... None -(yeto-rl-island-0, pid=3393) data_per_class_fraction ......................... 1.0 -(yeto-rl-island-0, pid=3393) data_sharding ................................... True -(yeto-rl-island-0, pid=3393) data_source_path ................................ miles.rollout.data_source.RolloutDataSourceWithBuffer -(yeto-rl-island-0, pid=3393) dataloader_defer_npy_index_mmap ................. False -(yeto-rl-island-0, pid=3393) dataloader_fast_cache_load ...................... False -(yeto-rl-island-0, pid=3393) dataloader_inter_document_masking ............... False -(yeto-rl-island-0, pid=3393) dataloader_type ................................. single -(yeto-rl-island-0, pid=3393) ddp_average_in_collective ....................... False -(yeto-rl-island-0, pid=3393) ddp_bucket_size ................................. None -(yeto-rl-island-0, pid=3393) ddp_num_buckets ................................. None -(yeto-rl-island-0, pid=3393) ddp_pad_buckets_for_high_nccl_busbw ............. False -(yeto-rl-island-0, pid=3393) ddp_param_name_patterns_for_fp32_local_accumulation [] -(yeto-rl-island-0, pid=3393) ddp_reduce_scatter_with_fp32_accumulation ....... False -(yeto-rl-island-0, pid=3393) debug_deterministic_collective .................. False -(yeto-rl-island-0, pid=3393) debug_disable_optimizer ......................... False -(yeto-rl-island-0, pid=3393) debug_exit_after_rollout ........................ None -(yeto-rl-island-0, pid=3393) debug_rollout_only .............................. False -(yeto-rl-island-0, pid=3393) debug_skip_weight_update ........................ False -(yeto-rl-island-0, pid=3393) debug_train_only ................................ False -(yeto-rl-island-0, pid=3393) debug_unified_grad_fused_logprob ................ False -(yeto-rl-island-0, pid=3393) decode_only_cuda_graphs ......................... False -(yeto-rl-island-0, pid=3393) decoder_first_pipeline_num_layers ............... None -(yeto-rl-island-0, pid=3393) decoder_last_pipeline_num_layers ................ None -(yeto-rl-island-0, pid=3393) decoder_num_layers .............................. None -(yeto-rl-island-0, pid=3393) decoder_seq_length .............................. None -(yeto-rl-island-0, pid=3393) decoupled_lr .................................... None -(yeto-rl-island-0, pid=3393) decoupled_min_lr ................................ None -(yeto-rl-island-0, pid=3393) decrease_batch_size_if_needed ................... False -(yeto-rl-island-0, pid=3393) defer_embedding_wgrad_compute ................... False -(yeto-rl-island-0, pid=3393) delay_offload_until_cuda_graph .................. False -(yeto-rl-island-0, pid=3393) delay_split_train_data_by_dp .................... False -(yeto-rl-island-0, pid=3393) delay_wgrad_compute ............................. False -(yeto-rl-island-0, pid=3393) delta_offload_bytes_across_pp_ranks ............. 0 -(yeto-rl-island-0, pid=3393) dense_grouped_gemm .............................. False -(yeto-rl-island-0, pid=3393) deploy_component ................................ all -(yeto-rl-island-0, pid=3393) deploy_instance_id .............................. None -(yeto-rl-island-0, pid=3393) deprecated_use_mcore_models ..................... False -(yeto-rl-island-0, pid=3393) deterministic_mode .............................. False -(yeto-rl-island-0, pid=3393) dino_bottleneck_size ............................ 256 -(yeto-rl-island-0, pid=3393) dino_freeze_last_layer .......................... 1 -(yeto-rl-island-0, pid=3393) dino_head_hidden_size ........................... 2048 -(yeto-rl-island-0, pid=3393) dino_local_crops_number ......................... 10 -(yeto-rl-island-0, pid=3393) dino_local_img_size ............................. 96 -(yeto-rl-island-0, pid=3393) dino_norm_last_layer ............................ False -(yeto-rl-island-0, pid=3393) dino_teacher_temp ............................... 0.07 -(yeto-rl-island-0, pid=3393) dino_warmup_teacher_temp ........................ 0.04 -(yeto-rl-island-0, pid=3393) dino_warmup_teacher_temp_epochs ................. 30 -(yeto-rl-island-0, pid=3393) disable_bf16_reduced_precision_matmul ........... False -(yeto-rl-island-0, pid=3393) disable_grad_buffers_cpu_backup ................. True -(yeto-rl-island-0, pid=3393) disable_jit_fuser ............................... False -(yeto-rl-island-0, pid=3393) disable_msc_deprecated .......................... False -(yeto-rl-island-0, pid=3393) disable_param_buffers_cpu_backup ................ True -(yeto-rl-island-0, pid=3393) disable_rollout_trim_samples .................... False -(yeto-rl-island-0, pid=3393) disable_straggler_on_startup .................... False -(yeto-rl-island-0, pid=3393) disable_symmetric_registration .................. False -(yeto-rl-island-0, pid=3393) dist_ckpt_format_deprecated ..................... None -(yeto-rl-island-0, pid=3393) dist_ckpt_optim_fully_reshardable ............... False -(yeto-rl-island-0, pid=3393) dist_ckpt_save_pre_mcore_014 .................... False -(yeto-rl-island-0, pid=3393) dist_ckpt_strictness ............................ assume_ok_unexpected -(yeto-rl-island-0, pid=3393) dist_ckpt_workers ............................... 1 -(yeto-rl-island-0, pid=3393) distrib_optim_fully_reshardable_mem_efficient ... False -(yeto-rl-island-0, pid=3393) distribute_saved_activations .................... False -(yeto-rl-island-0, pid=3393) distributed_backend ............................. nccl -(yeto-rl-island-0, pid=3393) distributed_timeout_minutes ..................... 60 -(yeto-rl-island-0, pid=3393) distributed_timeout_seconds_after_init .......... None -(yeto-rl-island-0, pid=3393) dsa_attention_backend ........................... tilelang -(yeto-rl-island-0, pid=3393) dsa_cp_balance_indexer .......................... False -(yeto-rl-island-0, pid=3393) dsa_indexer_head_dim ............................ None -(yeto-rl-island-0, pid=3393) dsa_indexer_k_norm_epsilon ...................... None -(yeto-rl-island-0, pid=3393) dsa_indexer_k_norm_fp32 ......................... False -(yeto-rl-island-0, pid=3393) dsa_indexer_loss_coeff .......................... None -(yeto-rl-island-0, pid=3393) dsa_indexer_n_heads ............................. None -(yeto-rl-island-0, pid=3393) dsa_indexer_precision ........................... bf16 -(yeto-rl-island-0, pid=3393) dsa_indexer_rope_interleaved .................... False -(yeto-rl-island-0, pid=3393) dsa_indexer_rotate_activation ................... True -(yeto-rl-island-0, pid=3393) dsa_indexer_scoring_relu ........................ True -(yeto-rl-island-0, pid=3393) dsa_indexer_skip_topk_offset .................... 0 -(yeto-rl-island-0, pid=3393) dsa_indexer_topk ................................ None -(yeto-rl-island-0, pid=3393) dsa_indexer_topk_freq ........................... 1 -(yeto-rl-island-0, pid=3393) dsa_indexer_use_sparse_loss ..................... False -(yeto-rl-island-0, pid=3393) dsa_indexer_weights_proj_output_dtype ........... bf16 -(yeto-rl-island-0, pid=3393) dsa_indexer_weights_proj_use_quantization ....... True -(yeto-rl-island-0, pid=3393) dsa_kernel_backend .............................. None -(yeto-rl-island-0, pid=3393) dsv4_impl ....................................... megatron -(yeto-rl-island-0, pid=3393) dump_details .................................... None -(yeto-rl-island-0, pid=3393) dump_param_to_param_group_map ................... None -(yeto-rl-island-0, pid=3393) dumper_dir ...................................... /tmp/dumper -(yeto-rl-island-0, pid=3393) dumper_enable ................................... False -(yeto-rl-island-0, pid=3393) dumper_fwd_bwd .................................. None -(yeto-rl-island-0, pid=3393) dumper_fwd_only ................................. None -(yeto-rl-island-0, pid=3393) dumper_inference ................................ None -(yeto-rl-island-0, pid=3393) dumper_source_patcher_config_inference .......... None -(yeto-rl-island-0, pid=3393) dumper_source_patcher_config_train .............. None -(yeto-rl-island-0, pid=3393) dynamic_context_parallel ........................ False -(yeto-rl-island-0, pid=3393) dynamic_sampling_filter_path .................... None -(yeto-rl-island-0, pid=3393) embedding_init_method_std ....................... None -(yeto-rl-island-0, pid=3393) embedding_path .................................. None -(yeto-rl-island-0, pid=3393) empty_unused_memory_level ....................... 0 -(yeto-rl-island-0, pid=3393) enable_chunked_prefill .......................... False -(yeto-rl-island-0, pid=3393) enable_cuda_graph ............................... False -(yeto-rl-island-0, pid=3393) enable_event_analyzer ........................... False -(yeto-rl-island-0, pid=3393) enable_experimental ............................. False -(yeto-rl-island-0, pid=3393) enable_ft_package ............................... False -(yeto-rl-island-0, pid=3393) enable_full_sharding_in_hsdp .................... False -(yeto-rl-island-0, pid=3393) enable_hyper_connections ........................ False -(yeto-rl-island-0, pid=3393) enable_msc ...................................... False -(yeto-rl-island-0, pid=3393) enable_mtp_training ............................. False -(yeto-rl-island-0, pid=3393) enable_one_logger ............................... True -(yeto-rl-island-0, pid=3393) enable_witness .................................. False -(yeto-rl-island-0, pid=3393) encoder_num_layers .............................. 4 -(yeto-rl-island-0, pid=3393) encoder_seq_length .............................. 2048 -(yeto-rl-island-0, pid=3393) end_weight_decay ................................ 0.01 -(yeto-rl-island-0, pid=3393) entropy_coef .................................... 0.0 -(yeto-rl-island-0, pid=3393) entry ........................................... train -(yeto-rl-island-0, pid=3393) env_report ...................................... -(yeto-rl-island-0, pid=3393) env_report_interval_seconds ..................... 3600.0 -(yeto-rl-island-0, pid=3393) eod_mask_loss ................................... False -(yeto-rl-island-0, pid=3393) ep_overlap_early_attn_memory_release ............ False -(yeto-rl-island-0, pid=3393) eps_clip ........................................ 0.2 -(yeto-rl-island-0, pid=3393) eps_clip_c ...................................... None -(yeto-rl-island-0, pid=3393) eps_clip_high ................................... 0.2 -(yeto-rl-island-0, pid=3393) error_injection_rate ............................ 0 -(yeto-rl-island-0, pid=3393) error_injection_type ............................ transient_error -(yeto-rl-island-0, pid=3393) eval_config ..................................... None -(yeto-rl-island-0, pid=3393) eval_datasets ................................... [] -(yeto-rl-island-0, pid=3393) eval_function_path .............................. miles.rollout.inference_rollout.inference_rollout_common.InferenceRolloutFn -(yeto-rl-island-0, pid=3393) eval_global_batch_size .......................... 64 -(yeto-rl-island-0, pid=3393) eval_hf_dir ..................................... None -(yeto-rl-island-0, pid=3393) eval_input_key .................................. None -(yeto-rl-island-0, pid=3393) eval_interval ................................... None -(yeto-rl-island-0, pid=3393) eval_iters ...................................... 100 -(yeto-rl-island-0, pid=3393) eval_keep_snapshots ............................. 2 -(yeto-rl-island-0, pid=3393) eval_label_key .................................. None -(yeto-rl-island-0, pid=3393) eval_max_context_len ............................ 2048 -(yeto-rl-island-0, pid=3393) eval_max_in_flight .............................. 2 -(yeto-rl-island-0, pid=3393) eval_max_prompt_len ............................. None -(yeto-rl-island-0, pid=3393) eval_max_response_len ........................... None -(yeto-rl-island-0, pid=3393) eval_micro_batch_size ........................... 1 -(yeto-rl-island-0, pid=3393) eval_min_new_tokens ............................. None -(yeto-rl-island-0, pid=3393) eval_num_gpus ................................... 0 -(yeto-rl-island-0, pid=3393) eval_num_gpus_per_engine ........................ 1 -(yeto-rl-island-0, pid=3393) eval_overflow_policy ............................ backpressure -(yeto-rl-island-0, pid=3393) eval_prompt_data ................................ None -(yeto-rl-island-0, pid=3393) eval_reward_key ................................. None -(yeto-rl-island-0, pid=3393) eval_temperature ................................ None -(yeto-rl-island-0, pid=3393) eval_tool_key ................................... None -(yeto-rl-island-0, pid=3393) eval_top_k ...................................... None -(yeto-rl-island-0, pid=3393) eval_top_p ...................................... None -(yeto-rl-island-0, pid=3393) eval_uses_snapshots ............................. False -(yeto-rl-island-0, pid=3393) evidence_data_path .............................. None -(yeto-rl-island-0, pid=3393) exclude_modules ................................. None -(yeto-rl-island-0, pid=3393) exit_duration_in_mins ........................... None -(yeto-rl-island-0, pid=3393) exit_interval ................................... None -(yeto-rl-island-0, pid=3393) exit_on_missing_checkpoint ...................... False -(yeto-rl-island-0, pid=3393) exit_signal ..................................... 15 -(yeto-rl-island-0, pid=3393) exit_signal_handler ............................. False -(yeto-rl-island-0, pid=3393) exit_signal_handler_for_dataloader .............. False -(yeto-rl-island-0, pid=3393) exit_signal_handler_for_training ................ False -(yeto-rl-island-0, pid=3393) exp_avg_dtype ................................... torch.float32 -(yeto-rl-island-0, pid=3393) exp_avg_sq_dtype ................................ torch.float32 -(yeto-rl-island-0, pid=3393) experimental_attention_variant .................. None -(yeto-rl-island-0, pid=3393) expert_model_parallel_size ...................... 4 -(yeto-rl-island-0, pid=3393) expert_tensor_parallel_size ..................... 1 -(yeto-rl-island-0, pid=3393) experts_shared_outer_loras ...................... False -(yeto-rl-island-0, pid=3393) external_cuda_graph ............................. False -(yeto-rl-island-0, pid=3393) extra_high_precision_layers_hf .................. () -(yeto-rl-island-0, pid=3393) extra_high_precision_layers_megatron ............ () -(yeto-rl-island-0, pid=3393) fake_process_group .............................. False -(yeto-rl-island-0, pid=3393) fallback_to_eager_attn .......................... False -(yeto-rl-island-0, pid=3393) fault_injector_delay_start_iteration ............ None -(yeto-rl-island-0, pid=3393) fault_injector_fault_delay ...................... None -(yeto-rl-island-0, pid=3393) fault_injector_fault_probabilities .............. None -(yeto-rl-island-0, pid=3393) fault_injector_fault_types ...................... None -(yeto-rl-island-0, pid=3393) fault_injector_mtti_seconds ..................... None -(yeto-rl-island-0, pid=3393) fault_injector_num_ranks ........................ None -(yeto-rl-island-0, pid=3393) fault_injector_offset_seconds ................... None -(yeto-rl-island-0, pid=3393) fault_injector_ranks ............................ None -(yeto-rl-island-0, pid=3393) fault_injector_seed ............................. None -(yeto-rl-island-0, pid=3393) fetch_trajectory_retry_times .................... -1 -(yeto-rl-island-0, pid=3393) ffn_hidden_size ................................. 640 -(yeto-rl-island-0, pid=3393) fim_data ........................................ False -(yeto-rl-island-0, pid=3393) fim_eod_token ................................... <|endoftext|> -(yeto-rl-island-0, pid=3393) fim_fragment_rate ............................... None -(yeto-rl-island-0, pid=3393) fim_middle_token ................................ -(yeto-rl-island-0, pid=3393) fim_no_prefix ................................... None -(yeto-rl-island-0, pid=3393) fim_pad_token ................................... -(yeto-rl-island-0, pid=3393) fim_prefix_token ................................ -(yeto-rl-island-0, pid=3393) fim_rate ........................................ 0.5 -(yeto-rl-island-0, pid=3393) fim_split_sample ................................ None -(yeto-rl-island-0, pid=3393) fim_spm_rate .................................... 0.5 -(yeto-rl-island-0, pid=3393) fim_suffix_token ................................ -(yeto-rl-island-0, pid=3393) fine_grained_activation_offloading .............. False -(yeto-rl-island-0, pid=3393) fine_grained_offloading_max_inflight_offloads ... None -(yeto-rl-island-0, pid=3393) finetune ........................................ True -(yeto-rl-island-0, pid=3393) first_last_layers_bf16 .......................... False -(yeto-rl-island-0, pid=3393) flash_decode .................................... False -(yeto-rl-island-0, pid=3393) flight_recorder_dump_on_timeout ................. True -(yeto-rl-island-0, pid=3393) flight_recorder_dump_path ....................... None -(yeto-rl-island-0, pid=3393) flight_recorder_extra_dump_on_exec .............. True -(yeto-rl-island-0, pid=3393) flight_recorder_include_only_active ............. True -(yeto-rl-island-0, pid=3393) flight_recorder_include_stack_trace ............. False -(yeto-rl-island-0, pid=3393) flight_recorder_trace_buffer_size ............... 2000 -(yeto-rl-island-0, pid=3393) fp16 ............................................ False -(yeto-rl-island-0, pid=3393) fp16_lm_cross_entropy ........................... False -(yeto-rl-island-0, pid=3393) fp32_residual_connection ........................ False -(yeto-rl-island-0, pid=3393) fp4 ............................................. None -(yeto-rl-island-0, pid=3393) fp4_param_gather ................................ False -(yeto-rl-island-0, pid=3393) fp4_quantizer_factory ........................... None -(yeto-rl-island-0, pid=3393) fp4_recipe ...................................... nvfp4 -(yeto-rl-island-0, pid=3393) fp8 ............................................. None -(yeto-rl-island-0, pid=3393) fp8_amax_compute_algo ........................... most_recent -(yeto-rl-island-0, pid=3393) fp8_amax_history_len ............................ 1 -(yeto-rl-island-0, pid=3393) fp8_interval .................................... 1 -(yeto-rl-island-0, pid=3393) fp8_margin ...................................... 0 -(yeto-rl-island-0, pid=3393) fp8_output_proj ................................. False -(yeto-rl-island-0, pid=3393) fp8_param_gather ................................ False -(yeto-rl-island-0, pid=3393) fp8_quantizer_factory ........................... None -(yeto-rl-island-0, pid=3393) fp8_recipe ...................................... delayed -(yeto-rl-island-0, pid=3393) fp8_wgrad ....................................... True -(yeto-rl-island-0, pid=3393) freeze_all_layers ............................... False -(yeto-rl-island-0, pid=3393) freeze_e_score_correction_bias .................. False -(yeto-rl-island-0, pid=3393) freeze_indexer .................................. False -(yeto-rl-island-0, pid=3393) fsdp_db_use_persist_buf_on_alloc_fail ........... False -(yeto-rl-island-0, pid=3393) fsdp_double_buffer .............................. False -(yeto-rl-island-0, pid=3393) fsdp_manual_registration ........................ False -(yeto-rl-island-0, pid=3393) ft_components ................................... [] -(yeto-rl-island-0, pid=3393) ft_num_warmup_iters ............................. 5 -(yeto-rl-island-0, pid=3393) full_validation ................................. False -(yeto-rl-island-0, pid=3393) fully_async ..................................... False -(yeto-rl-island-0, pid=3393) fused_residual_rmsnorm .......................... False -(yeto-rl-island-0, pid=3393) gamma ........................................... 1.0 -(yeto-rl-island-0, pid=3393) gdn_conv_pad_alignment .......................... None -(yeto-rl-island-0, pid=3393) gdn_pre_gated_delta_rule_fusion ................. False -(yeto-rl-island-0, pid=3393) get_mismatch_metrics ............................ False -(yeto-rl-island-0, pid=3393) global_batch_size ............................... 64 -(yeto-rl-island-0, pid=3393) glu_linear_offset ............................... 0.0 -(yeto-rl-island-0, pid=3393) gmpo_log_clip_high .............................. 0.4 -(yeto-rl-island-0, pid=3393) gmpo_log_clip_low ............................... 0.4 -(yeto-rl-island-0, pid=3393) gpu_sniff_test_interval ......................... None -(yeto-rl-island-0, pid=3393) grad_reduce_in_bf16 ............................. False -(yeto-rl-island-0, pid=3393) gradient_accumulation_fusion .................... False -(yeto-rl-island-0, pid=3393) gradient_reduce_div_fusion ...................... True -(yeto-rl-island-0, pid=3393) group_query_attention ........................... True -(yeto-rl-island-0, pid=3393) group_rm ........................................ False -(yeto-rl-island-0, pid=3393) grpo_clamp_eps_lower ............................ 0.01 -(yeto-rl-island-0, pid=3393) grpo_clamp_eps_upper ............................ 0.01 -(yeto-rl-island-0, pid=3393) grpo_entropy_term_weight ........................ 0.0 -(yeto-rl-island-0, pid=3393) grpo_filter_groups_with_same_reward ............. False -(yeto-rl-island-0, pid=3393) grpo_group_size ................................. 2 -(yeto-rl-island-0, pid=3393) grpo_iterations ................................. 2 -(yeto-rl-island-0, pid=3393) grpo_kl_beta .................................... 0.001 -(yeto-rl-island-0, pid=3393) grpo_prompts_per_step ........................... 32 -(yeto-rl-island-0, pid=3393) grpo_std_normalization .......................... True -(yeto-rl-island-0, pid=3393) head_lr_mult .................................... 1.0 -(yeto-rl-island-0, pid=3393) head_wise_attn_gate ............................. False -(yeto-rl-island-0, pid=3393) heterogeneous_layers_config_encoded_json ........ None -(yeto-rl-island-0, pid=3393) heterogeneous_layers_config_path ................ None -(yeto-rl-island-0, pid=3393) hf_checkpoint ................................... /mnt/yeto-models/hub/models--CharyZeng--Qwen3.8-Flash-Next-4layer/snapshots/d19a6b60c0df8f90faf92c7c592b37df2e15b060 -(yeto-rl-island-0, pid=3393) hf_lora_targets ................................. ['model.language_model.layers.*.self_attn.q_proj', 'model.language_model.layers.*.self_attn.k_proj', 'model.language_model.layers.*.self_attn.v_proj', 'model.language_model.layers.*.self_attn.o_proj', 'model.language_model.layers.*.linear_attn.in_proj_qkv', 'model.language_model.layers.*.linear_attn.in_proj_z', 'model.language_model.layers.*.linear_attn.in_proj_b', 'model.language_model.layers.*.linear_attn.in_proj_a', 'model.language_model.layers.*.linear_attn.out_proj', 'model.language_model.layers.*.mlp.experts.gate_up_proj', 'model.language_model.layers.*.mlp.experts.down_proj', 'model.language_model.layers.*.mlp.shared_expert.gate_proj', 'model.language_model.layers.*.mlp.shared_expert.up_proj', 'model.language_model.layers.*.mlp.shared_expert.down_proj'] -(yeto-rl-island-0, pid=3393) hidden_dropout .................................. 0.0 -(yeto-rl-island-0, pid=3393) hidden_size ..................................... 2560 -(yeto-rl-island-0, pid=3393) hierarchical_context_parallel_sizes ............. None -(yeto-rl-island-0, pid=3393) high_priority_a2a_comm_stream ................... False -(yeto-rl-island-0, pid=3393) high_priority_stream_groups ..................... [] -(yeto-rl-island-0, pid=3393) http_proxy ...................................... None -(yeto-rl-island-0, pid=3393) hybrid_context_parallel ......................... False -(yeto-rl-island-0, pid=3393) hybrid_layer_pattern ............................ None -(yeto-rl-island-0, pid=3393) hybrid_override_pattern ......................... None -(yeto-rl-island-0, pid=3393) hysteresis ...................................... 2 -(yeto-rl-island-0, pid=3393) ict_head_size ................................... None -(yeto-rl-island-0, pid=3393) ict_load ........................................ None -(yeto-rl-island-0, pid=3393) img_h ........................................... 224 -(yeto-rl-island-0, pid=3393) img_w ........................................... 224 -(yeto-rl-island-0, pid=3393) indep_dp ........................................ False -(yeto-rl-island-0, pid=3393) indexer_batch_size .............................. 128 -(yeto-rl-island-0, pid=3393) indexer_log_interval ............................ 1000 -(yeto-rl-island-0, pid=3393) inference_batch_times_seqlen_threshold .......... -1 -(yeto-rl-island-0, pid=3393) inference_controller_addr ....................... None -(yeto-rl-island-0, pid=3393) inference_coordinator_port ...................... None -(yeto-rl-island-0, pid=3393) inference_cuda_graph_all_prefills ............... False -(yeto-rl-island-0, pid=3393) inference_cuda_graph_scope ...................... InferenceCudaGraphScope.none -(yeto-rl-island-0, pid=3393) inference_disable_ep_consensus .................. False -(yeto-rl-island-0, pid=3393) inference_disable_triton_nvls_kernels ........... False -(yeto-rl-island-0, pid=3393) inference_dynamic_batching ...................... False -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_async_sched_mode ..... legacy -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_block_size ........... 256 -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_buffer_size_gb ....... 40.0 -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_cuda_graph_max_tokens 16384 -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_cuda_graph_mixed_prefill_count 16 -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_cuda_graph_sizing_distribution exponential -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_enable_prefix_caching False -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_logprobs_mode ........ raw_logprobs -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_mamba_memory_ratio ... None -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_max_requests ......... None -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_max_tokens ........... None -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_num_cuda_graphs ...... 16 -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_paused_buffer_size_gb None -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_prefix_caching_coordinator_policy first_prefix_block -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_prefix_caching_eviction_policy ref_zero -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_prefix_caching_mamba_gb None -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_prefix_caching_routing_alpha 0.5 -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_sampling_backend ..... torch -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_track_generated_token_events False -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_track_paused_request_events False -(yeto-rl-island-0, pid=3393) inference_dynamic_batching_unified_memory_level . 0 -(yeto-rl-island-0, pid=3393) inference_fuse_tp_communication ................. False -(yeto-rl-island-0, pid=3393) inference_grouped_gemm_backend .................. vllm -(yeto-rl-island-0, pid=3393) inference_logging_step_interval ................. 0 -(yeto-rl-island-0, pid=3393) inference_max_requests .......................... 8 -(yeto-rl-island-0, pid=3393) inference_max_seq_length ........................ 2560 -(yeto-rl-island-0, pid=3393) inference_moe_disable_fused_quant_kernels ....... False -(yeto-rl-island-0, pid=3393) inference_moe_token_dispatcher_type ............. nvls -(yeto-rl-island-0, pid=3393) inference_rng_tracker ........................... False -(yeto-rl-island-0, pid=3393) inference_shards ................................ None -(yeto-rl-island-0, pid=3393) inference_text_gen_server_logging ............... False -(yeto-rl-island-0, pid=3393) inference_use_synchronous_zmq_collectives ....... False -(yeto-rl-island-0, pid=3393) inference_wandb_logging ......................... False -(yeto-rl-island-0, pid=3393) init_expected_num_cells ......................... None -(yeto-rl-island-0, pid=3393) init_method_std ................................. 0.02 -(yeto-rl-island-0, pid=3393) init_method_xavier_uniform ...................... False -(yeto-rl-island-0, pid=3393) init_model_with_meta_device ..................... False -(yeto-rl-island-0, pid=3393) initial_loss_scale .............................. 4294967296 -(yeto-rl-island-0, pid=3393) inprocess_active_world_size ..................... 1 -(yeto-rl-island-0, pid=3393) inprocess_barrier_timeout ....................... 120 -(yeto-rl-island-0, pid=3393) inprocess_completion_timeout .................... 120 -(yeto-rl-island-0, pid=3393) inprocess_empty_cuda_cache ...................... False -(yeto-rl-island-0, pid=3393) inprocess_granularity ........................... node -(yeto-rl-island-0, pid=3393) inprocess_hard_timeout .......................... 90 -(yeto-rl-island-0, pid=3393) inprocess_heartbeat_interval .................... 30 -(yeto-rl-island-0, pid=3393) inprocess_heartbeat_timeout ..................... 60 -(yeto-rl-island-0, pid=3393) inprocess_last_call_wait ........................ 1 -(yeto-rl-island-0, pid=3393) inprocess_max_iterations ........................ None -(yeto-rl-island-0, pid=3393) inprocess_monitor_process_interval .............. 1.0 -(yeto-rl-island-0, pid=3393) inprocess_monitor_thread_interval ............... 1.0 -(yeto-rl-island-0, pid=3393) inprocess_progress_watchdog_interval ............ 1.0 -(yeto-rl-island-0, pid=3393) inprocess_restart ............................... False -(yeto-rl-island-0, pid=3393) inprocess_soft_timeout .......................... 60 -(yeto-rl-island-0, pid=3393) inprocess_termination_grace_time ................ 1 -(yeto-rl-island-0, pid=3393) input_key ....................................... messages -(yeto-rl-island-0, pid=3393) is_hybrid_model ................................. False -(yeto-rl-island-0, pid=3393) iter_per_epoch .................................. 1250 -(yeto-rl-island-0, pid=3393) iterations_to_skip .............................. [] -(yeto-rl-island-0, pid=3393) keep_fp8_transpose_cache ........................ False -(yeto-rl-island-0, pid=3393) keep_old_actor .................................. False -(yeto-rl-island-0, pid=3393) kitchen_attention_backend ....................... sdpa -(yeto-rl-island-0, pid=3393) kl_coef ......................................... 0.0 -(yeto-rl-island-0, pid=3393) kl_loss_coef .................................... 0.0 -(yeto-rl-island-0, pid=3393) kl_loss_type .................................... k1 -(yeto-rl-island-0, pid=3393) kv_channels ..................................... 256 -(yeto-rl-island-0, pid=3393) kv_lora_rank .................................... 32 -(yeto-rl-island-0, pid=3393) label_key ....................................... label -(yeto-rl-island-0, pid=3393) lambd ........................................... 1.0 -(yeto-rl-island-0, pid=3393) langrl_env_config ............................... None -(yeto-rl-island-0, pid=3393) layernorm_epsilon ............................... 1e-06 -(yeto-rl-island-0, pid=3393) layernorm_zero_centered_gamma ................... True -(yeto-rl-island-0, pid=3393) lazy_mpu_init ................................... False -(yeto-rl-island-0, pid=3393) linear_attention_backend ........................ flashqla -(yeto-rl-island-0, pid=3393) linear_attention_freq ........................... None -(yeto-rl-island-0, pid=3393) linear_attention_type ........................... None -(yeto-rl-island-0, pid=3393) linear_conv_kernel_dim .......................... 4 -(yeto-rl-island-0, pid=3393) linear_cp_mode .................................. chunkwise -(yeto-rl-island-0, pid=3393) linear_key_head_dim ............................. 128 -(yeto-rl-island-0, pid=3393) linear_num_key_heads ............................ 16 -(yeto-rl-island-0, pid=3393) linear_num_value_heads .......................... 32 -(yeto-rl-island-0, pid=3393) linear_value_head_dim ........................... 128 -(yeto-rl-island-0, pid=3393) lion_beta1 ...................................... 0.95 -(yeto-rl-island-0, pid=3393) lion_beta2 ...................................... 0.98 -(yeto-rl-island-0, pid=3393) load ............................................ /mnt/yeto-models/torch_dist/qwen3.8-flash-next-4layer_torch_dist -(yeto-rl-island-0, pid=3393) load_debug_rollout_data ......................... None -(yeto-rl-island-0, pid=3393) load_debug_rollout_data_subsample ............... None -(yeto-rl-island-0, pid=3393) load_main_params_from_ckpt ...................... False -(yeto-rl-island-0, pid=3393) local_rank ...................................... 0 -(yeto-rl-island-0, pid=3393) log_correct_samples ............................. False -(yeto-rl-island-0, pid=3393) log_device_memory_used .......................... False -(yeto-rl-island-0, pid=3393) log_energy ...................................... False -(yeto-rl-island-0, pid=3393) log_interval .................................... 100 -(yeto-rl-island-0, pid=3393) log_loss_scale_to_tensorboard ................... True -(yeto-rl-island-0, pid=3393) log_max_attention_logit ......................... False -(yeto-rl-island-0, pid=3393) log_memory_interval ............................. None -(yeto-rl-island-0, pid=3393) log_memory_to_tensorboard ....................... False -(yeto-rl-island-0, pid=3393) log_moe_overload_factor ......................... False -(yeto-rl-island-0, pid=3393) log_multi_turn .................................. False -(yeto-rl-island-0, pid=3393) log_num_zeros_in_grad ........................... False -(yeto-rl-island-0, pid=3393) log_params_norm ................................. False -(yeto-rl-island-0, pid=3393) log_passrate .................................... False -(yeto-rl-island-0, pid=3393) log_probs_chunk_size ............................ -1 -(yeto-rl-island-0, pid=3393) log_probs_max_tokens_per_gpu .................... None -(yeto-rl-island-0, pid=3393) log_progress .................................... False -(yeto-rl-island-0, pid=3393) log_reward_category ............................. None -(yeto-rl-island-0, pid=3393) log_straggler ................................... False -(yeto-rl-island-0, pid=3393) log_throughput .................................. False -(yeto-rl-island-0, pid=3393) log_timers_to_tensorboard ....................... False -(yeto-rl-island-0, pid=3393) log_validation_ppl_to_tensorboard ............... False -(yeto-rl-island-0, pid=3393) log_world_size_to_tensorboard ................... False -(yeto-rl-island-0, pid=3393) logging_level ................................... None -(yeto-rl-island-0, pid=3393) logits_load_decode_threads ...................... 4 -(yeto-rl-island-0, pid=3393) logits_load_dir ................................. None -(yeto-rl-island-0, pid=3393) logits_load_ignore_errors ....................... False -(yeto-rl-island-0, pid=3393) logits_load_kd_loss_alpha ....................... 1.0 -(yeto-rl-island-0, pid=3393) logits_load_msc_prefetch_depth .................. 2 -(yeto-rl-island-0, pid=3393) logits_load_prefetch_factor ..................... 3 -(yeto-rl-island-0, pid=3393) logits_save_dir ................................. None -(yeto-rl-island-0, pid=3393) logits_save_dtype ............................... fp16 -(yeto-rl-island-0, pid=3393) logits_save_top_k ............................... None -(yeto-rl-island-0, pid=3393) logits_save_top_p ............................... None -(yeto-rl-island-0, pid=3393) logits_save_top_p_min_k ......................... 1 -(yeto-rl-island-0, pid=3393) lora_adapter_path ............................... None -(yeto-rl-island-0, pid=3393) lora_adapter_targets ............................ ['model.language_model.layers.*.self_attn.q_proj', 'model.language_model.layers.*.self_attn.k_proj', 'model.language_model.layers.*.self_attn.v_proj', 'model.language_model.layers.*.self_attn.o_proj', 'model.language_model.layers.*.linear_attn.in_proj_qkv', 'model.language_model.layers.*.linear_attn.in_proj_z', 'model.language_model.layers.*.linear_attn.in_proj_b', 'model.language_model.layers.*.linear_attn.in_proj_a', 'model.language_model.layers.*.linear_attn.out_proj', 'model.language_model.layers.*.mlp.experts.gate_up_proj', 'model.language_model.layers.*.mlp.experts.down_proj', 'model.language_model.layers.*.mlp.shared_expert.gate_proj', 'model.language_model.layers.*.mlp.shared_expert.up_proj', 'model.language_model.layers.*.mlp.shared_expert.down_proj'] -(yeto-rl-island-0, pid=3393) lora_alpha ...................................... 16 -(yeto-rl-island-0, pid=3393) lora_base_cpu_backup ............................ False -(yeto-rl-island-0, pid=3393) lora_dp_invariant_rng ........................... exact -(yeto-rl-island-0, pid=3393) lora_dp_invariant_state ......................... False -(yeto-rl-island-0, pid=3393) lora_dropout .................................... 0.0 -(yeto-rl-island-0, pid=3393) lora_expert_rank ................................ 8 -(yeto-rl-island-0, pid=3393) lora_rank ....................................... 16 -(yeto-rl-island-0, pid=3393) lora_sync_from_tensor ........................... False -(yeto-rl-island-0, pid=3393) lora_train_only ................................. False -(yeto-rl-island-0, pid=3393) lora_type ....................................... lora -(yeto-rl-island-0, pid=3393) loss_mask_type .................................. qwen -(yeto-rl-island-0, pid=3393) loss_scale ...................................... None -(yeto-rl-island-0, pid=3393) loss_scale_window ............................... 1000 -(yeto-rl-island-0, pid=3393) loss_type ....................................... policy_loss -(yeto-rl-island-0, pid=3393) low_memory_resume ............................... False -(yeto-rl-island-0, pid=3393) lr .............................................. 1e-05 -(yeto-rl-island-0, pid=3393) lr_decay_iters .................................. 6 -(yeto-rl-island-0, pid=3393) lr_decay_samples ................................ None -(yeto-rl-island-0, pid=3393) lr_decay_style .................................. linear -(yeto-rl-island-0, pid=3393) lr_warmup_fraction .............................. None -(yeto-rl-island-0, pid=3393) lr_warmup_init .................................. 0.0 -(yeto-rl-island-0, pid=3393) lr_warmup_iters ................................. 0 -(yeto-rl-island-0, pid=3393) lr_warmup_samples ............................... 0 -(yeto-rl-island-0, pid=3393) lr_wsd_decay_iters .............................. None -(yeto-rl-island-0, pid=3393) lr_wsd_decay_samples ............................ None -(yeto-rl-island-0, pid=3393) lr_wsd_decay_style .............................. exponential -(yeto-rl-island-0, pid=3393) main_grads_dtype ................................ torch.float32 -(yeto-rl-island-0, pid=3393) main_params_dtype ............................... torch.float32 -(yeto-rl-island-0, pid=3393) make_vocab_size_divisible_by .................... 128 -(yeto-rl-island-0, pid=3393) mamba_head_dim .................................. 64 -(yeto-rl-island-0, pid=3393) mamba_inference_conv_states_dtype ............... torch.bfloat16 -(yeto-rl-island-0, pid=3393) mamba_inference_ssm_states_dtype ................ torch.bfloat16 -(yeto-rl-island-0, pid=3393) mamba_num_groups ................................ 8 -(yeto-rl-island-0, pid=3393) mamba_num_heads ................................. None -(yeto-rl-island-0, pid=3393) mamba_state_dim ................................. 128 -(yeto-rl-island-0, pid=3393) mamba_training_ssm_states_dtype ................. None -(yeto-rl-island-0, pid=3393) manual_gc ....................................... False -(yeto-rl-island-0, pid=3393) manual_gc_eval .................................. True -(yeto-rl-island-0, pid=3393) manual_gc_interval .............................. 0 -(yeto-rl-island-0, pid=3393) mask_factor ..................................... 1.0 -(yeto-rl-island-0, pid=3393) mask_offpolicy_in_partial_rollout ............... False -(yeto-rl-island-0, pid=3393) mask_prob ....................................... 0.15 -(yeto-rl-island-0, pid=3393) mask_type ....................................... random -(yeto-rl-island-0, pid=3393) masked_softmax_fusion ........................... True -(yeto-rl-island-0, pid=3393) max_position_embeddings ......................... 2048 -(yeto-rl-island-0, pid=3393) max_seqlen_per_dp_cp_rank ....................... None -(yeto-rl-island-0, pid=3393) max_tokens_per_gpu .............................. 8192 -(yeto-rl-island-0, pid=3393) max_tokens_to_oom ............................... 12000 -(yeto-rl-island-0, pid=3393) max_weight_staleness ............................ None -(yeto-rl-island-0, pid=3393) megatron_config ................................. None -(yeto-rl-island-0, pid=3393) megatron_fsdp_cache_param_bucket_views .......... False -(yeto-rl-island-0, pid=3393) megatron_fsdp_enable_fine_grained_param_gather .. False -(yeto-rl-island-0, pid=3393) megatron_fsdp_grad_comm_dtype ................... None -(yeto-rl-island-0, pid=3393) megatron_fsdp_main_grads_dtype .................. None -(yeto-rl-island-0, pid=3393) megatron_fsdp_main_params_dtype ................. torch.float32 -(yeto-rl-island-0, pid=3393) megatron_fsdp_max_pool_double_buffer ............ False -(yeto-rl-island-0, pid=3393) megatron_fsdp_prefetch_recompute_forward_weights False -(yeto-rl-island-0, pid=3393) megatron_to_hf_mode ............................. raw -(yeto-rl-island-0, pid=3393) memory_recorder ................................. torch -(yeto-rl-island-0, pid=3393) memory_snapshot_dir ............................. . -(yeto-rl-island-0, pid=3393) memory_snapshot_num_steps ....................... None -(yeto-rl-island-0, pid=3393) memory_snapshot_path ............................ snapshot.pickle -(yeto-rl-island-0, pid=3393) merge_file ...................................... None -(yeto-rl-island-0, pid=3393) metadata_key .................................... metadata -(yeto-rl-island-0, pid=3393) metadata_path ................................... None -(yeto-rl-island-0, pid=3393) mfu_peak_tflops ................................. None -(yeto-rl-island-0, pid=3393) mhc_init_gating_factor .......................... 0.01 -(yeto-rl-island-0, pid=3393) mhc_recompute_layer_num ......................... None -(yeto-rl-island-0, pid=3393) mhc_sinkhorn_iterations ......................... 20 -(yeto-rl-island-0, pid=3393) micro_batch_size ................................ 1 -(yeto-rl-island-0, pid=3393) microbatch_group_size_per_vp_stage .............. None -(yeto-rl-island-0, pid=3393) mid_level_dataset_surplus ....................... 0.005 -(yeto-rl-island-0, pid=3393) miles_dsa_topk_backend .......................... torch -(yeto-rl-island-0, pid=3393) miles_router_health_check_failure_threshold ..... 3 -(yeto-rl-island-0, pid=3393) miles_router_max_connections .................... None -(yeto-rl-island-0, pid=3393) miles_router_timeout ............................ None -(yeto-rl-island-0, pid=3393) min_batch_collection_ratio ...................... 1 -(yeto-rl-island-0, pid=3393) min_dynamic_context_parallel_size ............... 1 -(yeto-rl-island-0, pid=3393) min_loss_scale .................................. 1.0 -(yeto-rl-island-0, pid=3393) min_lr .......................................... 0.0 -(yeto-rl-island-0, pid=3393) min_offloaded_tensor_size ....................... 1048576 -(yeto-rl-island-0, pid=3393) mini_ft_controller_enable ....................... False -(yeto-rl-island-0, pid=3393) mini_ft_controller_poll_interval ................ 10.0 -(yeto-rl-island-0, pid=3393) mini_ft_controller_resume_delay ................. 10.0 -(yeto-rl-island-0, pid=3393) mla_down_proj_fusion ............................ False -(yeto-rl-island-0, pid=3393) mlflow_experiment_name .......................... miles -(yeto-rl-island-0, pid=3393) mlflow_run_id ................................... None -(yeto-rl-island-0, pid=3393) mlflow_run_name ................................. None -(yeto-rl-island-0, pid=3393) mlflow_tracking_uri ............................. None -(yeto-rl-island-0, pid=3393) mlp_chunks_for_prefill .......................... 1 -(yeto-rl-island-0, pid=3393) mlp_chunks_for_training ......................... 1 -(yeto-rl-island-0, pid=3393) mmap_bin_files .................................. True -(yeto-rl-island-0, pid=3393) mock_data ....................................... False -(yeto-rl-island-0, pid=3393) model_name ...................................... qwen4_exp -(yeto-rl-island-0, pid=3393) moe_activation_in_fp32 .......................... False -(yeto-rl-island-0, pid=3393) moe_apply_probs_on_input ........................ False -(yeto-rl-island-0, pid=3393) moe_aux_loss_coeff .............................. 0.0 -(yeto-rl-island-0, pid=3393) moe_combine_in_fp32 ............................. False -(yeto-rl-island-0, pid=3393) moe_deepep_num_sms .............................. None -(yeto-rl-island-0, pid=3393) moe_enable_deepep ............................... False -(yeto-rl-island-0, pid=3393) moe_enable_routing_replay ....................... False -(yeto-rl-island-0, pid=3393) moe_expert_capacity_factor ...................... None -(yeto-rl-island-0, pid=3393) moe_expert_rank_capacity_factor ................. None -(yeto-rl-island-0, pid=3393) moe_ffn_hidden_size ............................. 640 -(yeto-rl-island-0, pid=3393) moe_flex_dispatcher_backend ..................... deepep -(yeto-rl-island-0, pid=3393) moe_flex_dispatcher_num_sms ..................... None -(yeto-rl-island-0, pid=3393) moe_grouped_gemm ................................ True -(yeto-rl-island-0, pid=3393) moe_hybridep_num_blocks_permute ................. None -(yeto-rl-island-0, pid=3393) moe_hybridep_num_blocks_unpermute ............... None -(yeto-rl-island-0, pid=3393) moe_hybridep_num_sms ............................ None -(yeto-rl-island-0, pid=3393) moe_hybridep_num_sms_preprocessing .............. 108 -(yeto-rl-island-0, pid=3393) moe_hybridep_pad_variable_tokens ................ False -(yeto-rl-island-0, pid=3393) moe_input_jitter_eps ............................ None -(yeto-rl-island-0, pid=3393) moe_latent_size ................................. None -(yeto-rl-island-0, pid=3393) moe_latent_use_norm ............................. False -(yeto-rl-island-0, pid=3393) moe_layer_freq .................................. [1, 1, 1, 1] -(yeto-rl-island-0, pid=3393) moe_layer_recompute ............................. False -(yeto-rl-island-0, pid=3393) moe_mlp_glu_interleave_size ..................... None -(yeto-rl-island-0, pid=3393) moe_n_hash_layers ............................... 0 -(yeto-rl-island-0, pid=3393) moe_ncclep_static_shape ......................... False -(yeto-rl-island-0, pid=3393) moe_ncclep_use_symm_mem ......................... False -(yeto-rl-island-0, pid=3393) moe_pad_expert_input_to_capacity ................ False -(yeto-rl-island-0, pid=3393) moe_pad_experts_for_cuda_graph_inference ........ False -(yeto-rl-island-0, pid=3393) moe_paged_stash ................................. False -(yeto-rl-island-0, pid=3393) moe_paged_stash_buffer_size_factor_cpu .......... 0.0 -(yeto-rl-island-0, pid=3393) moe_paged_stash_buffer_size_factor_cuda ......... 1.1 -(yeto-rl-island-0, pid=3393) moe_paged_stash_page_size ....................... 64 -(yeto-rl-island-0, pid=3393) moe_per_layer_logging ........................... False -(yeto-rl-island-0, pid=3393) moe_permute_fusion .............................. True -(yeto-rl-island-0, pid=3393) moe_permute_fusion_into_hybridep ................ False -(yeto-rl-island-0, pid=3393) moe_router_bias_update_rate ..................... 0.001 -(yeto-rl-island-0, pid=3393) moe_router_dtype ................................ fp32 -(yeto-rl-island-0, pid=3393) moe_router_enable_expert_bias ................... False -(yeto-rl-island-0, pid=3393) moe_router_force_biased ......................... None -(yeto-rl-island-0, pid=3393) moe_router_force_load_balancing ................. False -(yeto-rl-island-0, pid=3393) moe_router_freeze_gate .......................... False -(yeto-rl-island-0, pid=3393) moe_router_fusion ............................... False -(yeto-rl-island-0, pid=3393) moe_router_group_topk ........................... None -(yeto-rl-island-0, pid=3393) moe_router_load_balancing_type .................. aux_loss -(yeto-rl-island-0, pid=3393) moe_router_num_groups ........................... None -(yeto-rl-island-0, pid=3393) moe_router_padding_for_fp8 ...................... False -(yeto-rl-island-0, pid=3393) moe_router_padding_for_quantization ............. False -(yeto-rl-island-0, pid=3393) moe_router_pre_softmax .......................... False -(yeto-rl-island-0, pid=3393) moe_router_score_function ....................... softmax -(yeto-rl-island-0, pid=3393) moe_router_topk ................................. 10 -(yeto-rl-island-0, pid=3393) moe_router_topk_scaling_factor .................. None -(yeto-rl-island-0, pid=3393) moe_router_use_torch_mm ......................... True -(yeto-rl-island-0, pid=3393) moe_routing_trace_capture_hidden_states ......... False -(yeto-rl-island-0, pid=3393) moe_routing_trace_capture_logits ................ False -(yeto-rl-island-0, pid=3393) moe_routing_trace_dump_weights .................. False -(yeto-rl-island-0, pid=3393) moe_routing_trace_max_training_iters ............ None -(yeto-rl-island-0, pid=3393) moe_routing_trace_path .......................... None -(yeto-rl-island-0, pid=3393) moe_shared_expert_gate .......................... True -(yeto-rl-island-0, pid=3393) moe_shared_expert_glu_interleave_size ........... None -(yeto-rl-island-0, pid=3393) moe_shared_expert_intermediate_size ............. 640 -(yeto-rl-island-0, pid=3393) moe_shared_expert_overlap ....................... False -(yeto-rl-island-0, pid=3393) moe_single_grouped_bias ......................... False -(yeto-rl-island-0, pid=3393) moe_single_grouped_weight ....................... False -(yeto-rl-island-0, pid=3393) moe_token_dispatcher_type ....................... alltoall -(yeto-rl-island-0, pid=3393) moe_token_drop_policy ........................... probs -(yeto-rl-island-0, pid=3393) moe_upcycling_granularity ....................... 1 -(yeto-rl-island-0, pid=3393) moe_use_upcycling ............................... False -(yeto-rl-island-0, pid=3393) moe_z_loss_coeff ................................ None -(yeto-rl-island-0, pid=3393) mooncake_replica_num ............................ 1 -(yeto-rl-island-0, pid=3393) mooncake_store_init_kwargs ...................... None -(yeto-rl-island-0, pid=3393) mrope_interleaved ............................... False -(yeto-rl-island-0, pid=3393) mrope_section ................................... None -(yeto-rl-island-0, pid=3393) mscale .......................................... 1.0 -(yeto-rl-island-0, pid=3393) mscale_all_dim .................................. 0.0 -(yeto-rl-island-0, pid=3393) mtp_detach_heads ................................ False -(yeto-rl-island-0, pid=3393) mtp_hybrid_override_pattern ..................... None -(yeto-rl-island-0, pid=3393) mtp_loss_scaling_factor ......................... 0.2 -(yeto-rl-island-0, pid=3393) mtp_num_layers .................................. None -(yeto-rl-island-0, pid=3393) mtp_standalone .................................. False -(yeto-rl-island-0, pid=3393) mtp_use_repeated_layer .......................... False -(yeto-rl-island-0, pid=3393) multi_latent_attention .......................... False -(yeto-rl-island-0, pid=3393) multi_lora ...................................... False -(yeto-rl-island-0, pid=3393) multi_lora_n_adapters ........................... 0 -(yeto-rl-island-0, pid=3393) multimodal_keys ................................. None -(yeto-rl-island-0, pid=3393) multiple_validation_sets ........................ False -(yeto-rl-island-0, pid=3393) muon_coefficient_type ........................... quintic -(yeto-rl-island-0, pid=3393) muon_extra_scale_factor ......................... 1.0 -(yeto-rl-island-0, pid=3393) muon_fp32_matmul_prec ........................... medium -(yeto-rl-island-0, pid=3393) muon_momentum ................................... 0.9 -(yeto-rl-island-0, pid=3393) muon_nesterov ................................... False -(yeto-rl-island-0, pid=3393) muon_num_ns_steps ............................... 5 -(yeto-rl-island-0, pid=3393) muon_scalar_optimizer ........................... adam -(yeto-rl-island-0, pid=3393) muon_scale_mode ................................. spectral -(yeto-rl-island-0, pid=3393) muon_split_qkv .................................. True -(yeto-rl-island-0, pid=3393) muon_tp_mode .................................... blockwise -(yeto-rl-island-0, pid=3393) mup_attn_scale_power ............................ 1.0 -(yeto-rl-island-0, pid=3393) mup_base_head_dim ............................... None -(yeto-rl-island-0, pid=3393) mup_base_hidden_size ............................ None -(yeto-rl-island-0, pid=3393) mup_embedding_mult .............................. 1.0 -(yeto-rl-island-0, pid=3393) mup_output_mult ................................. 1.0 -(yeto-rl-island-0, pid=3393) mup_width_mult .................................. 1.0 -(yeto-rl-island-0, pid=3393) n_samples_per_eval_prompt ....................... 1 -(yeto-rl-island-0, pid=3393) n_samples_per_prompt ............................ 8 -(yeto-rl-island-0, pid=3393) nccl_all_reduce_for_prefill ..................... False -(yeto-rl-island-0, pid=3393) nccl_communicator_config_path ................... None -(yeto-rl-island-0, pid=3393) nccl_ub ......................................... False -(yeto-rl-island-0, pid=3393) no_load_optim ................................... True -(yeto-rl-island-0, pid=3393) no_load_rng ..................................... True -(yeto-rl-island-0, pid=3393) no_persist_layer_norm ........................... False -(yeto-rl-island-0, pid=3393) no_rope_freq .................................... None -(yeto-rl-island-0, pid=3393) no_save_optim ................................... True -(yeto-rl-island-0, pid=3393) no_save_rng ..................................... True -(yeto-rl-island-0, pid=3393) no_weight_decay_cond_type ....................... None -(yeto-rl-island-0, pid=3393) non_persistent_ckpt_type ........................ None -(yeto-rl-island-0, pid=3393) non_persistent_global_ckpt_dir .................. None -(yeto-rl-island-0, pid=3393) non_persistent_local_ckpt_algo .................. fully_parallel -(yeto-rl-island-0, pid=3393) non_persistent_local_ckpt_dir ................... None -(yeto-rl-island-0, pid=3393) non_persistent_save_interval .................... None -(yeto-rl-island-0, pid=3393) normalization ................................... RMSNorm -(yeto-rl-island-0, pid=3393) normalize_advantages ............................ False -(yeto-rl-island-0, pid=3393) null_tokenizer_eod_id ........................... None -(yeto-rl-island-0, pid=3393) null_tokenizer_pad_id ........................... -1 -(yeto-rl-island-0, pid=3393) num_attention_heads ............................. 24 -(yeto-rl-island-0, pid=3393) num_channels .................................... 3 -(yeto-rl-island-0, pid=3393) num_classes ..................................... 1000 -(yeto-rl-island-0, pid=3393) num_critic_only_steps ........................... 0 -(yeto-rl-island-0, pid=3393) num_dataset_builder_threads ..................... 1 -(yeto-rl-island-0, pid=3393) num_distributed_optimizer_instances ............. 1 -(yeto-rl-island-0, pid=3393) num_epoch ....................................... None -(yeto-rl-island-0, pid=3393) num_experts ..................................... 512 -(yeto-rl-island-0, pid=3393) num_gpus_per_node ............................... 8 -(yeto-rl-island-0, pid=3393) num_layers ...................................... 4 -(yeto-rl-island-0, pid=3393) num_layers_at_end_in_bf16 ....................... 1 -(yeto-rl-island-0, pid=3393) num_layers_at_start_in_bf16 ..................... 1 -(yeto-rl-island-0, pid=3393) num_layers_per_virtual_pipeline_stage ........... None -(yeto-rl-island-0, pid=3393) num_query_groups ................................ 2 -(yeto-rl-island-0, pid=3393) num_residual_streams ............................ 4 -(yeto-rl-island-0, pid=3393) num_rollout ..................................... 6 -(yeto-rl-island-0, pid=3393) num_speculative_tokens .......................... 0 -(yeto-rl-island-0, pid=3393) num_steps_per_rollout ........................... 1 -(yeto-rl-island-0, pid=3393) num_virtual_stages_per_pipeline_rank ............ None -(yeto-rl-island-0, pid=3393) num_workers ..................................... 2 -(yeto-rl-island-0, pid=3393) nvtx_ranges ..................................... False -(yeto-rl-island-0, pid=3393) o_groups ........................................ 8 -(yeto-rl-island-0, pid=3393) o_lora_rank ..................................... 1024 -(yeto-rl-island-0, pid=3393) object_storage_cache_path ....................... None -(yeto-rl-island-0, pid=3393) object_store_backend ............................ ray -(yeto-rl-island-0, pid=3393) observe_training_entropy ........................ False -(yeto-rl-island-0, pid=3393) offload_modules ................................. [] -(yeto-rl-island-0, pid=3393) offload_optimizer_states ........................ False -(yeto-rl-island-0, pid=3393) offload_rollout ................................. True -(yeto-rl-island-0, pid=3393) offload_rollout_level ........................... ['kv_cache', 'weight'] -(yeto-rl-island-0, pid=3393) offload_train ................................... True -(yeto-rl-island-0, pid=3393) offload_train_disk_chunk_mb ..................... 256 -(yeto-rl-island-0, pid=3393) offload_train_disk_dir .......................... None -(yeto-rl-island-0, pid=3393) offload_train_target ............................ cpu -(yeto-rl-island-0, pid=3393) one_logger_async ................................ False -(yeto-rl-island-0, pid=3393) one_logger_project .............................. megatron-lm -(yeto-rl-island-0, pid=3393) one_logger_run_name ............................. None -(yeto-rl-island-0, pid=3393) onnx_safe ....................................... None -(yeto-rl-island-0, pid=3393) opd_kl_coef ..................................... 1.0 -(yeto-rl-island-0, pid=3393) opd_log_prob_top_k .............................. 0 -(yeto-rl-island-0, pid=3393) opd_reward_weight_mode .......................... student_p -(yeto-rl-island-0, pid=3393) opd_teacher_ckpt_step ........................... None -(yeto-rl-island-0, pid=3393) opd_teacher_key ................................. opd_teacher -(yeto-rl-island-0, pid=3393) opd_teacher_load ................................ None -(yeto-rl-island-0, pid=3393) opd_teacher_urls ................................ None -(yeto-rl-island-0, pid=3393) opd_top_k_strategy .............................. only-student -(yeto-rl-island-0, pid=3393) opd_topk_per_position ........................... False -(yeto-rl-island-0, pid=3393) opd_type ........................................ None -(yeto-rl-island-0, pid=3393) openai_gelu ..................................... False -(yeto-rl-island-0, pid=3393) opsm_delta ...................................... 0.0001 -(yeto-rl-island-0, pid=3393) optimizer ....................................... adam -(yeto-rl-island-0, pid=3393) optimizer_cpu_offload ........................... False -(yeto-rl-island-0, pid=3393) optimizer_cuda_graph ............................ False -(yeto-rl-island-0, pid=3393) optimizer_offload_fraction ...................... 1.0 -(yeto-rl-island-0, pid=3393) optimizer_state_offload_chunk_size_mb ........... 0 -(yeto-rl-island-0, pid=3393) optimizer_state_offload_fraction ................ 1.0 -(yeto-rl-island-0, pid=3393) original_max_position_embeddings ................ 4096 -(yeto-rl-island-0, pid=3393) outer_dp_sharding_strategy ...................... no_shard -(yeto-rl-island-0, pid=3393) output_bert_embeddings .......................... False -(yeto-rl-island-0, pid=3393) over_sampling_batch_size ........................ 8 -(yeto-rl-island-0, pid=3393) overlap_cpu_optimizer_d2h_h2d ................... False -(yeto-rl-island-0, pid=3393) overlap_dispatch_backward_with_experts_wgrad .... False -(yeto-rl-island-0, pid=3393) overlap_grad_reduce ............................. False -(yeto-rl-island-0, pid=3393) overlap_moe_expert_parallel_comm ................ False -(yeto-rl-island-0, pid=3393) overlap_p2p_comm ................................ False -(yeto-rl-island-0, pid=3393) overlap_p2p_comm_warmup_flush ................... False -(yeto-rl-island-0, pid=3393) overlap_param_gather ............................ False -(yeto-rl-island-0, pid=3393) overlap_param_gather_with_optimizer_step ........ False -(yeto-rl-island-0, pid=3393) override_opt_param_scheduler .................... False -(yeto-rl-island-0, pid=3393) p2p_transfer_num_workers ........................ 4 -(yeto-rl-island-0, pid=3393) p2p_transfer_timeout ............................ 30.0 -(yeto-rl-island-0, pid=3393) pad_packed_seq_alignment ........................ None -(yeto-rl-island-0, pid=3393) pad_vocab_size .................................. True -(yeto-rl-island-0, pid=3393) padded_vocab_size ............................... 248320 -(yeto-rl-island-0, pid=3393) params_dtype .................................... torch.bfloat16 -(yeto-rl-island-0, pid=3393) partial_rollout ................................. False -(yeto-rl-island-0, pid=3393) patch_dim ....................................... 16 -(yeto-rl-island-0, pid=3393) pause_generation_mode ........................... retract -(yeto-rl-island-0, pid=3393) per_dataset_sequences_path ...................... None -(yeto-rl-island-0, pid=3393) per_split_data_args_path ........................ None -(yeto-rl-island-0, pid=3393) perform_initialization .......................... True -(yeto-rl-island-0, pid=3393) perform_rl_step ................................. False -(yeto-rl-island-0, pid=3393) phase_transition_iterations ..................... None -(yeto-rl-island-0, pid=3393) pin_cpu_grads ................................... True -(yeto-rl-island-0, pid=3393) pin_cpu_params .................................. True -(yeto-rl-island-0, pid=3393) pin_rollout_manager_to_head ..................... True -(yeto-rl-island-0, pid=3393) pipeline_model_parallel_comm_backend ............ None -(yeto-rl-island-0, pid=3393) pipeline_model_parallel_layout .................. None -(yeto-rl-island-0, pid=3393) pipeline_model_parallel_size .................... 2 -(yeto-rl-island-0, pid=3393) policy_loss_variant ............................. policy_loss -(yeto-rl-island-0, pid=3393) position_embedding_type ......................... rope -(yeto-rl-island-0, pid=3393) post_mlp_layernorm .............................. False -(yeto-rl-island-0, pid=3393) post_self_attn_layernorm ........................ False -(yeto-rl-island-0, pid=3393) prefill_num_servers ............................. None -(yeto-rl-island-0, pid=3393) pretrained_checkpoint ........................... None -(yeto-rl-island-0, pid=3393) profile ......................................... False -(yeto-rl-island-0, pid=3393) profile_ranks ................................... [] -(yeto-rl-island-0, pid=3393) profile_step_end ................................ 12 -(yeto-rl-island-0, pid=3393) profile_step_start .............................. 10 -(yeto-rl-island-0, pid=3393) profile_target .................................. ['train_overall'] -(yeto-rl-island-0, pid=3393) prometheus_port ................................. 9090 -(yeto-rl-island-0, pid=3393) prometheus_run_name ............................. None -(yeto-rl-island-0, pid=3393) prompt_data ..................................... /root/yeto-rl/prompts.jsonl -(yeto-rl-island-0, pid=3393) pytorch_profiler_collect_callstack .............. False -(yeto-rl-island-0, pid=3393) pytorch_profiler_collect_chakra ................. False -(yeto-rl-island-0, pid=3393) pytorch_profiler_collect_shapes ................. False -(yeto-rl-island-0, pid=3393) q_lora_rank ..................................... None -(yeto-rl-island-0, pid=3393) qk_clip ......................................... False -(yeto-rl-island-0, pid=3393) qk_clip_alpha ................................... 0.5 -(yeto-rl-island-0, pid=3393) qk_clip_threshold ............................... 100 -(yeto-rl-island-0, pid=3393) qk_head_dim ..................................... 128 -(yeto-rl-island-0, pid=3393) qk_l2_norm ...................................... False -(yeto-rl-island-0, pid=3393) qk_layernorm .................................... True -(yeto-rl-island-0, pid=3393) qk_pos_emb_head_dim ............................. 64 -(yeto-rl-island-0, pid=3393) qkv_format ...................................... thd -(yeto-rl-island-0, pid=3393) query_in_block_prob ............................. 0.1 -(yeto-rl-island-0, pid=3393) quick_geglu ..................................... False -(yeto-rl-island-0, pid=3393) rampup_batch_size ............................... None -(yeto-rl-island-0, pid=3393) rank ............................................ 0 -(yeto-rl-island-0, pid=3393) recompute_granularity ........................... full -(yeto-rl-island-0, pid=3393) recompute_logprobs_via_prefill .................. False -(yeto-rl-island-0, pid=3393) recompute_loss_function ......................... False -(yeto-rl-island-0, pid=3393) recompute_method ................................ uniform -(yeto-rl-island-0, pid=3393) recompute_modules ............................... None -(yeto-rl-island-0, pid=3393) recompute_num_layers ............................ 1 -(yeto-rl-island-0, pid=3393) record_memory_history ........................... False -(yeto-rl-island-0, pid=3393) record_shapes ................................... False -(yeto-rl-island-0, pid=3393) ref_ckpt_step ................................... None -(yeto-rl-island-0, pid=3393) ref_load ........................................ /mnt/yeto-models/torch_dist/qwen3.8-flash-next-4layer_torch_dist -(yeto-rl-island-0, pid=3393) ref_update_interval ............................. None -(yeto-rl-island-0, pid=3393) refit_method .................................... gloo -(yeto-rl-island-0, pid=3393) relative_attention_max_distance ................. 128 -(yeto-rl-island-0, pid=3393) relative_attention_num_buckets .................. 32 -(yeto-rl-island-0, pid=3393) rematerialize_param_from_master_weight .......... False -(yeto-rl-island-0, pid=3393) replication ..................................... False -(yeto-rl-island-0, pid=3393) replication_factor .............................. 2 -(yeto-rl-island-0, pid=3393) replication_jump ................................ None -(yeto-rl-island-0, pid=3393) requested_load .................................. None -(yeto-rl-island-0, pid=3393) rerun_mode ...................................... validate_results -(yeto-rl-island-0, pid=3393) reset_attention_mask ............................ False -(yeto-rl-island-0, pid=3393) reset_optimizer_states .......................... False -(yeto-rl-island-0, pid=3393) reset_position_ids .............................. False -(yeto-rl-island-0, pid=3393) result_rejected_tracker_filename ................ None -(yeto-rl-island-0, pid=3393) retriever_report_topk_accuracies ................ [] -(yeto-rl-island-0, pid=3393) retriever_score_scaling ......................... False -(yeto-rl-island-0, pid=3393) reuse_grad_buf_for_mxfp8_param_ag ............... False -(yeto-rl-island-0, pid=3393) reward_key ...................................... None -(yeto-rl-island-0, pid=3393) rewards_normalization ........................... True -(yeto-rl-island-0, pid=3393) rl_consumption_granularity ...................... B -(yeto-rl-island-0, pid=3393) rl_default_temperature .......................... 1.0 -(yeto-rl-island-0, pid=3393) rl_default_top_k ................................ -1 -(yeto-rl-island-0, pid=3393) rl_default_top_p ................................ 0 -(yeto-rl-island-0, pid=3393) rl_generation_lag ............................... 0 -(yeto-rl-island-0, pid=3393) rl_importance_sampling_truncation_coef .......... None -(yeto-rl-island-0, pid=3393) rl_inference_expert_model_parallel_size ......... None -(yeto-rl-island-0, pid=3393) rl_inference_expert_tensor_model_parallel_size .. None -(yeto-rl-island-0, pid=3393) rl_inference_logprobs_is_correction ............. False -(yeto-rl-island-0, pid=3393) rl_inference_model_unified_memory_level ......... 0 -(yeto-rl-island-0, pid=3393) rl_inference_parsers ............................ [] -(yeto-rl-island-0, pid=3393) rl_inference_pipeline_model_parallel_size ....... None -(yeto-rl-island-0, pid=3393) rl_inference_tensor_model_parallel_size ......... None -(yeto-rl-island-0, pid=3393) rl_kv_cache_management_mode ..................... persist -(yeto-rl-island-0, pid=3393) rl_offload_inference_model_weights_when_idle .... False -(yeto-rl-island-0, pid=3393) rl_offload_optimizer_during_inference ........... False -(yeto-rl-island-0, pid=3393) rl_partial_rollouts ............................. False -(yeto-rl-island-0, pid=3393) rl_persist_cuda_graphs .......................... False -(yeto-rl-island-0, pid=3393) rl_profile ...................................... False -(yeto-rl-island-0, pid=3393) rl_profile_dir .................................. None -(yeto-rl-island-0, pid=3393) rl_prompts_per_eval ............................. 32 -(yeto-rl-island-0, pid=3393) rl_sequence_packing_algo ........................ fifo -(yeto-rl-island-0, pid=3393) rl_sequence_packing_max_sequences_per_bin ....... 50 -(yeto-rl-island-0, pid=3393) rl_skip_bos_token ............................... False -(yeto-rl-island-0, pid=3393) rl_submission_granularity ....................... B -(yeto-rl-island-0, pid=3393) rl_training_cuda_graphs ......................... False -(yeto-rl-island-0, pid=3393) rl_use_sequence_packing ......................... False -(yeto-rl-island-0, pid=3393) rl_verify_model_weights_swap .................... False -(yeto-rl-island-0, pid=3393) rm_type ......................................... None -(yeto-rl-island-0, pid=3393) rm_url .......................................... None -(yeto-rl-island-0, pid=3393) rollout_all_samples_process_path ................ yeto.rl.engine.miles_adapter.rollout_meta_hook.extract_rollout_metadata -(yeto-rl-island-0, pid=3393) rollout_batch_size .............................. 8 -(yeto-rl-island-0, pid=3393) rollout_buffer_url .............................. None -(yeto-rl-island-0, pid=3393) rollout_data_postprocess_path ................... None -(yeto-rl-island-0, pid=3393) rollout_external ................................ False -(yeto-rl-island-0, pid=3393) rollout_external_engine_addrs ................... None -(yeto-rl-island-0, pid=3393) rollout_external_router_pd ...................... False -(yeto-rl-island-0, pid=3393) rollout_function_path ........................... miles.rollout.inference_rollout.inference_rollout_common.InferenceRolloutFn -(yeto-rl-island-0, pid=3393) rollout_global_dataset .......................... True -(yeto-rl-island-0, pid=3393) rollout_health_check_failure_threshold .......... 1 -(yeto-rl-island-0, pid=3393) rollout_health_check_first_wait ................. 0.0 -(yeto-rl-island-0, pid=3393) rollout_health_check_interval ................... 300.0 -(yeto-rl-island-0, pid=3393) rollout_health_check_timeout .................... 300.0 -(yeto-rl-island-0, pid=3393) rollout_indexer_topk_num_streams ................ None -(yeto-rl-island-0, pid=3393) rollout_max_context_len ......................... 2048 -(yeto-rl-island-0, pid=3393) rollout_max_prompt_len .......................... 2047 -(yeto-rl-island-0, pid=3393) rollout_max_response_len ........................ 1024 -(yeto-rl-island-0, pid=3393) rollout_num_gpus ................................ 8 -(yeto-rl-island-0, pid=3393) rollout_num_gpus_per_engine ..................... 4 -(yeto-rl-island-0, pid=3393) rollout_sample_filter_path ...................... yeto.rl.engine.miles_adapter.rollout_meta_hook.record_trained_groups -(yeto-rl-island-0, pid=3393) rollout_seed .................................... 17 -(yeto-rl-island-0, pid=3393) rollout_shuffle ................................. False -(yeto-rl-island-0, pid=3393) rollout_skip_special_tokens ..................... False -(yeto-rl-island-0, pid=3393) rollout_stop .................................... None -(yeto-rl-island-0, pid=3393) rollout_stop_token_ids .......................... None -(yeto-rl-island-0, pid=3393) rollout_submission_granularity .................. None -(yeto-rl-island-0, pid=3393) rollout_task_type ............................... math -(yeto-rl-island-0, pid=3393) rollout_temperature ............................. 1.0 -(yeto-rl-island-0, pid=3393) rollout_top_k ................................... -1 -(yeto-rl-island-0, pid=3393) rollout_top_p ................................... 1.0 -(yeto-rl-island-0, pid=3393) rope_scaling_factor ............................. 8.0 -(yeto-rl-island-0, pid=3393) rope_type ....................................... rope -(yeto-rl-island-0, pid=3393) rotary_base ..................................... 10000000 -(yeto-rl-island-0, pid=3393) rotary_base_per_layer ........................... None -(yeto-rl-island-0, pid=3393) rotary_interleaved .............................. False -(yeto-rl-island-0, pid=3393) rotary_percent .................................. 0.25 -(yeto-rl-island-0, pid=3393) rotary_scaling_factor ........................... 1.0 -(yeto-rl-island-0, pid=3393) rotary_seq_len_interpolation_factor ............. None -(yeto-rl-island-0, pid=3393) router_allow_requests_without_routing_key ....... False -(yeto-rl-island-0, pid=3393) router_api_key .................................. None -(yeto-rl-island-0, pid=3393) router_assignment_mode .......................... random -(yeto-rl-island-0, pid=3393) router_backend .................................. sglang -(yeto-rl-island-0, pid=3393) router_balance_abs_threshold .................... 64 -(yeto-rl-island-0, pid=3393) router_balance_rel_threshold .................... 1.5 -(yeto-rl-island-0, pid=3393) router_bucket_adjust_interval_secs .............. 5 -(yeto-rl-island-0, pid=3393) router_ca_cert_paths ............................ [] -(yeto-rl-island-0, pid=3393) router_cache_threshold .......................... 0.3 -(yeto-rl-island-0, pid=3393) router_cb_failure_threshold ..................... 10 -(yeto-rl-island-0, pid=3393) router_cb_success_threshold ..................... 3 -(yeto-rl-island-0, pid=3393) router_cb_timeout_duration_secs ................. 60 -(yeto-rl-island-0, pid=3393) router_cb_window_duration_secs .................. 120 -(yeto-rl-island-0, pid=3393) router_chat_template ............................ None -(yeto-rl-island-0, pid=3393) router_client_cert_path ......................... None -(yeto-rl-island-0, pid=3393) router_client_key_path .......................... None -(yeto-rl-island-0, pid=3393) router_control_plane_api_keys ................... [] -(yeto-rl-island-0, pid=3393) router_control_plane_audit_enabled .............. False -(yeto-rl-island-0, pid=3393) router_cors_allowed_origins ..................... [] -(yeto-rl-island-0, pid=3393) router_decode ................................... None -(yeto-rl-island-0, pid=3393) router_decode_policy ............................ None -(yeto-rl-island-0, pid=3393) router_decode_selector .......................... {} -(yeto-rl-island-0, pid=3393) router_disable_circuit_breaker .................. False -(yeto-rl-island-0, pid=3393) router_disable_health_check ..................... False -(yeto-rl-island-0, pid=3393) router_disable_retries .......................... False -(yeto-rl-island-0, pid=3393) router_dp_aware ................................. False -(yeto-rl-island-0, pid=3393) router_enable_igw ............................... False -(yeto-rl-island-0, pid=3393) router_enable_trace ............................. False -(yeto-rl-island-0, pid=3393) router_eviction_interval_secs ................... 60 -(yeto-rl-island-0, pid=3393) router_health_check_endpoint .................... /health -(yeto-rl-island-0, pid=3393) router_health_check_interval_secs ............... 15 -(yeto-rl-island-0, pid=3393) router_health_check_timeout_secs ................ 5 -(yeto-rl-island-0, pid=3393) router_health_failure_threshold ................. 40 -(yeto-rl-island-0, pid=3393) router_health_success_threshold ................. 1 -(yeto-rl-island-0, pid=3393) router_history_backend .......................... memory -(yeto-rl-island-0, pid=3393) router_json_log ................................. False -(yeto-rl-island-0, pid=3393) router_jwt_audience ............................. None -(yeto-rl-island-0, pid=3393) router_jwt_issuer ............................... None -(yeto-rl-island-0, pid=3393) router_jwt_jwks_uri ............................. None -(yeto-rl-island-0, pid=3393) router_jwt_role_mapping ......................... [] -(yeto-rl-island-0, pid=3393) router_log_dir .................................. None -(yeto-rl-island-0, pid=3393) router_log_level ................................ info -(yeto-rl-island-0, pid=3393) router_max_concurrent_requests .................. -1 -(yeto-rl-island-0, pid=3393) router_max_idle_secs ............................ 14400 -(yeto-rl-island-0, pid=3393) router_max_payload_size ......................... 536870912 -(yeto-rl-island-0, pid=3393) router_max_tree_size ............................ 67108864 -(yeto-rl-island-0, pid=3393) router_mcp_config_path .......................... None -(yeto-rl-island-0, pid=3393) router_mini_lb .................................. False -(yeto-rl-island-0, pid=3393) router_model_path ............................... None -(yeto-rl-island-0, pid=3393) router_oracle_connect_descriptor ................ None -(yeto-rl-island-0, pid=3393) router_oracle_password .......................... None -(yeto-rl-island-0, pid=3393) router_oracle_pool_max .......................... 16 -(yeto-rl-island-0, pid=3393) router_oracle_pool_min .......................... 1 -(yeto-rl-island-0, pid=3393) router_oracle_pool_timeout_secs ................. 30 -(yeto-rl-island-0, pid=3393) router_oracle_tns_alias ......................... None -(yeto-rl-island-0, pid=3393) router_oracle_username .......................... None -(yeto-rl-island-0, pid=3393) router_oracle_wallet_path ....................... None -(yeto-rl-island-0, pid=3393) router_otlp_traces_endpoint ..................... localhost:4317 -(yeto-rl-island-0, pid=3393) router_pd_disaggregation ........................ False -(yeto-rl-island-0, pid=3393) router_policy ................................... cache_aware -(yeto-rl-island-0, pid=3393) router_postgres_db_url .......................... None -(yeto-rl-island-0, pid=3393) router_postgres_pool_max ........................ 16 -(yeto-rl-island-0, pid=3393) router_prefill .................................. None -(yeto-rl-island-0, pid=3393) router_prefill_policy ........................... None -(yeto-rl-island-0, pid=3393) router_prefill_selector ......................... {} -(yeto-rl-island-0, pid=3393) router_prometheus_duration_buckets .............. None -(yeto-rl-island-0, pid=3393) router_prometheus_host .......................... 0.0.0.0 -(yeto-rl-island-0, pid=3393) router_prometheus_port .......................... 29000 -(yeto-rl-island-0, pid=3393) router_queue_size ............................... 100 -(yeto-rl-island-0, pid=3393) router_queue_timeout_secs ....................... 60 -(yeto-rl-island-0, pid=3393) router_rate_limit_tokens_per_second ............. None -(yeto-rl-island-0, pid=3393) router_reasoning_parser ......................... None -(yeto-rl-island-0, pid=3393) router_redis_pool_max ........................... 16 -(yeto-rl-island-0, pid=3393) router_redis_retention_days ..................... 30 -(yeto-rl-island-0, pid=3393) router_redis_url ................................ None -(yeto-rl-island-0, pid=3393) router_request_id_headers ....................... None -(yeto-rl-island-0, pid=3393) router_request_timeout_secs ..................... 1800 -(yeto-rl-island-0, pid=3393) router_retry_backoff_multiplier ................. 1.5 -(yeto-rl-island-0, pid=3393) router_retry_initial_backoff_ms ................. 50 -(yeto-rl-island-0, pid=3393) router_retry_jitter_factor ...................... 0.2 -(yeto-rl-island-0, pid=3393) router_retry_max_backoff_ms ..................... 30000 -(yeto-rl-island-0, pid=3393) router_retry_max_retries ........................ 5 -(yeto-rl-island-0, pid=3393) router_selector ................................. {} -(yeto-rl-island-0, pid=3393) router_service_discovery ........................ False -(yeto-rl-island-0, pid=3393) router_service_discovery_namespace .............. None -(yeto-rl-island-0, pid=3393) router_service_discovery_port ................... 80 -(yeto-rl-island-0, pid=3393) router_shutdown_grace_period_secs ............... 180 -(yeto-rl-island-0, pid=3393) router_test_external_dp_routing ................. False -(yeto-rl-island-0, pid=3393) router_tls_cert_path ............................ None -(yeto-rl-island-0, pid=3393) router_tls_key_path ............................. None -(yeto-rl-island-0, pid=3393) router_tokenizer_cache_enable_l0 ................ False -(yeto-rl-island-0, pid=3393) router_tokenizer_cache_enable_l1 ................ False -(yeto-rl-island-0, pid=3393) router_tokenizer_cache_l0_max_entries ........... 10000 -(yeto-rl-island-0, pid=3393) router_tokenizer_cache_l1_max_memory ............ 52428800 -(yeto-rl-island-0, pid=3393) router_tokenizer_path ........................... None -(yeto-rl-island-0, pid=3393) router_tool_call_parser ......................... None -(yeto-rl-island-0, pid=3393) router_worker_startup_check_interval ............ 30 -(yeto-rl-island-0, pid=3393) router_worker_startup_timeout_secs .............. 1800 -(yeto-rl-island-0, pid=3393) run_uuid ........................................ de7315256fe1483d -(yeto-rl-island-0, pid=3393) run_workload_inspector_server ................... False -(yeto-rl-island-0, pid=3393) sample_rate ..................................... 1.0 -(yeto-rl-island-0, pid=3393) sapo_tau_neg .................................... 1.05 -(yeto-rl-island-0, pid=3393) sapo_tau_pos .................................... 1.0 -(yeto-rl-island-0, pid=3393) save ............................................ None -(yeto-rl-island-0, pid=3393) save_activations_interval ....................... None -(yeto-rl-island-0, pid=3393) save_debug_event_data ........................... None -(yeto-rl-island-0, pid=3393) save_debug_rollout_data ......................... None -(yeto-rl-island-0, pid=3393) save_debug_train_data ........................... None -(yeto-rl-island-0, pid=3393) save_debug_trajectory_data ...................... None -(yeto-rl-island-0, pid=3393) save_dgrads_interval ............................ None -(yeto-rl-island-0, pid=3393) save_hf ......................................... None -(yeto-rl-island-0, pid=3393) save_interval ................................... None -(yeto-rl-island-0, pid=3393) save_local_weight_checksum ...................... False -(yeto-rl-island-0, pid=3393) save_params_interval ............................ None -(yeto-rl-island-0, pid=3393) save_retain_interval ............................ None -(yeto-rl-island-0, pid=3393) save_tokens_per_expert_interval ................. None -(yeto-rl-island-0, pid=3393) save_trigger_sentinel ........................... None -(yeto-rl-island-0, pid=3393) save_wgrads_interval ............................ None -(yeto-rl-island-0, pid=3393) seed ............................................ 17 -(yeto-rl-island-0, pid=3393) seq_length ...................................... 2048 -(yeto-rl-island-0, pid=3393) sequence_packing_scheduler ...................... None -(yeto-rl-island-0, pid=3393) sequence_parallel ............................... True -(yeto-rl-island-0, pid=3393) session_message_matcher ......................... strict -(yeto-rl-island-0, pid=3393) session_sample_picker_path ...................... miles.rollout.session.v2.picker_hub.drop_same_prompt_retries -(yeto-rl-island-0, pid=3393) session_sample_postprocessor_path ............... miles.rollout.session.v2.postprocessor_hub.default_postprocess -(yeto-rl-island-0, pid=3393) session_server_external_host .................... None -(yeto-rl-island-0, pid=3393) session_server_ip ............................... None -(yeto-rl-island-0, pid=3393) session_server_port ............................. None -(yeto-rl-island-0, pid=3393) session_server_workers .......................... 32 -(yeto-rl-island-0, pid=3393) sft ............................................. False -(yeto-rl-island-0, pid=3393) sft_mock_dataset_config_json .................... None -(yeto-rl-island-0, pid=3393) sft_tokenizer_prompt_format ..................... nemotron-h-aligned -(yeto-rl-island-0, pid=3393) sgd_momentum .................................... 0.9 -(yeto-rl-island-0, pid=3393) sglang_abort_on_priority_when_disabled .......... False -(yeto-rl-island-0, pid=3393) sglang_admin_api_key ............................ None -(yeto-rl-island-0, pid=3393) sglang_allow_auto_truncate ...................... False -(yeto-rl-island-0, pid=3393) sglang_allowed_media_domains .................... [] -(yeto-rl-island-0, pid=3393) sglang_api_key .................................. None -(yeto-rl-island-0, pid=3393) sglang_asr_max_buffer_seconds ................... 60 -(yeto-rl-island-0, pid=3393) sglang_asr_max_concurrent_sessions .............. 32 -(yeto-rl-island-0, pid=3393) sglang_attention_backend ........................ None -(yeto-rl-island-0, pid=3393) sglang_attn_cp_size ............................. 1 -(yeto-rl-island-0, pid=3393) sglang_batch_notify_size ........................ 16 -(yeto-rl-island-0, pid=3393) sglang_bf16_gemm_backend ........................ auto -(yeto-rl-island-0, pid=3393) sglang_bucket_e2e_request_latency ............... None -(yeto-rl-island-0, pid=3393) sglang_bucket_inter_token_latency ............... None -(yeto-rl-island-0, pid=3393) sglang_bucket_time_to_first_token ............... None -(yeto-rl-island-0, pid=3393) sglang_c128_page_size ........................... 16 -(yeto-rl-island-0, pid=3393) sglang_chat_template ............................ None -(yeto-rl-island-0, pid=3393) sglang_checkpoint_engine_wait_weights_before_ready False -(yeto-rl-island-0, pid=3393) sglang_chunked_prefill_size ..................... 8192 -(yeto-rl-island-0, pid=3393) sglang_completion_template ...................... None -(yeto-rl-island-0, pid=3393) sglang_config ................................... None -(yeto-rl-island-0, pid=3393) sglang_constrained_json_disable_any_whitespace .. False -(yeto-rl-island-0, pid=3393) sglang_constrained_json_whitespace_pattern ...... None -(yeto-rl-island-0, pid=3393) sglang_context_length ........................... None -(yeto-rl-island-0, pid=3393) sglang_cp_strategy .............................. None -(yeto-rl-island-0, pid=3393) sglang_cpu_offload_gb ........................... 0 -(yeto-rl-island-0, pid=3393) sglang_crash_dump_folder ........................ None -(yeto-rl-island-0, pid=3393) sglang_cuda_graph_backend_decode ................ None -(yeto-rl-island-0, pid=3393) sglang_cuda_graph_backend_prefill ............... disabled -(yeto-rl-island-0, pid=3393) sglang_cuda_graph_bs_decode ..................... None -(yeto-rl-island-0, pid=3393) sglang_cuda_graph_bs_prefill .................... None -(yeto-rl-island-0, pid=3393) sglang_cuda_graph_config ........................ None -(yeto-rl-island-0, pid=3393) sglang_cuda_graph_max_bs_decode ................. None -(yeto-rl-island-0, pid=3393) sglang_cuda_graph_max_bs_prefill ................ None -(yeto-rl-island-0, pid=3393) sglang_cuda_graph_tc_compiler ................... None -(yeto-rl-island-0, pid=3393) sglang_custom_pull_weights_pre_read_hook ........ None -(yeto-rl-island-0, pid=3393) sglang_custom_weight_loader ..................... None -(yeto-rl-island-0, pid=3393) sglang_dcp_comm_backend ......................... None -(yeto-rl-island-0, pid=3393) sglang_dcp_replicate_q_proj ..................... None -(yeto-rl-island-0, pid=3393) sglang_dcp_size ................................. 1 -(yeto-rl-island-0, pid=3393) sglang_debug_cuda_graph ......................... False -(yeto-rl-island-0, pid=3393) sglang_debug_tensor_dump_input_file ............. None -(yeto-rl-island-0, pid=3393) sglang_debug_tensor_dump_layers ................. None -(yeto-rl-island-0, pid=3393) sglang_debug_tensor_dump_output_folder .......... None -(yeto-rl-island-0, pid=3393) sglang_decode_attention_backend ................. None -(yeto-rl-island-0, pid=3393) sglang_decode_log_interval ...................... 40 -(yeto-rl-island-0, pid=3393) sglang_decoupled_spec_bind_endpoint ............. None -(yeto-rl-island-0, pid=3393) sglang_decoupled_spec_connect_endpoints ......... None -(yeto-rl-island-0, pid=3393) sglang_decoupled_spec_rank ...................... None -(yeto-rl-island-0, pid=3393) sglang_decoupled_spec_role ...................... null -(yeto-rl-island-0, pid=3393) sglang_decrypted_config_file .................... None -(yeto-rl-island-0, pid=3393) sglang_decrypted_draft_config_file .............. None -(yeto-rl-island-0, pid=3393) sglang_deepep_config ............................ None -(yeto-rl-island-0, pid=3393) sglang_deepep_dispatcher_output_dtype ........... auto -(yeto-rl-island-0, pid=3393) sglang_deepep_mode .............................. auto -(yeto-rl-island-0, pid=3393) sglang_deepep_v2_mode ........................... direct -(yeto-rl-island-0, pid=3393) sglang_default_chat_template_kwargs ............. None -(yeto-rl-island-0, pid=3393) sglang_default_priority_value ................... None -(yeto-rl-island-0, pid=3393) sglang_delete_ckpt_after_loading ................ False -(yeto-rl-island-0, pid=3393) sglang_detokenizer_worker_num ................... 1 -(yeto-rl-island-0, pid=3393) sglang_device ................................... None -(yeto-rl-island-0, pid=3393) sglang_disable_attn_tp_gather ................... False -(yeto-rl-island-0, pid=3393) sglang_disable_chunked_prefix_cache ............. False -(yeto-rl-island-0, pid=3393) sglang_disable_cuda_graph ....................... False -(yeto-rl-island-0, pid=3393) sglang_disable_cuda_graph_padding ............... False -(yeto-rl-island-0, pid=3393) sglang_disable_custom_all_reduce ................ False -(yeto-rl-island-0, pid=3393) sglang_disable_decode_cuda_graph ................ False -(yeto-rl-island-0, pid=3393) sglang_disable_draft_cuda_graph ................. False -(yeto-rl-island-0, pid=3393) sglang_disable_fast_image_processor ............. False -(yeto-rl-island-0, pid=3393) sglang_disable_flashinfer_autotune .............. False -(yeto-rl-island-0, pid=3393) sglang_disable_flashinfer_cutlass_moe_fp4_allgather False -(yeto-rl-island-0, pid=3393) sglang_disable_hybrid_swa_memory ................ False -(yeto-rl-island-0, pid=3393) sglang_disable_outlines_disk_cache .............. False -(yeto-rl-island-0, pid=3393) sglang_disable_overlap_schedule ................. False -(yeto-rl-island-0, pid=3393) sglang_disable_prefill_cuda_graph ............... False -(yeto-rl-island-0, pid=3393) sglang_disable_priority_preemption .............. False -(yeto-rl-island-0, pid=3393) sglang_disable_radix_cache ...................... True -(yeto-rl-island-0, pid=3393) sglang_disable_shared_experts_fusion ............ False -(yeto-rl-island-0, pid=3393) sglang_disable_tokenizer_batch_decode ........... False -(yeto-rl-island-0, pid=3393) sglang_disaggregation_bootstrap_port ............ 8998 -(yeto-rl-island-0, pid=3393) sglang_disaggregation_decode_enable_offload_kvcache False -(yeto-rl-island-0, pid=3393) sglang_disaggregation_decode_enable_radix_cache . False -(yeto-rl-island-0, pid=3393) sglang_disaggregation_decode_extra_slots ........ None -(yeto-rl-island-0, pid=3393) sglang_disaggregation_decode_polling_interval ... 1 -(yeto-rl-island-0, pid=3393) sglang_disaggregation_decode_retraction_backup .. None -(yeto-rl-island-0, pid=3393) sglang_disaggregation_ib_device ................. None -(yeto-rl-island-0, pid=3393) sglang_disaggregation_mode ...................... null -(yeto-rl-island-0, pid=3393) sglang_disaggregation_transfer_backend .......... mooncake -(yeto-rl-island-0, pid=3393) sglang_dist_timeout ............................. None -(yeto-rl-island-0, pid=3393) sglang_dllm_algorithm ........................... None -(yeto-rl-island-0, pid=3393) sglang_dllm_algorithm_config .................... None -(yeto-rl-island-0, pid=3393) sglang_dllm_fdfo ................................ True -(yeto-rl-island-0, pid=3393) sglang_download_dir ............................. None -(yeto-rl-island-0, pid=3393) sglang_dp_size .................................. 1 -(yeto-rl-island-0, pid=3393) sglang_dsa_decode_backend ....................... None -(yeto-rl-island-0, pid=3393) sglang_dsa_paged_mqa_logits_backend ............. auto -(yeto-rl-island-0, pid=3393) sglang_dsa_prefill_backend ...................... None -(yeto-rl-island-0, pid=3393) sglang_dsa_topk_backend ......................... sgl-kernel -(yeto-rl-island-0, pid=3393) sglang_dsv4_attn_backend ........................ auto -(yeto-rl-island-0, pid=3393) sglang_dsv4_prefill_backend ..................... auto -(yeto-rl-island-0, pid=3393) sglang_dtype .................................... auto -(yeto-rl-island-0, pid=3393) sglang_dwdp_size ................................ 1 -(yeto-rl-island-0, pid=3393) sglang_dynamic_batch_tokenizer_batch_size ....... 32 -(yeto-rl-island-0, pid=3393) sglang_dynamic_batch_tokenizer_batch_timeout .... 0.002 -(yeto-rl-island-0, pid=3393) sglang_elastic_ep_backend ....................... None -(yeto-rl-island-0, pid=3393) sglang_elastic_ep_initial_size .................. None -(yeto-rl-island-0, pid=3393) sglang_elastic_ep_rejoin ........................ False -(yeto-rl-island-0, pid=3393) sglang_elastic_ep_scale_timeout ................. 600 -(yeto-rl-island-0, pid=3393) sglang_enable_adaptive_dispatch_to_encoder ...... False -(yeto-rl-island-0, pid=3393) sglang_enable_aiter_allreduce_fusion ............ False -(yeto-rl-island-0, pid=3393) sglang_enable_attn_tp_input_scattered ........... False -(yeto-rl-island-0, pid=3393) sglang_enable_broadcast_mm_inputs_process ....... False -(yeto-rl-island-0, pid=3393) sglang_enable_cache_report ...................... False -(yeto-rl-island-0, pid=3393) sglang_enable_cp_decode_attn_tp ................. False -(yeto-rl-island-0, pid=3393) sglang_enable_cudagraph_gc ...................... False -(yeto-rl-island-0, pid=3393) sglang_enable_custom_logit_processor ............ False -(yeto-rl-island-0, pid=3393) sglang_enable_deepseek_v4_fp4_indexer ........... False -(yeto-rl-island-0, pid=3393) sglang_enable_dense_mlp_attn_tp ................. False -(yeto-rl-island-0, pid=3393) sglang_enable_deterministic_inference ........... False -(yeto-rl-island-0, pid=3393) sglang_enable_dp_attention ...................... False -(yeto-rl-island-0, pid=3393) sglang_enable_dp_attention_local_control_broadcast False -(yeto-rl-island-0, pid=3393) sglang_enable_dp_lm_head ........................ False -(yeto-rl-island-0, pid=3393) sglang_enable_draft_weights_cpu_backup .......... False -(yeto-rl-island-0, pid=3393) sglang_enable_dsa_cache_layer_split ............. False -(yeto-rl-island-0, pid=3393) sglang_enable_dynamic_batch_tokenizer ........... False -(yeto-rl-island-0, pid=3393) sglang_enable_dynamic_chunking .................. False -(yeto-rl-island-0, pid=3393) sglang_enable_elastic_expert_backup ............. False -(yeto-rl-island-0, pid=3393) sglang_enable_engine_info_bootstrap ............. False -(yeto-rl-island-0, pid=3393) sglang_enable_eplb .............................. False -(yeto-rl-island-0, pid=3393) sglang_enable_flashinfer_allreduce_fusion ....... False -(yeto-rl-island-0, pid=3393) sglang_enable_flexkv ............................ False -(yeto-rl-island-0, pid=3393) sglang_enable_forward_pass_metrics .............. False -(yeto-rl-island-0, pid=3393) sglang_enable_fp32_lm_head ...................... False -(yeto-rl-island-0, pid=3393) sglang_enable_fused_moe_sum_all_reduce .......... False -(yeto-rl-island-0, pid=3393) sglang_enable_fused_qk_norm_rope ................ False -(yeto-rl-island-0, pid=3393) sglang_enable_hierarchical_cache ................ False -(yeto-rl-island-0, pid=3393) sglang_enable_hisparse .......................... False -(yeto-rl-island-0, pid=3393) sglang_enable_http2 ............................. False -(yeto-rl-island-0, pid=3393) sglang_enable_int8_mamba_checkpoint ............. False -(yeto-rl-island-0, pid=3393) sglang_enable_layernorm_sp ...................... False -(yeto-rl-island-0, pid=3393) sglang_enable_layerwise_nvtx_marker ............. False -(yeto-rl-island-0, pid=3393) sglang_enable_lean_attention .................... None -(yeto-rl-island-0, pid=3393) sglang_enable_linear_replayssm .................. False -(yeto-rl-island-0, pid=3393) sglang_enable_linear_replayssm_spec ............. False -(yeto-rl-island-0, pid=3393) sglang_enable_lmcache ........................... False -(yeto-rl-island-0, pid=3393) sglang_enable_lora .............................. None -(yeto-rl-island-0, pid=3393) sglang_enable_lora_overlap_loading .............. None -(yeto-rl-island-0, pid=3393) sglang_enable_mamba_cache_stochastic_rounding ... False -(yeto-rl-island-0, pid=3393) sglang_enable_metrics ........................... False -(yeto-rl-island-0, pid=3393) sglang_enable_metrics_for_all_schedulers ........ False -(yeto-rl-island-0, pid=3393) sglang_enable_mfu_metrics ....................... False -(yeto-rl-island-0, pid=3393) sglang_enable_mis ............................... False -(yeto-rl-island-0, pid=3393) sglang_enable_mixed_chunk ....................... False -(yeto-rl-island-0, pid=3393) sglang_enable_mm_global_cache ................... False -(yeto-rl-island-0, pid=3393) sglang_enable_mscclpp ........................... False -(yeto-rl-island-0, pid=3393) sglang_enable_multi_layer_eagle ................. False -(yeto-rl-island-0, pid=3393) sglang_enable_multimodal ........................ None -(yeto-rl-island-0, pid=3393) sglang_enable_nccl_nvls ......................... False -(yeto-rl-island-0, pid=3393) sglang_enable_p2p_check ......................... False -(yeto-rl-island-0, pid=3393) sglang_enable_page_major_kv_layout .............. False -(yeto-rl-island-0, pid=3393) sglang_enable_pdmux ............................. False -(yeto-rl-island-0, pid=3393) sglang_enable_precise_embedding_interpolation ... False -(yeto-rl-island-0, pid=3393) sglang_enable_prefill_cp ........................ False -(yeto-rl-island-0, pid=3393) sglang_enable_prefill_delayer ................... False -(yeto-rl-island-0, pid=3393) sglang_enable_prefill_only_deterministic_inference False -(yeto-rl-island-0, pid=3393) sglang_enable_prefix_mm_cache ................... False -(yeto-rl-island-0, pid=3393) sglang_enable_priority_scheduling ............... False -(yeto-rl-island-0, pid=3393) sglang_enable_profile_cuda_graph ................ False -(yeto-rl-island-0, pid=3393) sglang_enable_quant_communications .............. False -(yeto-rl-island-0, pid=3393) sglang_enable_rdt_weight_sync ................... False -(yeto-rl-island-0, pid=3393) sglang_enable_request_time_stats_logging ........ False -(yeto-rl-island-0, pid=3393) sglang_enable_response_store .................... False -(yeto-rl-island-0, pid=3393) sglang_enable_return_hidden_states .............. False -(yeto-rl-island-0, pid=3393) sglang_enable_scattered_sconv ................... False -(yeto-rl-island-0, pid=3393) sglang_enable_session_radix_cache ............... False -(yeto-rl-island-0, pid=3393) sglang_enable_shared_experts_attn_tp ............ False -(yeto-rl-island-0, pid=3393) sglang_enable_single_batch_overlap .............. False -(yeto-rl-island-0, pid=3393) sglang_enable_ssl_refresh ....................... False -(yeto-rl-island-0, pid=3393) sglang_enable_streaming_session ................. False -(yeto-rl-island-0, pid=3393) sglang_enable_strict_thinking ................... False -(yeto-rl-island-0, pid=3393) sglang_enable_symm_mem .......................... False -(yeto-rl-island-0, pid=3393) sglang_enable_tf32_matmul ....................... False -(yeto-rl-island-0, pid=3393) sglang_enable_tokenizer_batch_encode ............ False -(yeto-rl-island-0, pid=3393) sglang_enable_torch_compile ..................... False -(yeto-rl-island-0, pid=3393) sglang_enable_torch_compile_debug_mode .......... False -(yeto-rl-island-0, pid=3393) sglang_enable_torch_symm_mem .................... False -(yeto-rl-island-0, pid=3393) sglang_enable_tp_lm_head_all_to_all ............. None -(yeto-rl-island-0, pid=3393) sglang_enable_trace ............................. False -(yeto-rl-island-0, pid=3393) sglang_enable_two_batch_overlap ................. False -(yeto-rl-island-0, pid=3393) sglang_enable_unified_cache_external_linker ..... False -(yeto-rl-island-0, pid=3393) sglang_enable_unified_memory .................... False -(yeto-rl-island-0, pid=3393) sglang_enable_w4a4_mxfp4_megamoe ................ False -(yeto-rl-island-0, pid=3393) sglang_enable_waterfill ......................... False -(yeto-rl-island-0, pid=3393) sglang_enable_weights_cpu_backup ................ False -(yeto-rl-island-0, pid=3393) sglang_enable_weights_disk_backup ............... False -(yeto-rl-island-0, pid=3393) sglang_encoder_bootstrap_port ................... 8997 -(yeto-rl-island-0, pid=3393) sglang_encoder_only ............................. False -(yeto-rl-island-0, pid=3393) sglang_encoder_register_urls .................... [] -(yeto-rl-island-0, pid=3393) sglang_encoder_transfer_backend ................. auto -(yeto-rl-island-0, pid=3393) sglang_encoder_urls ............................. [] -(yeto-rl-island-0, pid=3393) sglang_enforce_disable_flashinfer_allreduce_fusion False -(yeto-rl-island-0, pid=3393) sglang_enforce_shared_experts_fusion ............ False -(yeto-rl-island-0, pid=3393) sglang_engine_info_bootstrap_port ............... 6789 -(yeto-rl-island-0, pid=3393) sglang_ep_dispatch_algorithm .................... None -(yeto-rl-island-0, pid=3393) sglang_ep_join_mode ............................. None -(yeto-rl-island-0, pid=3393) sglang_ep_join_rank_offset ...................... 0 -(yeto-rl-island-0, pid=3393) sglang_ep_num_redundant_experts ................. 0 -(yeto-rl-island-0, pid=3393) sglang_ep_size .................................. 4 -(yeto-rl-island-0, pid=3393) sglang_eplb_algorithm ........................... auto -(yeto-rl-island-0, pid=3393) sglang_eplb_min_rebalancing_utilization_threshold 1.0 -(yeto-rl-island-0, pid=3393) sglang_eplb_rebalance_layers_per_chunk .......... None -(yeto-rl-island-0, pid=3393) sglang_eplb_rebalance_num_iterations ............ 1000 -(yeto-rl-island-0, pid=3393) sglang_expert_balancedness_report_mode .......... off -(yeto-rl-island-0, pid=3393) sglang_expert_distribution_recorder_buffer_size . None -(yeto-rl-island-0, pid=3393) sglang_expert_distribution_recorder_mode ........ None -(yeto-rl-island-0, pid=3393) sglang_experts_shared_outer_loras ............... None -(yeto-rl-island-0, pid=3393) sglang_export_metrics_to_file ................... False -(yeto-rl-island-0, pid=3393) sglang_export_metrics_to_file_dir ............... None -(yeto-rl-island-0, pid=3393) sglang_extra_metric_labels ...................... None -(yeto-rl-island-0, pid=3393) sglang_fastapi_root_path ........................ -(yeto-rl-island-0, pid=3393) sglang_file_storage_path ........................ sglang_storage -(yeto-rl-island-0, pid=3393) sglang_flashinfer_a2a_dispatch_type ............. None -(yeto-rl-island-0, pid=3393) sglang_flashinfer_allreduce_fusion_backend ...... None -(yeto-rl-island-0, pid=3393) sglang_flashinfer_autotune_skip_ops ............. None -(yeto-rl-island-0, pid=3393) sglang_flashinfer_mla_disable_ragged ............ False -(yeto-rl-island-0, pid=3393) sglang_flashinfer_mxfp4_moe_precision ........... default -(yeto-rl-island-0, pid=3393) sglang_flexkv_config_file ....................... None -(yeto-rl-island-0, pid=3393) sglang_forward_hooks ............................ None -(yeto-rl-island-0, pid=3393) sglang_forward_pass_metrics_ipc_name ............ None -(yeto-rl-island-0, pid=3393) sglang_forward_pass_metrics_worker_id ........... -(yeto-rl-island-0, pid=3393) sglang_fp4_gemm_runner_backend .................. auto -(yeto-rl-island-0, pid=3393) sglang_fp8_gemm_runner_backend .................. auto -(yeto-rl-island-0, pid=3393) sglang_fuseep_mode .............................. 2 -(yeto-rl-island-0, pid=3393) sglang_gc_threshold ............................. None -(yeto-rl-island-0, pid=3393) sglang_gc_warning_threshold_secs ................ 0.0 -(yeto-rl-island-0, pid=3393) sglang_generation_tokens_buckets ................ None -(yeto-rl-island-0, pid=3393) sglang_grammar_backend .......................... None -(yeto-rl-island-0, pid=3393) sglang_grpc_mode ................................ False -(yeto-rl-island-0, pid=3393) sglang_grpc_port ................................ None -(yeto-rl-island-0, pid=3393) sglang_hf_chat_template_name .................... None -(yeto-rl-island-0, pid=3393) sglang_hicache_host_memory_mode ................. cache -(yeto-rl-island-0, pid=3393) sglang_hicache_io_backend ....................... kernel -(yeto-rl-island-0, pid=3393) sglang_hicache_mem_layout ....................... page_first -(yeto-rl-island-0, pid=3393) sglang_hicache_ratio ............................ None -(yeto-rl-island-0, pid=3393) sglang_hicache_size ............................. 0 -(yeto-rl-island-0, pid=3393) sglang_hicache_storage_backend .................. None -(yeto-rl-island-0, pid=3393) sglang_hicache_storage_backend_extra_config ..... None -(yeto-rl-island-0, pid=3393) sglang_hicache_storage_prefetch_policy .......... timeout -(yeto-rl-island-0, pid=3393) sglang_hicache_storage_prefetch_retry_max_attempts 8 -(yeto-rl-island-0, pid=3393) sglang_hicache_storage_prefetch_retry_poll_interval 8 -(yeto-rl-island-0, pid=3393) sglang_hicache_write_policy ..................... write_through -(yeto-rl-island-0, pid=3393) sglang_hisparse_config .......................... None -(yeto-rl-island-0, pid=3393) sglang_host ..................................... 127.0.0.1 -(yeto-rl-island-0, pid=3393) sglang_http2_initial_connection_window_size ..... 1048576 -(yeto-rl-island-0, pid=3393) sglang_http2_max_concurrent_streams ............. 200 -(yeto-rl-island-0, pid=3393) sglang_image_processor_backend .................. auto -(yeto-rl-island-0, pid=3393) sglang_incremental_streaming_output ............. False -(yeto-rl-island-0, pid=3393) sglang_init_expert_location ..................... trivial -(yeto-rl-island-0, pid=3393) sglang_int8_mamba_ckpt_size ..................... None -(yeto-rl-island-0, pid=3393) sglang_is_embedding ............................. False -(yeto-rl-island-0, pid=3393) sglang_json_model_override_args ................. {} -(yeto-rl-island-0, pid=3393) sglang_keep_mm_feature_on_device ................ False -(yeto-rl-island-0, pid=3393) sglang_kt_cpuinfer .............................. None -(yeto-rl-island-0, pid=3393) sglang_kt_max_deferred_experts_per_token ........ None -(yeto-rl-island-0, pid=3393) sglang_kt_method ................................ AMXINT4 -(yeto-rl-island-0, pid=3393) sglang_kt_num_gpu_experts ....................... None -(yeto-rl-island-0, pid=3393) sglang_kt_threadpool_count ...................... 2 -(yeto-rl-island-0, pid=3393) sglang_kt_weight_path ........................... None -(yeto-rl-island-0, pid=3393) sglang_kv_cache_dtype ........................... auto -(yeto-rl-island-0, pid=3393) sglang_kv_canary ................................ none -(yeto-rl-island-0, pid=3393) sglang_kv_canary_real_data ...................... none -(yeto-rl-island-0, pid=3393) sglang_kv_canary_sweep_interval ................. 0 -(yeto-rl-island-0, pid=3393) sglang_kv_events_config ......................... None -(yeto-rl-island-0, pid=3393) sglang_language_model_only ...................... False -(yeto-rl-island-0, pid=3393) sglang_language_only ............................ False -(yeto-rl-island-0, pid=3393) sglang_limit_mm_data_per_request ................ None -(yeto-rl-island-0, pid=3393) sglang_linear_attn_backend ...................... triton -(yeto-rl-island-0, pid=3393) sglang_linear_attn_decode_backend ............... None -(yeto-rl-island-0, pid=3393) sglang_linear_attn_prefill_backend .............. flashinfer -(yeto-rl-island-0, pid=3393) sglang_linear_attn_verify_backend ............... None -(yeto-rl-island-0, pid=3393) sglang_linear_replayssm_cache_len ............... 16 -(yeto-rl-island-0, pid=3393) sglang_lmcache_config_file ...................... None -(yeto-rl-island-0, pid=3393) sglang_load_balance_method ...................... auto -(yeto-rl-island-0, pid=3393) sglang_load_format .............................. auto -(yeto-rl-island-0, pid=3393) sglang_load_publish_endpoint .................... None -(yeto-rl-island-0, pid=3393) sglang_load_snapshot_publish_interval ........... 15 -(yeto-rl-island-0, pid=3393) sglang_log_level ................................ info -(yeto-rl-island-0, pid=3393) sglang_log_level_http ........................... None -(yeto-rl-island-0, pid=3393) sglang_log_requests ............................. False -(yeto-rl-island-0, pid=3393) sglang_log_requests_format ...................... text -(yeto-rl-island-0, pid=3393) sglang_log_requests_level ....................... 2 -(yeto-rl-island-0, pid=3393) sglang_log_requests_target ...................... None -(yeto-rl-island-0, pid=3393) sglang_lora_backend ............................. triton -(yeto-rl-island-0, pid=3393) sglang_lora_drain_wait_threshold ................ 0.0 -(yeto-rl-island-0, pid=3393) sglang_lora_eviction_policy ..................... lru -(yeto-rl-island-0, pid=3393) sglang_lora_no_cpu_backup ....................... False -(yeto-rl-island-0, pid=3393) sglang_lora_paths ............................... None -(yeto-rl-island-0, pid=3393) sglang_lora_strict_loading ...................... True -(yeto-rl-island-0, pid=3393) sglang_lora_target_modules ...................... None -(yeto-rl-island-0, pid=3393) sglang_lora_use_virtual_experts ................. True -(yeto-rl-island-0, pid=3393) sglang_mamba_backend ............................ triton -(yeto-rl-island-0, pid=3393) sglang_mamba_cache_philox_rounds ................ 0 -(yeto-rl-island-0, pid=3393) sglang_mamba_full_memory_ratio .................. None -(yeto-rl-island-0, pid=3393) sglang_mamba_max_states_per_path ................ -1 -(yeto-rl-island-0, pid=3393) sglang_mamba_radix_cache_strategy ............... auto -(yeto-rl-island-0, pid=3393) sglang_mamba_ssm_dtype .......................... None -(yeto-rl-island-0, pid=3393) sglang_mamba_track_interval ..................... 256 -(yeto-rl-island-0, pid=3393) sglang_max_ep_size .............................. None -(yeto-rl-island-0, pid=3393) sglang_max_loaded_loras ......................... None -(yeto-rl-island-0, pid=3393) sglang_max_lora_chunk_size ...................... 16 -(yeto-rl-island-0, pid=3393) sglang_max_lora_rank ............................ 16 -(yeto-rl-island-0, pid=3393) sglang_max_loras_per_batch ...................... 8 -(yeto-rl-island-0, pid=3393) sglang_max_mamba_cache_size ..................... None -(yeto-rl-island-0, pid=3393) sglang_max_prefill_tokens ....................... 16384 -(yeto-rl-island-0, pid=3393) sglang_max_queued_requests ...................... None -(yeto-rl-island-0, pid=3393) sglang_max_running_requests ..................... None -(yeto-rl-island-0, pid=3393) sglang_max_total_tokens ......................... None -(yeto-rl-island-0, pid=3393) sglang_media_url_max_file_size_mb ............... 64 -(yeto-rl-island-0, pid=3393) sglang_mem_fraction_static ...................... 0.7 -(yeto-rl-island-0, pid=3393) sglang_min_free_slots_delay ..................... None -(yeto-rl-island-0, pid=3393) sglang_mlx_enable_sampling ...................... False -(yeto-rl-island-0, pid=3393) sglang_mm_attention_backend ..................... None -(yeto-rl-island-0, pid=3393) sglang_mm_enable_dp_encoder ..................... False -(yeto-rl-island-0, pid=3393) sglang_mm_feature_transport ..................... None -(yeto-rl-island-0, pid=3393) sglang_mm_global_cache_backend .................. mooncake -(yeto-rl-island-0, pid=3393) sglang_mm_io_worker_num ......................... 0 -(yeto-rl-island-0, pid=3393) sglang_mm_preprocess_cache_size_mb .............. None -(yeto-rl-island-0, pid=3393) sglang_mm_process_config ........................ None -(yeto-rl-island-0, pid=3393) sglang_mm_processor_worker_num .................. 0 -(yeto-rl-island-0, pid=3393) sglang_model_checksum ........................... None -(yeto-rl-island-0, pid=3393) sglang_model_config_parser ...................... auto -(yeto-rl-island-0, pid=3393) sglang_model_impl ............................... auto -(yeto-rl-island-0, pid=3393) sglang_model_loader_extra_config ................ {} -(yeto-rl-island-0, pid=3393) sglang_model_routers ............................ None -(yeto-rl-island-0, pid=3393) sglang_modelexpress_config ...................... None -(yeto-rl-island-0, pid=3393) sglang_modelopt_checkpoint_restore_path ......... None -(yeto-rl-island-0, pid=3393) sglang_modelopt_checkpoint_save_path ............ None -(yeto-rl-island-0, pid=3393) sglang_modelopt_export_path ..................... None -(yeto-rl-island-0, pid=3393) sglang_modelopt_quant ........................... None -(yeto-rl-island-0, pid=3393) sglang_moe_a2a_backend .......................... none -(yeto-rl-island-0, pid=3393) sglang_moe_dense_tp_size ........................ None -(yeto-rl-island-0, pid=3393) sglang_moe_dp_size .............................. 1 -(yeto-rl-island-0, pid=3393) sglang_moe_runner_backend ....................... triton -(yeto-rl-island-0, pid=3393) sglang_mooncake_ib_device ....................... None -(yeto-rl-island-0, pid=3393) sglang_msprobe_dump_config ...................... None -(yeto-rl-island-0, pid=3393) sglang_num_continuous_decode_steps .............. 1 -(yeto-rl-island-0, pid=3393) sglang_num_reserved_decode_tokens ............... 512 -(yeto-rl-island-0, pid=3393) sglang_numa_node ................................ None -(yeto-rl-island-0, pid=3393) sglang_offload_group_size ....................... -1 -(yeto-rl-island-0, pid=3393) sglang_offload_mode ............................. cpu -(yeto-rl-island-0, pid=3393) sglang_offload_num_in_group ..................... 1 -(yeto-rl-island-0, pid=3393) sglang_offload_prefetch_step .................... 1 -(yeto-rl-island-0, pid=3393) sglang_optimistic_prefill_attempts .............. 0 -(yeto-rl-island-0, pid=3393) sglang_otlp_traces_endpoint ..................... localhost:4317 -(yeto-rl-island-0, pid=3393) sglang_page_size ................................ None -(yeto-rl-island-0, pid=3393) sglang_pdmux_config_path ........................ None -(yeto-rl-island-0, pid=3393) sglang_ple_offload_backend ...................... pinned -(yeto-rl-island-0, pid=3393) sglang_ple_offload_dir .......................... None -(yeto-rl-island-0, pid=3393) sglang_ple_offload_embedding .................... None -(yeto-rl-island-0, pid=3393) sglang_pp_async_batch_depth ..................... 0 -(yeto-rl-island-0, pid=3393) sglang_pp_max_micro_batch_size .................. None -(yeto-rl-island-0, pid=3393) sglang_pp_size .................................. 1 -(yeto-rl-island-0, pid=3393) sglang_pre_warm_nccl ............................ False -(yeto-rl-island-0, pid=3393) sglang_preferred_sampling_params ................ None -(yeto-rl-island-0, pid=3393) sglang_prefill_attention_backend ................ None -(yeto-rl-island-0, pid=3393) sglang_prefill_decode_interval .................. None -(yeto-rl-island-0, pid=3393) sglang_prefill_delayer_forward_passes_buckets ... None -(yeto-rl-island-0, pid=3393) sglang_prefill_delayer_max_delay_ms ............. None -(yeto-rl-island-0, pid=3393) sglang_prefill_delayer_max_delay_passes ......... 30 -(yeto-rl-island-0, pid=3393) sglang_prefill_delayer_queue_min_ratio .......... None -(yeto-rl-island-0, pid=3393) sglang_prefill_delayer_token_usage_low_watermark None -(yeto-rl-island-0, pid=3393) sglang_prefill_delayer_wait_seconds_buckets ..... None -(yeto-rl-island-0, pid=3393) sglang_prefill_max_requests ..................... None -(yeto-rl-island-0, pid=3393) sglang_prefill_only_disable_kv_cache ............ False -(yeto-rl-island-0, pid=3393) sglang_priority_scheduling_preemption_threshold . 10 -(yeto-rl-island-0, pid=3393) sglang_prompt_tokens_buckets .................... None -(yeto-rl-island-0, pid=3393) sglang_quantization ............................. None -(yeto-rl-island-0, pid=3393) sglang_quantization_param_path .................. None -(yeto-rl-island-0, pid=3393) sglang_quantize_and_serve ....................... False -(yeto-rl-island-0, pid=3393) sglang_radix_cache_backend ...................... None -(yeto-rl-island-0, pid=3393) sglang_radix_eviction_policy .................... lru -(yeto-rl-island-0, pid=3393) sglang_radix_eviction_policy_config ............. None -(yeto-rl-island-0, pid=3393) sglang_reasoning_parser ......................... None -(yeto-rl-island-0, pid=3393) sglang_remote_instance_weight_loader_backend .... nccl -(yeto-rl-island-0, pid=3393) sglang_remote_instance_weight_loader_seed_instance_ip None -(yeto-rl-island-0, pid=3393) sglang_remote_instance_weight_loader_seed_instance_service_port None -(yeto-rl-island-0, pid=3393) sglang_remote_instance_weight_loader_send_weights_group_ports None -(yeto-rl-island-0, pid=3393) sglang_remote_instance_weight_loader_start_seed_via_transfer_engine False -(yeto-rl-island-0, pid=3393) sglang_retraction_policy ........................ length -(yeto-rl-island-0, pid=3393) sglang_return_hidden_states_mode ................ None -(yeto-rl-island-0, pid=3393) sglang_return_input_ids ......................... False -(yeto-rl-island-0, pid=3393) sglang_return_output_ids ........................ False -(yeto-rl-island-0, pid=3393) sglang_revision ................................. None -(yeto-rl-island-0, pid=3393) sglang_rl_on_policy_target ...................... None -(yeto-rl-island-0, pid=3393) sglang_rl_quant_profile ......................... None -(yeto-rl-island-0, pid=3393) sglang_router_ip ................................ None -(yeto-rl-island-0, pid=3393) sglang_router_policy ............................ None -(yeto-rl-island-0, pid=3393) sglang_router_port .............................. None -(yeto-rl-island-0, pid=3393) sglang_router_request_timeout_secs .............. 14400 -(yeto-rl-island-0, pid=3393) sglang_sampling_backend ......................... None -(yeto-rl-island-0, pid=3393) sglang_sampling_defaults ........................ model -(yeto-rl-island-0, pid=3393) sglang_sampling_mask_max_tokens ................. 4096 -(yeto-rl-island-0, pid=3393) sglang_schedule_conservativeness ................ 1.0 -(yeto-rl-island-0, pid=3393) sglang_schedule_low_priority_values_first ....... False -(yeto-rl-island-0, pid=3393) sglang_schedule_policy .......................... fcfs -(yeto-rl-island-0, pid=3393) sglang_scheduler_recv_interval .................. 1 -(yeto-rl-island-0, pid=3393) sglang_served_model_name ........................ None -(yeto-rl-island-0, pid=3393) sglang_server_concurrency ....................... 512 -(yeto-rl-island-0, pid=3393) sglang_show_time_cost ........................... False -(yeto-rl-island-0, pid=3393) sglang_sidecar .................................. None -(yeto-rl-island-0, pid=3393) sglang_sidecar_args ............................. None -(yeto-rl-island-0, pid=3393) sglang_skip_tokenizer_init ...................... False -(yeto-rl-island-0, pid=3393) sglang_sleep_on_idle ............................ False -(yeto-rl-island-0, pid=3393) sglang_sm_group_num ............................. 8 -(yeto-rl-island-0, pid=3393) sglang_smg_grpc_mode ............................ False -(yeto-rl-island-0, pid=3393) sglang_smg_http_sidecar_port .................... None -(yeto-rl-island-0, pid=3393) sglang_soft_watchdog_timeout .................... None -(yeto-rl-island-0, pid=3393) sglang_spec_trace_dir ........................... None -(yeto-rl-island-0, pid=3393) sglang_speculative_accept_threshold_acc ......... 1.0 -(yeto-rl-island-0, pid=3393) sglang_speculative_accept_threshold_single ...... 1.0 -(yeto-rl-island-0, pid=3393) sglang_speculative_adaptive ..................... False -(yeto-rl-island-0, pid=3393) sglang_speculative_adaptive_config .............. None -(yeto-rl-island-0, pid=3393) sglang_speculative_algorithm .................... None -(yeto-rl-island-0, pid=3393) sglang_speculative_attention_mode ............... prefill -(yeto-rl-island-0, pid=3393) sglang_speculative_dflash_block_size ............ None -(yeto-rl-island-0, pid=3393) sglang_speculative_domino_candidate_pool_size ... 2048 -(yeto-rl-island-0, pid=3393) sglang_speculative_draft_attention_backend ...... None -(yeto-rl-island-0, pid=3393) sglang_speculative_draft_kv_cache_dtype ......... None -(yeto-rl-island-0, pid=3393) sglang_speculative_draft_load_format ............ None -(yeto-rl-island-0, pid=3393) sglang_speculative_draft_model_path ............. None -(yeto-rl-island-0, pid=3393) sglang_speculative_draft_model_quantization ..... None -(yeto-rl-island-0, pid=3393) sglang_speculative_draft_model_revision ......... None -(yeto-rl-island-0, pid=3393) sglang_speculative_draft_window_size ............ None -(yeto-rl-island-0, pid=3393) sglang_speculative_dsa_topk_backend ............. sgl-kernel -(yeto-rl-island-0, pid=3393) sglang_speculative_dspark_align_verify_tokens_to_graph_tier False -(yeto-rl-island-0, pid=3393) sglang_speculative_dspark_block_size ............ None -(yeto-rl-island-0, pid=3393) sglang_speculative_dspark_confidence_sts_path ... None -(yeto-rl-island-0, pid=3393) sglang_speculative_dspark_sps_table_path ........ None -(yeto-rl-island-0, pid=3393) sglang_speculative_eagle_topk ................... None -(yeto-rl-island-0, pid=3393) sglang_speculative_moe_a2a_backend .............. None -(yeto-rl-island-0, pid=3393) sglang_speculative_moe_runner_backend ........... None -(yeto-rl-island-0, pid=3393) sglang_speculative_ngram_capacity ............... 10000000 -(yeto-rl-island-0, pid=3393) sglang_speculative_ngram_external_corpus_max_tokens 10000000 -(yeto-rl-island-0, pid=3393) sglang_speculative_ngram_external_corpus_path ... None -(yeto-rl-island-0, pid=3393) sglang_speculative_ngram_external_sam_budget .... 0 -(yeto-rl-island-0, pid=3393) sglang_speculative_ngram_match_type ............. BFS -(yeto-rl-island-0, pid=3393) sglang_speculative_ngram_max_bfs_breadth ........ 10 -(yeto-rl-island-0, pid=3393) sglang_speculative_ngram_max_trie_depth ......... 18 -(yeto-rl-island-0, pid=3393) sglang_speculative_ngram_min_bfs_breadth ........ 1 -(yeto-rl-island-0, pid=3393) sglang_speculative_num_draft_tokens ............. None -(yeto-rl-island-0, pid=3393) sglang_speculative_num_steps .................... None -(yeto-rl-island-0, pid=3393) sglang_speculative_skip_dp_mlp_sync ............. False -(yeto-rl-island-0, pid=3393) sglang_speculative_token_map .................... None -(yeto-rl-island-0, pid=3393) sglang_speculative_use_rejection_sampling ....... False -(yeto-rl-island-0, pid=3393) sglang_ssl_ca_certs ............................. None -(yeto-rl-island-0, pid=3393) sglang_ssl_certfile ............................. None -(yeto-rl-island-0, pid=3393) sglang_ssl_keyfile .............................. None -(yeto-rl-island-0, pid=3393) sglang_ssl_keyfile_password ..................... None -(yeto-rl-island-0, pid=3393) sglang_startup_weight_load_mode ................. serial -(yeto-rl-island-0, pid=3393) sglang_stream_interval .......................... 1 -(yeto-rl-island-0, pid=3393) sglang_stream_response_default_include_usage .... False -(yeto-rl-island-0, pid=3393) sglang_strip_thinking_cache ..................... False -(yeto-rl-island-0, pid=3393) sglang_swa_full_tokens_ratio .................... None -(yeto-rl-island-0, pid=3393) sglang_tbo_token_distribution_threshold ......... 0.48 -(yeto-rl-island-0, pid=3393) sglang_tokenizer_backend ........................ huggingface -(yeto-rl-island-0, pid=3393) sglang_tokenizer_metrics_allowed_custom_labels .. None -(yeto-rl-island-0, pid=3393) sglang_tokenizer_metrics_custom_labels_header ... x-custom-labels -(yeto-rl-island-0, pid=3393) sglang_tokenizer_mode ........................... auto -(yeto-rl-island-0, pid=3393) sglang_tokenizer_path ........................... None -(yeto-rl-island-0, pid=3393) sglang_tokenizer_worker_num ..................... 1 -(yeto-rl-island-0, pid=3393) sglang_tool_call_parser ......................... None -(yeto-rl-island-0, pid=3393) sglang_tool_server .............................. None -(yeto-rl-island-0, pid=3393) sglang_torch_compile_max_bs ..................... 32 -(yeto-rl-island-0, pid=3393) sglang_tp_size .................................. 4 -(yeto-rl-island-0, pid=3393) sglang_trace_modules ............................ request -(yeto-rl-island-0, pid=3393) sglang_triton_attention_num_kv_splits ........... 8 -(yeto-rl-island-0, pid=3393) sglang_triton_attention_reduce_in_fp32 .......... False -(yeto-rl-island-0, pid=3393) sglang_triton_attention_split_tile_size ......... None -(yeto-rl-island-0, pid=3393) sglang_true_on_policy_contract .................. None -(yeto-rl-island-0, pid=3393) sglang_trust_mm_content_hashes .................. False -(yeto-rl-island-0, pid=3393) sglang_unified_cache_external_linker_backend .... mooncake -(yeto-rl-island-0, pid=3393) sglang_uno_lora_path ............................ None -(yeto-rl-island-0, pid=3393) sglang_use_ray .................................. False -(yeto-rl-island-0, pid=3393) sglang_uvicorn_access_log_exclude_prefixes ...... [] -(yeto-rl-island-0, pid=3393) sglang_warmups .................................. None -(yeto-rl-island-0, pid=3393) sglang_watchdog_timeout ......................... 300 -(yeto-rl-island-0, pid=3393) sglang_weight_cache_mode ........................ off -(yeto-rl-island-0, pid=3393) sglang_weight_cache_socket ...................... None -(yeto-rl-island-0, pid=3393) sglang_weight_cache_timeout ..................... 1800 -(yeto-rl-island-0, pid=3393) sglang_weight_loader_disable_mmap ............... False -(yeto-rl-island-0, pid=3393) sglang_weight_loader_drop_cache_after_load ...... False -(yeto-rl-island-0, pid=3393) sglang_weight_loader_prefetch_checkpoints ....... False -(yeto-rl-island-0, pid=3393) sglang_weight_loader_prefetch_num_threads ....... 4 -(yeto-rl-island-0, pid=3393) sglang_weight_version ........................... default -(yeto-rl-island-0, pid=3393) sharp_enabled_group ............................. None -(yeto-rl-island-0, pid=3393) short_seq_prob .................................. 0.1 -(yeto-rl-island-0, pid=3393) skip_actor_forward_only ......................... False -(yeto-rl-island-0, pid=3393) skip_eval_before_train .......................... False -(yeto-rl-island-0, pid=3393) skip_train ...................................... False -(yeto-rl-island-0, pid=3393) skipped_train_samples ........................... 0 -(yeto-rl-island-0, pid=3393) softmax_type .................................... vanilla -(yeto-rl-island-0, pid=3393) spec ............................................ ['miles_plugins.models.qwen3_8_next.qwen3_8_next', 'get_qwen3_8_next_spec'] -(yeto-rl-island-0, pid=3393) special_tokens .................................. None -(yeto-rl-island-0, pid=3393) split ........................................... None -(yeto-rl-island-0, pid=3393) squared_relu .................................... False -(yeto-rl-island-0, pid=3393) start_eval_at_iter .............................. None -(yeto-rl-island-0, pid=3393) start_rollout_id ................................ 0 -(yeto-rl-island-0, pid=3393) start_weight_decay .............................. 0.01 -(yeto-rl-island-0, pid=3393) starts_inference_engines ........................ True -(yeto-rl-island-0, pid=3393) step_batch_size_schedule ........................ None -(yeto-rl-island-0, pid=3393) straggler_ctrlr_port ............................ 65535 -(yeto-rl-island-0, pid=3393) straggler_minmax_count .......................... 1 -(yeto-rl-island-0, pid=3393) stream_optimizer_state_moment_dtype ............. fp32 -(yeto-rl-island-0, pid=3393) stream_optimizer_state_to_disk .................. False -(yeto-rl-island-0, pid=3393) strict_fsdp_dtensor_load ........................ True -(yeto-rl-island-0, pid=3393) suggested_communication_unit_size ............... None -(yeto-rl-island-0, pid=3393) swiglu .......................................... True -(yeto-rl-island-0, pid=3393) swin_backbone_type .............................. tiny -(yeto-rl-island-0, pid=3393) symmetric_ar_type ............................... None -(yeto-rl-island-0, pid=3393) target_modules .................................. model.language_model.layers.*.linear_attn.in_proj_qkv,model.language_model.layers.*.linear_attn.in_proj_z,model.language_model.layers.*.linear_attn.in_proj_b,model.language_model.layers.*.linear_attn.in_proj_a,model.language_model.layers.*.linear_attn.out_proj,model.language_model.layers.*.self_attn.q_proj,model.language_model.layers.*.self_attn.k_proj,model.language_model.layers.*.self_attn.v_proj,model.language_model.layers.*.self_attn.o_proj,model.language_model.layers.*.mlp.experts.gate_up_proj,model.language_model.layers.*.mlp.experts.down_proj,model.language_model.layers.*.mlp.shared_expert.gate_proj,model.language_model.layers.*.mlp.shared_expert.up_proj,model.language_model.layers.*.mlp.shared_expert.down_proj -(yeto-rl-island-0, pid=3393) tb_experiment_name .............................. None -(yeto-rl-island-0, pid=3393) tb_project_name ................................. None -(yeto-rl-island-0, pid=3393) te_precision_config_file ........................ None -(yeto-rl-island-0, pid=3393) te_rng_tracker .................................. False -(yeto-rl-island-0, pid=3393) tensor_model_parallel_size ...................... 2 -(yeto-rl-island-0, pid=3393) tensorboard_dir ................................. None -(yeto-rl-island-0, pid=3393) tensorboard_log_interval ........................ 1 -(yeto-rl-island-0, pid=3393) tensorboard_queue_size .......................... 1000 -(yeto-rl-island-0, pid=3393) test_data_path .................................. None -(yeto-rl-island-0, pid=3393) test_mode ....................................... False -(yeto-rl-island-0, pid=3393) thd_max_packed_sequences ........................ None -(yeto-rl-island-0, pid=3393) thd_tail_padding_policy ......................... None -(yeto-rl-island-0, pid=3393) tiktoken_num_special_tokens ..................... 1000 -(yeto-rl-island-0, pid=3393) tiktoken_pattern ................................ None -(yeto-rl-island-0, pid=3393) timing_log_level ................................ 0 -(yeto-rl-island-0, pid=3393) timing_log_option ............................... minmax -(yeto-rl-island-0, pid=3393) tis_clip ........................................ 2.0 -(yeto-rl-island-0, pid=3393) tis_clip_low .................................... 0 -(yeto-rl-island-0, pid=3393) titles_data_path ................................ None -(yeto-rl-island-0, pid=3393) tito_model ...................................... default -(yeto-rl-island-0, pid=3393) tokenizer_hf_no_include_special_tokens .......... False -(yeto-rl-island-0, pid=3393) tokenizer_hf_no_use_fast ........................ False -(yeto-rl-island-0, pid=3393) tokenizer_model ................................. /mnt/yeto-models/hub/models--CharyZeng--Qwen3.8-Flash-Next-4layer/snapshots/d19a6b60c0df8f90faf92c7c592b37df2e15b060 -(yeto-rl-island-0, pid=3393) tokenizer_sentencepiece_ignore_extra_whitespaces True -(yeto-rl-island-0, pid=3393) tokenizer_sentencepiece_legacy .................. False -(yeto-rl-island-0, pid=3393) tokenizer_type .................................. HuggingFaceTokenizer -(yeto-rl-island-0, pid=3393) tool_key ........................................ tools -(yeto-rl-island-0, pid=3393) torch_fsdp2_reshard_after_forward ............... True -(yeto-rl-island-0, pid=3393) tp_comm_bootstrap_backend ....................... nccl -(yeto-rl-island-0, pid=3393) tp_comm_bulk_dgrad .............................. True -(yeto-rl-island-0, pid=3393) tp_comm_bulk_wgrad .............................. True -(yeto-rl-island-0, pid=3393) tp_comm_overlap ................................. False -(yeto-rl-island-0, pid=3393) tp_comm_overlap_ag .............................. True -(yeto-rl-island-0, pid=3393) tp_comm_overlap_cfg ............................. None -(yeto-rl-island-0, pid=3393) tp_comm_overlap_rs .............................. True -(yeto-rl-island-0, pid=3393) tp_comm_overlap_rs_dgrad ........................ False -(yeto-rl-island-0, pid=3393) tp_comm_split_ag ................................ True -(yeto-rl-island-0, pid=3393) tp_comm_split_rs ................................ True -(yeto-rl-island-0, pid=3393) train_backend ................................... megatron -(yeto-rl-island-0, pid=3393) train_data_path ................................. None -(yeto-rl-island-0, pid=3393) train_env_vars .................................. {} -(yeto-rl-island-0, pid=3393) train_iters ..................................... None -(yeto-rl-island-0, pid=3393) train_memory_margin_bytes ....................... 3221225472 -(yeto-rl-island-0, pid=3393) train_samples ................................... None -(yeto-rl-island-0, pid=3393) train_sync_interval ............................. None -(yeto-rl-island-0, pid=3393) trainer_controller_addrs ........................ None -(yeto-rl-island-0, pid=3393) trainer_heartbeat_checker_failure_threshold ..... 3 -(yeto-rl-island-0, pid=3393) trainer_heartbeat_checker_first_wait ............ 300.0 -(yeto-rl-island-0, pid=3393) trainer_heartbeat_checker_interval .............. 10.0 -(yeto-rl-island-0, pid=3393) trainer_heartbeat_checker_timeout ............... 10.0 -(yeto-rl-island-0, pid=3393) trainer_id ...................................... actor -(yeto-rl-island-0, pid=3393) trainer_model_id ................................ None -(yeto-rl-island-0, pid=3393) transformer_impl ................................ transformer_engine -(yeto-rl-island-0, pid=3393) transformer_pipeline_model_parallel_size ........ 2 -(yeto-rl-island-0, pid=3393) true_on_policy_mode ............................. False -(yeto-rl-island-0, pid=3393) trust_remote_code ............................... True -(yeto-rl-island-0, pid=3393) untie_embeddings_and_output_weights ............. True -(yeto-rl-island-0, pid=3393) update_weight_buffer_size ....................... 1073741824 -(yeto-rl-island-0, pid=3393) update_weight_delta_checksum .................... xxh3-128 -(yeto-rl-island-0, pid=3393) update_weight_delta_encoding .................... xor -(yeto-rl-island-0, pid=3393) update_weight_disk_dir .......................... None -(yeto-rl-island-0, pid=3393) update_weight_group_timeout_s ................... None -(yeto-rl-island-0, pid=3393) update_weight_local_checkpoint_dir .............. None -(yeto-rl-island-0, pid=3393) update_weight_transfer_mode ..................... broadcast -(yeto-rl-island-0, pid=3393) update_weights_interval ......................... 1 -(yeto-rl-island-0, pid=3393) use_checkpoint_args ............................. False -(yeto-rl-island-0, pid=3393) use_checkpoint_opt_param_scheduler .............. False -(yeto-rl-island-0, pid=3393) use_cpu_initialization .......................... None -(yeto-rl-island-0, pid=3393) use_critic ...................................... False -(yeto-rl-island-0, pid=3393) use_dist_ckpt ................................... True -(yeto-rl-island-0, pid=3393) use_dist_ckpt_deprecated ........................ False -(yeto-rl-island-0, pid=3393) use_distributed_optimizer ....................... True -(yeto-rl-island-0, pid=3393) use_distributed_post ............................ False -(yeto-rl-island-0, pid=3393) use_dynamic_batch_size .......................... False -(yeto-rl-island-0, pid=3393) use_dynamic_global_batch_size ................... False -(yeto-rl-island-0, pid=3393) use_fault_tolerance ............................. False -(yeto-rl-island-0, pid=3393) use_flash_attn .................................. False -(yeto-rl-island-0, pid=3393) use_fused_mhc ................................... False -(yeto-rl-island-0, pid=3393) use_fused_weighted_squared_relu ................. False -(yeto-rl-island-0, pid=3393) use_gloo_process_groups ......................... True -(yeto-rl-island-0, pid=3393) use_grouped_gemm_for_dense_mlp .................. False -(yeto-rl-island-0, pid=3393) use_grouped_gemm_for_shared_expert .............. False -(yeto-rl-island-0, pid=3393) use_indexer_replay .............................. False -(yeto-rl-island-0, pid=3393) use_kitchen_attention ........................... False -(yeto-rl-island-0, pid=3393) use_kl_loss ..................................... False -(yeto-rl-island-0, pid=3393) use_layer_wise_distributed_optimizer ............ False -(yeto-rl-island-0, pid=3393) use_layer_wise_param_layout ..................... False -(yeto-rl-island-0, pid=3393) use_legacy_static_engine ........................ False -(yeto-rl-island-0, pid=3393) use_mamba_mem_eff_path .......................... True -(yeto-rl-island-0, pid=3393) use_megatron_fsdp ............................... False -(yeto-rl-island-0, pid=3393) use_miles_dashboard ............................. False -(yeto-rl-island-0, pid=3393) use_miles_router ................................ False -(yeto-rl-island-0, pid=3393) use_mlflow ...................................... False -(yeto-rl-island-0, pid=3393) use_mp_args_from_checkpoint_args ................ False -(yeto-rl-island-0, pid=3393) use_mup ......................................... False -(yeto-rl-island-0, pid=3393) use_one_sent_docs ............................... False -(yeto-rl-island-0, pid=3393) use_opd ......................................... False -(yeto-rl-island-0, pid=3393) use_opsm ........................................ False -(yeto-rl-island-0, pid=3393) use_persistent_ckpt_worker ...................... False -(yeto-rl-island-0, pid=3393) use_precision_aware_optimizer ................... False -(yeto-rl-island-0, pid=3393) use_prometheus .................................. False -(yeto-rl-island-0, pid=3393) use_pytorch_profiler ............................ False -(yeto-rl-island-0, pid=3393) use_ring_exchange_p2p ........................... False -(yeto-rl-island-0, pid=3393) use_rollout_entropy ............................. False -(yeto-rl-island-0, pid=3393) use_rollout_indexer_replay ...................... False -(yeto-rl-island-0, pid=3393) use_rollout_logprobs ............................ False -(yeto-rl-island-0, pid=3393) use_rollout_routing_replay ...................... False -(yeto-rl-island-0, pid=3393) use_rope_scaling ................................ False -(yeto-rl-island-0, pid=3393) use_rotary_position_embeddings .................. False -(yeto-rl-island-0, pid=3393) use_routing_replay .............................. False -(yeto-rl-island-0, pid=3393) use_sampling_support_replay ..................... False -(yeto-rl-island-0, pid=3393) use_session_server .............................. False -(yeto-rl-island-0, pid=3393) use_sharp ....................................... False -(yeto-rl-island-0, pid=3393) use_te_activation_func .......................... False -(yeto-rl-island-0, pid=3393) use_tensorboard ................................. False -(yeto-rl-island-0, pid=3393) use_tis ......................................... False -(yeto-rl-island-0, pid=3393) use_tokenizer_model_from_checkpoint_args ........ True -(yeto-rl-island-0, pid=3393) use_torch_fsdp2 ................................. False -(yeto-rl-island-0, pid=3393) use_torch_optimizer_for_cpu_offload ............. False -(yeto-rl-island-0, pid=3393) use_tp_pp_dp_mapping ............................ False -(yeto-rl-island-0, pid=3393) use_transformer_engine_op_fuser ................. False -(yeto-rl-island-0, pid=3393) use_unbiased_kl ................................. False -(yeto-rl-island-0, pid=3393) use_varlen_dataset .............................. False -(yeto-rl-island-0, pid=3393) use_wandb ....................................... False -(yeto-rl-island-0, pid=3393) v_head_dim ...................................... 128 -(yeto-rl-island-0, pid=3393) valid_data_path ................................. None -(yeto-rl-island-0, pid=3393) validation_set_names ............................ None -(yeto-rl-island-0, pid=3393) value_clip ...................................... 0.2 -(yeto-rl-island-0, pid=3393) varlen_mock_dataset_config_json ................. None -(yeto-rl-island-0, pid=3393) varlen_sbhd_validation .......................... False -(yeto-rl-island-0, pid=3393) verify_integrity ................................ False -(yeto-rl-island-0, pid=3393) virtual_pipeline_model_parallel_size ............ None -(yeto-rl-island-0, pid=3393) vision_backbone_type ............................ vit -(yeto-rl-island-0, pid=3393) vision_pretraining .............................. False -(yeto-rl-island-0, pid=3393) vision_pretraining_type ......................... classify -(yeto-rl-island-0, pid=3393) vocab_extra_ids ................................. 0 -(yeto-rl-island-0, pid=3393) vocab_file ...................................... None -(yeto-rl-island-0, pid=3393) vocab_size ...................................... 248320 -(yeto-rl-island-0, pid=3393) wandb_always_use_train_step ..................... False -(yeto-rl-island-0, pid=3393) wandb_dir ....................................... None -(yeto-rl-island-0, pid=3393) wandb_entity .................................... None -(yeto-rl-island-0, pid=3393) wandb_exp_name .................................. None -(yeto-rl-island-0, pid=3393) wandb_group ..................................... None -(yeto-rl-island-0, pid=3393) wandb_host ...................................... None -(yeto-rl-island-0, pid=3393) wandb_key ....................................... None -(yeto-rl-island-0, pid=3393) wandb_mode ...................................... None -(yeto-rl-island-0, pid=3393) wandb_project ................................... None -(yeto-rl-island-0, pid=3393) wandb_random_suffix ............................. True -(yeto-rl-island-0, pid=3393) wandb_run_id .................................... None -(yeto-rl-island-0, pid=3393) wandb_save_dir .................................. None -(yeto-rl-island-0, pid=3393) wandb_team ...................................... None -(yeto-rl-island-0, pid=3393) weight_decay .................................... 0.01 -(yeto-rl-island-0, pid=3393) weight_decay_incr_style ......................... constant -(yeto-rl-island-0, pid=3393) wgrad_deferral_limit ............................ 0 -(yeto-rl-island-0, pid=3393) window_attn_skip_freq ........................... None -(yeto-rl-island-0, pid=3393) window_size ..................................... None -(yeto-rl-island-0, pid=3393) witness_buffer_size ............................. 1048576 -(yeto-rl-island-0, pid=3393) worker_comm_backend ............................. ray -(yeto-rl-island-0, pid=3393) worker_dynamic_port_start ....................... 20000 -(yeto-rl-island-0, pid=3393) worker_urls ..................................... [] -(yeto-rl-island-0, pid=3393) world_size ...................................... 8 -(yeto-rl-island-0, pid=3393) yaml_cfg ........................................ None -(yeto-rl-island-0, pid=3393) yarn_beta_fast .................................. None -(yeto-rl-island-0, pid=3393) yarn_beta_slow .................................. None -(yeto-rl-island-0, pid=3393) yarn_correction_range_round_to_int .............. None -(yeto-rl-island-0, pid=3393) yarn_original_max_position_embeddings ........... None -(yeto-rl-island-0, pid=3393) yeto_placement_map .............................. None -(yeto-rl-island-0, pid=3393) yeto_rl_dynamic_sampling_max_replacements ....... None -(yeto-rl-island-0, pid=3393) -------------------- end of arguments --------------------- -(yeto-rl-island-0, pid=3393) {"event": "rl_boot_only", "learner_id": 0, "marker": "FN_BOOT_ONLY_OK", "miles_argc": 220, "model_recipe": "qwen3_8_next", "ref_load": {"configured": "/mnt/yeto-models/torch_dist/qwen3.8-flash-next-4layer_torch_dist", "present": true, "reason": null, "resolved": "/mnt/yeto-models/torch_dist/qwen3.8-flash-next-4layer_torch_dist"}, "ref_load_present": true, "runtime_fingerprint": "sha256:f70d5baffa871590366c1eb4164ea682b9947a050d79a6c2f9fc306dd61517e3"} -(yeto-rl-island-0, pid=3393) FN_BOOT_ONLY_OK ref_load_present=true recipe=qwen3_8_next - -==> /root/sky_logs/2-yeto-rl-island-0/run.log <== -2026-10-06 11:09:35,908 INFO worker.py:1544 -- Connecting to existing Ray cluster at address: 10.0.0.8:6380... -2026-10-06 11:09:35,915 INFO worker.py:1719 -- Connected to Ray cluster. View the dashboard at 127.0.0.1:8266  -├── Waiting for task resources on 1 node. -└── Job started. Streaming logs... (Ctrl-C to exit log streaming; job will not be killed) -(setup pid=4051) [yeto-setup] image provides miles c35702eefcf2862cee155e46870e6ad30568d2c6 (/opt/yeto/image-manifest.json); editable install kept -(setup pid=4051) [yeto-setup] image provides sglang 4e4148f1b4fe9f05973da0d1e5cfe237512d5155 -(yeto-rl-island-0, pid=4051) 2026-10-06 11:09:58,992 INFO usage_lib.py:448 -- Usage stats collection is disabled. -(yeto-rl-island-0, pid=4051) 2026-10-06 11:09:58,992 INFO scripts.py:1042 -- Local node IP: 10.0.0.8 -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 SUCC scripts.py:1081 -- -------------------- -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 SUCC scripts.py:1082 -- Ray runtime started. -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 SUCC scripts.py:1083 -- -------------------- -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1085 -- Next steps -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO cli_output_helpers.py:16 -- Note: The following commands are intended for use on -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO cli_output_helpers.py:17 -- the head node or within the cluster network. -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1089 -- To add another node to this Ray cluster, run -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1092 --  ray start --address='10.0.0.8:6379' -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1103 -- To connect to this Ray cluster: -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1105 -- import ray -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1106 -- ray.init(_node_ip_address='10.0.0.8') -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1120 -- To submit a Ray job using the Ray Jobs CLI: -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1121 --  RAY_API_SERVER_ADDRESS='http://127.0.0.1:8265' ray job submit --working-dir . -- python my_script.py -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1130 -- See https://docs.ray.io/en/latest/cluster/running-applications/job-submission/index.html -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1134 -- for more information on submitting Ray jobs to the Ray cluster. -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1139 -- To terminate the Ray runtime, run -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1140 --  ray stop -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1143 -- To view the status of the cluster, use -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1144 -- ray status -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,743 INFO scripts.py:1148 -- To monitor and debug Ray, view the dashboard at -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,744 INFO scripts.py:1149 -- 127.0.0.1:8265 -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:06,744 INFO scripts.py:1156 -- If connection to the dashboard fails, check your firewall settings and network configuration. -(yeto-rl-island-0, pid=4051) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (yeto-rl-island-0, pid=4051) Fetching 22 files: 0%| | 0/22 [00:00 failed for pad_packed_seq_alignment: typing.Union[int, typing.Literal['max'], NoneType]. -(yeto-rl-island-0, pid=4051) Deferring to attribute metadata. If the metadata is incomplete, 'parser.add_argument()' may fail. -(yeto-rl-island-0, pid=4051) Original failure: Unions not supported by argparse: typing.Union[int, typing.Literal['max'], NoneType] -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:30.539 main] arguments.py:49 - --tokenizer-model not set, use --hf-checkpoint as tokenizer model. -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:30.552 main] arguments.py:239 - MoE-expert LoRA layout: per-expert (--experts-shared-outer-loras). -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:30.553 main] arguments.py:3688 - Colocate mode: defaulting --sglang-cuda-graph-backend-prefill=disabled to avoid NVLS OOM. Set --sglang-cuda-graph-backend-prefill explicitly to override. -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:30.553 main] arguments.py:3698 - rollout_num_gpus None != actor_num_gpus_per_node 8 * actor_num_nodes 1, overriding rollout_num_gpus to match actor_num_gpus_per_node * actor_num_nodes. -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:30.553 main] arguments.py:3930 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 2048 as default value. -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:30.553 main] arguments.py:3938 - args.rollout_max_prompt_len is not set. Use args.rollout_max_context_len - 1 (2048 - 1) as default value so that there is at least one generated token to compute loss. -(yeto-rl-island-0, pid=4051) using world size: 8, data-parallel size: 2, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 2, pipeline-model-parallel size: 2 -(yeto-rl-island-0, pid=4051) WARNING: Setting args.overlap_p2p_comm and args.align_param_gather to False since non-interleaved schedule does not support overlapping p2p communication and aligned param AG -(yeto-rl-island-0, pid=4051) Number of virtual stages per pipeline stage: None -(yeto-rl-island-0, pid=4051) using torch.bfloat16 for parameters ... -(yeto-rl-island-0, pid=4051) ------------------------ arguments ------------------------ -(yeto-rl-island-0, pid=4051) _is_global_batch_size_explicitly_specified ...... True -(yeto-rl-island-0, pid=4051) account_for_embedding_in_pipeline_split ......... False -(yeto-rl-island-0, pid=4051) account_for_loss_in_pipeline_split .............. False -(yeto-rl-island-0, pid=4051) accumulate_allreduce_grads_in_fp32 .............. True -(yeto-rl-island-0, pid=4051) activation_func_clamp_shared_expert ............. True -(yeto-rl-island-0, pid=4051) activation_func_clamp_value ..................... None -(yeto-rl-island-0, pid=4051) activation_offload_fraction ..................... 1.0 -(yeto-rl-island-0, pid=4051) actor_num_gpus_per_node ......................... 8 -(yeto-rl-island-0, pid=4051) actor_num_nodes ................................. 1 -(yeto-rl-island-0, pid=4051) adam_beta1 ...................................... 0.9 -(yeto-rl-island-0, pid=4051) adam_beta2 ...................................... 0.999 -(yeto-rl-island-0, pid=4051) adam_eps ........................................ 1e-08 -(yeto-rl-island-0, pid=4051) add_bias_linear ................................. False -(yeto-rl-island-0, pid=4051) add_position_embedding .......................... True -(yeto-rl-island-0, pid=4051) add_qkv_bias .................................... False -(yeto-rl-island-0, pid=4051) adlr_autoresume ................................. False -(yeto-rl-island-0, pid=4051) adlr_autoresume_interval ........................ 1000 -(yeto-rl-island-0, pid=4051) advantage_estimator ............................. grpo -(yeto-rl-island-0, pid=4051) align_grad_reduce ............................... True -(yeto-rl-island-0, pid=4051) align_param_gather .............................. False -(yeto-rl-island-0, pid=4051) allgather_cp .................................... False -(yeto-rl-island-0, pid=4051) allow_ambiguous_pad_tokens ...................... False -(yeto-rl-island-0, pid=4051) allow_partial_train_step ........................ False -(yeto-rl-island-0, pid=4051) api_server_host ................................. 127.0.0.1 -(yeto-rl-island-0, pid=4051) api_server_port ................................. 0 -(yeto-rl-island-0, pid=4051) app_tag_run_name ................................ None -(yeto-rl-island-0, pid=4051) app_tag_run_version ............................. 0.0.0 -(yeto-rl-island-0, pid=4051) apply_chat_template ............................. True -(yeto-rl-island-0, pid=4051) apply_chat_template_kwargs ...................... {} -(yeto-rl-island-0, pid=4051) apply_dsa_kernel_fusion ......................... None -(yeto-rl-island-0, pid=4051) apply_query_key_layer_scaling ................... False -(yeto-rl-island-0, pid=4051) apply_residual_connection_post_layernorm ........ False -(yeto-rl-island-0, pid=4051) apply_rope_fusion ............................... True -(yeto-rl-island-0, pid=4051) apply_wd_to_qk_layernorm ........................ False -(yeto-rl-island-0, pid=4051) async_ckpt_cpu_priority ......................... 10 -(yeto-rl-island-0, pid=4051) async_ckpt_io_priority .......................... 3 -(yeto-rl-island-0, pid=4051) async_ckpt_use_cpu_shm .......................... False -(yeto-rl-island-0, pid=4051) async_data_buffer_capacity_factor ............... 2.0 -(yeto-rl-island-0, pid=4051) async_max_concurrent_samples .................... None -(yeto-rl-island-0, pid=4051) async_save ...................................... False -(yeto-rl-island-0, pid=4051) async_strategy .................................. mcore -(yeto-rl-island-0, pid=4051) async_unused_samples_handler .................... drop -(yeto-rl-island-0, pid=4051) attention_backend ............................... AttnBackend.auto -(yeto-rl-island-0, pid=4051) attention_dropout ............................... 0.0 -(yeto-rl-island-0, pid=4051) attention_output_gate ........................... True -(yeto-rl-island-0, pid=4051) attention_softmax_in_fp32 ....................... True -(yeto-rl-island-0, pid=4051) auto_detect_ckpt_format ......................... False -(yeto-rl-island-0, pid=4051) balance_by_flops ................................ False -(yeto-rl-island-0, pid=4051) balance_data .................................... True -(yeto-rl-island-0, pid=4051) barrier_with_L1_time ............................ True -(yeto-rl-island-0, pid=4051) batch_invariant_mode ............................ False -(yeto-rl-island-0, pid=4051) bert_binary_head ................................ True -(yeto-rl-island-0, pid=4051) bert_embedder_type .............................. megatron -(yeto-rl-island-0, pid=4051) bert_load ....................................... None -(yeto-rl-island-0, pid=4051) beta_fast ....................................... 32 -(yeto-rl-island-0, pid=4051) beta_slow ....................................... 1 -(yeto-rl-island-0, pid=4051) bf16 ............................................ True -(yeto-rl-island-0, pid=4051) bias_dropout_fusion ............................. True -(yeto-rl-island-0, pid=4051) bias_gelu_fusion ................................ False -(yeto-rl-island-0, pid=4051) bias_swiglu_fusion .............................. True -(yeto-rl-island-0, pid=4051) biencoder_projection_dim ........................ 0 -(yeto-rl-island-0, pid=4051) biencoder_shared_query_context_model ............ False -(yeto-rl-island-0, pid=4051) block_data_path ................................. None -(yeto-rl-island-0, pid=4051) buffer_filter_path .............................. yeto.rl.engine.miles_adapter.rollout_meta_hook.policy_buffer_filter -(yeto-rl-island-0, pid=4051) cache_mla_latents ............................... False -(yeto-rl-island-0, pid=4051) calc_ft_timeouts ................................ False -(yeto-rl-island-0, pid=4051) calculate_per_token_loss ........................ False -(yeto-rl-island-0, pid=4051) chat_template ................................... None -(yeto-rl-island-0, pid=4051) chat_template_path .............................. None -(yeto-rl-island-0, pid=4051) check_for_large_grads ........................... False -(yeto-rl-island-0, pid=4051) check_for_nan_in_loss_and_grad .................. True -(yeto-rl-island-0, pid=4051) check_for_spiky_loss ............................ False -(yeto-rl-island-0, pid=4051) check_lora_weight_equal ......................... True -(yeto-rl-island-0, pid=4051) check_rematerialize_param_from_master_weight .... False -(yeto-rl-island-0, pid=4051) check_weight_hash_across_dp_replicas_interval ... None -(yeto-rl-island-0, pid=4051) check_weight_update_allow_quant_error ........... False -(yeto-rl-island-0, pid=4051) check_weight_update_equal ....................... False -(yeto-rl-island-0, pid=4051) check_weight_update_selector .................... all -(yeto-rl-island-0, pid=4051) check_weight_update_skip_list ................... None -(yeto-rl-island-0, pid=4051) chunked_optimizer_state_offload ................. False -(yeto-rl-island-0, pid=4051) ci_check_model_hash ............................. False -(yeto-rl-island-0, pid=4051) ci_disable_kl_checker ........................... False -(yeto-rl-island-0, pid=4051) ci_disable_logprobs_checker ..................... False -(yeto-rl-island-0, pid=4051) ci_disable_weight_update_checker ................ False -(yeto-rl-island-0, pid=4051) ci_enable_metrics_capture ....................... False -(yeto-rl-island-0, pid=4051) ci_ft_test_actions .............................. None -(yeto-rl-island-0, pid=4051) ci_ft_test_actions_path ......................... None -(yeto-rl-island-0, pid=4051) ci_inject_rollout_data_min_match_ratio .......... 0.9 -(yeto-rl-island-0, pid=4051) ci_inject_rollout_data_path ..................... None -(yeto-rl-island-0, pid=4051) ci_inject_rollout_data_start_rollout_id ......... None -(yeto-rl-island-0, pid=4051) ci_load_grad_norm ............................... None -(yeto-rl-island-0, pid=4051) ci_metric_checker_expect_num .................... None -(yeto-rl-island-0, pid=4051) ci_metric_checker_key ........................... None -(yeto-rl-island-0, pid=4051) ci_metric_checker_threshold ..................... None -(yeto-rl-island-0, pid=4051) ci_save_grad_norm ............................... None -(yeto-rl-island-0, pid=4051) ci_save_model_hash .............................. False -(yeto-rl-island-0, pid=4051) ci_test ......................................... False -(yeto-rl-island-0, pid=4051) ci_tito_special_token_count_threshold ........... 0.0 -(yeto-rl-island-0, pid=4051) ckpt_assume_constant_structure .................. False -(yeto-rl-island-0, pid=4051) ckpt_convert_format ............................. None -(yeto-rl-island-0, pid=4051) ckpt_convert_save ............................... None -(yeto-rl-island-0, pid=4051) ckpt_convert_update_legacy_dist_opt_format ...... False -(yeto-rl-island-0, pid=4051) ckpt_format ..................................... torch_dist -(yeto-rl-island-0, pid=4051) ckpt_fully_parallel_load ........................ False -(yeto-rl-island-0, pid=4051) ckpt_fully_parallel_load_exchange_algo .......... broadcast -(yeto-rl-island-0, pid=4051) ckpt_fully_parallel_load_process_group .......... dp -(yeto-rl-island-0, pid=4051) ckpt_fully_parallel_save ........................ True -(yeto-rl-island-0, pid=4051) ckpt_fully_parallel_save_deprecated ............. False -(yeto-rl-island-0, pid=4051) ckpt_fully_parallel_save_process_group .......... dp -(yeto-rl-island-0, pid=4051) ckpt_load_validate_sharding_integrity ........... True -(yeto-rl-island-0, pid=4051) ckpt_step ....................................... None -(yeto-rl-island-0, pid=4051) classes_fraction ................................ 1.0 -(yeto-rl-island-0, pid=4051) clear_quantized_weight_workspaces_on_offload .... True -(yeto-rl-island-0, pid=4051) clip_grad ....................................... 1.0 -(yeto-rl-island-0, pid=4051) clone_scatter_output_in_embedding ............... True -(yeto-rl-island-0, pid=4051) cluster_backend ................................. ray -(yeto-rl-island-0, pid=4051) colocate ........................................ True -(yeto-rl-island-0, pid=4051) colocate_memory_peak_device ..................... cpu -(yeto-rl-island-0, pid=4051) compress_ratios ................................. None -(yeto-rl-island-0, pid=4051) compute_advantages_and_returns .................. True -(yeto-rl-island-0, pid=4051) config_logger_dir ............................... -(yeto-rl-island-0, pid=4051) consumed_train_samples .......................... 0 -(yeto-rl-island-0, pid=4051) consumed_valid_samples .......................... 0 -(yeto-rl-island-0, pid=4051) context_parallel_size ........................... 1 -(yeto-rl-island-0, pid=4051) cp_comm_type .................................... ['p2p'] -(yeto-rl-island-0, pid=4051) cp_partition_mode ............................... zigzag -(yeto-rl-island-0, pid=4051) cpu_offloading_num_layers ....................... 0 -(yeto-rl-island-0, pid=4051) cpu_offloading_retain_pinned_cpu_buffers ........ False -(yeto-rl-island-0, pid=4051) create_all_gather_group ......................... False -(yeto-rl-island-0, pid=4051) create_attention_mask_in_dataloader ............. True -(yeto-rl-island-0, pid=4051) critic_load ..................................... /mnt/yeto-models/torch_dist/qwen3.8-flash-next-4layer_torch_dist -(yeto-rl-island-0, pid=4051) critic_lr ....................................... 1e-05 -(yeto-rl-island-0, pid=4051) critic_lr_warmup_iters .......................... 0 -(yeto-rl-island-0, pid=4051) critic_num_gpus_per_node ........................ None -(yeto-rl-island-0, pid=4051) critic_num_nodes ................................ None -(yeto-rl-island-0, pid=4051) critic_save ..................................... None -(yeto-rl-island-0, pid=4051) cross_entropy_fusion_impl ....................... native -(yeto-rl-island-0, pid=4051) cross_entropy_loss_fusion ....................... False -(yeto-rl-island-0, pid=4051) csa_compress_ratios ............................. None -(yeto-rl-island-0, pid=4051) csa_compress_rotary_base ........................ 40000.0 -(yeto-rl-island-0, pid=4051) csa_dense_mode .................................. False -(yeto-rl-island-0, pid=4051) csa_window_size ................................. 128 -(yeto-rl-island-0, pid=4051) cuda_graph_dynamic_microbatches ................. False -(yeto-rl-island-0, pid=4051) cuda_graph_impl ................................. none -(yeto-rl-island-0, pid=4051) cuda_graph_modules .............................. [] -(yeto-rl-island-0, pid=4051) cuda_graph_warmup_steps ......................... 3 -(yeto-rl-island-0, pid=4051) custom_async_data_buffer_path ................... None -(yeto-rl-island-0, pid=4051) custom_config_path .............................. None -(yeto-rl-island-0, pid=4051) custom_convert_samples_to_train_data_path ....... None -(yeto-rl-island-0, pid=4051) custom_eval_rollout_log_function_path ........... None -(yeto-rl-island-0, pid=4051) custom_generate_function_path ................... None -(yeto-rl-island-0, pid=4051) custom_inference_engine_provider_path ........... miles.ray.specs.inference.backend_inference_engine_provider -(yeto-rl-island-0, pid=4051) custom_loss_function_path ....................... None -(yeto-rl-island-0, pid=4051) custom_megatron_before_log_prob_hook_path ....... None -(yeto-rl-island-0, pid=4051) custom_megatron_before_train_step_hook_path ..... None -(yeto-rl-island-0, pid=4051) custom_megatron_init_path ....................... None -(yeto-rl-island-0, pid=4051) custom_megatron_post_save_hook_path ............. None -(yeto-rl-island-0, pid=4051) custom_model_provider_path ...................... miles_plugins.models.qwen3_8_next.model_provider.get_qwen3_8_next_model_provider -(yeto-rl-island-0, pid=4051) custom_pg_loss_reducer_function_path ............ None -(yeto-rl-island-0, pid=4051) custom_reward_post_process_path ................. None -(yeto-rl-island-0, pid=4051) custom_rm_path .................................. gsm8k_reward.score -(yeto-rl-island-0, pid=4051) custom_rollout_log_function_path ................ None -(yeto-rl-island-0, pid=4051) custom_tis_function_path ........................ None -(yeto-rl-island-0, pid=4051) custom_update_weight_post_write_path ............ None -(yeto-rl-island-0, pid=4051) dashboard_flush_interval ........................ 5.0 -(yeto-rl-island-0, pid=4051) dashboard_forward_prometheus .................... False -(yeto-rl-island-0, pid=4051) dashboard_gpu_sample_interval ................... 1.0 -(yeto-rl-island-0, pid=4051) dashboard_sglang_metrics ........................ None -(yeto-rl-island-0, pid=4051) dashboard_sglang_scrape_interval ................ 2.0 -(yeto-rl-island-0, pid=4051) dashboard_sglang_scrape_mode .................... auto -(yeto-rl-island-0, pid=4051) data_args_path .................................. None -(yeto-rl-island-0, pid=4051) data_cache_path ................................. None -(yeto-rl-island-0, pid=4051) data_pad_size_multiplier ........................ 128 -(yeto-rl-island-0, pid=4051) data_parallel_random_init ....................... False -(yeto-rl-island-0, pid=4051) data_parallel_sharding_strategy ................. optim_grads_params -(yeto-rl-island-0, pid=4051) data_parallel_size .............................. 2 -(yeto-rl-island-0, pid=4051) data_path ....................................... None -(yeto-rl-island-0, pid=4051) data_per_class_fraction ......................... 1.0 -(yeto-rl-island-0, pid=4051) data_sharding ................................... True -(yeto-rl-island-0, pid=4051) data_source_path ................................ miles.rollout.data_source.RolloutDataSourceWithBuffer -(yeto-rl-island-0, pid=4051) dataloader_defer_npy_index_mmap ................. False -(yeto-rl-island-0, pid=4051) dataloader_fast_cache_load ...................... False -(yeto-rl-island-0, pid=4051) dataloader_inter_document_masking ............... False -(yeto-rl-island-0, pid=4051) dataloader_type ................................. single -(yeto-rl-island-0, pid=4051) ddp_average_in_collective ....................... False -(yeto-rl-island-0, pid=4051) ddp_bucket_size ................................. None -(yeto-rl-island-0, pid=4051) ddp_num_buckets ................................. None -(yeto-rl-island-0, pid=4051) ddp_pad_buckets_for_high_nccl_busbw ............. False -(yeto-rl-island-0, pid=4051) ddp_param_name_patterns_for_fp32_local_accumulation [] -(yeto-rl-island-0, pid=4051) ddp_reduce_scatter_with_fp32_accumulation ....... False -(yeto-rl-island-0, pid=4051) debug_deterministic_collective .................. False -(yeto-rl-island-0, pid=4051) debug_disable_optimizer ......................... False -(yeto-rl-island-0, pid=4051) debug_exit_after_rollout ........................ None -(yeto-rl-island-0, pid=4051) debug_rollout_only .............................. False -(yeto-rl-island-0, pid=4051) debug_skip_weight_update ........................ False -(yeto-rl-island-0, pid=4051) debug_train_only ................................ False -(yeto-rl-island-0, pid=4051) debug_unified_grad_fused_logprob ................ False -(yeto-rl-island-0, pid=4051) decode_only_cuda_graphs ......................... False -(yeto-rl-island-0, pid=4051) decoder_first_pipeline_num_layers ............... None -(yeto-rl-island-0, pid=4051) decoder_last_pipeline_num_layers ................ None -(yeto-rl-island-0, pid=4051) decoder_num_layers .............................. None -(yeto-rl-island-0, pid=4051) decoder_seq_length .............................. None -(yeto-rl-island-0, pid=4051) decoupled_lr .................................... None -(yeto-rl-island-0, pid=4051) decoupled_min_lr ................................ None -(yeto-rl-island-0, pid=4051) decrease_batch_size_if_needed ................... False -(yeto-rl-island-0, pid=4051) defer_embedding_wgrad_compute ................... False -(yeto-rl-island-0, pid=4051) delay_offload_until_cuda_graph .................. False -(yeto-rl-island-0, pid=4051) delay_split_train_data_by_dp .................... False -(yeto-rl-island-0, pid=4051) delay_wgrad_compute ............................. False -(yeto-rl-island-0, pid=4051) delta_offload_bytes_across_pp_ranks ............. 0 -(yeto-rl-island-0, pid=4051) dense_grouped_gemm .............................. False -(yeto-rl-island-0, pid=4051) deploy_component ................................ all -(yeto-rl-island-0, pid=4051) deploy_instance_id .............................. None -(yeto-rl-island-0, pid=4051) deprecated_use_mcore_models ..................... False -(yeto-rl-island-0, pid=4051) deterministic_mode .............................. False -(yeto-rl-island-0, pid=4051) dino_bottleneck_size ............................ 256 -(yeto-rl-island-0, pid=4051) dino_freeze_last_layer .......................... 1 -(yeto-rl-island-0, pid=4051) dino_head_hidden_size ........................... 2048 -(yeto-rl-island-0, pid=4051) dino_local_crops_number ......................... 10 -(yeto-rl-island-0, pid=4051) dino_local_img_size ............................. 96 -(yeto-rl-island-0, pid=4051) dino_norm_last_layer ............................ False -(yeto-rl-island-0, pid=4051) dino_teacher_temp ............................... 0.07 -(yeto-rl-island-0, pid=4051) dino_warmup_teacher_temp ........................ 0.04 -(yeto-rl-island-0, pid=4051) dino_warmup_teacher_temp_epochs ................. 30 -(yeto-rl-island-0, pid=4051) disable_bf16_reduced_precision_matmul ........... False -(yeto-rl-island-0, pid=4051) disable_grad_buffers_cpu_backup ................. True -(yeto-rl-island-0, pid=4051) disable_jit_fuser ............................... False -(yeto-rl-island-0, pid=4051) disable_msc_deprecated .......................... False -(yeto-rl-island-0, pid=4051) disable_param_buffers_cpu_backup ................ True -(yeto-rl-island-0, pid=4051) disable_rollout_trim_samples .................... False -(yeto-rl-island-0, pid=4051) disable_straggler_on_startup .................... False -(yeto-rl-island-0, pid=4051) disable_symmetric_registration .................. False -(yeto-rl-island-0, pid=4051) dist_ckpt_format_deprecated ..................... None -(yeto-rl-island-0, pid=4051) dist_ckpt_optim_fully_reshardable ............... False -(yeto-rl-island-0, pid=4051) dist_ckpt_save_pre_mcore_014 .................... False -(yeto-rl-island-0, pid=4051) dist_ckpt_strictness ............................ assume_ok_unexpected -(yeto-rl-island-0, pid=4051) dist_ckpt_workers ............................... 1 -(yeto-rl-island-0, pid=4051) distrib_optim_fully_reshardable_mem_efficient ... False -(yeto-rl-island-0, pid=4051) distribute_saved_activations .................... False -(yeto-rl-island-0, pid=4051) distributed_backend ............................. nccl -(yeto-rl-island-0, pid=4051) distributed_timeout_minutes ..................... 60 -(yeto-rl-island-0, pid=4051) distributed_timeout_seconds_after_init .......... None -(yeto-rl-island-0, pid=4051) dsa_attention_backend ........................... tilelang -(yeto-rl-island-0, pid=4051) dsa_cp_balance_indexer .......................... False -(yeto-rl-island-0, pid=4051) dsa_indexer_head_dim ............................ None -(yeto-rl-island-0, pid=4051) dsa_indexer_k_norm_epsilon ...................... None -(yeto-rl-island-0, pid=4051) dsa_indexer_k_norm_fp32 ......................... False -(yeto-rl-island-0, pid=4051) dsa_indexer_loss_coeff .......................... None -(yeto-rl-island-0, pid=4051) dsa_indexer_n_heads ............................. None -(yeto-rl-island-0, pid=4051) dsa_indexer_precision ........................... bf16 -(yeto-rl-island-0, pid=4051) dsa_indexer_rope_interleaved .................... False -(yeto-rl-island-0, pid=4051) dsa_indexer_rotate_activation ................... True -(yeto-rl-island-0, pid=4051) dsa_indexer_scoring_relu ........................ True -(yeto-rl-island-0, pid=4051) dsa_indexer_skip_topk_offset .................... 0 -(yeto-rl-island-0, pid=4051) dsa_indexer_topk ................................ None -(yeto-rl-island-0, pid=4051) dsa_indexer_topk_freq ........................... 1 -(yeto-rl-island-0, pid=4051) dsa_indexer_use_sparse_loss ..................... False -(yeto-rl-island-0, pid=4051) dsa_indexer_weights_proj_output_dtype ........... bf16 -(yeto-rl-island-0, pid=4051) dsa_indexer_weights_proj_use_quantization ....... True -(yeto-rl-island-0, pid=4051) dsa_kernel_backend .............................. None -(yeto-rl-island-0, pid=4051) dsv4_impl ....................................... megatron -(yeto-rl-island-0, pid=4051) dump_details .................................... None -(yeto-rl-island-0, pid=4051) dump_param_to_param_group_map ................... None -(yeto-rl-island-0, pid=4051) dumper_dir ...................................... /tmp/dumper -(yeto-rl-island-0, pid=4051) dumper_enable ................................... False -(yeto-rl-island-0, pid=4051) dumper_fwd_bwd .................................. None -(yeto-rl-island-0, pid=4051) dumper_fwd_only ................................. None -(yeto-rl-island-0, pid=4051) dumper_inference ................................ None -(yeto-rl-island-0, pid=4051) dumper_source_patcher_config_inference .......... None -(yeto-rl-island-0, pid=4051) dumper_source_patcher_config_train .............. None -(yeto-rl-island-0, pid=4051) dynamic_context_parallel ........................ False -(yeto-rl-island-0, pid=4051) dynamic_sampling_filter_path .................... None -(yeto-rl-island-0, pid=4051) embedding_init_method_std ....................... None -(yeto-rl-island-0, pid=4051) embedding_path .................................. None -(yeto-rl-island-0, pid=4051) empty_unused_memory_level ....................... 0 -(yeto-rl-island-0, pid=4051) enable_chunked_prefill .......................... False -(yeto-rl-island-0, pid=4051) enable_cuda_graph ............................... False -(yeto-rl-island-0, pid=4051) enable_event_analyzer ........................... False -(yeto-rl-island-0, pid=4051) enable_experimental ............................. False -(yeto-rl-island-0, pid=4051) enable_ft_package ............................... False -(yeto-rl-island-0, pid=4051) enable_full_sharding_in_hsdp .................... False -(yeto-rl-island-0, pid=4051) enable_hyper_connections ........................ False -(yeto-rl-island-0, pid=4051) enable_msc ...................................... False -(yeto-rl-island-0, pid=4051) enable_mtp_training ............................. False -(yeto-rl-island-0, pid=4051) enable_one_logger ............................... True -(yeto-rl-island-0, pid=4051) enable_witness .................................. False -(yeto-rl-island-0, pid=4051) encoder_num_layers .............................. 4 -(yeto-rl-island-0, pid=4051) encoder_seq_length .............................. 2048 -(yeto-rl-island-0, pid=4051) end_weight_decay ................................ 0.01 -(yeto-rl-island-0, pid=4051) entropy_coef .................................... 0.0 -(yeto-rl-island-0, pid=4051) entry ........................................... train -(yeto-rl-island-0, pid=4051) env_report ...................................... -(yeto-rl-island-0, pid=4051) env_report_interval_seconds ..................... 3600.0 -(yeto-rl-island-0, pid=4051) eod_mask_loss ................................... False -(yeto-rl-island-0, pid=4051) ep_overlap_early_attn_memory_release ............ False -(yeto-rl-island-0, pid=4051) eps_clip ........................................ 0.2 -(yeto-rl-island-0, pid=4051) eps_clip_c ...................................... None -(yeto-rl-island-0, pid=4051) eps_clip_high ................................... 0.2 -(yeto-rl-island-0, pid=4051) error_injection_rate ............................ 0 -(yeto-rl-island-0, pid=4051) error_injection_type ............................ transient_error -(yeto-rl-island-0, pid=4051) eval_config ..................................... None -(yeto-rl-island-0, pid=4051) eval_datasets ................................... [] -(yeto-rl-island-0, pid=4051) eval_function_path .............................. miles.rollout.inference_rollout.inference_rollout_common.InferenceRolloutFn -(yeto-rl-island-0, pid=4051) eval_global_batch_size .......................... 64 -(yeto-rl-island-0, pid=4051) eval_hf_dir ..................................... None -(yeto-rl-island-0, pid=4051) eval_input_key .................................. None -(yeto-rl-island-0, pid=4051) eval_interval ................................... None -(yeto-rl-island-0, pid=4051) eval_iters ...................................... 100 -(yeto-rl-island-0, pid=4051) eval_keep_snapshots ............................. 2 -(yeto-rl-island-0, pid=4051) eval_label_key .................................. None -(yeto-rl-island-0, pid=4051) eval_max_context_len ............................ 2048 -(yeto-rl-island-0, pid=4051) eval_max_in_flight .............................. 2 -(yeto-rl-island-0, pid=4051) eval_max_prompt_len ............................. None -(yeto-rl-island-0, pid=4051) eval_max_response_len ........................... None -(yeto-rl-island-0, pid=4051) eval_micro_batch_size ........................... 1 -(yeto-rl-island-0, pid=4051) eval_min_new_tokens ............................. None -(yeto-rl-island-0, pid=4051) eval_num_gpus ................................... 0 -(yeto-rl-island-0, pid=4051) eval_num_gpus_per_engine ........................ 1 -(yeto-rl-island-0, pid=4051) eval_overflow_policy ............................ backpressure -(yeto-rl-island-0, pid=4051) eval_prompt_data ................................ None -(yeto-rl-island-0, pid=4051) eval_reward_key ................................. None -(yeto-rl-island-0, pid=4051) eval_temperature ................................ None -(yeto-rl-island-0, pid=4051) eval_tool_key ................................... None -(yeto-rl-island-0, pid=4051) eval_top_k ...................................... None -(yeto-rl-island-0, pid=4051) eval_top_p ...................................... None -(yeto-rl-island-0, pid=4051) eval_uses_snapshots ............................. False -(yeto-rl-island-0, pid=4051) evidence_data_path .............................. None -(yeto-rl-island-0, pid=4051) exclude_modules ................................. None -(yeto-rl-island-0, pid=4051) exit_duration_in_mins ........................... None -(yeto-rl-island-0, pid=4051) exit_interval ................................... None -(yeto-rl-island-0, pid=4051) exit_on_missing_checkpoint ...................... False -(yeto-rl-island-0, pid=4051) exit_signal ..................................... 15 -(yeto-rl-island-0, pid=4051) exit_signal_handler ............................. False -(yeto-rl-island-0, pid=4051) exit_signal_handler_for_dataloader .............. False -(yeto-rl-island-0, pid=4051) exit_signal_handler_for_training ................ False -(yeto-rl-island-0, pid=4051) exp_avg_dtype ................................... torch.float32 -(yeto-rl-island-0, pid=4051) exp_avg_sq_dtype ................................ torch.float32 -(yeto-rl-island-0, pid=4051) experimental_attention_variant .................. None -(yeto-rl-island-0, pid=4051) expert_model_parallel_size ...................... 4 -(yeto-rl-island-0, pid=4051) expert_tensor_parallel_size ..................... 1 -(yeto-rl-island-0, pid=4051) experts_shared_outer_loras ...................... False -(yeto-rl-island-0, pid=4051) external_cuda_graph ............................. False -(yeto-rl-island-0, pid=4051) extra_high_precision_layers_hf .................. () -(yeto-rl-island-0, pid=4051) extra_high_precision_layers_megatron ............ () -(yeto-rl-island-0, pid=4051) fake_process_group .............................. False -(yeto-rl-island-0, pid=4051) fallback_to_eager_attn .......................... False -(yeto-rl-island-0, pid=4051) fault_injector_delay_start_iteration ............ None -(yeto-rl-island-0, pid=4051) fault_injector_fault_delay ...................... None -(yeto-rl-island-0, pid=4051) fault_injector_fault_probabilities .............. None -(yeto-rl-island-0, pid=4051) fault_injector_fault_types ...................... None -(yeto-rl-island-0, pid=4051) fault_injector_mtti_seconds ..................... None -(yeto-rl-island-0, pid=4051) fault_injector_num_ranks ........................ None -(yeto-rl-island-0, pid=4051) fault_injector_offset_seconds ................... None -(yeto-rl-island-0, pid=4051) fault_injector_ranks ............................ None -(yeto-rl-island-0, pid=4051) fault_injector_seed ............................. None -(yeto-rl-island-0, pid=4051) fetch_trajectory_retry_times .................... -1 -(yeto-rl-island-0, pid=4051) ffn_hidden_size ................................. 640 -(yeto-rl-island-0, pid=4051) fim_data ........................................ False -(yeto-rl-island-0, pid=4051) fim_eod_token ................................... <|endoftext|> -(yeto-rl-island-0, pid=4051) fim_fragment_rate ............................... None -(yeto-rl-island-0, pid=4051) fim_middle_token ................................ -(yeto-rl-island-0, pid=4051) fim_no_prefix ................................... None -(yeto-rl-island-0, pid=4051) fim_pad_token ................................... -(yeto-rl-island-0, pid=4051) fim_prefix_token ................................ -(yeto-rl-island-0, pid=4051) fim_rate ........................................ 0.5 -(yeto-rl-island-0, pid=4051) fim_split_sample ................................ None -(yeto-rl-island-0, pid=4051) fim_spm_rate .................................... 0.5 -(yeto-rl-island-0, pid=4051) fim_suffix_token ................................ -(yeto-rl-island-0, pid=4051) fine_grained_activation_offloading .............. False -(yeto-rl-island-0, pid=4051) fine_grained_offloading_max_inflight_offloads ... None -(yeto-rl-island-0, pid=4051) finetune ........................................ True -(yeto-rl-island-0, pid=4051) first_last_layers_bf16 .......................... False -(yeto-rl-island-0, pid=4051) flash_decode .................................... False -(yeto-rl-island-0, pid=4051) flight_recorder_dump_on_timeout ................. True -(yeto-rl-island-0, pid=4051) flight_recorder_dump_path ....................... None -(yeto-rl-island-0, pid=4051) flight_recorder_extra_dump_on_exec .............. True -(yeto-rl-island-0, pid=4051) flight_recorder_include_only_active ............. True -(yeto-rl-island-0, pid=4051) flight_recorder_include_stack_trace ............. False -(yeto-rl-island-0, pid=4051) flight_recorder_trace_buffer_size ............... 2000 -(yeto-rl-island-0, pid=4051) fp16 ............................................ False -(yeto-rl-island-0, pid=4051) fp16_lm_cross_entropy ........................... False -(yeto-rl-island-0, pid=4051) fp32_residual_connection ........................ False -(yeto-rl-island-0, pid=4051) fp4 ............................................. None -(yeto-rl-island-0, pid=4051) fp4_param_gather ................................ False -(yeto-rl-island-0, pid=4051) fp4_quantizer_factory ........................... None -(yeto-rl-island-0, pid=4051) fp4_recipe ...................................... nvfp4 -(yeto-rl-island-0, pid=4051) fp8 ............................................. None -(yeto-rl-island-0, pid=4051) fp8_amax_compute_algo ........................... most_recent -(yeto-rl-island-0, pid=4051) fp8_amax_history_len ............................ 1 -(yeto-rl-island-0, pid=4051) fp8_interval .................................... 1 -(yeto-rl-island-0, pid=4051) fp8_margin ...................................... 0 -(yeto-rl-island-0, pid=4051) fp8_output_proj ................................. False -(yeto-rl-island-0, pid=4051) fp8_param_gather ................................ False -(yeto-rl-island-0, pid=4051) fp8_quantizer_factory ........................... None -(yeto-rl-island-0, pid=4051) fp8_recipe ...................................... delayed -(yeto-rl-island-0, pid=4051) fp8_wgrad ....................................... True -(yeto-rl-island-0, pid=4051) freeze_all_layers ............................... False -(yeto-rl-island-0, pid=4051) freeze_e_score_correction_bias .................. False -(yeto-rl-island-0, pid=4051) freeze_indexer .................................. False -(yeto-rl-island-0, pid=4051) fsdp_db_use_persist_buf_on_alloc_fail ........... False -(yeto-rl-island-0, pid=4051) fsdp_double_buffer .............................. False -(yeto-rl-island-0, pid=4051) fsdp_manual_registration ........................ False -(yeto-rl-island-0, pid=4051) ft_components ................................... [] -(yeto-rl-island-0, pid=4051) ft_num_warmup_iters ............................. 5 -(yeto-rl-island-0, pid=4051) full_validation ................................. False -(yeto-rl-island-0, pid=4051) fully_async ..................................... False -(yeto-rl-island-0, pid=4051) fused_residual_rmsnorm .......................... False -(yeto-rl-island-0, pid=4051) gamma ........................................... 1.0 -(yeto-rl-island-0, pid=4051) gdn_conv_pad_alignment .......................... None -(yeto-rl-island-0, pid=4051) gdn_pre_gated_delta_rule_fusion ................. False -(yeto-rl-island-0, pid=4051) get_mismatch_metrics ............................ False -(yeto-rl-island-0, pid=4051) global_batch_size ............................... 64 -(yeto-rl-island-0, pid=4051) glu_linear_offset ............................... 0.0 -(yeto-rl-island-0, pid=4051) gmpo_log_clip_high .............................. 0.4 -(yeto-rl-island-0, pid=4051) gmpo_log_clip_low ............................... 0.4 -(yeto-rl-island-0, pid=4051) gpu_sniff_test_interval ......................... None -(yeto-rl-island-0, pid=4051) grad_reduce_in_bf16 ............................. False -(yeto-rl-island-0, pid=4051) gradient_accumulation_fusion .................... False -(yeto-rl-island-0, pid=4051) gradient_reduce_div_fusion ...................... True -(yeto-rl-island-0, pid=4051) group_query_attention ........................... True -(yeto-rl-island-0, pid=4051) group_rm ........................................ False -(yeto-rl-island-0, pid=4051) grpo_clamp_eps_lower ............................ 0.01 -(yeto-rl-island-0, pid=4051) grpo_clamp_eps_upper ............................ 0.01 -(yeto-rl-island-0, pid=4051) grpo_entropy_term_weight ........................ 0.0 -(yeto-rl-island-0, pid=4051) grpo_filter_groups_with_same_reward ............. False -(yeto-rl-island-0, pid=4051) grpo_group_size ................................. 2 -(yeto-rl-island-0, pid=4051) grpo_iterations ................................. 2 -(yeto-rl-island-0, pid=4051) grpo_kl_beta .................................... 0.001 -(yeto-rl-island-0, pid=4051) grpo_prompts_per_step ........................... 32 -(yeto-rl-island-0, pid=4051) grpo_std_normalization .......................... True -(yeto-rl-island-0, pid=4051) head_lr_mult .................................... 1.0 -(yeto-rl-island-0, pid=4051) head_wise_attn_gate ............................. False -(yeto-rl-island-0, pid=4051) heterogeneous_layers_config_encoded_json ........ None -(yeto-rl-island-0, pid=4051) heterogeneous_layers_config_path ................ None -(yeto-rl-island-0, pid=4051) hf_checkpoint ................................... /mnt/yeto-models/hub/models--CharyZeng--Qwen3.8-Flash-Next-4layer/snapshots/d19a6b60c0df8f90faf92c7c592b37df2e15b060 -(yeto-rl-island-0, pid=4051) hf_lora_targets ................................. ['model.language_model.layers.*.self_attn.q_proj', 'model.language_model.layers.*.self_attn.k_proj', 'model.language_model.layers.*.self_attn.v_proj', 'model.language_model.layers.*.self_attn.o_proj', 'model.language_model.layers.*.linear_attn.in_proj_qkv', 'model.language_model.layers.*.linear_attn.in_proj_z', 'model.language_model.layers.*.linear_attn.in_proj_b', 'model.language_model.layers.*.linear_attn.in_proj_a', 'model.language_model.layers.*.linear_attn.out_proj', 'model.language_model.layers.*.mlp.experts.gate_up_proj', 'model.language_model.layers.*.mlp.experts.down_proj', 'model.language_model.layers.*.mlp.shared_expert.gate_proj', 'model.language_model.layers.*.mlp.shared_expert.up_proj', 'model.language_model.layers.*.mlp.shared_expert.down_proj'] -(yeto-rl-island-0, pid=4051) hidden_dropout .................................. 0.0 -(yeto-rl-island-0, pid=4051) hidden_size ..................................... 2560 -(yeto-rl-island-0, pid=4051) hierarchical_context_parallel_sizes ............. None -(yeto-rl-island-0, pid=4051) high_priority_a2a_comm_stream ................... False -(yeto-rl-island-0, pid=4051) high_priority_stream_groups ..................... [] -(yeto-rl-island-0, pid=4051) http_proxy ...................................... None -(yeto-rl-island-0, pid=4051) hybrid_context_parallel ......................... False -(yeto-rl-island-0, pid=4051) hybrid_layer_pattern ............................ None -(yeto-rl-island-0, pid=4051) hybrid_override_pattern ......................... None -(yeto-rl-island-0, pid=4051) hysteresis ...................................... 2 -(yeto-rl-island-0, pid=4051) ict_head_size ................................... None -(yeto-rl-island-0, pid=4051) ict_load ........................................ None -(yeto-rl-island-0, pid=4051) img_h ........................................... 224 -(yeto-rl-island-0, pid=4051) img_w ........................................... 224 -(yeto-rl-island-0, pid=4051) indep_dp ........................................ False -(yeto-rl-island-0, pid=4051) indexer_batch_size .............................. 128 -(yeto-rl-island-0, pid=4051) indexer_log_interval ............................ 1000 -(yeto-rl-island-0, pid=4051) inference_batch_times_seqlen_threshold .......... -1 -(yeto-rl-island-0, pid=4051) inference_controller_addr ....................... None -(yeto-rl-island-0, pid=4051) inference_coordinator_port ...................... None -(yeto-rl-island-0, pid=4051) inference_cuda_graph_all_prefills ............... False -(yeto-rl-island-0, pid=4051) inference_cuda_graph_scope ...................... InferenceCudaGraphScope.none -(yeto-rl-island-0, pid=4051) inference_disable_ep_consensus .................. False -(yeto-rl-island-0, pid=4051) inference_disable_triton_nvls_kernels ........... False -(yeto-rl-island-0, pid=4051) inference_dynamic_batching ...................... False -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_async_sched_mode ..... legacy -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_block_size ........... 256 -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_buffer_size_gb ....... 40.0 -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_cuda_graph_max_tokens 16384 -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_cuda_graph_mixed_prefill_count 16 -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_cuda_graph_sizing_distribution exponential -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_enable_prefix_caching False -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_logprobs_mode ........ raw_logprobs -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_mamba_memory_ratio ... None -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_max_requests ......... None -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_max_tokens ........... None -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_num_cuda_graphs ...... 16 -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_paused_buffer_size_gb None -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_prefix_caching_coordinator_policy first_prefix_block -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_prefix_caching_eviction_policy ref_zero -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_prefix_caching_mamba_gb None -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_prefix_caching_routing_alpha 0.5 -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_sampling_backend ..... torch -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_track_generated_token_events False -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_track_paused_request_events False -(yeto-rl-island-0, pid=4051) inference_dynamic_batching_unified_memory_level . 0 -(yeto-rl-island-0, pid=4051) inference_fuse_tp_communication ................. False -(yeto-rl-island-0, pid=4051) inference_grouped_gemm_backend .................. vllm -(yeto-rl-island-0, pid=4051) inference_logging_step_interval ................. 0 -(yeto-rl-island-0, pid=4051) inference_max_requests .......................... 8 -(yeto-rl-island-0, pid=4051) inference_max_seq_length ........................ 2560 -(yeto-rl-island-0, pid=4051) inference_moe_disable_fused_quant_kernels ....... False -(yeto-rl-island-0, pid=4051) inference_moe_token_dispatcher_type ............. nvls -(yeto-rl-island-0, pid=4051) inference_rng_tracker ........................... False -(yeto-rl-island-0, pid=4051) inference_shards ................................ None -(yeto-rl-island-0, pid=4051) inference_text_gen_server_logging ............... False -(yeto-rl-island-0, pid=4051) inference_use_synchronous_zmq_collectives ....... False -(yeto-rl-island-0, pid=4051) inference_wandb_logging ......................... False -(yeto-rl-island-0, pid=4051) init_expected_num_cells ......................... None -(yeto-rl-island-0, pid=4051) init_method_std ................................. 0.02 -(yeto-rl-island-0, pid=4051) init_method_xavier_uniform ...................... False -(yeto-rl-island-0, pid=4051) init_model_with_meta_device ..................... False -(yeto-rl-island-0, pid=4051) initial_loss_scale .............................. 4294967296 -(yeto-rl-island-0, pid=4051) inprocess_active_world_size ..................... 1 -(yeto-rl-island-0, pid=4051) inprocess_barrier_timeout ....................... 120 -(yeto-rl-island-0, pid=4051) inprocess_completion_timeout .................... 120 -(yeto-rl-island-0, pid=4051) inprocess_empty_cuda_cache ...................... False -(yeto-rl-island-0, pid=4051) inprocess_granularity ........................... node -(yeto-rl-island-0, pid=4051) inprocess_hard_timeout .......................... 90 -(yeto-rl-island-0, pid=4051) inprocess_heartbeat_interval .................... 30 -(yeto-rl-island-0, pid=4051) inprocess_heartbeat_timeout ..................... 60 -(yeto-rl-island-0, pid=4051) inprocess_last_call_wait ........................ 1 -(yeto-rl-island-0, pid=4051) inprocess_max_iterations ........................ None -(yeto-rl-island-0, pid=4051) inprocess_monitor_process_interval .............. 1.0 -(yeto-rl-island-0, pid=4051) inprocess_monitor_thread_interval ............... 1.0 -(yeto-rl-island-0, pid=4051) inprocess_progress_watchdog_interval ............ 1.0 -(yeto-rl-island-0, pid=4051) inprocess_restart ............................... False -(yeto-rl-island-0, pid=4051) inprocess_soft_timeout .......................... 60 -(yeto-rl-island-0, pid=4051) inprocess_termination_grace_time ................ 1 -(yeto-rl-island-0, pid=4051) input_key ....................................... messages -(yeto-rl-island-0, pid=4051) is_hybrid_model ................................. False -(yeto-rl-island-0, pid=4051) iter_per_epoch .................................. 1250 -(yeto-rl-island-0, pid=4051) iterations_to_skip .............................. [] -(yeto-rl-island-0, pid=4051) keep_fp8_transpose_cache ........................ False -(yeto-rl-island-0, pid=4051) keep_old_actor .................................. False -(yeto-rl-island-0, pid=4051) kitchen_attention_backend ....................... sdpa -(yeto-rl-island-0, pid=4051) kl_coef ......................................... 0.0 -(yeto-rl-island-0, pid=4051) kl_loss_coef .................................... 0.0 -(yeto-rl-island-0, pid=4051) kl_loss_type .................................... k1 -(yeto-rl-island-0, pid=4051) kv_channels ..................................... 256 -(yeto-rl-island-0, pid=4051) kv_lora_rank .................................... 32 -(yeto-rl-island-0, pid=4051) label_key ....................................... label -(yeto-rl-island-0, pid=4051) lambd ........................................... 1.0 -(yeto-rl-island-0, pid=4051) langrl_env_config ............................... None -(yeto-rl-island-0, pid=4051) layernorm_epsilon ............................... 1e-06 -(yeto-rl-island-0, pid=4051) layernorm_zero_centered_gamma ................... True -(yeto-rl-island-0, pid=4051) lazy_mpu_init ................................... False -(yeto-rl-island-0, pid=4051) linear_attention_backend ........................ flashqla -(yeto-rl-island-0, pid=4051) linear_attention_freq ........................... None -(yeto-rl-island-0, pid=4051) linear_attention_type ........................... None -(yeto-rl-island-0, pid=4051) linear_conv_kernel_dim .......................... 4 -(yeto-rl-island-0, pid=4051) linear_cp_mode .................................. chunkwise -(yeto-rl-island-0, pid=4051) linear_key_head_dim ............................. 128 -(yeto-rl-island-0, pid=4051) linear_num_key_heads ............................ 16 -(yeto-rl-island-0, pid=4051) linear_num_value_heads .......................... 32 -(yeto-rl-island-0, pid=4051) linear_value_head_dim ........................... 128 -(yeto-rl-island-0, pid=4051) lion_beta1 ...................................... 0.95 -(yeto-rl-island-0, pid=4051) lion_beta2 ...................................... 0.98 -(yeto-rl-island-0, pid=4051) load ............................................ /mnt/yeto-models/torch_dist/qwen3.8-flash-next-4layer_torch_dist -(yeto-rl-island-0, pid=4051) load_debug_rollout_data ......................... None -(yeto-rl-island-0, pid=4051) load_debug_rollout_data_subsample ............... None -(yeto-rl-island-0, pid=4051) load_main_params_from_ckpt ...................... False -(yeto-rl-island-0, pid=4051) local_rank ...................................... 0 -(yeto-rl-island-0, pid=4051) log_correct_samples ............................. False -(yeto-rl-island-0, pid=4051) log_device_memory_used .......................... False -(yeto-rl-island-0, pid=4051) log_energy ...................................... False -(yeto-rl-island-0, pid=4051) log_interval .................................... 100 -(yeto-rl-island-0, pid=4051) log_loss_scale_to_tensorboard ................... True -(yeto-rl-island-0, pid=4051) log_max_attention_logit ......................... False -(yeto-rl-island-0, pid=4051) log_memory_interval ............................. None -(yeto-rl-island-0, pid=4051) log_memory_to_tensorboard ....................... False -(yeto-rl-island-0, pid=4051) log_moe_overload_factor ......................... False -(yeto-rl-island-0, pid=4051) log_multi_turn .................................. False -(yeto-rl-island-0, pid=4051) log_num_zeros_in_grad ........................... False -(yeto-rl-island-0, pid=4051) log_params_norm ................................. False -(yeto-rl-island-0, pid=4051) log_passrate .................................... False -(yeto-rl-island-0, pid=4051) log_probs_chunk_size ............................ -1 -(yeto-rl-island-0, pid=4051) log_probs_max_tokens_per_gpu .................... None -(yeto-rl-island-0, pid=4051) log_progress .................................... False -(yeto-rl-island-0, pid=4051) log_reward_category ............................. None -(yeto-rl-island-0, pid=4051) log_straggler ................................... False -(yeto-rl-island-0, pid=4051) log_throughput .................................. False -(yeto-rl-island-0, pid=4051) log_timers_to_tensorboard ....................... False -(yeto-rl-island-0, pid=4051) log_validation_ppl_to_tensorboard ............... False -(yeto-rl-island-0, pid=4051) log_world_size_to_tensorboard ................... False -(yeto-rl-island-0, pid=4051) logging_level ................................... None -(yeto-rl-island-0, pid=4051) logits_load_decode_threads ...................... 4 -(yeto-rl-island-0, pid=4051) logits_load_dir ................................. None -(yeto-rl-island-0, pid=4051) logits_load_ignore_errors ....................... False -(yeto-rl-island-0, pid=4051) logits_load_kd_loss_alpha ....................... 1.0 -(yeto-rl-island-0, pid=4051) logits_load_msc_prefetch_depth .................. 2 -(yeto-rl-island-0, pid=4051) logits_load_prefetch_factor ..................... 3 -(yeto-rl-island-0, pid=4051) logits_save_dir ................................. None -(yeto-rl-island-0, pid=4051) logits_save_dtype ............................... fp16 -(yeto-rl-island-0, pid=4051) logits_save_top_k ............................... None -(yeto-rl-island-0, pid=4051) logits_save_top_p ............................... None -(yeto-rl-island-0, pid=4051) logits_save_top_p_min_k ......................... 1 -(yeto-rl-island-0, pid=4051) lora_adapter_path ............................... None -(yeto-rl-island-0, pid=4051) lora_adapter_targets ............................ ['model.language_model.layers.*.self_attn.q_proj', 'model.language_model.layers.*.self_attn.k_proj', 'model.language_model.layers.*.self_attn.v_proj', 'model.language_model.layers.*.self_attn.o_proj', 'model.language_model.layers.*.linear_attn.in_proj_qkv', 'model.language_model.layers.*.linear_attn.in_proj_z', 'model.language_model.layers.*.linear_attn.in_proj_b', 'model.language_model.layers.*.linear_attn.in_proj_a', 'model.language_model.layers.*.linear_attn.out_proj', 'model.language_model.layers.*.mlp.experts.gate_up_proj', 'model.language_model.layers.*.mlp.experts.down_proj', 'model.language_model.layers.*.mlp.shared_expert.gate_proj', 'model.language_model.layers.*.mlp.shared_expert.up_proj', 'model.language_model.layers.*.mlp.shared_expert.down_proj'] -(yeto-rl-island-0, pid=4051) lora_alpha ...................................... 16 -(yeto-rl-island-0, pid=4051) lora_base_cpu_backup ............................ False -(yeto-rl-island-0, pid=4051) lora_dp_invariant_rng ........................... exact -(yeto-rl-island-0, pid=4051) lora_dp_invariant_state ......................... False -(yeto-rl-island-0, pid=4051) lora_dropout .................................... 0.0 -(yeto-rl-island-0, pid=4051) lora_expert_rank ................................ 8 -(yeto-rl-island-0, pid=4051) lora_rank ....................................... 16 -(yeto-rl-island-0, pid=4051) lora_sync_from_tensor ........................... False -(yeto-rl-island-0, pid=4051) lora_train_only ................................. False -(yeto-rl-island-0, pid=4051) lora_type ....................................... lora -(yeto-rl-island-0, pid=4051) loss_mask_type .................................. qwen -(yeto-rl-island-0, pid=4051) loss_scale ...................................... None -(yeto-rl-island-0, pid=4051) loss_scale_window ............................... 1000 -(yeto-rl-island-0, pid=4051) loss_type ....................................... policy_loss -(yeto-rl-island-0, pid=4051) low_memory_resume ............................... False -(yeto-rl-island-0, pid=4051) lr .............................................. 1e-05 -(yeto-rl-island-0, pid=4051) lr_decay_iters .................................. 6 -(yeto-rl-island-0, pid=4051) lr_decay_samples ................................ None -(yeto-rl-island-0, pid=4051) lr_decay_style .................................. linear -(yeto-rl-island-0, pid=4051) lr_warmup_fraction .............................. None -(yeto-rl-island-0, pid=4051) lr_warmup_init .................................. 0.0 -(yeto-rl-island-0, pid=4051) lr_warmup_iters ................................. 0 -(yeto-rl-island-0, pid=4051) lr_warmup_samples ............................... 0 -(yeto-rl-island-0, pid=4051) lr_wsd_decay_iters .............................. None -(yeto-rl-island-0, pid=4051) lr_wsd_decay_samples ............................ None -(yeto-rl-island-0, pid=4051) lr_wsd_decay_style .............................. exponential -(yeto-rl-island-0, pid=4051) main_grads_dtype ................................ torch.float32 -(yeto-rl-island-0, pid=4051) main_params_dtype ............................... torch.float32 -(yeto-rl-island-0, pid=4051) make_vocab_size_divisible_by .................... 128 -(yeto-rl-island-0, pid=4051) mamba_head_dim .................................. 64 -(yeto-rl-island-0, pid=4051) mamba_inference_conv_states_dtype ............... torch.bfloat16 -(yeto-rl-island-0, pid=4051) mamba_inference_ssm_states_dtype ................ torch.bfloat16 -(yeto-rl-island-0, pid=4051) mamba_num_groups ................................ 8 -(yeto-rl-island-0, pid=4051) mamba_num_heads ................................. None -(yeto-rl-island-0, pid=4051) mamba_state_dim ................................. 128 -(yeto-rl-island-0, pid=4051) mamba_training_ssm_states_dtype ................. None -(yeto-rl-island-0, pid=4051) manual_gc ....................................... False -(yeto-rl-island-0, pid=4051) manual_gc_eval .................................. True -(yeto-rl-island-0, pid=4051) manual_gc_interval .............................. 0 -(yeto-rl-island-0, pid=4051) mask_factor ..................................... 1.0 -(yeto-rl-island-0, pid=4051) mask_offpolicy_in_partial_rollout ............... False -(yeto-rl-island-0, pid=4051) mask_prob ....................................... 0.15 -(yeto-rl-island-0, pid=4051) mask_type ....................................... random -(yeto-rl-island-0, pid=4051) masked_softmax_fusion ........................... True -(yeto-rl-island-0, pid=4051) max_position_embeddings ......................... 2048 -(yeto-rl-island-0, pid=4051) max_seqlen_per_dp_cp_rank ....................... None -(yeto-rl-island-0, pid=4051) max_tokens_per_gpu .............................. 8192 -(yeto-rl-island-0, pid=4051) max_tokens_to_oom ............................... 12000 -(yeto-rl-island-0, pid=4051) max_weight_staleness ............................ None -(yeto-rl-island-0, pid=4051) megatron_config ................................. None -(yeto-rl-island-0, pid=4051) megatron_fsdp_cache_param_bucket_views .......... False -(yeto-rl-island-0, pid=4051) megatron_fsdp_enable_fine_grained_param_gather .. False -(yeto-rl-island-0, pid=4051) megatron_fsdp_grad_comm_dtype ................... None -(yeto-rl-island-0, pid=4051) megatron_fsdp_main_grads_dtype .................. None -(yeto-rl-island-0, pid=4051) megatron_fsdp_main_params_dtype ................. torch.float32 -(yeto-rl-island-0, pid=4051) megatron_fsdp_max_pool_double_buffer ............ False -(yeto-rl-island-0, pid=4051) megatron_fsdp_prefetch_recompute_forward_weights False -(yeto-rl-island-0, pid=4051) megatron_to_hf_mode ............................. raw -(yeto-rl-island-0, pid=4051) memory_recorder ................................. torch -(yeto-rl-island-0, pid=4051) memory_snapshot_dir ............................. . -(yeto-rl-island-0, pid=4051) memory_snapshot_num_steps ....................... None -(yeto-rl-island-0, pid=4051) memory_snapshot_path ............................ snapshot.pickle -(yeto-rl-island-0, pid=4051) merge_file ...................................... None -(yeto-rl-island-0, pid=4051) metadata_key .................................... metadata -(yeto-rl-island-0, pid=4051) metadata_path ................................... None -(yeto-rl-island-0, pid=4051) mfu_peak_tflops ................................. None -(yeto-rl-island-0, pid=4051) mhc_init_gating_factor .......................... 0.01 -(yeto-rl-island-0, pid=4051) mhc_recompute_layer_num ......................... None -(yeto-rl-island-0, pid=4051) mhc_sinkhorn_iterations ......................... 20 -(yeto-rl-island-0, pid=4051) micro_batch_size ................................ 1 -(yeto-rl-island-0, pid=4051) microbatch_group_size_per_vp_stage .............. None -(yeto-rl-island-0, pid=4051) mid_level_dataset_surplus ....................... 0.005 -(yeto-rl-island-0, pid=4051) miles_dsa_topk_backend .......................... torch -(yeto-rl-island-0, pid=4051) miles_router_health_check_failure_threshold ..... 3 -(yeto-rl-island-0, pid=4051) miles_router_max_connections .................... None -(yeto-rl-island-0, pid=4051) miles_router_timeout ............................ None -(yeto-rl-island-0, pid=4051) min_batch_collection_ratio ...................... 1 -(yeto-rl-island-0, pid=4051) min_dynamic_context_parallel_size ............... 1 -(yeto-rl-island-0, pid=4051) min_loss_scale .................................. 1.0 -(yeto-rl-island-0, pid=4051) min_lr .......................................... 0.0 -(yeto-rl-island-0, pid=4051) min_offloaded_tensor_size ....................... 1048576 -(yeto-rl-island-0, pid=4051) mini_ft_controller_enable ....................... False -(yeto-rl-island-0, pid=4051) mini_ft_controller_poll_interval ................ 10.0 -(yeto-rl-island-0, pid=4051) mini_ft_controller_resume_delay ................. 10.0 -(yeto-rl-island-0, pid=4051) mla_down_proj_fusion ............................ False -(yeto-rl-island-0, pid=4051) mlflow_experiment_name .......................... miles -(yeto-rl-island-0, pid=4051) mlflow_run_id ................................... None -(yeto-rl-island-0, pid=4051) mlflow_run_name ................................. None -(yeto-rl-island-0, pid=4051) mlflow_tracking_uri ............................. None -(yeto-rl-island-0, pid=4051) mlp_chunks_for_prefill .......................... 1 -(yeto-rl-island-0, pid=4051) mlp_chunks_for_training ......................... 1 -(yeto-rl-island-0, pid=4051) mmap_bin_files .................................. True -(yeto-rl-island-0, pid=4051) mock_data ....................................... False -(yeto-rl-island-0, pid=4051) model_name ...................................... qwen4_exp -(yeto-rl-island-0, pid=4051) moe_activation_in_fp32 .......................... False -(yeto-rl-island-0, pid=4051) moe_apply_probs_on_input ........................ False -(yeto-rl-island-0, pid=4051) moe_aux_loss_coeff .............................. 0.0 -(yeto-rl-island-0, pid=4051) moe_combine_in_fp32 ............................. False -(yeto-rl-island-0, pid=4051) moe_deepep_num_sms .............................. None -(yeto-rl-island-0, pid=4051) moe_enable_deepep ............................... False -(yeto-rl-island-0, pid=4051) moe_enable_routing_replay ....................... False -(yeto-rl-island-0, pid=4051) moe_expert_capacity_factor ...................... None -(yeto-rl-island-0, pid=4051) moe_expert_rank_capacity_factor ................. None -(yeto-rl-island-0, pid=4051) moe_ffn_hidden_size ............................. 640 -(yeto-rl-island-0, pid=4051) moe_flex_dispatcher_backend ..................... deepep -(yeto-rl-island-0, pid=4051) moe_flex_dispatcher_num_sms ..................... None -(yeto-rl-island-0, pid=4051) moe_grouped_gemm ................................ True -(yeto-rl-island-0, pid=4051) moe_hybridep_num_blocks_permute ................. None -(yeto-rl-island-0, pid=4051) moe_hybridep_num_blocks_unpermute ............... None -(yeto-rl-island-0, pid=4051) moe_hybridep_num_sms ............................ None -(yeto-rl-island-0, pid=4051) moe_hybridep_num_sms_preprocessing .............. 108 -(yeto-rl-island-0, pid=4051) moe_hybridep_pad_variable_tokens ................ False -(yeto-rl-island-0, pid=4051) moe_input_jitter_eps ............................ None -(yeto-rl-island-0, pid=4051) moe_latent_size ................................. None -(yeto-rl-island-0, pid=4051) moe_latent_use_norm ............................. False -(yeto-rl-island-0, pid=4051) moe_layer_freq .................................. [1, 1, 1, 1] -(yeto-rl-island-0, pid=4051) moe_layer_recompute ............................. False -(yeto-rl-island-0, pid=4051) moe_mlp_glu_interleave_size ..................... None -(yeto-rl-island-0, pid=4051) moe_n_hash_layers ............................... 0 -(yeto-rl-island-0, pid=4051) moe_ncclep_static_shape ......................... False -(yeto-rl-island-0, pid=4051) moe_ncclep_use_symm_mem ......................... False -(yeto-rl-island-0, pid=4051) moe_pad_expert_input_to_capacity ................ False -(yeto-rl-island-0, pid=4051) moe_pad_experts_for_cuda_graph_inference ........ False -(yeto-rl-island-0, pid=4051) moe_paged_stash ................................. False -(yeto-rl-island-0, pid=4051) moe_paged_stash_buffer_size_factor_cpu .......... 0.0 -(yeto-rl-island-0, pid=4051) moe_paged_stash_buffer_size_factor_cuda ......... 1.1 -(yeto-rl-island-0, pid=4051) moe_paged_stash_page_size ....................... 64 -(yeto-rl-island-0, pid=4051) moe_per_layer_logging ........................... False -(yeto-rl-island-0, pid=4051) moe_permute_fusion .............................. True -(yeto-rl-island-0, pid=4051) moe_permute_fusion_into_hybridep ................ False -(yeto-rl-island-0, pid=4051) moe_router_bias_update_rate ..................... 0.001 -(yeto-rl-island-0, pid=4051) moe_router_dtype ................................ fp32 -(yeto-rl-island-0, pid=4051) moe_router_enable_expert_bias ................... False -(yeto-rl-island-0, pid=4051) moe_router_force_biased ......................... None -(yeto-rl-island-0, pid=4051) moe_router_force_load_balancing ................. False -(yeto-rl-island-0, pid=4051) moe_router_freeze_gate .......................... False -(yeto-rl-island-0, pid=4051) moe_router_fusion ............................... False -(yeto-rl-island-0, pid=4051) moe_router_group_topk ........................... None -(yeto-rl-island-0, pid=4051) moe_router_load_balancing_type .................. aux_loss -(yeto-rl-island-0, pid=4051) moe_router_num_groups ........................... None -(yeto-rl-island-0, pid=4051) moe_router_padding_for_fp8 ...................... False -(yeto-rl-island-0, pid=4051) moe_router_padding_for_quantization ............. False -(yeto-rl-island-0, pid=4051) moe_router_pre_softmax .......................... False -(yeto-rl-island-0, pid=4051) moe_router_score_function ....................... softmax -(yeto-rl-island-0, pid=4051) moe_router_topk ................................. 10 -(yeto-rl-island-0, pid=4051) moe_router_topk_scaling_factor .................. None -(yeto-rl-island-0, pid=4051) moe_router_use_torch_mm ......................... True -(yeto-rl-island-0, pid=4051) moe_routing_trace_capture_hidden_states ......... False -(yeto-rl-island-0, pid=4051) moe_routing_trace_capture_logits ................ False -(yeto-rl-island-0, pid=4051) moe_routing_trace_dump_weights .................. False -(yeto-rl-island-0, pid=4051) moe_routing_trace_max_training_iters ............ None -(yeto-rl-island-0, pid=4051) moe_routing_trace_path .......................... None -(yeto-rl-island-0, pid=4051) moe_shared_expert_gate .......................... True -(yeto-rl-island-0, pid=4051) moe_shared_expert_glu_interleave_size ........... None -(yeto-rl-island-0, pid=4051) moe_shared_expert_intermediate_size ............. 640 -(yeto-rl-island-0, pid=4051) moe_shared_expert_overlap ....................... False -(yeto-rl-island-0, pid=4051) moe_single_grouped_bias ......................... False -(yeto-rl-island-0, pid=4051) moe_single_grouped_weight ....................... False -(yeto-rl-island-0, pid=4051) moe_token_dispatcher_type ....................... alltoall -(yeto-rl-island-0, pid=4051) moe_token_drop_policy ........................... probs -(yeto-rl-island-0, pid=4051) moe_upcycling_granularity ....................... 1 -(yeto-rl-island-0, pid=4051) moe_use_upcycling ............................... False -(yeto-rl-island-0, pid=4051) moe_z_loss_coeff ................................ None -(yeto-rl-island-0, pid=4051) mooncake_replica_num ............................ 1 -(yeto-rl-island-0, pid=4051) mooncake_store_init_kwargs ...................... None -(yeto-rl-island-0, pid=4051) mrope_interleaved ............................... False -(yeto-rl-island-0, pid=4051) mrope_section ................................... None -(yeto-rl-island-0, pid=4051) mscale .......................................... 1.0 -(yeto-rl-island-0, pid=4051) mscale_all_dim .................................. 0.0 -(yeto-rl-island-0, pid=4051) mtp_detach_heads ................................ False -(yeto-rl-island-0, pid=4051) mtp_hybrid_override_pattern ..................... None -(yeto-rl-island-0, pid=4051) mtp_loss_scaling_factor ......................... 0.2 -(yeto-rl-island-0, pid=4051) mtp_num_layers .................................. None -(yeto-rl-island-0, pid=4051) mtp_standalone .................................. False -(yeto-rl-island-0, pid=4051) mtp_use_repeated_layer .......................... False -(yeto-rl-island-0, pid=4051) multi_latent_attention .......................... False -(yeto-rl-island-0, pid=4051) multi_lora ...................................... False -(yeto-rl-island-0, pid=4051) multi_lora_n_adapters ........................... 0 -(yeto-rl-island-0, pid=4051) multimodal_keys ................................. None -(yeto-rl-island-0, pid=4051) multiple_validation_sets ........................ False -(yeto-rl-island-0, pid=4051) muon_coefficient_type ........................... quintic -(yeto-rl-island-0, pid=4051) muon_extra_scale_factor ......................... 1.0 -(yeto-rl-island-0, pid=4051) muon_fp32_matmul_prec ........................... medium -(yeto-rl-island-0, pid=4051) muon_momentum ................................... 0.9 -(yeto-rl-island-0, pid=4051) muon_nesterov ................................... False -(yeto-rl-island-0, pid=4051) muon_num_ns_steps ............................... 5 -(yeto-rl-island-0, pid=4051) muon_scalar_optimizer ........................... adam -(yeto-rl-island-0, pid=4051) muon_scale_mode ................................. spectral -(yeto-rl-island-0, pid=4051) muon_split_qkv .................................. True -(yeto-rl-island-0, pid=4051) muon_tp_mode .................................... blockwise -(yeto-rl-island-0, pid=4051) mup_attn_scale_power ............................ 1.0 -(yeto-rl-island-0, pid=4051) mup_base_head_dim ............................... None -(yeto-rl-island-0, pid=4051) mup_base_hidden_size ............................ None -(yeto-rl-island-0, pid=4051) mup_embedding_mult .............................. 1.0 -(yeto-rl-island-0, pid=4051) mup_output_mult ................................. 1.0 -(yeto-rl-island-0, pid=4051) mup_width_mult .................................. 1.0 -(yeto-rl-island-0, pid=4051) n_samples_per_eval_prompt ....................... 1 -(yeto-rl-island-0, pid=4051) n_samples_per_prompt ............................ 8 -(yeto-rl-island-0, pid=4051) nccl_all_reduce_for_prefill ..................... False -(yeto-rl-island-0, pid=4051) nccl_communicator_config_path ................... None -(yeto-rl-island-0, pid=4051) nccl_ub ......................................... False -(yeto-rl-island-0, pid=4051) no_load_optim ................................... True -(yeto-rl-island-0, pid=4051) no_load_rng ..................................... True -(yeto-rl-island-0, pid=4051) no_persist_layer_norm ........................... False -(yeto-rl-island-0, pid=4051) no_rope_freq .................................... None -(yeto-rl-island-0, pid=4051) no_save_optim ................................... True -(yeto-rl-island-0, pid=4051) no_save_rng ..................................... True -(yeto-rl-island-0, pid=4051) no_weight_decay_cond_type ....................... None -(yeto-rl-island-0, pid=4051) non_persistent_ckpt_type ........................ None -(yeto-rl-island-0, pid=4051) non_persistent_global_ckpt_dir .................. None -(yeto-rl-island-0, pid=4051) non_persistent_local_ckpt_algo .................. fully_parallel -(yeto-rl-island-0, pid=4051) non_persistent_local_ckpt_dir ................... None -(yeto-rl-island-0, pid=4051) non_persistent_save_interval .................... None -(yeto-rl-island-0, pid=4051) normalization ................................... RMSNorm -(yeto-rl-island-0, pid=4051) normalize_advantages ............................ False -(yeto-rl-island-0, pid=4051) null_tokenizer_eod_id ........................... None -(yeto-rl-island-0, pid=4051) null_tokenizer_pad_id ........................... -1 -(yeto-rl-island-0, pid=4051) num_attention_heads ............................. 24 -(yeto-rl-island-0, pid=4051) num_channels .................................... 3 -(yeto-rl-island-0, pid=4051) num_classes ..................................... 1000 -(yeto-rl-island-0, pid=4051) num_critic_only_steps ........................... 0 -(yeto-rl-island-0, pid=4051) num_dataset_builder_threads ..................... 1 -(yeto-rl-island-0, pid=4051) num_distributed_optimizer_instances ............. 1 -(yeto-rl-island-0, pid=4051) num_epoch ....................................... None -(yeto-rl-island-0, pid=4051) num_experts ..................................... 512 -(yeto-rl-island-0, pid=4051) num_gpus_per_node ............................... 8 -(yeto-rl-island-0, pid=4051) num_layers ...................................... 4 -(yeto-rl-island-0, pid=4051) num_layers_at_end_in_bf16 ....................... 1 -(yeto-rl-island-0, pid=4051) num_layers_at_start_in_bf16 ..................... 1 -(yeto-rl-island-0, pid=4051) num_layers_per_virtual_pipeline_stage ........... None -(yeto-rl-island-0, pid=4051) num_query_groups ................................ 2 -(yeto-rl-island-0, pid=4051) num_residual_streams ............................ 4 -(yeto-rl-island-0, pid=4051) num_rollout ..................................... 6 -(yeto-rl-island-0, pid=4051) num_speculative_tokens .......................... 0 -(yeto-rl-island-0, pid=4051) num_steps_per_rollout ........................... 1 -(yeto-rl-island-0, pid=4051) num_virtual_stages_per_pipeline_rank ............ None -(yeto-rl-island-0, pid=4051) num_workers ..................................... 2 -(yeto-rl-island-0, pid=4051) nvtx_ranges ..................................... False -(yeto-rl-island-0, pid=4051) o_groups ........................................ 8 -(yeto-rl-island-0, pid=4051) o_lora_rank ..................................... 1024 -(yeto-rl-island-0, pid=4051) object_storage_cache_path ....................... None -(yeto-rl-island-0, pid=4051) object_store_backend ............................ ray -(yeto-rl-island-0, pid=4051) observe_training_entropy ........................ False -(yeto-rl-island-0, pid=4051) offload_modules ................................. [] -(yeto-rl-island-0, pid=4051) offload_optimizer_states ........................ False -(yeto-rl-island-0, pid=4051) offload_rollout ................................. True -(yeto-rl-island-0, pid=4051) offload_rollout_level ........................... ['kv_cache', 'weight'] -(yeto-rl-island-0, pid=4051) offload_train ................................... True -(yeto-rl-island-0, pid=4051) offload_train_disk_chunk_mb ..................... 256 -(yeto-rl-island-0, pid=4051) offload_train_disk_dir .......................... None -(yeto-rl-island-0, pid=4051) offload_train_target ............................ cpu -(yeto-rl-island-0, pid=4051) one_logger_async ................................ False -(yeto-rl-island-0, pid=4051) one_logger_project .............................. megatron-lm -(yeto-rl-island-0, pid=4051) one_logger_run_name ............................. None -(yeto-rl-island-0, pid=4051) onnx_safe ....................................... None -(yeto-rl-island-0, pid=4051) opd_kl_coef ..................................... 1.0 -(yeto-rl-island-0, pid=4051) opd_log_prob_top_k .............................. 0 -(yeto-rl-island-0, pid=4051) opd_reward_weight_mode .......................... student_p -(yeto-rl-island-0, pid=4051) opd_teacher_ckpt_step ........................... None -(yeto-rl-island-0, pid=4051) opd_teacher_key ................................. opd_teacher -(yeto-rl-island-0, pid=4051) opd_teacher_load ................................ None -(yeto-rl-island-0, pid=4051) opd_teacher_urls ................................ None -(yeto-rl-island-0, pid=4051) opd_top_k_strategy .............................. only-student -(yeto-rl-island-0, pid=4051) opd_topk_per_position ........................... False -(yeto-rl-island-0, pid=4051) opd_type ........................................ None -(yeto-rl-island-0, pid=4051) openai_gelu ..................................... False -(yeto-rl-island-0, pid=4051) opsm_delta ...................................... 0.0001 -(yeto-rl-island-0, pid=4051) optimizer ....................................... adam -(yeto-rl-island-0, pid=4051) optimizer_cpu_offload ........................... False -(yeto-rl-island-0, pid=4051) optimizer_cuda_graph ............................ False -(yeto-rl-island-0, pid=4051) optimizer_offload_fraction ...................... 1.0 -(yeto-rl-island-0, pid=4051) optimizer_state_offload_chunk_size_mb ........... 0 -(yeto-rl-island-0, pid=4051) optimizer_state_offload_fraction ................ 1.0 -(yeto-rl-island-0, pid=4051) original_max_position_embeddings ................ 4096 -(yeto-rl-island-0, pid=4051) outer_dp_sharding_strategy ...................... no_shard -(yeto-rl-island-0, pid=4051) output_bert_embeddings .......................... False -(yeto-rl-island-0, pid=4051) over_sampling_batch_size ........................ 8 -(yeto-rl-island-0, pid=4051) overlap_cpu_optimizer_d2h_h2d ................... False -(yeto-rl-island-0, pid=4051) overlap_dispatch_backward_with_experts_wgrad .... False -(yeto-rl-island-0, pid=4051) overlap_grad_reduce ............................. False -(yeto-rl-island-0, pid=4051) overlap_moe_expert_parallel_comm ................ False -(yeto-rl-island-0, pid=4051) overlap_p2p_comm ................................ False -(yeto-rl-island-0, pid=4051) overlap_p2p_comm_warmup_flush ................... False -(yeto-rl-island-0, pid=4051) overlap_param_gather ............................ False -(yeto-rl-island-0, pid=4051) overlap_param_gather_with_optimizer_step ........ False -(yeto-rl-island-0, pid=4051) override_opt_param_scheduler .................... False -(yeto-rl-island-0, pid=4051) p2p_transfer_num_workers ........................ 4 -(yeto-rl-island-0, pid=4051) p2p_transfer_timeout ............................ 30.0 -(yeto-rl-island-0, pid=4051) pad_packed_seq_alignment ........................ None -(yeto-rl-island-0, pid=4051) pad_vocab_size .................................. True -(yeto-rl-island-0, pid=4051) padded_vocab_size ............................... 248320 -(yeto-rl-island-0, pid=4051) params_dtype .................................... torch.bfloat16 -(yeto-rl-island-0, pid=4051) partial_rollout ................................. False -(yeto-rl-island-0, pid=4051) patch_dim ....................................... 16 -(yeto-rl-island-0, pid=4051) pause_generation_mode ........................... retract -(yeto-rl-island-0, pid=4051) per_dataset_sequences_path ...................... None -(yeto-rl-island-0, pid=4051) per_split_data_args_path ........................ None -(yeto-rl-island-0, pid=4051) perform_initialization .......................... True -(yeto-rl-island-0, pid=4051) perform_rl_step ................................. False -(yeto-rl-island-0, pid=4051) phase_transition_iterations ..................... None -(yeto-rl-island-0, pid=4051) pin_cpu_grads ................................... True -(yeto-rl-island-0, pid=4051) pin_cpu_params .................................. True -(yeto-rl-island-0, pid=4051) pin_rollout_manager_to_head ..................... True -(yeto-rl-island-0, pid=4051) pipeline_model_parallel_comm_backend ............ None -(yeto-rl-island-0, pid=4051) pipeline_model_parallel_layout .................. None -(yeto-rl-island-0, pid=4051) pipeline_model_parallel_size .................... 2 -(yeto-rl-island-0, pid=4051) policy_loss_variant ............................. policy_loss -(yeto-rl-island-0, pid=4051) position_embedding_type ......................... rope -(yeto-rl-island-0, pid=4051) post_mlp_layernorm .............................. False -(yeto-rl-island-0, pid=4051) post_self_attn_layernorm ........................ False -(yeto-rl-island-0, pid=4051) prefill_num_servers ............................. None -(yeto-rl-island-0, pid=4051) pretrained_checkpoint ........................... None -(yeto-rl-island-0, pid=4051) profile ......................................... False -(yeto-rl-island-0, pid=4051) profile_ranks ................................... [] -(yeto-rl-island-0, pid=4051) profile_step_end ................................ 12 -(yeto-rl-island-0, pid=4051) profile_step_start .............................. 10 -(yeto-rl-island-0, pid=4051) profile_target .................................. ['train_overall'] -(yeto-rl-island-0, pid=4051) prometheus_port ................................. 9090 -(yeto-rl-island-0, pid=4051) prometheus_run_name ............................. None -(yeto-rl-island-0, pid=4051) prompt_data ..................................... /root/yeto-rl/prompts.jsonl -(yeto-rl-island-0, pid=4051) pytorch_profiler_collect_callstack .............. False -(yeto-rl-island-0, pid=4051) pytorch_profiler_collect_chakra ................. False -(yeto-rl-island-0, pid=4051) pytorch_profiler_collect_shapes ................. False -(yeto-rl-island-0, pid=4051) q_lora_rank ..................................... None -(yeto-rl-island-0, pid=4051) qk_clip ......................................... False -(yeto-rl-island-0, pid=4051) qk_clip_alpha ................................... 0.5 -(yeto-rl-island-0, pid=4051) qk_clip_threshold ............................... 100 -(yeto-rl-island-0, pid=4051) qk_head_dim ..................................... 128 -(yeto-rl-island-0, pid=4051) qk_l2_norm ...................................... False -(yeto-rl-island-0, pid=4051) qk_layernorm .................................... True -(yeto-rl-island-0, pid=4051) qk_pos_emb_head_dim ............................. 64 -(yeto-rl-island-0, pid=4051) qkv_format ...................................... thd -(yeto-rl-island-0, pid=4051) query_in_block_prob ............................. 0.1 -(yeto-rl-island-0, pid=4051) quick_geglu ..................................... False -(yeto-rl-island-0, pid=4051) rampup_batch_size ............................... None -(yeto-rl-island-0, pid=4051) rank ............................................ 0 -(yeto-rl-island-0, pid=4051) recompute_granularity ........................... full -(yeto-rl-island-0, pid=4051) recompute_logprobs_via_prefill .................. False -(yeto-rl-island-0, pid=4051) recompute_loss_function ......................... False -(yeto-rl-island-0, pid=4051) recompute_method ................................ uniform -(yeto-rl-island-0, pid=4051) recompute_modules ............................... None -(yeto-rl-island-0, pid=4051) recompute_num_layers ............................ 1 -(yeto-rl-island-0, pid=4051) record_memory_history ........................... False -(yeto-rl-island-0, pid=4051) record_shapes ................................... False -(yeto-rl-island-0, pid=4051) ref_ckpt_step ................................... None -(yeto-rl-island-0, pid=4051) ref_load ........................................ /mnt/yeto-models/torch_dist/qwen3.8-flash-next-4layer_torch_dist -(yeto-rl-island-0, pid=4051) ref_update_interval ............................. None -(yeto-rl-island-0, pid=4051) refit_method .................................... gloo -(yeto-rl-island-0, pid=4051) relative_attention_max_distance ................. 128 -(yeto-rl-island-0, pid=4051) relative_attention_num_buckets .................. 32 -(yeto-rl-island-0, pid=4051) rematerialize_param_from_master_weight .......... False -(yeto-rl-island-0, pid=4051) replication ..................................... False -(yeto-rl-island-0, pid=4051) replication_factor .............................. 2 -(yeto-rl-island-0, pid=4051) replication_jump ................................ None -(yeto-rl-island-0, pid=4051) requested_load .................................. None -(yeto-rl-island-0, pid=4051) rerun_mode ...................................... validate_results -(yeto-rl-island-0, pid=4051) reset_attention_mask ............................ False -(yeto-rl-island-0, pid=4051) reset_optimizer_states .......................... False -(yeto-rl-island-0, pid=4051) reset_position_ids .............................. False -(yeto-rl-island-0, pid=4051) result_rejected_tracker_filename ................ None -(yeto-rl-island-0, pid=4051) retriever_report_topk_accuracies ................ [] -(yeto-rl-island-0, pid=4051) retriever_score_scaling ......................... False -(yeto-rl-island-0, pid=4051) reuse_grad_buf_for_mxfp8_param_ag ............... False -(yeto-rl-island-0, pid=4051) reward_key ...................................... None -(yeto-rl-island-0, pid=4051) rewards_normalization ........................... True -(yeto-rl-island-0, pid=4051) rl_consumption_granularity ...................... B -(yeto-rl-island-0, pid=4051) rl_default_temperature .......................... 1.0 -(yeto-rl-island-0, pid=4051) rl_default_top_k ................................ -1 -(yeto-rl-island-0, pid=4051) rl_default_top_p ................................ 0 -(yeto-rl-island-0, pid=4051) rl_generation_lag ............................... 0 -(yeto-rl-island-0, pid=4051) rl_importance_sampling_truncation_coef .......... None -(yeto-rl-island-0, pid=4051) rl_inference_expert_model_parallel_size ......... None -(yeto-rl-island-0, pid=4051) rl_inference_expert_tensor_model_parallel_size .. None -(yeto-rl-island-0, pid=4051) rl_inference_logprobs_is_correction ............. False -(yeto-rl-island-0, pid=4051) rl_inference_model_unified_memory_level ......... 0 -(yeto-rl-island-0, pid=4051) rl_inference_parsers ............................ [] -(yeto-rl-island-0, pid=4051) rl_inference_pipeline_model_parallel_size ....... None -(yeto-rl-island-0, pid=4051) rl_inference_tensor_model_parallel_size ......... None -(yeto-rl-island-0, pid=4051) rl_kv_cache_management_mode ..................... persist -(yeto-rl-island-0, pid=4051) rl_offload_inference_model_weights_when_idle .... False -(yeto-rl-island-0, pid=4051) rl_offload_optimizer_during_inference ........... False -(yeto-rl-island-0, pid=4051) rl_partial_rollouts ............................. False -(yeto-rl-island-0, pid=4051) rl_persist_cuda_graphs .......................... False -(yeto-rl-island-0, pid=4051) rl_profile ...................................... False -(yeto-rl-island-0, pid=4051) rl_profile_dir .................................. None -(yeto-rl-island-0, pid=4051) rl_prompts_per_eval ............................. 32 -(yeto-rl-island-0, pid=4051) rl_sequence_packing_algo ........................ fifo -(yeto-rl-island-0, pid=4051) rl_sequence_packing_max_sequences_per_bin ....... 50 -(yeto-rl-island-0, pid=4051) rl_skip_bos_token ............................... False -(yeto-rl-island-0, pid=4051) rl_submission_granularity ....................... B -(yeto-rl-island-0, pid=4051) rl_training_cuda_graphs ......................... False -(yeto-rl-island-0, pid=4051) rl_use_sequence_packing ......................... False -(yeto-rl-island-0, pid=4051) rl_verify_model_weights_swap .................... False -(yeto-rl-island-0, pid=4051) rm_type ......................................... None -(yeto-rl-island-0, pid=4051) rm_url .......................................... None -(yeto-rl-island-0, pid=4051) rollout_all_samples_process_path ................ yeto.rl.engine.miles_adapter.rollout_meta_hook.extract_rollout_metadata -(yeto-rl-island-0, pid=4051) rollout_batch_size .............................. 8 -(yeto-rl-island-0, pid=4051) rollout_buffer_url .............................. None -(yeto-rl-island-0, pid=4051) rollout_data_postprocess_path ................... None -(yeto-rl-island-0, pid=4051) rollout_external ................................ False -(yeto-rl-island-0, pid=4051) rollout_external_engine_addrs ................... None -(yeto-rl-island-0, pid=4051) rollout_external_router_pd ...................... False -(yeto-rl-island-0, pid=4051) rollout_function_path ........................... miles.rollout.inference_rollout.inference_rollout_common.InferenceRolloutFn -(yeto-rl-island-0, pid=4051) rollout_global_dataset .......................... True -(yeto-rl-island-0, pid=4051) rollout_health_check_failure_threshold .......... 1 -(yeto-rl-island-0, pid=4051) rollout_health_check_first_wait ................. 0.0 -(yeto-rl-island-0, pid=4051) rollout_health_check_interval ................... 300.0 -(yeto-rl-island-0, pid=4051) rollout_health_check_timeout .................... 300.0 -(yeto-rl-island-0, pid=4051) rollout_indexer_topk_num_streams ................ None -(yeto-rl-island-0, pid=4051) rollout_max_context_len ......................... 2048 -(yeto-rl-island-0, pid=4051) rollout_max_prompt_len .......................... 2047 -(yeto-rl-island-0, pid=4051) rollout_max_response_len ........................ 1024 -(yeto-rl-island-0, pid=4051) rollout_num_gpus ................................ 8 -(yeto-rl-island-0, pid=4051) rollout_num_gpus_per_engine ..................... 4 -(yeto-rl-island-0, pid=4051) rollout_sample_filter_path ...................... yeto.rl.engine.miles_adapter.rollout_meta_hook.record_trained_groups -(yeto-rl-island-0, pid=4051) rollout_seed .................................... 17 -(yeto-rl-island-0, pid=4051) rollout_shuffle ................................. False -(yeto-rl-island-0, pid=4051) rollout_skip_special_tokens ..................... False -(yeto-rl-island-0, pid=4051) rollout_stop .................................... None -(yeto-rl-island-0, pid=4051) rollout_stop_token_ids .......................... None -(yeto-rl-island-0, pid=4051) rollout_submission_granularity .................. None -(yeto-rl-island-0, pid=4051) rollout_task_type ............................... math -(yeto-rl-island-0, pid=4051) rollout_temperature ............................. 1.0 -(yeto-rl-island-0, pid=4051) rollout_top_k ................................... -1 -(yeto-rl-island-0, pid=4051) rollout_top_p ................................... 1.0 -(yeto-rl-island-0, pid=4051) rope_scaling_factor ............................. 8.0 -(yeto-rl-island-0, pid=4051) rope_type ....................................... rope -(yeto-rl-island-0, pid=4051) rotary_base ..................................... 10000000 -(yeto-rl-island-0, pid=4051) rotary_base_per_layer ........................... None -(yeto-rl-island-0, pid=4051) rotary_interleaved .............................. False -(yeto-rl-island-0, pid=4051) rotary_percent .................................. 0.25 -(yeto-rl-island-0, pid=4051) rotary_scaling_factor ........................... 1.0 -(yeto-rl-island-0, pid=4051) rotary_seq_len_interpolation_factor ............. None -(yeto-rl-island-0, pid=4051) router_allow_requests_without_routing_key ....... False -(yeto-rl-island-0, pid=4051) router_api_key .................................. None -(yeto-rl-island-0, pid=4051) router_assignment_mode .......................... random -(yeto-rl-island-0, pid=4051) router_backend .................................. sglang -(yeto-rl-island-0, pid=4051) router_balance_abs_threshold .................... 64 -(yeto-rl-island-0, pid=4051) router_balance_rel_threshold .................... 1.5 -(yeto-rl-island-0, pid=4051) router_bucket_adjust_interval_secs .............. 5 -(yeto-rl-island-0, pid=4051) router_ca_cert_paths ............................ [] -(yeto-rl-island-0, pid=4051) router_cache_threshold .......................... 0.3 -(yeto-rl-island-0, pid=4051) router_cb_failure_threshold ..................... 10 -(yeto-rl-island-0, pid=4051) router_cb_success_threshold ..................... 3 -(yeto-rl-island-0, pid=4051) router_cb_timeout_duration_secs ................. 60 -(yeto-rl-island-0, pid=4051) router_cb_window_duration_secs .................. 120 -(yeto-rl-island-0, pid=4051) router_chat_template ............................ None -(yeto-rl-island-0, pid=4051) router_client_cert_path ......................... None -(yeto-rl-island-0, pid=4051) router_client_key_path .......................... None -(yeto-rl-island-0, pid=4051) router_control_plane_api_keys ................... [] -(yeto-rl-island-0, pid=4051) router_control_plane_audit_enabled .............. False -(yeto-rl-island-0, pid=4051) router_cors_allowed_origins ..................... [] -(yeto-rl-island-0, pid=4051) router_decode ................................... None -(yeto-rl-island-0, pid=4051) router_decode_policy ............................ None -(yeto-rl-island-0, pid=4051) router_decode_selector .......................... {} -(yeto-rl-island-0, pid=4051) router_disable_circuit_breaker .................. False -(yeto-rl-island-0, pid=4051) router_disable_health_check ..................... False -(yeto-rl-island-0, pid=4051) router_disable_retries .......................... False -(yeto-rl-island-0, pid=4051) router_dp_aware ................................. False -(yeto-rl-island-0, pid=4051) router_enable_igw ............................... False -(yeto-rl-island-0, pid=4051) router_enable_trace ............................. False -(yeto-rl-island-0, pid=4051) router_eviction_interval_secs ................... 60 -(yeto-rl-island-0, pid=4051) router_health_check_endpoint .................... /health -(yeto-rl-island-0, pid=4051) router_health_check_interval_secs ............... 15 -(yeto-rl-island-0, pid=4051) router_health_check_timeout_secs ................ 5 -(yeto-rl-island-0, pid=4051) router_health_failure_threshold ................. 40 -(yeto-rl-island-0, pid=4051) router_health_success_threshold ................. 1 -(yeto-rl-island-0, pid=4051) router_history_backend .......................... memory -(yeto-rl-island-0, pid=4051) router_json_log ................................. False -(yeto-rl-island-0, pid=4051) router_jwt_audience ............................. None -(yeto-rl-island-0, pid=4051) router_jwt_issuer ............................... None -(yeto-rl-island-0, pid=4051) router_jwt_jwks_uri ............................. None -(yeto-rl-island-0, pid=4051) router_jwt_role_mapping ......................... [] -(yeto-rl-island-0, pid=4051) router_log_dir .................................. None -(yeto-rl-island-0, pid=4051) router_log_level ................................ info -(yeto-rl-island-0, pid=4051) router_max_concurrent_requests .................. -1 -(yeto-rl-island-0, pid=4051) router_max_idle_secs ............................ 14400 -(yeto-rl-island-0, pid=4051) router_max_payload_size ......................... 536870912 -(yeto-rl-island-0, pid=4051) router_max_tree_size ............................ 67108864 -(yeto-rl-island-0, pid=4051) router_mcp_config_path .......................... None -(yeto-rl-island-0, pid=4051) router_mini_lb .................................. False -(yeto-rl-island-0, pid=4051) router_model_path ............................... None -(yeto-rl-island-0, pid=4051) router_oracle_connect_descriptor ................ None -(yeto-rl-island-0, pid=4051) router_oracle_password .......................... None -(yeto-rl-island-0, pid=4051) router_oracle_pool_max .......................... 16 -(yeto-rl-island-0, pid=4051) router_oracle_pool_min .......................... 1 -(yeto-rl-island-0, pid=4051) router_oracle_pool_timeout_secs ................. 30 -(yeto-rl-island-0, pid=4051) router_oracle_tns_alias ......................... None -(yeto-rl-island-0, pid=4051) router_oracle_username .......................... None -(yeto-rl-island-0, pid=4051) router_oracle_wallet_path ....................... None -(yeto-rl-island-0, pid=4051) router_otlp_traces_endpoint ..................... localhost:4317 -(yeto-rl-island-0, pid=4051) router_pd_disaggregation ........................ False -(yeto-rl-island-0, pid=4051) router_policy ................................... cache_aware -(yeto-rl-island-0, pid=4051) router_postgres_db_url .......................... None -(yeto-rl-island-0, pid=4051) router_postgres_pool_max ........................ 16 -(yeto-rl-island-0, pid=4051) router_prefill .................................. None -(yeto-rl-island-0, pid=4051) router_prefill_policy ........................... None -(yeto-rl-island-0, pid=4051) router_prefill_selector ......................... {} -(yeto-rl-island-0, pid=4051) router_prometheus_duration_buckets .............. None -(yeto-rl-island-0, pid=4051) router_prometheus_host .......................... 0.0.0.0 -(yeto-rl-island-0, pid=4051) router_prometheus_port .......................... 29000 -(yeto-rl-island-0, pid=4051) router_queue_size ............................... 100 -(yeto-rl-island-0, pid=4051) router_queue_timeout_secs ....................... 60 -(yeto-rl-island-0, pid=4051) router_rate_limit_tokens_per_second ............. None -(yeto-rl-island-0, pid=4051) router_reasoning_parser ......................... None -(yeto-rl-island-0, pid=4051) router_redis_pool_max ........................... 16 -(yeto-rl-island-0, pid=4051) router_redis_retention_days ..................... 30 -(yeto-rl-island-0, pid=4051) router_redis_url ................................ None -(yeto-rl-island-0, pid=4051) router_request_id_headers ....................... None -(yeto-rl-island-0, pid=4051) router_request_timeout_secs ..................... 1800 -(yeto-rl-island-0, pid=4051) router_retry_backoff_multiplier ................. 1.5 -(yeto-rl-island-0, pid=4051) router_retry_initial_backoff_ms ................. 50 -(yeto-rl-island-0, pid=4051) router_retry_jitter_factor ...................... 0.2 -(yeto-rl-island-0, pid=4051) router_retry_max_backoff_ms ..................... 30000 -(yeto-rl-island-0, pid=4051) router_retry_max_retries ........................ 5 -(yeto-rl-island-0, pid=4051) router_selector ................................. {} -(yeto-rl-island-0, pid=4051) router_service_discovery ........................ False -(yeto-rl-island-0, pid=4051) router_service_discovery_namespace .............. None -(yeto-rl-island-0, pid=4051) router_service_discovery_port ................... 80 -(yeto-rl-island-0, pid=4051) router_shutdown_grace_period_secs ............... 180 -(yeto-rl-island-0, pid=4051) router_test_external_dp_routing ................. False -(yeto-rl-island-0, pid=4051) router_tls_cert_path ............................ None -(yeto-rl-island-0, pid=4051) router_tls_key_path ............................. None -(yeto-rl-island-0, pid=4051) router_tokenizer_cache_enable_l0 ................ False -(yeto-rl-island-0, pid=4051) router_tokenizer_cache_enable_l1 ................ False -(yeto-rl-island-0, pid=4051) router_tokenizer_cache_l0_max_entries ........... 10000 -(yeto-rl-island-0, pid=4051) router_tokenizer_cache_l1_max_memory ............ 52428800 -(yeto-rl-island-0, pid=4051) router_tokenizer_path ........................... None -(yeto-rl-island-0, pid=4051) router_tool_call_parser ......................... None -(yeto-rl-island-0, pid=4051) router_worker_startup_check_interval ............ 30 -(yeto-rl-island-0, pid=4051) router_worker_startup_timeout_secs .............. 1800 -(yeto-rl-island-0, pid=4051) run_uuid ........................................ bf845e8d7da54983 -(yeto-rl-island-0, pid=4051) run_workload_inspector_server ................... False -(yeto-rl-island-0, pid=4051) sample_rate ..................................... 1.0 -(yeto-rl-island-0, pid=4051) sapo_tau_neg .................................... 1.05 -(yeto-rl-island-0, pid=4051) sapo_tau_pos .................................... 1.0 -(yeto-rl-island-0, pid=4051) save ............................................ None -(yeto-rl-island-0, pid=4051) save_activations_interval ....................... None -(yeto-rl-island-0, pid=4051) save_debug_event_data ........................... None -(yeto-rl-island-0, pid=4051) save_debug_rollout_data ......................... None -(yeto-rl-island-0, pid=4051) save_debug_train_data ........................... None -(yeto-rl-island-0, pid=4051) save_debug_trajectory_data ...................... None -(yeto-rl-island-0, pid=4051) save_dgrads_interval ............................ None -(yeto-rl-island-0, pid=4051) save_hf ......................................... None -(yeto-rl-island-0, pid=4051) save_interval ................................... None -(yeto-rl-island-0, pid=4051) save_local_weight_checksum ...................... False -(yeto-rl-island-0, pid=4051) save_params_interval ............................ None -(yeto-rl-island-0, pid=4051) save_retain_interval ............................ None -(yeto-rl-island-0, pid=4051) save_tokens_per_expert_interval ................. None -(yeto-rl-island-0, pid=4051) save_trigger_sentinel ........................... None -(yeto-rl-island-0, pid=4051) save_wgrads_interval ............................ None -(yeto-rl-island-0, pid=4051) seed ............................................ 17 -(yeto-rl-island-0, pid=4051) seq_length ...................................... 2048 -(yeto-rl-island-0, pid=4051) sequence_packing_scheduler ...................... None -(yeto-rl-island-0, pid=4051) sequence_parallel ............................... True -(yeto-rl-island-0, pid=4051) session_message_matcher ......................... strict -(yeto-rl-island-0, pid=4051) session_sample_picker_path ...................... miles.rollout.session.v2.picker_hub.drop_same_prompt_retries -(yeto-rl-island-0, pid=4051) session_sample_postprocessor_path ............... miles.rollout.session.v2.postprocessor_hub.default_postprocess -(yeto-rl-island-0, pid=4051) session_server_external_host .................... None -(yeto-rl-island-0, pid=4051) session_server_ip ............................... None -(yeto-rl-island-0, pid=4051) session_server_port ............................. None -(yeto-rl-island-0, pid=4051) session_server_workers .......................... 32 -(yeto-rl-island-0, pid=4051) sft ............................................. False -(yeto-rl-island-0, pid=4051) sft_mock_dataset_config_json .................... None -(yeto-rl-island-0, pid=4051) sft_tokenizer_prompt_format ..................... nemotron-h-aligned -(yeto-rl-island-0, pid=4051) sgd_momentum .................................... 0.9 -(yeto-rl-island-0, pid=4051) sglang_abort_on_priority_when_disabled .......... False -(yeto-rl-island-0, pid=4051) sglang_admin_api_key ............................ None -(yeto-rl-island-0, pid=4051) sglang_allow_auto_truncate ...................... False -(yeto-rl-island-0, pid=4051) sglang_allowed_media_domains .................... [] -(yeto-rl-island-0, pid=4051) sglang_api_key .................................. None -(yeto-rl-island-0, pid=4051) sglang_asr_max_buffer_seconds ................... 60 -(yeto-rl-island-0, pid=4051) sglang_asr_max_concurrent_sessions .............. 32 -(yeto-rl-island-0, pid=4051) sglang_attention_backend ........................ None -(yeto-rl-island-0, pid=4051) sglang_attn_cp_size ............................. 1 -(yeto-rl-island-0, pid=4051) sglang_batch_notify_size ........................ 16 -(yeto-rl-island-0, pid=4051) sglang_bf16_gemm_backend ........................ auto -(yeto-rl-island-0, pid=4051) sglang_bucket_e2e_request_latency ............... None -(yeto-rl-island-0, pid=4051) sglang_bucket_inter_token_latency ............... None -(yeto-rl-island-0, pid=4051) sglang_bucket_time_to_first_token ............... None -(yeto-rl-island-0, pid=4051) sglang_c128_page_size ........................... 16 -(yeto-rl-island-0, pid=4051) sglang_chat_template ............................ None -(yeto-rl-island-0, pid=4051) sglang_checkpoint_engine_wait_weights_before_ready False -(yeto-rl-island-0, pid=4051) sglang_chunked_prefill_size ..................... 8192 -(yeto-rl-island-0, pid=4051) sglang_completion_template ...................... None -(yeto-rl-island-0, pid=4051) sglang_config ................................... None -(yeto-rl-island-0, pid=4051) sglang_constrained_json_disable_any_whitespace .. False -(yeto-rl-island-0, pid=4051) sglang_constrained_json_whitespace_pattern ...... None -(yeto-rl-island-0, pid=4051) sglang_context_length ........................... None -(yeto-rl-island-0, pid=4051) sglang_cp_strategy .............................. None -(yeto-rl-island-0, pid=4051) sglang_cpu_offload_gb ........................... 0 -(yeto-rl-island-0, pid=4051) sglang_crash_dump_folder ........................ None -(yeto-rl-island-0, pid=4051) sglang_cuda_graph_backend_decode ................ None -(yeto-rl-island-0, pid=4051) sglang_cuda_graph_backend_prefill ............... disabled -(yeto-rl-island-0, pid=4051) sglang_cuda_graph_bs_decode ..................... None -(yeto-rl-island-0, pid=4051) sglang_cuda_graph_bs_prefill .................... None -(yeto-rl-island-0, pid=4051) sglang_cuda_graph_config ........................ None -(yeto-rl-island-0, pid=4051) sglang_cuda_graph_max_bs_decode ................. None -(yeto-rl-island-0, pid=4051) sglang_cuda_graph_max_bs_prefill ................ None -(yeto-rl-island-0, pid=4051) sglang_cuda_graph_tc_compiler ................... None -(yeto-rl-island-0, pid=4051) sglang_custom_pull_weights_pre_read_hook ........ None -(yeto-rl-island-0, pid=4051) sglang_custom_weight_loader ..................... None -(yeto-rl-island-0, pid=4051) sglang_dcp_comm_backend ......................... None -(yeto-rl-island-0, pid=4051) sglang_dcp_replicate_q_proj ..................... None -(yeto-rl-island-0, pid=4051) sglang_dcp_size ................................. 1 -(yeto-rl-island-0, pid=4051) sglang_debug_cuda_graph ......................... False -(yeto-rl-island-0, pid=4051) sglang_debug_tensor_dump_input_file ............. None -(yeto-rl-island-0, pid=4051) sglang_debug_tensor_dump_layers ................. None -(yeto-rl-island-0, pid=4051) sglang_debug_tensor_dump_output_folder .......... None -(yeto-rl-island-0, pid=4051) sglang_decode_attention_backend ................. None -(yeto-rl-island-0, pid=4051) sglang_decode_log_interval ...................... 40 -(yeto-rl-island-0, pid=4051) sglang_decoupled_spec_bind_endpoint ............. None -(yeto-rl-island-0, pid=4051) sglang_decoupled_spec_connect_endpoints ......... None -(yeto-rl-island-0, pid=4051) sglang_decoupled_spec_rank ...................... None -(yeto-rl-island-0, pid=4051) sglang_decoupled_spec_role ...................... null -(yeto-rl-island-0, pid=4051) sglang_decrypted_config_file .................... None -(yeto-rl-island-0, pid=4051) sglang_decrypted_draft_config_file .............. None -(yeto-rl-island-0, pid=4051) sglang_deepep_config ............................ None -(yeto-rl-island-0, pid=4051) sglang_deepep_dispatcher_output_dtype ........... auto -(yeto-rl-island-0, pid=4051) sglang_deepep_mode .............................. auto -(yeto-rl-island-0, pid=4051) sglang_deepep_v2_mode ........................... direct -(yeto-rl-island-0, pid=4051) sglang_default_chat_template_kwargs ............. None -(yeto-rl-island-0, pid=4051) sglang_default_priority_value ................... None -(yeto-rl-island-0, pid=4051) sglang_delete_ckpt_after_loading ................ False -(yeto-rl-island-0, pid=4051) sglang_detokenizer_worker_num ................... 1 -(yeto-rl-island-0, pid=4051) sglang_device ................................... None -(yeto-rl-island-0, pid=4051) sglang_disable_attn_tp_gather ................... False -(yeto-rl-island-0, pid=4051) sglang_disable_chunked_prefix_cache ............. False -(yeto-rl-island-0, pid=4051) sglang_disable_cuda_graph ....................... False -(yeto-rl-island-0, pid=4051) sglang_disable_cuda_graph_padding ............... False -(yeto-rl-island-0, pid=4051) sglang_disable_custom_all_reduce ................ False -(yeto-rl-island-0, pid=4051) sglang_disable_decode_cuda_graph ................ False -(yeto-rl-island-0, pid=4051) sglang_disable_draft_cuda_graph ................. False -(yeto-rl-island-0, pid=4051) sglang_disable_fast_image_processor ............. False -(yeto-rl-island-0, pid=4051) sglang_disable_flashinfer_autotune .............. False -(yeto-rl-island-0, pid=4051) sglang_disable_flashinfer_cutlass_moe_fp4_allgather False -(yeto-rl-island-0, pid=4051) sglang_disable_hybrid_swa_memory ................ False -(yeto-rl-island-0, pid=4051) sglang_disable_outlines_disk_cache .............. False -(yeto-rl-island-0, pid=4051) sglang_disable_overlap_schedule ................. False -(yeto-rl-island-0, pid=4051) sglang_disable_prefill_cuda_graph ............... False -(yeto-rl-island-0, pid=4051) sglang_disable_priority_preemption .............. False -(yeto-rl-island-0, pid=4051) sglang_disable_radix_cache ...................... True -(yeto-rl-island-0, pid=4051) sglang_disable_shared_experts_fusion ............ False -(yeto-rl-island-0, pid=4051) sglang_disable_tokenizer_batch_decode ........... False -(yeto-rl-island-0, pid=4051) sglang_disaggregation_bootstrap_port ............ 8998 -(yeto-rl-island-0, pid=4051) sglang_disaggregation_decode_enable_offload_kvcache False -(yeto-rl-island-0, pid=4051) sglang_disaggregation_decode_enable_radix_cache . False -(yeto-rl-island-0, pid=4051) sglang_disaggregation_decode_extra_slots ........ None -(yeto-rl-island-0, pid=4051) sglang_disaggregation_decode_polling_interval ... 1 -(yeto-rl-island-0, pid=4051) sglang_disaggregation_decode_retraction_backup .. None -(yeto-rl-island-0, pid=4051) sglang_disaggregation_ib_device ................. None -(yeto-rl-island-0, pid=4051) sglang_disaggregation_mode ...................... null -(yeto-rl-island-0, pid=4051) sglang_disaggregation_transfer_backend .......... mooncake -(yeto-rl-island-0, pid=4051) sglang_dist_timeout ............................. None -(yeto-rl-island-0, pid=4051) sglang_dllm_algorithm ........................... None -(yeto-rl-island-0, pid=4051) sglang_dllm_algorithm_config .................... None -(yeto-rl-island-0, pid=4051) sglang_dllm_fdfo ................................ True -(yeto-rl-island-0, pid=4051) sglang_download_dir ............................. None -(yeto-rl-island-0, pid=4051) sglang_dp_size .................................. 1 -(yeto-rl-island-0, pid=4051) sglang_dsa_decode_backend ....................... None -(yeto-rl-island-0, pid=4051) sglang_dsa_paged_mqa_logits_backend ............. auto -(yeto-rl-island-0, pid=4051) sglang_dsa_prefill_backend ...................... None -(yeto-rl-island-0, pid=4051) sglang_dsa_topk_backend ......................... sgl-kernel -(yeto-rl-island-0, pid=4051) sglang_dsv4_attn_backend ........................ auto -(yeto-rl-island-0, pid=4051) sglang_dsv4_prefill_backend ..................... auto -(yeto-rl-island-0, pid=4051) sglang_dtype .................................... auto -(yeto-rl-island-0, pid=4051) sglang_dwdp_size ................................ 1 -(yeto-rl-island-0, pid=4051) sglang_dynamic_batch_tokenizer_batch_size ....... 32 -(yeto-rl-island-0, pid=4051) sglang_dynamic_batch_tokenizer_batch_timeout .... 0.002 -(yeto-rl-island-0, pid=4051) sglang_elastic_ep_backend ....................... None -(yeto-rl-island-0, pid=4051) sglang_elastic_ep_initial_size .................. None -(yeto-rl-island-0, pid=4051) sglang_elastic_ep_rejoin ........................ False -(yeto-rl-island-0, pid=4051) sglang_elastic_ep_scale_timeout ................. 600 -(yeto-rl-island-0, pid=4051) sglang_enable_adaptive_dispatch_to_encoder ...... False -(yeto-rl-island-0, pid=4051) sglang_enable_aiter_allreduce_fusion ............ False -(yeto-rl-island-0, pid=4051) sglang_enable_attn_tp_input_scattered ........... False -(yeto-rl-island-0, pid=4051) sglang_enable_broadcast_mm_inputs_process ....... False -(yeto-rl-island-0, pid=4051) sglang_enable_cache_report ...................... False -(yeto-rl-island-0, pid=4051) sglang_enable_cp_decode_attn_tp ................. False -(yeto-rl-island-0, pid=4051) sglang_enable_cudagraph_gc ...................... False -(yeto-rl-island-0, pid=4051) sglang_enable_custom_logit_processor ............ False -(yeto-rl-island-0, pid=4051) sglang_enable_deepseek_v4_fp4_indexer ........... False -(yeto-rl-island-0, pid=4051) sglang_enable_dense_mlp_attn_tp ................. False -(yeto-rl-island-0, pid=4051) sglang_enable_deterministic_inference ........... False -(yeto-rl-island-0, pid=4051) sglang_enable_dp_attention ...................... False -(yeto-rl-island-0, pid=4051) sglang_enable_dp_attention_local_control_broadcast False -(yeto-rl-island-0, pid=4051) sglang_enable_dp_lm_head ........................ False -(yeto-rl-island-0, pid=4051) sglang_enable_draft_weights_cpu_backup .......... False -(yeto-rl-island-0, pid=4051) sglang_enable_dsa_cache_layer_split ............. False -(yeto-rl-island-0, pid=4051) sglang_enable_dynamic_batch_tokenizer ........... False -(yeto-rl-island-0, pid=4051) sglang_enable_dynamic_chunking .................. False -(yeto-rl-island-0, pid=4051) sglang_enable_elastic_expert_backup ............. False -(yeto-rl-island-0, pid=4051) sglang_enable_engine_info_bootstrap ............. False -(yeto-rl-island-0, pid=4051) sglang_enable_eplb .............................. False -(yeto-rl-island-0, pid=4051) sglang_enable_flashinfer_allreduce_fusion ....... False -(yeto-rl-island-0, pid=4051) sglang_enable_flexkv ............................ False -(yeto-rl-island-0, pid=4051) sglang_enable_forward_pass_metrics .............. False -(yeto-rl-island-0, pid=4051) sglang_enable_fp32_lm_head ...................... False -(yeto-rl-island-0, pid=4051) sglang_enable_fused_moe_sum_all_reduce .......... False -(yeto-rl-island-0, pid=4051) sglang_enable_fused_qk_norm_rope ................ False -(yeto-rl-island-0, pid=4051) sglang_enable_hierarchical_cache ................ False -(yeto-rl-island-0, pid=4051) sglang_enable_hisparse .......................... False -(yeto-rl-island-0, pid=4051) sglang_enable_http2 ............................. False -(yeto-rl-island-0, pid=4051) sglang_enable_int8_mamba_checkpoint ............. False -(yeto-rl-island-0, pid=4051) sglang_enable_layernorm_sp ...................... False -(yeto-rl-island-0, pid=4051) sglang_enable_layerwise_nvtx_marker ............. False -(yeto-rl-island-0, pid=4051) sglang_enable_lean_attention .................... None -(yeto-rl-island-0, pid=4051) sglang_enable_linear_replayssm .................. False -(yeto-rl-island-0, pid=4051) sglang_enable_linear_replayssm_spec ............. False -(yeto-rl-island-0, pid=4051) sglang_enable_lmcache ........................... False -(yeto-rl-island-0, pid=4051) sglang_enable_lora .............................. None -(yeto-rl-island-0, pid=4051) sglang_enable_lora_overlap_loading .............. None -(yeto-rl-island-0, pid=4051) sglang_enable_mamba_cache_stochastic_rounding ... False -(yeto-rl-island-0, pid=4051) sglang_enable_metrics ........................... False -(yeto-rl-island-0, pid=4051) sglang_enable_metrics_for_all_schedulers ........ False -(yeto-rl-island-0, pid=4051) sglang_enable_mfu_metrics ....................... False -(yeto-rl-island-0, pid=4051) sglang_enable_mis ............................... False -(yeto-rl-island-0, pid=4051) sglang_enable_mixed_chunk ....................... False -(yeto-rl-island-0, pid=4051) sglang_enable_mm_global_cache ................... False -(yeto-rl-island-0, pid=4051) sglang_enable_mscclpp ........................... False -(yeto-rl-island-0, pid=4051) sglang_enable_multi_layer_eagle ................. False -(yeto-rl-island-0, pid=4051) sglang_enable_multimodal ........................ None -(yeto-rl-island-0, pid=4051) sglang_enable_nccl_nvls ......................... False -(yeto-rl-island-0, pid=4051) sglang_enable_p2p_check ......................... False -(yeto-rl-island-0, pid=4051) sglang_enable_page_major_kv_layout .............. False -(yeto-rl-island-0, pid=4051) sglang_enable_pdmux ............................. False -(yeto-rl-island-0, pid=4051) sglang_enable_precise_embedding_interpolation ... False -(yeto-rl-island-0, pid=4051) sglang_enable_prefill_cp ........................ False -(yeto-rl-island-0, pid=4051) sglang_enable_prefill_delayer ................... False -(yeto-rl-island-0, pid=4051) sglang_enable_prefill_only_deterministic_inference False -(yeto-rl-island-0, pid=4051) sglang_enable_prefix_mm_cache ................... False -(yeto-rl-island-0, pid=4051) sglang_enable_priority_scheduling ............... False -(yeto-rl-island-0, pid=4051) sglang_enable_profile_cuda_graph ................ False -(yeto-rl-island-0, pid=4051) sglang_enable_quant_communications .............. False -(yeto-rl-island-0, pid=4051) sglang_enable_rdt_weight_sync ................... False -(yeto-rl-island-0, pid=4051) sglang_enable_request_time_stats_logging ........ False -(yeto-rl-island-0, pid=4051) sglang_enable_response_store .................... False -(yeto-rl-island-0, pid=4051) sglang_enable_return_hidden_states .............. False -(yeto-rl-island-0, pid=4051) sglang_enable_scattered_sconv ................... False -(yeto-rl-island-0, pid=4051) sglang_enable_session_radix_cache ............... False -(yeto-rl-island-0, pid=4051) sglang_enable_shared_experts_attn_tp ............ False -(yeto-rl-island-0, pid=4051) sglang_enable_single_batch_overlap .............. False -(yeto-rl-island-0, pid=4051) sglang_enable_ssl_refresh ....................... False -(yeto-rl-island-0, pid=4051) sglang_enable_streaming_session ................. False -(yeto-rl-island-0, pid=4051) sglang_enable_strict_thinking ................... False -(yeto-rl-island-0, pid=4051) sglang_enable_symm_mem .......................... False -(yeto-rl-island-0, pid=4051) sglang_enable_tf32_matmul ....................... False -(yeto-rl-island-0, pid=4051) sglang_enable_tokenizer_batch_encode ............ False -(yeto-rl-island-0, pid=4051) sglang_enable_torch_compile ..................... False -(yeto-rl-island-0, pid=4051) sglang_enable_torch_compile_debug_mode .......... False -(yeto-rl-island-0, pid=4051) sglang_enable_torch_symm_mem .................... False -(yeto-rl-island-0, pid=4051) sglang_enable_tp_lm_head_all_to_all ............. None -(yeto-rl-island-0, pid=4051) sglang_enable_trace ............................. False -(yeto-rl-island-0, pid=4051) sglang_enable_two_batch_overlap ................. False -(yeto-rl-island-0, pid=4051) sglang_enable_unified_cache_external_linker ..... False -(yeto-rl-island-0, pid=4051) sglang_enable_unified_memory .................... False -(yeto-rl-island-0, pid=4051) sglang_enable_w4a4_mxfp4_megamoe ................ False -(yeto-rl-island-0, pid=4051) sglang_enable_waterfill ......................... False -(yeto-rl-island-0, pid=4051) sglang_enable_weights_cpu_backup ................ False -(yeto-rl-island-0, pid=4051) sglang_enable_weights_disk_backup ............... False -(yeto-rl-island-0, pid=4051) sglang_encoder_bootstrap_port ................... 8997 -(yeto-rl-island-0, pid=4051) sglang_encoder_only ............................. False -(yeto-rl-island-0, pid=4051) sglang_encoder_register_urls .................... [] -(yeto-rl-island-0, pid=4051) sglang_encoder_transfer_backend ................. auto -(yeto-rl-island-0, pid=4051) sglang_encoder_urls ............................. [] -(yeto-rl-island-0, pid=4051) sglang_enforce_disable_flashinfer_allreduce_fusion False -(yeto-rl-island-0, pid=4051) sglang_enforce_shared_experts_fusion ............ False -(yeto-rl-island-0, pid=4051) sglang_engine_info_bootstrap_port ............... 6789 -(yeto-rl-island-0, pid=4051) sglang_ep_dispatch_algorithm .................... None -(yeto-rl-island-0, pid=4051) sglang_ep_join_mode ............................. None -(yeto-rl-island-0, pid=4051) sglang_ep_join_rank_offset ...................... 0 -(yeto-rl-island-0, pid=4051) sglang_ep_num_redundant_experts ................. 0 -(yeto-rl-island-0, pid=4051) sglang_ep_size .................................. 4 -(yeto-rl-island-0, pid=4051) sglang_eplb_algorithm ........................... auto -(yeto-rl-island-0, pid=4051) sglang_eplb_min_rebalancing_utilization_threshold 1.0 -(yeto-rl-island-0, pid=4051) sglang_eplb_rebalance_layers_per_chunk .......... None -(yeto-rl-island-0, pid=4051) sglang_eplb_rebalance_num_iterations ............ 1000 -(yeto-rl-island-0, pid=4051) sglang_expert_balancedness_report_mode .......... off -(yeto-rl-island-0, pid=4051) sglang_expert_distribution_recorder_buffer_size . None -(yeto-rl-island-0, pid=4051) sglang_expert_distribution_recorder_mode ........ None -(yeto-rl-island-0, pid=4051) sglang_experts_shared_outer_loras ............... None -(yeto-rl-island-0, pid=4051) sglang_export_metrics_to_file ................... False -(yeto-rl-island-0, pid=4051) sglang_export_metrics_to_file_dir ............... None -(yeto-rl-island-0, pid=4051) sglang_extra_metric_labels ...................... None -(yeto-rl-island-0, pid=4051) sglang_fastapi_root_path ........................ -(yeto-rl-island-0, pid=4051) sglang_file_storage_path ........................ sglang_storage -(yeto-rl-island-0, pid=4051) sglang_flashinfer_a2a_dispatch_type ............. None -(yeto-rl-island-0, pid=4051) sglang_flashinfer_allreduce_fusion_backend ...... None -(yeto-rl-island-0, pid=4051) sglang_flashinfer_autotune_skip_ops ............. None -(yeto-rl-island-0, pid=4051) sglang_flashinfer_mla_disable_ragged ............ False -(yeto-rl-island-0, pid=4051) sglang_flashinfer_mxfp4_moe_precision ........... default -(yeto-rl-island-0, pid=4051) sglang_flexkv_config_file ....................... None -(yeto-rl-island-0, pid=4051) sglang_forward_hooks ............................ None -(yeto-rl-island-0, pid=4051) sglang_forward_pass_metrics_ipc_name ............ None -(yeto-rl-island-0, pid=4051) sglang_forward_pass_metrics_worker_id ........... -(yeto-rl-island-0, pid=4051) sglang_fp4_gemm_runner_backend .................. auto -(yeto-rl-island-0, pid=4051) sglang_fp8_gemm_runner_backend .................. auto -(yeto-rl-island-0, pid=4051) sglang_fuseep_mode .............................. 2 -(yeto-rl-island-0, pid=4051) sglang_gc_threshold ............................. None -(yeto-rl-island-0, pid=4051) sglang_gc_warning_threshold_secs ................ 0.0 -(yeto-rl-island-0, pid=4051) sglang_generation_tokens_buckets ................ None -(yeto-rl-island-0, pid=4051) sglang_grammar_backend .......................... None -(yeto-rl-island-0, pid=4051) sglang_grpc_mode ................................ False -(yeto-rl-island-0, pid=4051) sglang_grpc_port ................................ None -(yeto-rl-island-0, pid=4051) sglang_hf_chat_template_name .................... None -(yeto-rl-island-0, pid=4051) sglang_hicache_host_memory_mode ................. cache -(yeto-rl-island-0, pid=4051) sglang_hicache_io_backend ....................... kernel -(yeto-rl-island-0, pid=4051) sglang_hicache_mem_layout ....................... page_first -(yeto-rl-island-0, pid=4051) sglang_hicache_ratio ............................ None -(yeto-rl-island-0, pid=4051) sglang_hicache_size ............................. 0 -(yeto-rl-island-0, pid=4051) sglang_hicache_storage_backend .................. None -(yeto-rl-island-0, pid=4051) sglang_hicache_storage_backend_extra_config ..... None -(yeto-rl-island-0, pid=4051) sglang_hicache_storage_prefetch_policy .......... timeout -(yeto-rl-island-0, pid=4051) sglang_hicache_storage_prefetch_retry_max_attempts 8 -(yeto-rl-island-0, pid=4051) sglang_hicache_storage_prefetch_retry_poll_interval 8 -(yeto-rl-island-0, pid=4051) sglang_hicache_write_policy ..................... write_through -(yeto-rl-island-0, pid=4051) sglang_hisparse_config .......................... None -(yeto-rl-island-0, pid=4051) sglang_host ..................................... 127.0.0.1 -(yeto-rl-island-0, pid=4051) sglang_http2_initial_connection_window_size ..... 1048576 -(yeto-rl-island-0, pid=4051) sglang_http2_max_concurrent_streams ............. 200 -(yeto-rl-island-0, pid=4051) sglang_image_processor_backend .................. auto -(yeto-rl-island-0, pid=4051) sglang_incremental_streaming_output ............. False -(yeto-rl-island-0, pid=4051) sglang_init_expert_location ..................... trivial -(yeto-rl-island-0, pid=4051) sglang_int8_mamba_ckpt_size ..................... None -(yeto-rl-island-0, pid=4051) sglang_is_embedding ............................. False -(yeto-rl-island-0, pid=4051) sglang_json_model_override_args ................. {} -(yeto-rl-island-0, pid=4051) sglang_keep_mm_feature_on_device ................ False -(yeto-rl-island-0, pid=4051) sglang_kt_cpuinfer .............................. None -(yeto-rl-island-0, pid=4051) sglang_kt_max_deferred_experts_per_token ........ None -(yeto-rl-island-0, pid=4051) sglang_kt_method ................................ AMXINT4 -(yeto-rl-island-0, pid=4051) sglang_kt_num_gpu_experts ....................... None -(yeto-rl-island-0, pid=4051) sglang_kt_threadpool_count ...................... 2 -(yeto-rl-island-0, pid=4051) sglang_kt_weight_path ........................... None -(yeto-rl-island-0, pid=4051) sglang_kv_cache_dtype ........................... auto -(yeto-rl-island-0, pid=4051) sglang_kv_canary ................................ none -(yeto-rl-island-0, pid=4051) sglang_kv_canary_real_data ...................... none -(yeto-rl-island-0, pid=4051) sglang_kv_canary_sweep_interval ................. 0 -(yeto-rl-island-0, pid=4051) sglang_kv_events_config ......................... None -(yeto-rl-island-0, pid=4051) sglang_language_model_only ...................... False -(yeto-rl-island-0, pid=4051) sglang_language_only ............................ False -(yeto-rl-island-0, pid=4051) sglang_limit_mm_data_per_request ................ None -(yeto-rl-island-0, pid=4051) sglang_linear_attn_backend ...................... triton -(yeto-rl-island-0, pid=4051) sglang_linear_attn_decode_backend ............... None -(yeto-rl-island-0, pid=4051) sglang_linear_attn_prefill_backend .............. flashinfer -(yeto-rl-island-0, pid=4051) sglang_linear_attn_verify_backend ............... None -(yeto-rl-island-0, pid=4051) sglang_linear_replayssm_cache_len ............... 16 -(yeto-rl-island-0, pid=4051) sglang_lmcache_config_file ...................... None -(yeto-rl-island-0, pid=4051) sglang_load_balance_method ...................... auto -(yeto-rl-island-0, pid=4051) sglang_load_format .............................. auto -(yeto-rl-island-0, pid=4051) sglang_load_publish_endpoint .................... None -(yeto-rl-island-0, pid=4051) sglang_load_snapshot_publish_interval ........... 15 -(yeto-rl-island-0, pid=4051) sglang_log_level ................................ info -(yeto-rl-island-0, pid=4051) sglang_log_level_http ........................... None -(yeto-rl-island-0, pid=4051) sglang_log_requests ............................. False -(yeto-rl-island-0, pid=4051) sglang_log_requests_format ...................... text -(yeto-rl-island-0, pid=4051) sglang_log_requests_level ....................... 2 -(yeto-rl-island-0, pid=4051) sglang_log_requests_target ...................... None -(yeto-rl-island-0, pid=4051) sglang_lora_backend ............................. triton -(yeto-rl-island-0, pid=4051) sglang_lora_drain_wait_threshold ................ 0.0 -(yeto-rl-island-0, pid=4051) sglang_lora_eviction_policy ..................... lru -(yeto-rl-island-0, pid=4051) sglang_lora_no_cpu_backup ....................... False -(yeto-rl-island-0, pid=4051) sglang_lora_paths ............................... None -(yeto-rl-island-0, pid=4051) sglang_lora_strict_loading ...................... True -(yeto-rl-island-0, pid=4051) sglang_lora_target_modules ...................... None -(yeto-rl-island-0, pid=4051) sglang_lora_use_virtual_experts ................. True -(yeto-rl-island-0, pid=4051) sglang_mamba_backend ............................ triton -(yeto-rl-island-0, pid=4051) sglang_mamba_cache_philox_rounds ................ 0 -(yeto-rl-island-0, pid=4051) sglang_mamba_full_memory_ratio .................. None -(yeto-rl-island-0, pid=4051) sglang_mamba_max_states_per_path ................ -1 -(yeto-rl-island-0, pid=4051) sglang_mamba_radix_cache_strategy ............... auto -(yeto-rl-island-0, pid=4051) sglang_mamba_ssm_dtype .......................... None -(yeto-rl-island-0, pid=4051) sglang_mamba_track_interval ..................... 256 -(yeto-rl-island-0, pid=4051) sglang_max_ep_size .............................. None -(yeto-rl-island-0, pid=4051) sglang_max_loaded_loras ......................... None -(yeto-rl-island-0, pid=4051) sglang_max_lora_chunk_size ...................... 16 -(yeto-rl-island-0, pid=4051) sglang_max_lora_rank ............................ 16 -(yeto-rl-island-0, pid=4051) sglang_max_loras_per_batch ...................... 8 -(yeto-rl-island-0, pid=4051) sglang_max_mamba_cache_size ..................... None -(yeto-rl-island-0, pid=4051) sglang_max_prefill_tokens ....................... 16384 -(yeto-rl-island-0, pid=4051) sglang_max_queued_requests ...................... None -(yeto-rl-island-0, pid=4051) sglang_max_running_requests ..................... None -(yeto-rl-island-0, pid=4051) sglang_max_total_tokens ......................... None -(yeto-rl-island-0, pid=4051) sglang_media_url_max_file_size_mb ............... 64 -(yeto-rl-island-0, pid=4051) sglang_mem_fraction_static ...................... 0.7 -(yeto-rl-island-0, pid=4051) sglang_min_free_slots_delay ..................... None -(yeto-rl-island-0, pid=4051) sglang_mlx_enable_sampling ...................... False -(yeto-rl-island-0, pid=4051) sglang_mm_attention_backend ..................... None -(yeto-rl-island-0, pid=4051) sglang_mm_enable_dp_encoder ..................... False -(yeto-rl-island-0, pid=4051) sglang_mm_feature_transport ..................... None -(yeto-rl-island-0, pid=4051) sglang_mm_global_cache_backend .................. mooncake -(yeto-rl-island-0, pid=4051) sglang_mm_io_worker_num ......................... 0 -(yeto-rl-island-0, pid=4051) sglang_mm_preprocess_cache_size_mb .............. None -(yeto-rl-island-0, pid=4051) sglang_mm_process_config ........................ None -(yeto-rl-island-0, pid=4051) sglang_mm_processor_worker_num .................. 0 -(yeto-rl-island-0, pid=4051) sglang_model_checksum ........................... None -(yeto-rl-island-0, pid=4051) sglang_model_config_parser ...................... auto -(yeto-rl-island-0, pid=4051) sglang_model_impl ............................... auto -(yeto-rl-island-0, pid=4051) sglang_model_loader_extra_config ................ {} -(yeto-rl-island-0, pid=4051) sglang_model_routers ............................ None -(yeto-rl-island-0, pid=4051) sglang_modelexpress_config ...................... None -(yeto-rl-island-0, pid=4051) sglang_modelopt_checkpoint_restore_path ......... None -(yeto-rl-island-0, pid=4051) sglang_modelopt_checkpoint_save_path ............ None -(yeto-rl-island-0, pid=4051) sglang_modelopt_export_path ..................... None -(yeto-rl-island-0, pid=4051) sglang_modelopt_quant ........................... None -(yeto-rl-island-0, pid=4051) sglang_moe_a2a_backend .......................... none -(yeto-rl-island-0, pid=4051) sglang_moe_dense_tp_size ........................ None -(yeto-rl-island-0, pid=4051) sglang_moe_dp_size .............................. 1 -(yeto-rl-island-0, pid=4051) sglang_moe_runner_backend ....................... triton -(yeto-rl-island-0, pid=4051) sglang_mooncake_ib_device ....................... None -(yeto-rl-island-0, pid=4051) sglang_msprobe_dump_config ...................... None -(yeto-rl-island-0, pid=4051) sglang_num_continuous_decode_steps .............. 1 -(yeto-rl-island-0, pid=4051) sglang_num_reserved_decode_tokens ............... 512 -(yeto-rl-island-0, pid=4051) sglang_numa_node ................................ None -(yeto-rl-island-0, pid=4051) sglang_offload_group_size ....................... -1 -(yeto-rl-island-0, pid=4051) sglang_offload_mode ............................. cpu -(yeto-rl-island-0, pid=4051) sglang_offload_num_in_group ..................... 1 -(yeto-rl-island-0, pid=4051) sglang_offload_prefetch_step .................... 1 -(yeto-rl-island-0, pid=4051) sglang_optimistic_prefill_attempts .............. 0 -(yeto-rl-island-0, pid=4051) sglang_otlp_traces_endpoint ..................... localhost:4317 -(yeto-rl-island-0, pid=4051) sglang_page_size ................................ None -(yeto-rl-island-0, pid=4051) sglang_pdmux_config_path ........................ None -(yeto-rl-island-0, pid=4051) sglang_ple_offload_backend ...................... pinned -(yeto-rl-island-0, pid=4051) sglang_ple_offload_dir .......................... None -(yeto-rl-island-0, pid=4051) sglang_ple_offload_embedding .................... None -(yeto-rl-island-0, pid=4051) sglang_pp_async_batch_depth ..................... 0 -(yeto-rl-island-0, pid=4051) sglang_pp_max_micro_batch_size .................. None -(yeto-rl-island-0, pid=4051) sglang_pp_size .................................. 1 -(yeto-rl-island-0, pid=4051) sglang_pre_warm_nccl ............................ False -(yeto-rl-island-0, pid=4051) sglang_preferred_sampling_params ................ None -(yeto-rl-island-0, pid=4051) sglang_prefill_attention_backend ................ None -(yeto-rl-island-0, pid=4051) sglang_prefill_decode_interval .................. None -(yeto-rl-island-0, pid=4051) sglang_prefill_delayer_forward_passes_buckets ... None -(yeto-rl-island-0, pid=4051) sglang_prefill_delayer_max_delay_ms ............. None -(yeto-rl-island-0, pid=4051) sglang_prefill_delayer_max_delay_passes ......... 30 -(yeto-rl-island-0, pid=4051) sglang_prefill_delayer_queue_min_ratio .......... None -(yeto-rl-island-0, pid=4051) sglang_prefill_delayer_token_usage_low_watermark None -(yeto-rl-island-0, pid=4051) sglang_prefill_delayer_wait_seconds_buckets ..... None -(yeto-rl-island-0, pid=4051) sglang_prefill_max_requests ..................... None -(yeto-rl-island-0, pid=4051) sglang_prefill_only_disable_kv_cache ............ False -(yeto-rl-island-0, pid=4051) sglang_priority_scheduling_preemption_threshold . 10 -(yeto-rl-island-0, pid=4051) sglang_prompt_tokens_buckets .................... None -(yeto-rl-island-0, pid=4051) sglang_quantization ............................. None -(yeto-rl-island-0, pid=4051) sglang_quantization_param_path .................. None -(yeto-rl-island-0, pid=4051) sglang_quantize_and_serve ....................... False -(yeto-rl-island-0, pid=4051) sglang_radix_cache_backend ...................... None -(yeto-rl-island-0, pid=4051) sglang_radix_eviction_policy .................... lru -(yeto-rl-island-0, pid=4051) sglang_radix_eviction_policy_config ............. None -(yeto-rl-island-0, pid=4051) sglang_reasoning_parser ......................... None -(yeto-rl-island-0, pid=4051) sglang_remote_instance_weight_loader_backend .... nccl -(yeto-rl-island-0, pid=4051) sglang_remote_instance_weight_loader_seed_instance_ip None -(yeto-rl-island-0, pid=4051) sglang_remote_instance_weight_loader_seed_instance_service_port None -(yeto-rl-island-0, pid=4051) sglang_remote_instance_weight_loader_send_weights_group_ports None -(yeto-rl-island-0, pid=4051) sglang_remote_instance_weight_loader_start_seed_via_transfer_engine False -(yeto-rl-island-0, pid=4051) sglang_retraction_policy ........................ length -(yeto-rl-island-0, pid=4051) sglang_return_hidden_states_mode ................ None -(yeto-rl-island-0, pid=4051) sglang_return_input_ids ......................... False -(yeto-rl-island-0, pid=4051) sglang_return_output_ids ........................ False -(yeto-rl-island-0, pid=4051) sglang_revision ................................. None -(yeto-rl-island-0, pid=4051) sglang_rl_on_policy_target ...................... None -(yeto-rl-island-0, pid=4051) sglang_rl_quant_profile ......................... None -(yeto-rl-island-0, pid=4051) sglang_router_ip ................................ None -(yeto-rl-island-0, pid=4051) sglang_router_policy ............................ None -(yeto-rl-island-0, pid=4051) sglang_router_port .............................. None -(yeto-rl-island-0, pid=4051) sglang_router_request_timeout_secs .............. 14400 -(yeto-rl-island-0, pid=4051) sglang_sampling_backend ......................... None -(yeto-rl-island-0, pid=4051) sglang_sampling_defaults ........................ model -(yeto-rl-island-0, pid=4051) sglang_sampling_mask_max_tokens ................. 4096 -(yeto-rl-island-0, pid=4051) sglang_schedule_conservativeness ................ 1.0 -(yeto-rl-island-0, pid=4051) sglang_schedule_low_priority_values_first ....... False -(yeto-rl-island-0, pid=4051) sglang_schedule_policy .......................... fcfs -(yeto-rl-island-0, pid=4051) sglang_scheduler_recv_interval .................. 1 -(yeto-rl-island-0, pid=4051) sglang_served_model_name ........................ None -(yeto-rl-island-0, pid=4051) sglang_server_concurrency ....................... 512 -(yeto-rl-island-0, pid=4051) sglang_show_time_cost ........................... False -(yeto-rl-island-0, pid=4051) sglang_sidecar .................................. None -(yeto-rl-island-0, pid=4051) sglang_sidecar_args ............................. None -(yeto-rl-island-0, pid=4051) sglang_skip_tokenizer_init ...................... False -(yeto-rl-island-0, pid=4051) sglang_sleep_on_idle ............................ False -(yeto-rl-island-0, pid=4051) sglang_sm_group_num ............................. 8 -(yeto-rl-island-0, pid=4051) sglang_smg_grpc_mode ............................ False -(yeto-rl-island-0, pid=4051) sglang_smg_http_sidecar_port .................... None -(yeto-rl-island-0, pid=4051) sglang_soft_watchdog_timeout .................... None -(yeto-rl-island-0, pid=4051) sglang_spec_trace_dir ........................... None -(yeto-rl-island-0, pid=4051) sglang_speculative_accept_threshold_acc ......... 1.0 -(yeto-rl-island-0, pid=4051) sglang_speculative_accept_threshold_single ...... 1.0 -(yeto-rl-island-0, pid=4051) sglang_speculative_adaptive ..................... False -(yeto-rl-island-0, pid=4051) sglang_speculative_adaptive_config .............. None -(yeto-rl-island-0, pid=4051) sglang_speculative_algorithm .................... None -(yeto-rl-island-0, pid=4051) sglang_speculative_attention_mode ............... prefill -(yeto-rl-island-0, pid=4051) sglang_speculative_dflash_block_size ............ None -(yeto-rl-island-0, pid=4051) sglang_speculative_domino_candidate_pool_size ... 2048 -(yeto-rl-island-0, pid=4051) sglang_speculative_draft_attention_backend ...... None -(yeto-rl-island-0, pid=4051) sglang_speculative_draft_kv_cache_dtype ......... None -(yeto-rl-island-0, pid=4051) sglang_speculative_draft_load_format ............ None -(yeto-rl-island-0, pid=4051) sglang_speculative_draft_model_path ............. None -(yeto-rl-island-0, pid=4051) sglang_speculative_draft_model_quantization ..... None -(yeto-rl-island-0, pid=4051) sglang_speculative_draft_model_revision ......... None -(yeto-rl-island-0, pid=4051) sglang_speculative_draft_window_size ............ None -(yeto-rl-island-0, pid=4051) sglang_speculative_dsa_topk_backend ............. sgl-kernel -(yeto-rl-island-0, pid=4051) sglang_speculative_dspark_align_verify_tokens_to_graph_tier False -(yeto-rl-island-0, pid=4051) sglang_speculative_dspark_block_size ............ None -(yeto-rl-island-0, pid=4051) sglang_speculative_dspark_confidence_sts_path ... None -(yeto-rl-island-0, pid=4051) sglang_speculative_dspark_sps_table_path ........ None -(yeto-rl-island-0, pid=4051) sglang_speculative_eagle_topk ................... None -(yeto-rl-island-0, pid=4051) sglang_speculative_moe_a2a_backend .............. None -(yeto-rl-island-0, pid=4051) sglang_speculative_moe_runner_backend ........... None -(yeto-rl-island-0, pid=4051) sglang_speculative_ngram_capacity ............... 10000000 -(yeto-rl-island-0, pid=4051) sglang_speculative_ngram_external_corpus_max_tokens 10000000 -(yeto-rl-island-0, pid=4051) sglang_speculative_ngram_external_corpus_path ... None -(yeto-rl-island-0, pid=4051) sglang_speculative_ngram_external_sam_budget .... 0 -(yeto-rl-island-0, pid=4051) sglang_speculative_ngram_match_type ............. BFS -(yeto-rl-island-0, pid=4051) sglang_speculative_ngram_max_bfs_breadth ........ 10 -(yeto-rl-island-0, pid=4051) sglang_speculative_ngram_max_trie_depth ......... 18 -(yeto-rl-island-0, pid=4051) sglang_speculative_ngram_min_bfs_breadth ........ 1 -(yeto-rl-island-0, pid=4051) sglang_speculative_num_draft_tokens ............. None -(yeto-rl-island-0, pid=4051) sglang_speculative_num_steps .................... None -(yeto-rl-island-0, pid=4051) sglang_speculative_skip_dp_mlp_sync ............. False -(yeto-rl-island-0, pid=4051) sglang_speculative_token_map .................... None -(yeto-rl-island-0, pid=4051) sglang_speculative_use_rejection_sampling ....... False -(yeto-rl-island-0, pid=4051) sglang_ssl_ca_certs ............................. None -(yeto-rl-island-0, pid=4051) sglang_ssl_certfile ............................. None -(yeto-rl-island-0, pid=4051) sglang_ssl_keyfile .............................. None -(yeto-rl-island-0, pid=4051) sglang_ssl_keyfile_password ..................... None -(yeto-rl-island-0, pid=4051) sglang_startup_weight_load_mode ................. serial -(yeto-rl-island-0, pid=4051) sglang_stream_interval .......................... 1 -(yeto-rl-island-0, pid=4051) sglang_stream_response_default_include_usage .... False -(yeto-rl-island-0, pid=4051) sglang_strip_thinking_cache ..................... False -(yeto-rl-island-0, pid=4051) sglang_swa_full_tokens_ratio .................... None -(yeto-rl-island-0, pid=4051) sglang_tbo_token_distribution_threshold ......... 0.48 -(yeto-rl-island-0, pid=4051) sglang_tokenizer_backend ........................ huggingface -(yeto-rl-island-0, pid=4051) sglang_tokenizer_metrics_allowed_custom_labels .. None -(yeto-rl-island-0, pid=4051) sglang_tokenizer_metrics_custom_labels_header ... x-custom-labels -(yeto-rl-island-0, pid=4051) sglang_tokenizer_mode ........................... auto -(yeto-rl-island-0, pid=4051) sglang_tokenizer_path ........................... None -(yeto-rl-island-0, pid=4051) sglang_tokenizer_worker_num ..................... 1 -(yeto-rl-island-0, pid=4051) sglang_tool_call_parser ......................... None -(yeto-rl-island-0, pid=4051) sglang_tool_server .............................. None -(yeto-rl-island-0, pid=4051) sglang_torch_compile_max_bs ..................... 32 -(yeto-rl-island-0, pid=4051) sglang_tp_size .................................. 4 -(yeto-rl-island-0, pid=4051) sglang_trace_modules ............................ request -(yeto-rl-island-0, pid=4051) sglang_triton_attention_num_kv_splits ........... 8 -(yeto-rl-island-0, pid=4051) sglang_triton_attention_reduce_in_fp32 .......... False -(yeto-rl-island-0, pid=4051) sglang_triton_attention_split_tile_size ......... None -(yeto-rl-island-0, pid=4051) sglang_true_on_policy_contract .................. None -(yeto-rl-island-0, pid=4051) sglang_trust_mm_content_hashes .................. False -(yeto-rl-island-0, pid=4051) sglang_unified_cache_external_linker_backend .... mooncake -(yeto-rl-island-0, pid=4051) sglang_uno_lora_path ............................ None -(yeto-rl-island-0, pid=4051) sglang_use_ray .................................. False -(yeto-rl-island-0, pid=4051) sglang_uvicorn_access_log_exclude_prefixes ...... [] -(yeto-rl-island-0, pid=4051) sglang_warmups .................................. None -(yeto-rl-island-0, pid=4051) sglang_watchdog_timeout ......................... 300 -(yeto-rl-island-0, pid=4051) sglang_weight_cache_mode ........................ off -(yeto-rl-island-0, pid=4051) sglang_weight_cache_socket ...................... None -(yeto-rl-island-0, pid=4051) sglang_weight_cache_timeout ..................... 1800 -(yeto-rl-island-0, pid=4051) sglang_weight_loader_disable_mmap ............... False -(yeto-rl-island-0, pid=4051) sglang_weight_loader_drop_cache_after_load ...... False -(yeto-rl-island-0, pid=4051) sglang_weight_loader_prefetch_checkpoints ....... False -(yeto-rl-island-0, pid=4051) sglang_weight_loader_prefetch_num_threads ....... 4 -(yeto-rl-island-0, pid=4051) sglang_weight_version ........................... default -(yeto-rl-island-0, pid=4051) sharp_enabled_group ............................. None -(yeto-rl-island-0, pid=4051) short_seq_prob .................................. 0.1 -(yeto-rl-island-0, pid=4051) skip_actor_forward_only ......................... False -(yeto-rl-island-0, pid=4051) skip_eval_before_train .......................... False -(yeto-rl-island-0, pid=4051) skip_train ...................................... False -(yeto-rl-island-0, pid=4051) skipped_train_samples ........................... 0 -(yeto-rl-island-0, pid=4051) softmax_type .................................... vanilla -(yeto-rl-island-0, pid=4051) spec ............................................ ['miles_plugins.models.qwen3_8_next.qwen3_8_next', 'get_qwen3_8_next_spec'] -(yeto-rl-island-0, pid=4051) special_tokens .................................. None -(yeto-rl-island-0, pid=4051) split ........................................... None -(yeto-rl-island-0, pid=4051) squared_relu .................................... False -(yeto-rl-island-0, pid=4051) start_eval_at_iter .............................. None -(yeto-rl-island-0, pid=4051) start_rollout_id ................................ 0 -(yeto-rl-island-0, pid=4051) start_weight_decay .............................. 0.01 -(yeto-rl-island-0, pid=4051) starts_inference_engines ........................ True -(yeto-rl-island-0, pid=4051) step_batch_size_schedule ........................ None -(yeto-rl-island-0, pid=4051) straggler_ctrlr_port ............................ 65535 -(yeto-rl-island-0, pid=4051) straggler_minmax_count .......................... 1 -(yeto-rl-island-0, pid=4051) stream_optimizer_state_moment_dtype ............. fp32 -(yeto-rl-island-0, pid=4051) stream_optimizer_state_to_disk .................. False -(yeto-rl-island-0, pid=4051) strict_fsdp_dtensor_load ........................ True -(yeto-rl-island-0, pid=4051) suggested_communication_unit_size ............... None -(yeto-rl-island-0, pid=4051) swiglu .......................................... True -(yeto-rl-island-0, pid=4051) swin_backbone_type .............................. tiny -(yeto-rl-island-0, pid=4051) symmetric_ar_type ............................... None -(yeto-rl-island-0, pid=4051) target_modules .................................. model.language_model.layers.*.linear_attn.in_proj_qkv,model.language_model.layers.*.linear_attn.in_proj_z,model.language_model.layers.*.linear_attn.in_proj_b,model.language_model.layers.*.linear_attn.in_proj_a,model.language_model.layers.*.linear_attn.out_proj,model.language_model.layers.*.self_attn.q_proj,model.language_model.layers.*.self_attn.k_proj,model.language_model.layers.*.self_attn.v_proj,model.language_model.layers.*.self_attn.o_proj,model.language_model.layers.*.mlp.experts.gate_up_proj,model.language_model.layers.*.mlp.experts.down_proj,model.language_model.layers.*.mlp.shared_expert.gate_proj,model.language_model.layers.*.mlp.shared_expert.up_proj,model.language_model.layers.*.mlp.shared_expert.down_proj -(yeto-rl-island-0, pid=4051) tb_experiment_name .............................. None -(yeto-rl-island-0, pid=4051) tb_project_name ................................. None -(yeto-rl-island-0, pid=4051) te_precision_config_file ........................ None -(yeto-rl-island-0, pid=4051) te_rng_tracker .................................. False -(yeto-rl-island-0, pid=4051) tensor_model_parallel_size ...................... 2 -(yeto-rl-island-0, pid=4051) tensorboard_dir ................................. None -(yeto-rl-island-0, pid=4051) tensorboard_log_interval ........................ 1 -(yeto-rl-island-0, pid=4051) tensorboard_queue_size .......................... 1000 -(yeto-rl-island-0, pid=4051) test_data_path .................................. None -(yeto-rl-island-0, pid=4051) test_mode ....................................... False -(yeto-rl-island-0, pid=4051) thd_max_packed_sequences ........................ None -(yeto-rl-island-0, pid=4051) thd_tail_padding_policy ......................... None -(yeto-rl-island-0, pid=4051) tiktoken_num_special_tokens ..................... 1000 -(yeto-rl-island-0, pid=4051) tiktoken_pattern ................................ None -(yeto-rl-island-0, pid=4051) timing_log_level ................................ 0 -(yeto-rl-island-0, pid=4051) timing_log_option ............................... minmax -(yeto-rl-island-0, pid=4051) tis_clip ........................................ 2.0 -(yeto-rl-island-0, pid=4051) tis_clip_low .................................... 0 -(yeto-rl-island-0, pid=4051) titles_data_path ................................ None -(yeto-rl-island-0, pid=4051) tito_model ...................................... default -(yeto-rl-island-0, pid=4051) tokenizer_hf_no_include_special_tokens .......... False -(yeto-rl-island-0, pid=4051) tokenizer_hf_no_use_fast ........................ False -(yeto-rl-island-0, pid=4051) tokenizer_model ................................. /mnt/yeto-models/hub/models--CharyZeng--Qwen3.8-Flash-Next-4layer/snapshots/d19a6b60c0df8f90faf92c7c592b37df2e15b060 -(yeto-rl-island-0, pid=4051) tokenizer_sentencepiece_ignore_extra_whitespaces True -(yeto-rl-island-0, pid=4051) tokenizer_sentencepiece_legacy .................. False -(yeto-rl-island-0, pid=4051) tokenizer_type .................................. HuggingFaceTokenizer -(yeto-rl-island-0, pid=4051) tool_key ........................................ tools -(yeto-rl-island-0, pid=4051) torch_fsdp2_reshard_after_forward ............... True -(yeto-rl-island-0, pid=4051) tp_comm_bootstrap_backend ....................... nccl -(yeto-rl-island-0, pid=4051) tp_comm_bulk_dgrad .............................. True -(yeto-rl-island-0, pid=4051) tp_comm_bulk_wgrad .............................. True -(yeto-rl-island-0, pid=4051) tp_comm_overlap ................................. False -(yeto-rl-island-0, pid=4051) tp_comm_overlap_ag .............................. True -(yeto-rl-island-0, pid=4051) tp_comm_overlap_cfg ............................. None -(yeto-rl-island-0, pid=4051) tp_comm_overlap_rs .............................. True -(yeto-rl-island-0, pid=4051) tp_comm_overlap_rs_dgrad ........................ False -(yeto-rl-island-0, pid=4051) tp_comm_split_ag ................................ True -(yeto-rl-island-0, pid=4051) tp_comm_split_rs ................................ True -(yeto-rl-island-0, pid=4051) train_backend ................................... megatron -(yeto-rl-island-0, pid=4051) train_data_path ................................. None -(yeto-rl-island-0, pid=4051) train_env_vars .................................. {} -(yeto-rl-island-0, pid=4051) train_iters ..................................... None -(yeto-rl-island-0, pid=4051) train_memory_margin_bytes ....................... 3221225472 -(yeto-rl-island-0, pid=4051) train_samples ................................... None -(yeto-rl-island-0, pid=4051) train_sync_interval ............................. None -(yeto-rl-island-0, pid=4051) trainer_controller_addrs ........................ None -(yeto-rl-island-0, pid=4051) trainer_heartbeat_checker_failure_threshold ..... 3 -(yeto-rl-island-0, pid=4051) trainer_heartbeat_checker_first_wait ............ 300.0 -(yeto-rl-island-0, pid=4051) trainer_heartbeat_checker_interval .............. 10.0 -(yeto-rl-island-0, pid=4051) trainer_heartbeat_checker_timeout ............... 10.0 -(yeto-rl-island-0, pid=4051) trainer_id ...................................... actor -(yeto-rl-island-0, pid=4051) trainer_model_id ................................ None -(yeto-rl-island-0, pid=4051) transformer_impl ................................ transformer_engine -(yeto-rl-island-0, pid=4051) transformer_pipeline_model_parallel_size ........ 2 -(yeto-rl-island-0, pid=4051) true_on_policy_mode ............................. False -(yeto-rl-island-0, pid=4051) trust_remote_code ............................... True -(yeto-rl-island-0, pid=4051) untie_embeddings_and_output_weights ............. True -(yeto-rl-island-0, pid=4051) update_weight_buffer_size ....................... 1073741824 -(yeto-rl-island-0, pid=4051) update_weight_delta_checksum .................... xxh3-128 -(yeto-rl-island-0, pid=4051) update_weight_delta_encoding .................... xor -(yeto-rl-island-0, pid=4051) update_weight_disk_dir .......................... None -(yeto-rl-island-0, pid=4051) update_weight_group_timeout_s ................... None -(yeto-rl-island-0, pid=4051) update_weight_local_checkpoint_dir .............. None -(yeto-rl-island-0, pid=4051) update_weight_transfer_mode ..................... broadcast -(yeto-rl-island-0, pid=4051) update_weights_interval ......................... 1 -(yeto-rl-island-0, pid=4051) use_checkpoint_args ............................. False -(yeto-rl-island-0, pid=4051) use_checkpoint_opt_param_scheduler .............. False -(yeto-rl-island-0, pid=4051) use_cpu_initialization .......................... None -(yeto-rl-island-0, pid=4051) use_critic ...................................... False -(yeto-rl-island-0, pid=4051) use_dist_ckpt ................................... True -(yeto-rl-island-0, pid=4051) use_dist_ckpt_deprecated ........................ False -(yeto-rl-island-0, pid=4051) use_distributed_optimizer ....................... True -(yeto-rl-island-0, pid=4051) use_distributed_post ............................ False -(yeto-rl-island-0, pid=4051) use_dynamic_batch_size .......................... False -(yeto-rl-island-0, pid=4051) use_dynamic_global_batch_size ................... False -(yeto-rl-island-0, pid=4051) use_fault_tolerance ............................. False -(yeto-rl-island-0, pid=4051) use_flash_attn .................................. False -(yeto-rl-island-0, pid=4051) use_fused_mhc ................................... False -(yeto-rl-island-0, pid=4051) use_fused_weighted_squared_relu ................. False -(yeto-rl-island-0, pid=4051) use_gloo_process_groups ......................... True -(yeto-rl-island-0, pid=4051) use_grouped_gemm_for_dense_mlp .................. False -(yeto-rl-island-0, pid=4051) use_grouped_gemm_for_shared_expert .............. False -(yeto-rl-island-0, pid=4051) use_indexer_replay .............................. False -(yeto-rl-island-0, pid=4051) use_kitchen_attention ........................... False -(yeto-rl-island-0, pid=4051) use_kl_loss ..................................... False -(yeto-rl-island-0, pid=4051) use_layer_wise_distributed_optimizer ............ False -(yeto-rl-island-0, pid=4051) use_layer_wise_param_layout ..................... False -(yeto-rl-island-0, pid=4051) use_legacy_static_engine ........................ False -(yeto-rl-island-0, pid=4051) use_mamba_mem_eff_path .......................... True -(yeto-rl-island-0, pid=4051) use_megatron_fsdp ............................... False -(yeto-rl-island-0, pid=4051) use_miles_dashboard ............................. False -(yeto-rl-island-0, pid=4051) use_miles_router ................................ False -(yeto-rl-island-0, pid=4051) use_mlflow ...................................... False -(yeto-rl-island-0, pid=4051) use_mp_args_from_checkpoint_args ................ False -(yeto-rl-island-0, pid=4051) use_mup ......................................... False -(yeto-rl-island-0, pid=4051) use_one_sent_docs ............................... False -(yeto-rl-island-0, pid=4051) use_opd ......................................... False -(yeto-rl-island-0, pid=4051) use_opsm ........................................ False -(yeto-rl-island-0, pid=4051) use_persistent_ckpt_worker ...................... False -(yeto-rl-island-0, pid=4051) use_precision_aware_optimizer ................... False -(yeto-rl-island-0, pid=4051) use_prometheus .................................. False -(yeto-rl-island-0, pid=4051) use_pytorch_profiler ............................ False -(yeto-rl-island-0, pid=4051) use_ring_exchange_p2p ........................... False -(yeto-rl-island-0, pid=4051) use_rollout_entropy ............................. False -(yeto-rl-island-0, pid=4051) use_rollout_indexer_replay ...................... False -(yeto-rl-island-0, pid=4051) use_rollout_logprobs ............................ False -(yeto-rl-island-0, pid=4051) use_rollout_routing_replay ...................... False -(yeto-rl-island-0, pid=4051) use_rope_scaling ................................ False -(yeto-rl-island-0, pid=4051) use_rotary_position_embeddings .................. False -(yeto-rl-island-0, pid=4051) use_routing_replay .............................. False -(yeto-rl-island-0, pid=4051) use_sampling_support_replay ..................... False -(yeto-rl-island-0, pid=4051) use_session_server .............................. False -(yeto-rl-island-0, pid=4051) use_sharp ....................................... False -(yeto-rl-island-0, pid=4051) use_te_activation_func .......................... False -(yeto-rl-island-0, pid=4051) use_tensorboard ................................. False -(yeto-rl-island-0, pid=4051) use_tis ......................................... False -(yeto-rl-island-0, pid=4051) use_tokenizer_model_from_checkpoint_args ........ True -(yeto-rl-island-0, pid=4051) use_torch_fsdp2 ................................. False -(yeto-rl-island-0, pid=4051) use_torch_optimizer_for_cpu_offload ............. False -(yeto-rl-island-0, pid=4051) use_tp_pp_dp_mapping ............................ False -(yeto-rl-island-0, pid=4051) use_transformer_engine_op_fuser ................. False -(yeto-rl-island-0, pid=4051) use_unbiased_kl ................................. False -(yeto-rl-island-0, pid=4051) use_varlen_dataset .............................. False -(yeto-rl-island-0, pid=4051) use_wandb ....................................... False -(yeto-rl-island-0, pid=4051) v_head_dim ...................................... 128 -(yeto-rl-island-0, pid=4051) valid_data_path ................................. None -(yeto-rl-island-0, pid=4051) validation_set_names ............................ None -(yeto-rl-island-0, pid=4051) value_clip ...................................... 0.2 -(yeto-rl-island-0, pid=4051) varlen_mock_dataset_config_json ................. None -(yeto-rl-island-0, pid=4051) varlen_sbhd_validation .......................... False -(yeto-rl-island-0, pid=4051) verify_integrity ................................ False -(yeto-rl-island-0, pid=4051) virtual_pipeline_model_parallel_size ............ None -(yeto-rl-island-0, pid=4051) vision_backbone_type ............................ vit -(yeto-rl-island-0, pid=4051) vision_pretraining .............................. False -(yeto-rl-island-0, pid=4051) vision_pretraining_type ......................... classify -(yeto-rl-island-0, pid=4051) vocab_extra_ids ................................. 0 -(yeto-rl-island-0, pid=4051) vocab_file ...................................... None -(yeto-rl-island-0, pid=4051) vocab_size ...................................... 248320 -(yeto-rl-island-0, pid=4051) wandb_always_use_train_step ..................... False -(yeto-rl-island-0, pid=4051) wandb_dir ....................................... None -(yeto-rl-island-0, pid=4051) wandb_entity .................................... None -(yeto-rl-island-0, pid=4051) wandb_exp_name .................................. None -(yeto-rl-island-0, pid=4051) wandb_group ..................................... None -(yeto-rl-island-0, pid=4051) wandb_host ...................................... None -(yeto-rl-island-0, pid=4051) wandb_key ....................................... None -(yeto-rl-island-0, pid=4051) wandb_mode ...................................... None -(yeto-rl-island-0, pid=4051) wandb_project ................................... None -(yeto-rl-island-0, pid=4051) wandb_random_suffix ............................. True -(yeto-rl-island-0, pid=4051) wandb_run_id .................................... None -(yeto-rl-island-0, pid=4051) wandb_save_dir .................................. None -(yeto-rl-island-0, pid=4051) wandb_team ...................................... None -(yeto-rl-island-0, pid=4051) weight_decay .................................... 0.01 -(yeto-rl-island-0, pid=4051) weight_decay_incr_style ......................... constant -(yeto-rl-island-0, pid=4051) wgrad_deferral_limit ............................ 0 -(yeto-rl-island-0, pid=4051) window_attn_skip_freq ........................... None -(yeto-rl-island-0, pid=4051) window_size ..................................... None -(yeto-rl-island-0, pid=4051) witness_buffer_size ............................. 1048576 -(yeto-rl-island-0, pid=4051) worker_comm_backend ............................. ray -(yeto-rl-island-0, pid=4051) worker_dynamic_port_start ....................... 20000 -(yeto-rl-island-0, pid=4051) worker_urls ..................................... [] -(yeto-rl-island-0, pid=4051) world_size ...................................... 8 -(yeto-rl-island-0, pid=4051) yaml_cfg ........................................ None -(yeto-rl-island-0, pid=4051) yarn_beta_fast .................................. None -(yeto-rl-island-0, pid=4051) yarn_beta_slow .................................. None -(yeto-rl-island-0, pid=4051) yarn_correction_range_round_to_int .............. None -(yeto-rl-island-0, pid=4051) yarn_original_max_position_embeddings ........... None -(yeto-rl-island-0, pid=4051) yeto_placement_map .............................. None -(yeto-rl-island-0, pid=4051) yeto_rl_dynamic_sampling_max_replacements ....... None -(yeto-rl-island-0, pid=4051) -------------------- end of arguments --------------------- -(yeto-rl-island-0, pid=4051) [rl] LoRA layout: learned from the first qwen3_8_next native export (no predicted specs); targets=['down_proj', 'gate_proj', 'gate_up_proj', 'in_proj_a', 'in_proj_b', 'in_proj_qkv', 'in_proj_z', 'k_proj', 'o_proj', 'out_proj', 'q_proj', 'up_proj', 'v_proj'] -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:30,657 INFO worker.py:1833 -- Connecting to existing Ray cluster at address: 10.0.0.8:6379... -(yeto-rl-island-0, pid=4051) 2026-10-06 11:10:30,680 INFO worker.py:2015 -- Connected to Ray cluster. View the dashboard at http://127.0.0.1:8265  -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_engine_selected","island_id":0,"miles_commit":"c35702eefcf2862cee155e46870e6ad30568d2c6","placement":"colocated","rl/algorithm_absorbed_flags":{},"rl/algorithm_spec":"{\"advantage_estimator\":\"grpo\",\"dynamic_sampling_filter\":null,\"dynamic_sampling_max_replacements\":null,\"kl_coef\":null,\"loss\":\"policy_loss\",\"schema\":\"yeto-rl-algorithm-spec-v1\"}","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","rl/outer_sync":false,"rl_engine":"ports","time_unix":1791285031.9600205} -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:31.971 main] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=124 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:31.980 main] placement_group.py:311 - Creating placement group with 8 GPUs... -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:32.857 main] placement_group.py:118 - bundle 0, actual_bundle_index: 0, node: 10.0.0.8, gpu: 0 -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:32.857 main] placement_group.py:118 - bundle 1, actual_bundle_index: 1, node: 10.0.0.8, gpu: 1 -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:32.857 main] placement_group.py:118 - bundle 2, actual_bundle_index: 2, node: 10.0.0.8, gpu: 2 -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:32.857 main] placement_group.py:118 - bundle 3, actual_bundle_index: 3, node: 10.0.0.8, gpu: 3 -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:32.857 main] placement_group.py:118 - bundle 4, actual_bundle_index: 4, node: 10.0.0.8, gpu: 4 -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:32.857 main] placement_group.py:118 - bundle 5, actual_bundle_index: 5, node: 10.0.0.8, gpu: 5 -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:32.857 main] placement_group.py:118 - bundle 6, actual_bundle_index: 6, node: 10.0.0.8, gpu: 6 -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:32.857 main] placement_group.py:118 - bundle 7, actual_bundle_index: 7, node: 10.0.0.8, gpu: 7 -(yeto-rl-island-0, pid=4051) [2026-10-06 11:10:34.714 main] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=124 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (pid=26006) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26006) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26006) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=26006) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=26006) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:10:50.193 worker_manager] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=127 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:10:52.467 worker_manager] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=127 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (pid=26750) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26750) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26748) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26748) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26749) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26749) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26751) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26751) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26746) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26746) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26750) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=26752) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26752) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26748) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=26749) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=26751) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=26746) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=26750) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=26748) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=27296) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=27296) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26749) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=26752) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=26751) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=26750) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (pid=26746) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=26749) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (pid=26746) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (pid=26751) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (pid=26752) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=27296) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=26748) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (pid=27297) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=27297) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=26752) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (pid=27299) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=27299) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=27302) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=27302) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=27305) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=27305) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=27296) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=27304) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=27304) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=27303) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=27303) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:11:09.888 rollout_executor] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=127 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (pid=27297) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=27296) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (pid=27300) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=27300) Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers. -(yeto-rl-island-0, pid=4051) (pid=27299) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=27302) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=27303) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=27305) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=27304) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=27297) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=27299) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=27305) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=27302) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=27304) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=27303) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (pid=27297) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (pid=27300) LANG_RL Log directory: None -(yeto-rl-island-0, pid=4051) (pid=27299) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (pid=27302) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (pid=27303) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (pid=27305) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (pid=27304) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (pid=27300) WARNING:megatron.bridge.models.muse_glimmer:Muse Glimmer Hugging Face classes are unavailable with transformers 5.12.1. Upgrade transformers to a version that supports Muse Glimmer. -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:11:12.060 rollout_executor] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=127 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:12.246 actor_cell00000_rank00000] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=146 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (pid=27300) WARNING:miles_plugins.models.qwen3_vl:megatron-bridge Qwen3VLModel lacks native CP-local vision-embed selection; CP runs with vision tokens will mis-place vision embeddings. Apply the matching Megatron-Bridge patch (radixark/Megatron-Bridge PR #9). -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:11:13.284 actor_cell00000_rank00001] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=146 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:11:13.464 actor_cell00000_rank00003] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=146 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:11:13.481 actor_cell00000_rank00004] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=146 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:11:13.449 actor_cell00000_rank00007] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=146 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:11:13.578 actor_cell00000_rank00005] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=146 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:11:13.669 actor_cell00000_rank00006] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=146 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:14.401 actor_cell00000_rank00000] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=146 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:11:14.476 worker_manager] inference.py:186 - Launch router with args: {'worker_urls': [], 'policy': 'cache_aware', 'prefill_policy': None, 'decode_policy': None, 'cache_threshold': 0.3, 'balance_abs_threshold': 64, 'balance_rel_threshold': 1.5, 'bucket_adjust_interval_secs': 5, 'eviction_interval_secs': 60, 'max_tree_size': 67108864, 'max_idle_secs': 14400, 'assignment_mode': 'random', 'max_payload_size': 536870912, 'dp_aware': False, 'enable_igw': False, 'allow_requests_without_routing_key': False, 'mini_lb': False, 'test_external_dp_routing': False, 'pd_disaggregation': False, 'prefill': None, 'decode': None, 'worker_startup_timeout_secs': 1800, 'worker_startup_check_interval': 30, 'log_dir': None, 'log_level': 'warn', 'json_log': False, 'service_discovery': False, 'selector': {}, 'service_discovery_port': 80, 'service_discovery_namespace': None, 'prefill_selector': {}, 'decode_selector': {}, 'prometheus_port': 20036, 'prometheus_host': '0.0.0.0', 'prometheus_duration_buckets': None, 'request_id_headers': None, 'request_timeout_secs': 14400, 'shutdown_grace_period_secs': 180, 'cors_allowed_origins': [], 'max_concurrent_requests': -1, 'queue_size': 100, 'queue_timeout_secs': 60, 'rate_limit_tokens_per_second': None, 'retry_max_retries': 5, 'retry_initial_backoff_ms': 50, 'retry_max_backoff_ms': 30000, 'retry_backoff_multiplier': 1.5, 'retry_jitter_factor': 0.2, 'disable_retries': False, 'cb_failure_threshold': 10, 'cb_success_threshold': 3, 'cb_timeout_duration_secs': 60, 'cb_window_duration_secs': 120, 'disable_circuit_breaker': False, 'health_failure_threshold': 40, 'health_success_threshold': 1, 'health_check_timeout_secs': 5, 'health_check_interval_secs': 15, 'health_check_endpoint': '/health', 'disable_health_check': False, 'model_path': None, 'tokenizer_path': None, 'chat_template': None, 'tokenizer_cache_enable_l0': False, 'tokenizer_cache_l0_max_entries': 10000, 'tokenizer_cache_enable_l1': False, 'tokenizer_cache_l1_max_memory': 52428800, 'reasoning_parser': None, 'tool_call_parser': None, 'mcp_config_path': None, 'backend': 'sglang', 'history_backend': 'memory', 'oracle_wallet_path': None, 'oracle_tns_alias': None, 'oracle_connect_descriptor': None, 'oracle_username': None, 'oracle_password': None, 'oracle_pool_min': 1, 'oracle_pool_max': 16, 'oracle_pool_timeout_secs': 30, 'postgres_db_url': None, 'postgres_pool_max': 16, 'redis_url': None, 'redis_pool_max': 16, 'redis_retention_days': 30, 'client_cert_path': None, 'client_key_path': None, 'ca_cert_paths': [], 'tls_cert_path': None, 'tls_key_path': None, 'enable_trace': False, 'otlp_traces_endpoint': 'localhost:4317', 'api_key': None, 'control_plane_api_keys': [], 'control_plane_audit_enabled': False, 'jwt_issuer': None, 'jwt_audience': None, 'jwt_jwks_uri': None, 'jwt_role_mapping': [], 'host': '10.0.0.8', 'port': 20035} -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:11:14.480 worker_manager] sglang_engine.py:169 - No pre-trained LoRA adapter_path provided, will use random initial weights -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:11:14.512 worker_manager] sglang_engine.py:169 - No pre-trained LoRA adapter_path provided, will use random initial weights -(yeto-rl-island-0, pid=4051) (CommandActor pid=26749) [process_trampoline pid=28481] bound to parent 26749; exec ['/bin/sh', '-c', 'exec /opt/sglang/bin/python3 -m sglang_router.launch_router --log-level warn --prometheus-port 20036 --request-timeout-secs 14400 --health-failure-threshold 40 --health-success-threshold 1 --health-check-interval-secs 15 --host 10.0.0.8 --port 20035'] -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [process_trampoline pid=28483] bound to parent 26750; exec ['/bin/sh', '-c', "exec /opt/sglang/bin/python3 -m sglang.launch_server --trust-remote-code --model-path /mnt/yeto-models/hub/models--CharyZeng--Qwen3.8-Flash-Next-4layer/snapshots/d19a6b60c0df8f90faf92c7c592b37df2e15b060 --host 10.0.0.8 --port 20000 --device cuda --random-seed 17 --enable-memory-saver --nccl-port 20032 --dist-init-addr 10.0.0.8:20001 --gated-launch-port 20034 --tp-size 4 --ep-size 4 --skip-server-warmup --enable-draft-weights-cpu-backup --enable-metrics --engine-info-bootstrap-port 20033 --enable-lora --max-loras-per-batch 1 --max-lora-rank 16 --lora-target-modules 'model.language_model.layers.*.self_attn.q_proj' 'model.language_model.layers.*.self_attn.k_proj' 'model.language_model.layers.*.self_attn.v_proj' 'model.language_model.layers.*.self_attn.o_proj' 'model.language_model.layers.*.linear_attn.in_proj_qkv' 'model.language_model.layers.*.linear_attn.in_proj_z' 'model.language_model.layers.*.linear_attn.in_proj_b' 'model.language_model.layers.*.linear_attn.in_proj_a' 'model.language_model.layers.*.linear_attn.out_proj' 'model.language_model.layers.*.mlp.experts.gate_up_proj' 'model.language_model.layers.*.mlp.experts.down_proj' 'model.language_model.layers.*.mlp.shared_expert.gate_proj' 'model.language_model.layers.*.mlp.shared_expert.up_proj' 'model.language_model.layers.*.mlp.shared_expert.down_proj' --mem-fraction-static 0.7 --chunked-prefill-size 8192 --disable-radix-cache --cuda-graph-backend-prefill disabled --moe-runner-backend triton --linear-attn-prefill-backend flashinfer --lora-backend triton --lora-use-virtual-experts --lora-strict-loading"] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:11:14.462 actor_cell00000_rank00002] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=146 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [process_trampoline pid=28485] bound to parent 26751; exec ['/bin/sh', '-c', "exec /opt/sglang/bin/python3 -m sglang.launch_server --trust-remote-code --model-path /mnt/yeto-models/hub/models--CharyZeng--Qwen3.8-Flash-Next-4layer/snapshots/d19a6b60c0df8f90faf92c7c592b37df2e15b060 --host 10.0.0.8 --port 20037 --device cuda --random-seed 18 --enable-memory-saver --nccl-port 20069 --dist-init-addr 10.0.0.8:20038 --base-gpu-id 4 --gated-launch-port 20071 --tp-size 4 --ep-size 4 --skip-server-warmup --enable-draft-weights-cpu-backup --enable-metrics --engine-info-bootstrap-port 20070 --enable-lora --max-loras-per-batch 1 --max-lora-rank 16 --lora-target-modules 'model.language_model.layers.*.self_attn.q_proj' 'model.language_model.layers.*.self_attn.k_proj' 'model.language_model.layers.*.self_attn.v_proj' 'model.language_model.layers.*.self_attn.o_proj' 'model.language_model.layers.*.linear_attn.in_proj_qkv' 'model.language_model.layers.*.linear_attn.in_proj_z' 'model.language_model.layers.*.linear_attn.in_proj_b' 'model.language_model.layers.*.linear_attn.in_proj_a' 'model.language_model.layers.*.linear_attn.out_proj' 'model.language_model.layers.*.mlp.experts.gate_up_proj' 'model.language_model.layers.*.mlp.experts.down_proj' 'model.language_model.layers.*.mlp.shared_expert.gate_proj' 'model.language_model.layers.*.mlp.shared_expert.up_proj' 'model.language_model.layers.*.mlp.shared_expert.down_proj' --mem-fraction-static 0.7 --chunked-prefill-size 8192 --disable-radix-cache --cuda-graph-backend-prefill disabled --moe-runner-backend triton --linear-attn-prefill-backend flashinfer --lora-backend triton --lora-use-virtual-experts --lora-strict-loading"] -(yeto-rl-island-0, pid=4051) [2026-10-06 11:11:15.057 main] router_manager.py:64 - Router ready at host='10.0.0.8' port=20035 external_host=None -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:11:15.091 inference_controller] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=127 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:11:15.112 rollout_executor] router_manager.py:64 - Router ready at host='10.0.0.8' port=20035 external_host=None -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:11:15.103 inference_controller] router_manager.py:64 - Router ready at host='10.0.0.8' port=20035 external_host=None -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:11:15.456 actor_cell00000_rank00001] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=146 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:11:15.662 actor_cell00000_rank00003] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=146 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:11:15.665 actor_cell00000_rank00004] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=146 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:11:15.659 actor_cell00000_rank00007] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=146 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:11:15.767 actor_cell00000_rank00005] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=146 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:11:15.871 actor_cell00000_rank00006] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=146 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:11:16.711 actor_cell00000_rank00002] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=146 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:11:17.219 inference_controller] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=127 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:11:19.116 rollout_executor] data.py:133 - Filtered 0 samples longer than max_length=2047. -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:11:19.714 rollout_executor] rollout_executor.py:117 - import miles.rollout.inference_rollout.inference_rollout_common.InferenceRolloutFn as generate_rollout function. -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:11:19.714 rollout_executor] rollout_executor.py:118 - import miles.rollout.inference_rollout.inference_rollout_common.InferenceRolloutFn as eval_generate_rollout function. -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:11:19.751 trainer_controller_actor] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=127 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) Using blocking ray.get inside async actor. This blocks the event loop. Please use `await` on object ref with asyncio.gather if you want to yield execution to the event loop instead. -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:11:20.846 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=configure_master_addr_and_port n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:11:20.851 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=configure_master_addr_and_port ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:11:20.851 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=init n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:11:20.855 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=init phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:11:20.856 actor_cell00000_rank00001] reloadable_process_group.py:21 - Applying monkey patch to torch.distributed -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:20.854 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=init phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:20.855 actor_cell00000_rank00000] reloadable_process_group.py:21 - Applying monkey patch to torch.distributed -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:11:20.857 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=init phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:11:20.857 actor_cell00000_rank00003] reloadable_process_group.py:21 - Applying monkey patch to torch.distributed -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:11:20.856 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=init phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:11:20.856 actor_cell00000_rank00002] reloadable_process_group.py:21 - Applying monkey patch to torch.distributed -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:11:20.857 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=init phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:11:20.857 actor_cell00000_rank00004] reloadable_process_group.py:21 - Applying monkey patch to torch.distributed -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:11:20.858 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=init phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:11:20.858 actor_cell00000_rank00006] reloadable_process_group.py:21 - Applying monkey patch to torch.distributed -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:11:20.858 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=init phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:11:20.858 actor_cell00000_rank00005] reloadable_process_group.py:21 - Applying monkey patch to torch.distributed -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:11:20.859 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=init phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:11:20.859 actor_cell00000_rank00007] reloadable_process_group.py:21 - Applying monkey patch to torch.distributed -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:11:21.902 trainer_controller_actor] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=127 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:11:21.994 actor_cell00000_rank00001] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=148 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:21.994 actor_cell00000_rank00000] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=148 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:11:21.994 actor_cell00000_rank00003] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=148 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:11:21.994 actor_cell00000_rank00002] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=148 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:11:21.994 actor_cell00000_rank00004] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=148 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:11:21.994 actor_cell00000_rank00006] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=148 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:11:21.995 actor_cell00000_rank00005] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=148 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:11:21.994 actor_cell00000_rank00007] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{}" num_packages=0 num_env_vars=148 packages_probed=false stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:11:22.625 actor_cell00000_rank00001] train_actor.py:155 - Set NUMA affinity for GPU 1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:22.626 actor_cell00000_rank00000] train_actor.py:155 - Set NUMA affinity for GPU 0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:11:22.603 actor_cell00000_rank00003] train_actor.py:155 - Set NUMA affinity for GPU 3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:11:22.613 actor_cell00000_rank00002] train_actor.py:155 - Set NUMA affinity for GPU 2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:11:22.586 actor_cell00000_rank00004] train_actor.py:155 - Set NUMA affinity for GPU 4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:11:22.621 actor_cell00000_rank00006] train_actor.py:155 - Set NUMA affinity for GPU 6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:11:22.623 actor_cell00000_rank00005] train_actor.py:155 - Set NUMA affinity for GPU 5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:11:22.626 actor_cell00000_rank00007] train_actor.py:155 - Set NUMA affinity for GPU 7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:22.724 actor_cell00000_rank00000] initialize.py:46 - > setting random seeds to 17 ... -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:23] Setting page size to 64 for compressed QSA (full//ratio compressed addressing). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:23] Auto-enabling FlashInfer AllReduce Fusion on SM90/SM10X for Qwen4ExpForConditionalGeneration -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:23] Attention backend not specified. Use fa3 backend by default. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:23] Setting page size to 64 for compressed QSA (full//ratio compressed addressing). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:23] Auto-enabling FlashInfer AllReduce Fusion on SM90/SM10X for Qwen4ExpForConditionalGeneration -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:23] Attention backend not specified. Use fa3 backend by default. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:26] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/root/.cache/sglang). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.triton, /root/.cache/flashinfer. Remove them yourself if nothing else needs them. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:26] Virtual expert computation enabled. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:26] server_args={'model_path': '/mnt/yeto-models/hub/models--CharyZeng--Qwen3.8-Flash-Next-4layer/snapshots/d19a6b60c0df8f90faf92c7c592b37df2e15b060', 'tokenizer_path': '/mnt/yeto-models/hub/models--CharyZeng--Qwen3.8-Flash-Next-4layer/snapshots/d19a6b60c0df8f90faf92c7c592b37df2e15b060', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'auto', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.7, 'max_running_requests': None, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'radix_eviction_policy_config': None, 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': False, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': 20069, 'dist_timeout': None, 'dist_init_addr': '10.0.0.8:20038', 'gated_launch_port': 20071, 'nnodes': 1, 'node_rank': 0, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 1, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'round_robin', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'enable_layernorm_sp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 4, 'gpu_id_step': 1, 'random_seed': 18, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '10.0.0.8', 'port': 20037, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': True, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': '/mnt/yeto-models/hub/models--CharyZeng--Qwen3.8-Flash-Next-4layer/snapshots/d19a6b60c0df8f90faf92c7c592b37df2e15b060', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'return_input_ids': False, 'return_output_ids': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'fa3', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'full', 'max_bs': 512, 'bs': [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'disabled', 'max_bs': 8192, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': 'disabled', 'cuda_graph_max_bs_decode': None, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': 'auto', 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': None, 'uno_lora_path': None, 'speculative_draft_model_path': None, 'speculative_draft_model_revision': None, 'speculative_draft_load_format': None, 'speculative_num_steps': None, 'speculative_eagle_topk': None, 'speculative_num_draft_tokens': None, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': None, 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': None, 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'triton', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'triton', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.9, 'mamba_radix_cache_strategy': 'auto', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': 'flashinfer', 'linear_attn_verify_backend': None, 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 8, 'hicache_storage_prefetch_retry_max_attempts': 8, 'enable_unified_cache_external_linker': False, 'unified_cache_external_linker_backend': 'mooncake', 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': True, 'enable_lora_overlap_loading': False, 'max_lora_rank': 16, 'lora_target_modules': {'model.language_model.layers.*.mlp.experts.gate_up_proj', 'model.language_model.layers.*.linear_attn.in_proj_a', 'model.language_model.layers.*.self_attn.q_proj', 'model.language_model.layers.*.self_attn.o_proj', 'model.language_model.layers.*.self_attn.k_proj', 'model.language_model.layers.*.linear_attn.in_proj_z', 'model.language_model.layers.*.self_attn.v_proj', 'model.language_model.layers.*.linear_attn.in_proj_qkv', 'model.language_model.layers.*.linear_attn.out_proj', 'model.language_model.layers.*.mlp.shared_expert.down_proj', 'model.language_model.layers.*.mlp.shared_expert.up_proj', 'model.language_model.layers.*.linear_attn.in_proj_b', 'model.language_model.layers.*.mlp.shared_expert.gate_proj', 'model.language_model.layers.*.mlp.experts.down_proj'}, 'lora_paths': [], 'max_loaded_loras': None, 'max_loras_per_batch': 1, 'lora_eviction_policy': 'lru', 'lora_backend': 'triton', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': True, 'lora_strict_loading': True, 'lora_no_cpu_backup': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'null', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 8998, 'disaggregation_ib_device': None, 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_scheduler', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 20070, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': True, 'enable_weights_cpu_backup': False, 'enable_weights_disk_backup': False, 'enable_draft_weights_cpu_backup': True, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'sampling_mask_max_tokens': 4096, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None, 'custom_pull_weights_pre_read_hook': None, 'enable_engine_info_bootstrap': False, 'enable_rdt_weight_sync': False, 'enable_prefill_only_deterministic_inference': False, 'true_on_policy_contract': None, 'ple_offload_embedding': True, 'ple_offload_backend': 'pinned', 'ple_offload_dir': None, 'speculative_domino_candidate_pool_size': 2048, 'flashinfer_a2a_dispatch_type': None, 'disable_draft_cuda_graph': False, 'dsv4_attn_backend': 'auto', '_radix_eviction_policy_explicitly_set': False, 'enable_response_store': False} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:26] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:26] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:26] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:26] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:26] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/root/.cache/sglang). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.triton, /root/.cache/flashinfer. Remove them yourself if nothing else needs them. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:26] Virtual expert computation enabled. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:26] server_args={'model_path': '/mnt/yeto-models/hub/models--CharyZeng--Qwen3.8-Flash-Next-4layer/snapshots/d19a6b60c0df8f90faf92c7c592b37df2e15b060', 'tokenizer_path': '/mnt/yeto-models/hub/models--CharyZeng--Qwen3.8-Flash-Next-4layer/snapshots/d19a6b60c0df8f90faf92c7c592b37df2e15b060', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'auto', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.7, 'max_running_requests': None, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'radix_eviction_policy_config': None, 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': False, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': 20032, 'dist_timeout': None, 'dist_init_addr': '10.0.0.8:20001', 'gated_launch_port': 20034, 'nnodes': 1, 'node_rank': 0, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 1, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'round_robin', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'enable_layernorm_sp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 17, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '10.0.0.8', 'port': 20000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': True, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': '/mnt/yeto-models/hub/models--CharyZeng--Qwen3.8-Flash-Next-4layer/snapshots/d19a6b60c0df8f90faf92c7c592b37df2e15b060', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'return_input_ids': False, 'return_output_ids': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': False, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'fa3', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'full', 'max_bs': 512, 'bs': [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'disabled', 'max_bs': 8192, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': 'disabled', 'cuda_graph_max_bs_decode': None, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': 'auto', 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': None, 'uno_lora_path': None, 'speculative_draft_model_path': None, 'speculative_draft_model_revision': None, 'speculative_draft_load_format': None, 'speculative_num_steps': None, 'speculative_eagle_topk': None, 'speculative_num_draft_tokens': None, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': None, 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': None, 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'triton', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'triton', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.9, 'mamba_radix_cache_strategy': 'auto', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': 'flashinfer', 'linear_attn_verify_backend': None, 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 8, 'hicache_storage_prefetch_retry_max_attempts': 8, 'enable_unified_cache_external_linker': False, 'unified_cache_external_linker_backend': 'mooncake', 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': True, 'enable_lora_overlap_loading': False, 'max_lora_rank': 16, 'lora_target_modules': {'model.language_model.layers.*.self_attn.o_proj', 'model.language_model.layers.*.linear_attn.in_proj_b', 'model.language_model.layers.*.mlp.shared_expert.down_proj', 'model.language_model.layers.*.self_attn.k_proj', 'model.language_model.layers.*.mlp.shared_expert.gate_proj', 'model.language_model.layers.*.self_attn.q_proj', 'model.language_model.layers.*.mlp.experts.gate_up_proj', 'model.language_model.layers.*.mlp.experts.down_proj', 'model.language_model.layers.*.mlp.shared_expert.up_proj', 'model.language_model.layers.*.self_attn.v_proj', 'model.language_model.layers.*.linear_attn.out_proj', 'model.language_model.layers.*.linear_attn.in_proj_a', 'model.language_model.layers.*.linear_attn.in_proj_z', 'model.language_model.layers.*.linear_attn.in_proj_qkv'}, 'lora_paths': [], 'max_loaded_loras': None, 'max_loras_per_batch': 1, 'lora_eviction_policy': 'lru', 'lora_backend': 'triton', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': True, 'lora_strict_loading': True, 'lora_no_cpu_backup': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'null', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 8998, 'disaggregation_ib_device': None, 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_scheduler', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 20033, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': True, 'enable_weights_cpu_backup': False, 'enable_weights_disk_backup': False, 'enable_draft_weights_cpu_backup': True, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'sampling_mask_max_tokens': 4096, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None, 'custom_pull_weights_pre_read_hook': None, 'enable_engine_info_bootstrap': False, 'enable_rdt_weight_sync': False, 'enable_prefill_only_deterministic_inference': False, 'true_on_policy_contract': None, 'ple_offload_embedding': True, 'ple_offload_backend': 'pinned', 'ple_offload_dir': None, 'speculative_domino_candidate_pool_size': 2048, 'flashinfer_a2a_dispatch_type': None, 'disable_draft_cuda_graph': False, 'dsv4_attn_backend': 'auto', '_radix_eviction_policy_explicitly_set': False, 'enable_response_store': False} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:26] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:26] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:26] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:26] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:27] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:27] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:27] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:27] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:11:31.629 actor_cell00000_rank00001] actor.py:184 - Set torch_memory_saver.memory_margin_bytes to 3221225472 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:11:31.631 actor_cell00000_rank00001] utils.py:122 - Patched _ParamAndGradBuffer.__init__ for LoRA colocate mode (disable cpu backup) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:31.629 actor_cell00000_rank00000] actor.py:184 - Set torch_memory_saver.memory_margin_bytes to 3221225472 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:31.631 actor_cell00000_rank00000] utils.py:122 - Patched _ParamAndGradBuffer.__init__ for LoRA colocate mode (disable cpu backup) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:11:31.629 actor_cell00000_rank00003] actor.py:184 - Set torch_memory_saver.memory_margin_bytes to 3221225472 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:11:31.632 actor_cell00000_rank00003] utils.py:122 - Patched _ParamAndGradBuffer.__init__ for LoRA colocate mode (disable cpu backup) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:11:31.629 actor_cell00000_rank00002] actor.py:184 - Set torch_memory_saver.memory_margin_bytes to 3221225472 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:11:31.631 actor_cell00000_rank00002] utils.py:122 - Patched _ParamAndGradBuffer.__init__ for LoRA colocate mode (disable cpu backup) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:11:31.629 actor_cell00000_rank00004] actor.py:184 - Set torch_memory_saver.memory_margin_bytes to 3221225472 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:11:31.632 actor_cell00000_rank00004] utils.py:122 - Patched _ParamAndGradBuffer.__init__ for LoRA colocate mode (disable cpu backup) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:11:31.629 actor_cell00000_rank00006] actor.py:184 - Set torch_memory_saver.memory_margin_bytes to 3221225472 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:11:31.632 actor_cell00000_rank00006] utils.py:122 - Patched _ParamAndGradBuffer.__init__ for LoRA colocate mode (disable cpu backup) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:11:31.630 actor_cell00000_rank00005] actor.py:184 - Set torch_memory_saver.memory_margin_bytes to 3221225472 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:11:31.637 actor_cell00000_rank00005] utils.py:122 - Patched _ParamAndGradBuffer.__init__ for LoRA colocate mode (disable cpu backup) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:11:31.629 actor_cell00000_rank00007] actor.py:184 - Set torch_memory_saver.memory_margin_bytes to 3221225472 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:11:31.632 actor_cell00000_rank00007] utils.py:122 - Patched _ParamAndGradBuffer.__init__ for LoRA colocate mode (disable cpu backup) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) > number of parameters on (tensor, pipeline) model parallel rank (0, 1): 1716583904 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:11:34.297 actor_cell00000_rank00004] lora.py:410 - Qwen3.8-Next native LoRA applied: rank=16 expert_rank=8 alpha=16 trainable=15416832 total=1716583904 ratio=0.898111% -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) > number of parameters on (tensor, pipeline) model parallel rank (1, 1): 1716583904 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:11:34.321 actor_cell00000_rank00005] lora.py:410 - Qwen3.8-Next native LoRA applied: rank=16 expert_rank=8 alpha=16 trainable=15416832 total=1716583904 ratio=0.898111% -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:11:34.605 actor_cell00000_rank00003] model_provider.py:49 - PLE n-gram context hooks installed on the stage hosting the PLE layer -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:11:34.617 actor_cell00000_rank00003] lora.py:410 - Qwen3.8-Next native LoRA applied: rank=16 expert_rank=8 alpha=16 trainable=15746048 total=1774604736 ratio=0.887299% -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:11:34.593 actor_cell00000_rank00002] model_provider.py:49 - PLE n-gram context hooks installed on the stage hosting the PLE layer -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:11:34.606 actor_cell00000_rank00002] lora.py:410 - Qwen3.8-Next native LoRA applied: rank=16 expert_rank=8 alpha=16 trainable=15746048 total=1774604736 ratio=0.887299% -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:11:34.633 actor_cell00000_rank00001] model_provider.py:49 - PLE n-gram context hooks installed on the stage hosting the PLE layer -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:11:34.647 actor_cell00000_rank00001] lora.py:410 - Qwen3.8-Next native LoRA applied: rank=16 expert_rank=8 alpha=16 trainable=15746048 total=1774604736 ratio=0.887299% -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 1774604736 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:11:34.900 actor_cell00000_rank00006] lora.py:410 - Qwen3.8-Next native LoRA applied: rank=16 expert_rank=8 alpha=16 trainable=15416832 total=1716583904 ratio=0.898111% -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:11:34.870 actor_cell00000_rank00007] lora.py:410 - Qwen3.8-Next native LoRA applied: rank=16 expert_rank=8 alpha=16 trainable=15416832 total=1716583904 ratio=0.898111% -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:35.255 actor_cell00000_rank00000] model_provider.py:49 - PLE n-gram context hooks installed on the stage hosting the PLE layer -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:35.268 actor_cell00000_rank00000] lora.py:410 - Qwen3.8-Next native LoRA applied: rank=16 expert_rank=8 alpha=16 trainable=15746048 total=1774604736 ratio=0.887299% -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 1774604736 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:35.353 actor_cell00000_rank00000] fp32_param_utils.py:51 - Enforced marked parameter dtypes for 2 tensors. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_section', 'mrope_interleaved'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_section', 'mrope_interleaved'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_section', 'mrope_interleaved'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_section', 'mrope_interleaved'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_section', 'mrope_interleaved'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_section', 'mrope_interleaved'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_section', 'mrope_interleaved'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_section', 'mrope_interleaved'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_section', 'mrope_interleaved'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_section', 'mrope_interleaved'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_section', 'mrope_interleaved'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_section', 'mrope_interleaved'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_section', 'mrope_interleaved'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:38 TP1 EP1] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:38 TP1 EP1] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:38 TP0 EP0] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:38 TP0 EP0] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:11:38.854 actor_cell00000_rank00001] utils.py:996 - make_tp_sharded_tensor_for_checkpoint received extra kwargs: ['allow_shape_mismatch'] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:38.854 actor_cell00000_rank00000] utils.py:996 - make_tp_sharded_tensor_for_checkpoint received extra kwargs: ['allow_shape_mismatch'] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) (TP, PP) mismatch after resume ((2, 2) vs (2, 1) from checkpoint): RNG state will be ignored -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) sharded_state_dict metadata loaded from the checkpoint: {'distrib_optim_sharding_type': 'dp_reshardable', 'singleton_local_shards': False, 'chained_optim_avoid_prefix': True} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) Job sharding has changed: Rerun state will be ignored -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) loading release distributed checkpoint from /mnt/yeto-models/torch_dist/qwen3.8-flash-next-4layer_torch_dist -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:11:38.853 actor_cell00000_rank00003] utils.py:996 - make_tp_sharded_tensor_for_checkpoint received extra kwargs: ['allow_shape_mismatch'] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:11:38.855 actor_cell00000_rank00002] utils.py:996 - make_tp_sharded_tensor_for_checkpoint received extra kwargs: ['allow_shape_mismatch'] -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:39 TP2 EP2] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:39 TP1 EP1] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:39 TP2 EP2] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:39 TP1 EP1] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:39 TP3 EP3] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:39 TP3 EP3] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:39 TP3 EP3] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:39 TP3 EP3] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:39 TP2 EP2] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:39 TP2 EP2] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:39 TP0 EP0] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:39 TP0 EP0] Detected Megatron Core, using Megatron-FSDP with Megatron. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [transformers] Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:41 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:41 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:42 TP1 EP1] Init torch distributed begin. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:42 TP0 EP0] Init torch distributed begin. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:42 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:42 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:42] Multimodal data loading enabled with 16 worker threads (auto). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:42] Using default HuggingFace chat template with detected content format: openai -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:42 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:42] Multimodal data loading enabled with 16 worker threads (auto). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:42] Using default HuggingFace chat template with detected content format: openai -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:43 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:43 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:43 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:43] Auto-detected template features: reasoning_config=ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None), reasoning_parser=qwen3, tool_call_parser=qwen3_coder -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:43] Auto-detected template features: reasoning_config=ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None), reasoning_parser=qwen3, tool_call_parser=qwen3_coder -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:43 TP3 EP3] Init torch distributed begin. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:43 TP1 EP1] Init torch distributed begin. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:44 TP2 EP2] Init torch distributed begin. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:44 TP0 EP0] sglang is using nccl==2.29.7 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:44 TP3 EP3] Init torch distributed begin. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:44 TP2 EP2] Init torch distributed begin. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:44 TP0 EP0] Init torch distributed begin. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:44 TP0 EP0] sglang is using nccl==2.29.7 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:45 TP3 EP3] Gated launch waiting for activation. rank=3 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:45 TP2 EP2] Gated launch waiting for activation. rank=2 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:45 TP1 EP1] Gated launch waiting for activation. rank=1 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:45 TP0 EP0] Gated launch control server started on 10.0.0.8:20034 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:45 TP0 EP0] Gated launch waiting for activation. rank=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:46 TP3 EP3] Gated launch waiting for activation. rank=3 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:46 TP1 EP1] Gated launch waiting for activation. rank=1 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:46 TP2 EP2] Gated launch waiting for activation. rank=2 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:46 TP0 EP0] Gated launch control server started on 10.0.0.8:20071 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:46 TP0 EP0] Gated launch waiting for activation. rank=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:55 TP0 EP0] Gated launch still waiting for activation. rank=0 elapsed=10 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:55 TP2 EP2] Gated launch still waiting for activation. rank=2 elapsed=10 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:55 TP1 EP1] Gated launch still waiting for activation. rank=1 elapsed=10 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:11:55 TP3 EP3] Gated launch still waiting for activation. rank=3 elapsed=10 s -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) load_return: None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:56 TP0 EP0] Gated launch still waiting for activation. rank=0 elapsed=10 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:56 TP2 EP2] Gated launch still waiting for activation. rank=2 elapsed=10 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:56 TP1 EP1] Gated launch still waiting for activation. rank=1 elapsed=10 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:11:56 TP3 EP3] Gated launch still waiting for activation. rank=3 elapsed=10 s -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) load_return: None -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) load_return: None -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) could not find arguments in the checkpoint ... -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) load_return: None -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) checkpoint version 3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) load_return: None -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) load_return: None -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) successfully loaded checkpoint from /mnt/yeto-models/torch_dist/qwen3.8-flash-next-4layer_torch_dist [ t 1/2, p 1/2 ] at iteration 0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) load_return: None -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) load_return: None -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:11:59.812 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:59.811 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:11:59.812 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:11:59.812 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:11:59.811 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:11:59.812 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:11:59.812 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:11:59.812 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:11:59.812 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:00.368 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 71.04, 'used_GB': 8.14, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:00.404 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 71.15, 'used_GB': 8.02, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:00.403 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 71.62, 'used_GB': 7.56, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:00.406 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.14, 'used_GB': 8.04, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:00.405 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 71.2, 'used_GB': 7.98, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:00.424 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 71.04, 'used_GB': 8.14, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:00.454 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 71.28, 'used_GB': 7.9, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:00.433 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 71.15, 'used_GB': 8.02, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:02.355 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 75.34, 'used_GB': 3.84, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:02.355 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=2.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:02.355 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=init phase=end ok=true elapsed_s=41.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:02.338 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayAct -(yeto-rl-island-0, pid=4051) or pid=27304) [2026-10-06 11:12:02.338 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=2.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:02.338 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=init phase=end ok=true elapsed_s=41.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:02.340 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 75.76, 'used_GB': 3.42, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:02.340 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=2.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:02.340 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=init phase=end ok=true elapsed_s=41.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:02.439 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 75.38, 'used_GB': 3.8, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:02.439 actor_cell00000_rank00000] timer.py:42 - Timer sleep end (elapsed: 2.6s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:02.439 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=2.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:02.439 actor_cell00000_rank00000] timer.py:31 - Timer train_wait start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:02.439 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=init phase=end ok=true elapsed_s=41.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:02.474 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:02.474 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=2.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:02.474 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=init phase=end ok=true elapsed_s=41.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:02.508 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:02.508 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=2.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:02.508 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=init phase=end ok=true elapsed_s=41.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:02.681 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:02.682 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=2.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:02.682 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=init phase=end ok=true elapsed_s=41.8 -(yeto-rl-island-0, pid=4051) [2026-10-06 11:12:02.707 main] placement_group.py:384 - trainer 'actor' restored rollout 1, and --start-rollout-id 0 was asked for, so it starts at 0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:12:02.715 rollout_executor] data_source.py:154 - no dataset state under /mnt/yeto-models/torch_dist/qwen3.8-flash-next-4layer_torch_dist/rollout/global_dataset_state_dict_-1.pt: the dataset starts where a fresh run's would -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:12:02.705 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=init ok=true elapsed_s=41.9 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:12:02.706 trainer_controller_actor] cell.py:194 - ft op=state phase=start name=_mark_as_alive cell=trainer-engine-actor-00000 from_state=StateAllocatedUninitialized -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:12:02.706 trainer_controller_actor] cell.py:205 - ft op=state phase=end name=_mark_as_alive cell=trainer-engine-actor-00000 to_state=StateAllocatedAlive -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:12:02.708 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=get_train_parallel_config n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:12:02.710 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=get_train_parallel_config ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:02.704 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 75.52, 'used_GB': 3.65, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:02.704 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=2.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:02.705 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=init phase=end ok=true elapsed_s=41.8 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_driver_start","execution_mode":"colocated-serial","island_id":0,"placement":"colocated","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/algorithm_spec_sha256":"27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea","runtime_fingerprint":"sha256:f70d5baffa871590366c1eb4164ea682b9947a050d79a6c2f9fc306dd61517e3","time_unix":1791285122.8190894} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:12:02.820 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:02.824 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:02.825 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:02.824 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:02.824 actor_cell00000_rank00000] timer.py:31 - Timer wake_up start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:02.825 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 75.38, 'used_GB': 3.8, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:02.824 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:02.825 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 75.52, 'used_GB': 3.65, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:02.824 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:02.825 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:02.824 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:02.825 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 75.34, 'used_GB': 3.84, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:02.825 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:02.825 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:02.825 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:02.825 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:02.825 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:02.825 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 75.76, 'used_GB': 3.42, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:04.146 actor_cell00000_rank00001] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27297 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:04.148 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 71.64, 'used_GB': 7.54, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:04.149 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:04.155 actor_cell00000_rank00000] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27296 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:04.158 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.73, 'used_GB': 7.45, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:04.158 actor_cell00000_rank00000] timer.py:42 - Timer wake_up end (elapsed: 1.3s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:04.158 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:04.152 actor_cell00000_rank00003] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27299 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:04.154 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 71.87, 'used_GB': 7.31, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:04.155 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:04.121 actor_cell00000_rank00002] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27300 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:04.124 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 71.64, 'used_GB': 7.54, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:04.124 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:04.153 actor_cell00000_rank00004] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27302 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:04.155 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 71.8, 'used_GB': 7.38, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:04.155 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:04.128 actor_cell00000_rank00006] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27304 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:04.130 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 71.75, 'used_GB': 7.43, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:04.131 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:04.140 actor_cell00000_rank00007] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27305 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:04.142 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 72.22, 'used_GB': 6.96, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:04.142 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:04.171 actor_cell00000_rank00005] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27303 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:04.173 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 71.75, 'used_GB': 7.43, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:04.174 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:05.084 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:05.084 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:05 TP0 EP0] Gated launch still waiting for activation. rank=0 elapsed=20 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:05 TP2 EP2] Gated launch still waiting for activation. rank=2 elapsed=20 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:05 TP1 EP1] Gated launch still waiting for activation. rank=1 elapsed=20 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:05 TP3 EP3] Gated launch still waiting for activation. rank=3 elapsed=20 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:06 TP2 EP2] Gated launch still waiting for activation. rank=2 elapsed=20 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:06 TP1 EP1] Gated launch still waiting for activation. rank=1 elapsed=20 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:06 TP0 EP0] Gated launch still waiting for activation. rank=0 elapsed=20 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:06 TP3 EP3] Gated launch still waiting for activation. rank=3 elapsed=20 s -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:07.278 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:07.278 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:07.278 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:07.363 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:07.362 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:07.515 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:07.859 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 70.57, 'used_GB': 8.61, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:07.946 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 70.1, 'used_GB': 9.08, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:07.915 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 70.24, 'used_GB': 8.94, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:07.940 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 70.1, 'used_GB': 9.08, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:08.172 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 70.21, 'used_GB': 8.97, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:08.152 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 70.91, 'used_GB': 8.26, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:08.371 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:08.396 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 70.21, 'used_GB': 8.97, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:08.939 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 70.31, 'used_GB': 8.87, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:10.078 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:10.078 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:10.119 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:10.119 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=2.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:10.255 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 75.38, 'used_GB': 3.8, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:10.255 actor_cell00000_rank00000] timer.py:42 - Timer sleep end (elapsed: 3.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:10.256 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:10.404 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:10.404 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=3.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:10.396 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 75.52, 'used_GB': 3.66, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:10.396 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:10.402 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:10.402 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=2.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:10.411 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 75.76, 'used_GB': 3.42, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:10.412 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:11.361 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 75.34, 'used_GB': 3.84, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:11.361 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:12:12.845 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) [rl] learned LoRA layout from first export: 38 tensors, hash=261ae606309341c7f21e5481558516204befc39a01c7aa3af3b7506e284d608b -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:15 TP0 EP0] Gated launch still waiting for activation. rank=0 elapsed=30 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:15 TP2 EP2] Gated launch still waiting for activation. rank=2 elapsed=30 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:15 TP1 EP1] Gated launch still waiting for activation. rank=1 elapsed=30 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:15 TP3 EP3] Gated launch still waiting for activation. rank=3 elapsed=30 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:16 TP0 EP0] Gated launch still waiting for activation. rank=0 elapsed=30 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:16 TP2 EP2] Gated launch still waiting for activation. rank=2 elapsed=30 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:16 TP1 EP1] Gated launch still waiting for activation. rank=1 elapsed=30 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:16 TP3 EP3] Gated launch still waiting for activation. rank=3 elapsed=30 s -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1791285136.3302248} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:12:17.203 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:17.205 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:17.205 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:17.205 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:17.205 actor_cell00000_rank00000] timer.py:31 - Timer wake_up start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:17.205 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 75.38, 'used_GB': 3.8, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:17.205 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:17.205 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 75.52, 'used_GB': 3.66, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:17.205 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:17.205 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:17.205 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:17.206 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 75.34, 'used_GB': 3.84, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:17.205 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:17.206 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:17.205 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:17.206 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:17.205 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:17.206 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 75.76, 'used_GB': 3.42, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:18.383 actor_cell00000_rank00004] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27302 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:18.385 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 71.8, 'used_GB': 7.38, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:18.385 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:18.381 actor_cell00000_rank00006] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27304 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:18.384 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 71.75, 'used_GB': 7.43, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:18.384 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:18.453 actor_cell00000_rank00001] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27297 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:18.456 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 71.64, 'used_GB': 7.54, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:18.456 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:18.435 actor_cell00000_rank00000] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27296 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:18.437 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.73, 'used_GB': 7.45, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:18.437 actor_cell00000_rank00000] timer.py:42 - Timer wake_up end (elapsed: 1.2s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:18.437 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:18.404 actor_cell00000_rank00003] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27299 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:18.406 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 71.87, 'used_GB': 7.31, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:18.406 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:18.399 actor_cell00000_rank00002] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27300 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:18.402 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 71.64, 'used_GB': 7.54, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:18.402 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:18.412 actor_cell00000_rank00005] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27303 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:18.415 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 71.75, 'used_GB': 7.43, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:18.415 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:18.417 actor_cell00000_rank00007] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27305 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:18.419 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 72.21, 'used_GB': 6.97, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:18.419 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:19.276 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:19.276 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:19.668 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:19.931 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:19.931 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:19.932 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:19.879 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:19.879 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:20.282 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 70.21, 'used_GB': 8.97, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:20.249 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 70.91, 'used_GB': 8.27, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:20.491 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 70.1, 'used_GB': 9.08, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:20.483 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 70.57, 'used_GB': 8.61, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:20.506 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 70.1, 'used_GB': 9.08, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:20.516 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:20.530 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 70.24, 'used_GB': 8.94, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:20.606 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 70.21, 'used_GB': 8.97, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:21.144 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 70.31, 'used_GB': 8.87, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:22.393 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 75.76, 'used_GB': 3.42, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:12:22.393 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=3.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:22.694 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:12:22.694 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=3.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:22.808 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:12:22.808 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=3.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:22.929 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:12:22.929 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:22.951 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 75.38, 'used_GB': 3.8, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:22.951 actor_cell00000_rank00000] timer.py:42 - Timer sleep end (elapsed: 3.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:12:22.951 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:22.956 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 75.52, 'used_GB': 3.66, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:12:22.956 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=3.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:22.962 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 75.29, 'used_GB': 3.89, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:12:22.962 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=3.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:23.405 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 75.34, 'used_GB': 3.84, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:12:23.405 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=2.9 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:12:24.580 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=7.4 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:25 TP2 EP2] Gated launch still waiting for activation. rank=2 elapsed=40 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:25 TP1 EP1] Gated launch still waiting for activation. rank=1 elapsed=40 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:25 TP0 EP0] Gated launch still waiting for activation. rank=0 elapsed=40 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:25 TP3 EP3] Gated launch still waiting for activation. rank=3 elapsed=40 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:26 TP0 EP0] Gated launch still waiting for activation. rank=0 elapsed=40 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:26 TP2 EP2] Gated launch still waiting for activation. rank=2 elapsed=40 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:26 TP1 EP1] Gated launch still waiting for activation. rank=1 elapsed=40 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:26 TP3 EP3] Gated launch still waiting for activation. rank=3 elapsed=40 s -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:12:27.828 inference_controller] server_cell.py:315 - Cell inference-engine-all-0-0-00000 init start old= -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:12:27.828 inference_controller] server_cell.py:318 - Cell inference-engine-all-0-0-00000 init end new=addr_info=CellAddrInfo(server_url='http://10.0.0.8:20000', bootstrap_port=None, gate_url='http://10.0.0.8:20034') start_time=990.316308463 -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:12:27.828 inference_controller] server_cell.py:315 - Cell inference-engine-all-0-0-00001 init start old= -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:12:27.828 inference_controller] server_cell.py:318 - Cell inference-engine-all-0-0-00001 init end new=addr_info=CellAddrInfo(server_url='http://10.0.0.8:20037', bootstrap_port=None, gate_url='http://10.0.0.8:20071') start_time=990.316940997 -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:12:27.829 inference_controller] inference_controller.py:815 - Waiting for 2/2 cells to become ready... -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP2 EP2] Gated launch activated. elapsed=42.03 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP1 EP1] Gated launch activated. elapsed=42.03 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP0 EP0] Gated launch activated. elapsed=42.02 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP3 EP3] Gated launch activated. elapsed=42.03 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP0 EP0] Init torch distributed ends. elapsed=44.16 s, mem usage=-2.29 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP3 EP3] Init torch distributed ends. elapsed=44.17 s, mem usage=-2.48 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP2 EP2] Init torch distributed ends. elapsed=44.16 s, mem usage=-2.25 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP1 EP1] Init torch distributed ends. elapsed=45.76 s, mem usage=-2.25 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP0 EP0] Load weight begin. avail mem=75.34 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP3 EP3] Load weight begin. avail mem=75.76 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP1 EP1] Load weight begin. avail mem=75.29 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP2 EP2] Load weight begin. avail mem=75.29 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP0 EP0] Gated launch activated. elapsed=43.02 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP3 EP3] Gated launch activated. elapsed=43.03 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP1 EP1] Gated launch activated. elapsed=43.02 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP2 EP2] Gated launch activated. elapsed=43.03 s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP0 EP0] Init torch distributed ends. elapsed=45.96 s, mem usage=-2.40 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP3 EP3] Init torch distributed ends. elapsed=44.66 s, mem usage=-2.59 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP2 EP2] Init torch distributed ends. elapsed=44.54 s, mem usage=-2.36 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP1 EP1] Init torch distributed ends. elapsed=44.55 s, mem usage=-2.36 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP0 EP0] Multimodal attention backend not set. Use fa3. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP0 EP0] Using fa3 as multimodal attention backend. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP3 EP3] Multimodal attention backend not set. Use fa3. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP3 EP3] Using fa3 as multimodal attention backend. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP2 EP2] Multimodal attention backend not set. Use fa3. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP2 EP2] Using fa3 as multimodal attention backend. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP1 EP1] Multimodal attention backend not set. Use fa3. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP1 EP1] Using fa3 as multimodal attention backend. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP0 EP0] using attn output gate! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP2 EP2] using attn output gate! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP1 EP1] using attn output gate! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:12:28 TP3 EP3] using attn output gate! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP0 EP0] Load weight begin. avail mem=75.38 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP1 EP1] Load weight begin. avail mem=75.29 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP3 EP3] Load weight begin. avail mem=75.52 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP2 EP2] Load weight begin. avail mem=75.29 GB -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP0 EP0] Multimodal attention backend not set. Use fa3. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP0 EP0] Using fa3 as multimodal attention backend. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP1 EP1] Multimodal attention backend not set. Use fa3. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP1 EP1] Using fa3 as multimodal attention backend. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP3 EP3] Multimodal attention backend not set. Use fa3. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP3 EP3] Using fa3 as multimodal attention backend. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP2 EP2] Multimodal attention backend not set. Use fa3. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP2 EP2] Using fa3 as multimodal attention backend. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP0 EP0] using attn output gate! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP1 EP1] using attn output gate! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP3 EP3] using attn output gate! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:12:28 TP2 EP2] using attn output gate! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) Multi-thread loading shards: 0% Completed | 0/11 [00:00 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:33 TP1 EP1] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:33 TP3 EP3] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:33 TP2 EP2] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33] INFO: 10.0.0.8:43196 - "POST /end_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33] INFO: 10.0.0.8:43196 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33] INFO: 10.0.0.8:43196 - "POST /continue_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33 TP3 EP3] LoRA adapter loading from tensors completes: LoRARef(lora_id=7d347176e0914e4f85e83a43247f90cb, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33 TP2 EP2] LoRA adapter loading from tensors completes: LoRARef(lora_id=7d347176e0914e4f85e83a43247f90cb, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33 TP0 EP0] LoRA adapter loading from tensors completes: LoRARef(lora_id=7d347176e0914e4f85e83a43247f90cb, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33 TP1 EP1] LoRA adapter loading from tensors completes: LoRARef(lora_id=7d347176e0914e4f85e83a43247f90cb, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33 TP0 EP0] Weight version changed. old_version='default' new_version='1' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33 TP2 EP2] Weight version changed. old_version='default' new_version='1' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33 TP1 EP1] Weight version changed. old_version='default' new_version='1' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33 TP3 EP3] Weight version changed. old_version='default' new_version='1' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33 TP3 EP3] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33 TP2 EP2] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33 TP1 EP1] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:33 TP0 EP0] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [miles-colocate] Update weights: 23it [00:05, 4.00it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:14:33.175 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 55.07, 'used_GB': 24.11, 'allocated_GB': 3.42, 'reserved_GB': 11.15} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:33.173 actor_cell00000_rank00000] timer.py:42 - Timer finalize_and_resume_engines end (elapsed: 1.1s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [miles-colocate] Update weights: 23it [00:05, 4.00it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:33.175 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 56.44, 'used_GB': 22.74, 'allocated_GB': 3.42, 'reserved_GB': 9.97} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [miles-colocate] Update weights: 23it [00:05, 4.00it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:14:33.175 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after update_weights: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 55.76, 'used_GB': 23.42, 'allocated_GB': 3.42, 'reserved_GB': 11.15} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [miles-colocate] Update weights: 23it [00:05, 4.00it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:14:33.175 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after update_weights: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 55.11, 'used_GB': 24.07, 'allocated_GB': 3.42, 'reserved_GB': 11.15} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [miles-colocate] Update weights: 23it [00:05, 4.00it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:14:33.175 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after update_weights: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 56.39, 'used_GB': 22.79, 'allocated_GB': 3.31, 'reserved_GB': 9.86} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [miles-colocate] Update weights: 23it [00:05, 4.00it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:14:33.175 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after update_weights: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 55.06, 'used_GB': 24.12, 'allocated_GB': 3.31, 'reserved_GB': 11.04} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [miles-colocate] Update weights: 23it [00:05, 4.00it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:14:33.175 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after update_weights: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 55.06, 'used_GB': 24.12, 'allocated_GB': 3.31, 'reserved_GB': 11.04} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [miles-colocate] Update weights: 23it [00:05, 4.00it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:14:33.175 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after update_weights: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 56.0, 'used_GB': 23.18, 'allocated_GB': 3.31, 'reserved_GB': 11.04} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:14:33.995 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=6.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:14:34.216 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=6.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:14:34.215 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=6.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:14:34.219 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=6.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:14:34.227 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=6.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:14:34.221 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=6.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:14:34.288 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=6.9 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34] INFO: 10.0.0.8:39316 - "PRI %2A HTTP/2.0" 404 Not Found -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34] INFO: 10.0.0.8:39314 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34] INFO: 10.0.0.8:39330 - "PRI %2A HTTP/2.0" 404 Not Found -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34] WARNING: Invalid HTTP request received. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34] WARNING: Invalid HTTP request received. -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:14:34.524 inference_controller] server_cell.py:315 - Cell inference-engine-all-0-0-00001 mark_serving start old=addr_info=CellAddrInfo(server_url='http://10.0.0.8:20037', bootstrap_port=None, gate_url='http://10.0.0.8:20071') -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:14:34.524 inference_controller] server_cell.py:318 - Cell inference-engine-all-0-0-00001 mark_serving end new=addr_info=CellAddrInfo(server_url='http://10.0.0.8:20037', bootstrap_port=None, gate_url='http://10.0.0.8:20071') -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:14:34.524 inference_controller] server_cell.py:315 - Cell inference-engine-all-0-0-00000 mark_serving start old=addr_info=CellAddrInfo(server_url='http://10.0.0.8:20000', bootstrap_port=None, gate_url='http://10.0.0.8:20034') -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:14:34.525 inference_controller] server_cell.py:318 - Cell inference-engine-all-0-0-00000 mark_serving end new=addr_info=CellAddrInfo(server_url='http://10.0.0.8:20000', bootstrap_port=None, gate_url='http://10.0.0.8:20034') -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34] INFO: 10.0.0.8:60864 - "PRI %2A HTTP/2.0" 404 Not Found -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34] INFO: 10.0.0.8:60860 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34] INFO: 10.0.0.8:60870 - "PRI %2A HTTP/2.0" 404 Not Found -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34] WARNING: Invalid HTTP request received. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34] WARNING: Invalid HTTP request received. -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:14:34.517 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=update_weights ok=true elapsed_s=7.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:34.516 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 7.2s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:34.516 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=7.2 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34] INFO: 10.0.0.8:39342 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34] INFO: 10.0.0.8:39354 - "GET /server_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34] INFO: 10.0.0.8:39354 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34] INFO: 10.0.0.8:39360 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34] INFO: 10.0.0.8:39360 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34 TP0 EP0] Weight version changed. old_version='1' new_version='yeto:0:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34 TP1 EP1] Weight version changed. old_version='1' new_version='yeto:0:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34 TP2 EP2] Weight version changed. old_version='1' new_version='yeto:0:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34 TP3 EP3] Weight version changed. old_version='1' new_version='yeto:0:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:34 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34] INFO: 10.0.0.8:60886 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34] INFO: 10.0.0.8:60892 - "GET /server_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34] INFO: 10.0.0.8:60892 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34] INFO: 10.0.0.8:60898 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34] INFO: 10.0.0.8:60898 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34 TP0 EP0] Weight version changed. old_version='1' new_version='yeto:0:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34 TP2 EP2] Weight version changed. old_version='1' new_version='yeto:0:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34 TP1 EP1] Weight version changed. old_version='1' new_version='yeto:0:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34 TP3 EP3] Weight version changed. old_version='1' new_version='yeto:0:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:34 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:36 TP1 EP1] Triton kernel 'hash_tiles32_kernel_blocked' took 1.38 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:36 TP3 EP3] Triton kernel 'hash_tiles32_kernel_blocked' took 1.39 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:36 TP0 EP0] Triton kernel 'hash_tiles32_kernel_blocked' took 1.39 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:36 TP2 EP2] Triton kernel 'hash_tiles32_kernel_blocked' took 1.41 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:36 TP3 EP3] Triton kernel 'hash_tiles32_kernel_blocked' took 1.38 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:36 TP1 EP1] Triton kernel 'hash_tiles32_kernel_blocked' took 1.38 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:36 TP2 EP2] Triton kernel 'hash_tiles32_kernel_blocked' took 1.39 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:36 TP0 EP0] Triton kernel 'hash_tiles32_kernel_blocked' took 1.40 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:38 TP1 EP1] Triton kernel 'add_tree_reduce_u64_kernel' took 2.72 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:38 TP3 EP3] Triton kernel 'add_tree_reduce_u64_kernel' took 2.72 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:38 TP0 EP0] Triton kernel 'add_tree_reduce_u64_kernel' took 2.73 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:38 TP2 EP2] Triton kernel 'add_tree_reduce_u64_kernel' took 2.74 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:38 TP1 EP1] Triton kernel 'add_tree_reduce_u64_kernel' took 2.67 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:38 TP2 EP2] Triton kernel 'add_tree_reduce_u64_kernel' took 2.67 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:38 TP3 EP3] Triton kernel 'add_tree_reduce_u64_kernel' took 2.69 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:38 TP0 EP0] Triton kernel 'add_tree_reduce_u64_kernel' took 2.72 s to compile after serving started. Serving-time compilation can stall the engine; pre-compile it during engine init. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:39] INFO: 10.0.0.8:39342 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:39 TP1 EP1] [WeightChecker] checksum computed for 437 tensors in 4.316s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:39 TP3 EP3] [WeightChecker] checksum computed for 437 tensors in 4.321s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:39 TP0 EP0] [WeightChecker] checksum computed for 437 tensors in 4.334s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:39 TP2 EP2] [WeightChecker] checksum computed for 437 tensors in 4.360s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:39] INFO: 10.0.0.8:60886 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:39 TP1 EP1] [WeightChecker] checksum computed for 437 tensors in 4.268s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:39 TP2 EP2] [WeightChecker] checksum computed for 437 tensors in 4.278s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:39 TP3 EP3] [WeightChecker] checksum computed for 437 tensors in 4.286s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:39 TP0 EP0] [WeightChecker] checksum computed for 437 tensors in 4.337s -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_publication","island_id":0,"policy_version":0,"rl/policy_token":"yeto:0:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285279.1716428} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1121.659985907,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":978.818225393,"task":"publish","time_unix":1791285279.1720073,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":0,"time_unix":1791285279.1721241} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1121.665593291,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":0,"start":1121.660211319,"task":"offload","time_unix":1791285279.1776223,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1791285279.17777} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:14:39.173 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=sleep n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:14:39.176 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=sleep ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:14:39.174 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:14:39.174 actor_cell00000_rank00001] actor.py:452 - sleep() called while already offloaded; skipping -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:14:39.174 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:39.174 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:39.174 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:39.174 actor_cell00000_rank00000] actor.py:452 - sleep() called while already offloaded; skipping -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:39.174 actor_cell00000_rank00000] timer.py:42 - Timer sleep end (elapsed: 0.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:39.174 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:14:39.174 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:14:39.174 actor_cell00000_rank00003] actor.py:452 - sleep() called while already offloaded; skipping -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:14:39.175 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:14:39.174 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:14:39.174 actor_cell00000_rank00002] actor.py:452 - sleep() called while already offloaded; skipping -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:14:39.174 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:14:39.174 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:14:39.175 actor_cell00000_rank00004] actor.py:452 - sleep() called while already offloaded; skipping -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:14:39.175 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:14:39.175 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:14:39.175 actor_cell00000_rank00006] actor.py:452 - sleep() called while already offloaded; skipping -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:14:39.175 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:14:39.174 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:14:39.175 actor_cell00000_rank00005] actor.py:452 - sleep() called while already offloaded; skipping -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:14:39.175 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:14:39.175 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:14:39.175 actor_cell00000_rank00007] actor.py:452 - sleep() called while already offloaded; skipping -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:14:39.175 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Rollout generation: 0%| | 0/64 [00:00system\nReasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.\n\nYou are a helpful assistant. Please put the answer within \boxed{}.<|im_end|>\n<|im_start|>user\nNatalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May?<|im_end|>\n<|im_start|>assis..., label=72, reward_summary=0.0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Rollout generation: 12%|█▎ | 8/64 [00:17<02:02, 2.19s/it] Rollout generation: 100%|██████████| 64/64 [00:17<00:00, 3.66it/s] -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:14:57.216 rollout_executor] inference_rollout_train.py:161 - Finish rollout: text_preview=<|im_start|>system\nReasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.\n\nYou are a helpful assistant. Please put the answer within \boxed{}.<|im_end|>\n<|im_start|>user\nWeng earns $12 an hour for babysitting. Yesterday, she just did 50 minutes of babysitting. How much did she earn?<|im_end|>\n<|im_start|>assistant\n\n_CMthink Salvini形色 solidari..., label=10, reward_summary=0.0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:14:57.220 rollout_executor] inference_rollout_train.py:33 - Abort request for ['http://10.0.0.8:20037', 'http://10.0.0.8:20000'] -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Using blocking ray.get inside async actor. This blocks the event loop. Please use `await` on object ref with asyncio.gather if you want to yield execution to the event loop instead. -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:14:57.258 rollout_executor] rollout_data_conversion.py:39 - Final collected 64 samples from rollout to train -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:14:57.258 rollout_executor] metrics.py:93 - perf 0: {'rollout/raw_reward_unfiltered': 0.0, 'rollout/num_training_samples': 64, 'rollout/episode_raw_reward': 0.0, 'rollout/episode_response_length/mean': 1000.828125, 'rollout/episode_response_length/median': 1024.0, 'rollout/episode_response_length/max': 1024, 'rollout/episode_response_length/min': 75, 'rollout/episode_total_response_length/mean': 1000.828125, 'rollout/response_len/mean': 1000.828125, 'rollout/response_len/median': 1024.0, 'rollout/response_len/max': 1024, 'rollout/response_len/min': 75, 'rollout/zero_std/count_0.0': 8, 'rollout/zero_std/all_zero_percentage': 1.0, 'rollout/zero_std/all_one_percentage': 0.0, 'rollout/prefix_cache_hit_rate': 0.0, 'rollout/avg_cached_tokens_per_sample': 0.0, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.953125, 'perf/rollout_time': 17.93143320083618, 'perf/tokens_per_gpu_per_sec': 446.51338854646787, 'perf/longest_sample_tokens_per_sec': 57.10642247783343, 'perf/effective_tokens_per_gpu_per_sec': 446.51338854646787, 'perf/longest_effective_sample_tokens_per_sec': 57.10642247783343} -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:14:57.261 rollout_executor] train_data_conversion.py:334 - Rollout-side DP schedule: num_samples=64, num_rollouts=[64], num_microbatches=[32] -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:57] INFO: 10.0.0.8:53520 - "POST /abort_request HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:57] INFO: 10.0.0.8:54176 - "POST /abort_request HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:57 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:57 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:57 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:57 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:57] INFO: 10.0.0.8:54184 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:57 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:57 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:57 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:57 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:57 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:57 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:57 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:57 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:57 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:57 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:57] INFO: 10.0.0.8:53524 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:57 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:57 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:57 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:57 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:57 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:57 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1140.45609471,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"rollout","rollout_id":0,"start":1121.665857615,"task":"generate","time_unix":1791285297.9681737,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":0,"time_unix":1791285297.9683726} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:57] INFO: 10.0.0.8:53524 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:57] INFO: 10.0.0.8:54184 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:14:57.969 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=wake_up n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:14:57.970 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:14:57.971 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 70.83, 'used_GB': 8.35, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:57.970 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:57.970 actor_cell00000_rank00000] timer.py:31 - Timer wake_up start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:57.971 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 71.01, 'used_GB': 8.17, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:14:57.970 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:14:57.971 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 71.33, 'used_GB': 7.84, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:14:57.970 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:14:57.971 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 70.87, 'used_GB': 8.31, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:14:57.971 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:14:57.971 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 70.86, 'used_GB': 8.32, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:14:57.971 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:14:57.971 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 70.76, 'used_GB': 8.41, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:14:57.971 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:14:57.971 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 70.76, 'used_GB': 8.41, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:14:57.971 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:14:57.972 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 71.47, 'used_GB': 7.71, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:14:59.221 actor_cell00000_rank00001] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27297 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:14:59.224 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 67.21, 'used_GB': 11.97, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:14:59.224 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:14:59.180 actor_cell00000_rank00002] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27300 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:14:59.183 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 67.21, 'used_GB': 11.97, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:14:59.183 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:14:59.237 actor_cell00000_rank00004] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27302 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:14:59.240 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 67.31, 'used_GB': 11.86, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:14:59.240 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:14:59.258 actor_cell00000_rank00006] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27304 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:14:59.260 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 67.22, 'used_GB': 11.96, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:14:59.261 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:14:59.244 actor_cell00000_rank00007] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27305 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:14:59.247 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 67.92, 'used_GB': 11.26, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:14:59.248 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:59.352 actor_cell00000_rank00000] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27296 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:59.355 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 67.35, 'used_GB': 11.82, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:59.355 actor_cell00000_rank00000] timer.py:42 - Timer wake_up end (elapsed: 1.4s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:59.355 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:14:59.302 actor_cell00000_rank00003] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27299 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:14:59.304 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 67.68, 'used_GB': 11.5, 'allocated_GB': 3.42, 'reserved_GB': 3.71} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:14:59.304 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1141.889917275,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":0,"start":1140.456458337,"task":"onload","time_unix":1791285299.4020224,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1791285299.402256} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:14:59.400 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=wake_up ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:14:59.404 trainer_controller_actor] group.py:191 - ft op=train phase=start rollout=0 attempt=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:14:59.404 trainer_controller_actor] group.py:540 - ft op=refresh phase=start rollout=0 alive=0 healing= all_states="(0, 'StateAllocatedAlive')" quorum=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:14:59.404 trainer_controller_actor] group.py:563 - ft op=refresh phase=decision rollout=0 needs_reconfigure=false reason=alive_config_unchanged,no_healing quorum=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:14:59.404 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=train n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:14:59.406 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:14:59.417 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:14:59.418 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:59.406 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:59.406 actor_cell00000_rank00000] timer.py:31 - Timer data_preprocess start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:59.417 actor_cell00000_rank00000] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:59.417 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:59.417 actor_cell00000_rank00000] timer.py:42 - Timer train_wait end (elapsed: 177.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:59.417 actor_cell00000_rank00000] timer.py:31 - Timer train start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:59.417 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:14:59.417 actor_cell00000_rank00000] timer.py:31 - Timer log_probs start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:14:59.406 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:14:59.417 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:14:59.418 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:14:59.406 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:14:59.418 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:14:59.418 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:14:59.407 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:14:59.419 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:14:59.419 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:14:59.407 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:14:59.418 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:14:59.418 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:14:59.396 actor_cell00000_rank00005] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27303 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:14:59.399 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 67.22, 'used_GB': 11.96, 'allocated_GB': 3.31, 'reserved_GB': 3.59} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:14:59.399 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:14:59.407 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:14:59.418 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:14:59.418 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:14:59.407 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:14:59.418 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:14:59.418 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:14:59] INFO: 10.0.0.8:32996 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:14:59] INFO: 10.0.0.8:41004 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":0,"rounds_completed":0,"seq":6,"time_unix":1791285302.7600424,"trained_version":null,"uptime_s":180.003,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"available":true,"config_epoch":0,"cpu_rss_bytes":3355287552,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":13592.625,"power_w":220.123,"util_pct":83,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":13738.625,"power_w":239.64,"util_pct":70,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":13736.625,"power_w":239.441,"util_pct":82,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":13256.625,"power_w":230.92,"util_pct":39,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":12628.625,"power_w":123.401,"util_pct":0,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":12724.625,"power_w":119.548,"util_pct":0,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":12724.625,"power_w":123.401,"util_pct":0,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":12006.625,"power_w":119.573,"util_pct":0,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1145.400624592,"time_unix":1791285302.912751,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) 2026-10-06 11:15:06 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_chunk_local_cumsum_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) 2026-10-06 11:15:06 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_chunk_local_cumsum_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) 2026-10-06 11:15:06 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_chunk_local_cumsum_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) 2026-10-06 11:15:06 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_chunk_local_cumsum_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) 2026-10-06 11:15:08 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_chunk_local_cumsum_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) 2026-10-06 11:15:09 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_chunk_local_cumsum_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) 2026-10-06 11:15:09 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_chunk_local_cumsum_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) 2026-10-06 11:15:08 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_chunk_local_cumsum_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) 2026-10-06 11:15:09 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_kkt_solve_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) 2026-10-06 11:15:09 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_kkt_solve_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) 2026-10-06 11:15:09 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_kkt_solve_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) 2026-10-06 11:15:09 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_kkt_solve_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) 2026-10-06 11:15:14 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_kkt_solve_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) 2026-10-06 11:15:14 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_kkt_solve_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) 2026-10-06 11:15:14 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_kkt_solve_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) 2026-10-06 11:15:14 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_kkt_solve_kernel` -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:15:14] INFO: 10.0.0.8:57084 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:15:14] INFO: 10.0.0.8:40712 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) 2026-10-06 11:15:15 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_fused_chunk_gdr_fwd_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) 2026-10-06 11:15:15 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_fused_chunk_gdr_fwd_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) 2026-10-06 11:15:15 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_fused_chunk_gdr_fwd_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) 2026-10-06 11:15:15 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_fused_chunk_gdr_fwd_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) 2026-10-06 11:15:27 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_fused_chunk_gdr_fwd_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) 2026-10-06 11:15:27 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_fused_chunk_gdr_fwd_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) 2026-10-06 11:15:27 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_fused_chunk_gdr_fwd_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) 2026-10-06 11:15:27 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_fused_chunk_gdr_fwd_kernel` -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:15:29] INFO: 10.0.0.8:40122 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:15:29] INFO: 10.0.0.8:34764 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":0,"rounds_completed":0,"seq":7,"time_unix":1791285332.7605896,"trained_version":null,"uptime_s":210.004,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) [2026-10-06 11:15:36.872 main] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=124 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:15:44] INFO: 10.0.0.8:33854 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:15:44] INFO: 10.0.0.8:44556 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:15:54.645 worker_manager] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=130 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:15:59] INFO: 10.0.0.8:54964 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:15:59] INFO: 10.0.0.8:41458 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:16:02.245 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=62.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:16:02.245 actor_cell00000_rank00000] timer.py:42 - Timer log_probs end (elapsed: 62.8s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:16:02.245 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=62.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:16:02.245 actor_cell00000_rank00000] timer.py:31 - Timer actor_train start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:16:02.244 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=62.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:16:02.244 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=62.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:16:02.349 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=62.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:16:02.357 actor_cell00000_rank00004] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:16:02.359 actor_cell00000_rank00004] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:16:02.359 actor_cell00000_rank00004] log_utils.py:126 - rollout 0: {'rollout/response_lengths': 1000.828125, 'rollout/rewards': 0.0, 'rollout/truncated': 0.953125, 'rollout/rollout_log_probs': -11.998551845550537, 'rollout/raw_reward': 0.0, 'rollout/total_lengths': 1122.578125, 'rollout/log_probs': -12.000439167022705, 'rollout/advantages': 0.0, 'rollout/returns': 0.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:16:02.349 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=62.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:16:02.357 actor_cell00000_rank00006] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:16:02.359 actor_cell00000_rank00006] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=1 success=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:16:02.350 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=62.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:16:02.349 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=62.9 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":0,"rounds_completed":0,"seq":8,"time_unix":1791285362.7611294,"trained_version":null,"uptime_s":240.004,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"available":true,"config_epoch":0,"cpu_rss_bytes":3337641984,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":18152.625,"power_w":162.666,"util_pct":0,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":18394.625,"power_w":173.579,"util_pct":0,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":18390.625,"power_w":170.784,"util_pct":0,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":17430.625,"power_w":165.541,"util_pct":0,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":19970.625,"power_w":202.606,"util_pct":0,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":19904.625,"power_w":191.969,"util_pct":0,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":20162.625,"power_w":203.501,"util_pct":0,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":18964.625,"power_w":200.51,"util_pct":2,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1205.401699242,"time_unix":1791285362.9137716,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:16:14.219 rollout_executor] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=130 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:16:14] INFO: 10.0.0.8:54638 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:16:14] INFO: 10.0.0.8:59290 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:16:16.661 actor_cell00000_rank00000] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=154 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) 2026-10-06 11:16:17 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_prepare_h_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) 2026-10-06 11:16:17 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_prepare_h_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) 2026-10-06 11:16:17 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_prepare_h_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) 2026-10-06 11:16:17 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_prepare_h_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:16:17.685 actor_cell00000_rank00001] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=154 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:16:17.930 actor_cell00000_rank00003] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=154 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:16:17.932 actor_cell00000_rank00004] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=154 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:16:17.932 actor_cell00000_rank00007] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=154 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:16:18.125 actor_cell00000_rank00006] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=154 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:16:18.111 actor_cell00000_rank00005] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=154 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:16:18.856 actor_cell00000_rank00002] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=154 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:16:19.298 inference_controller] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=127 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:16:24.078 trainer_controller_actor] reporter.py:90 - audit op=env_report_summary hostname=computeinstance-e00qaqh66cs45p24y9 versions="{\"python\":\"3.12.3\",\"platform\":\"Linux-6.11.0-1016-nvidia-x86_64-with-glibc2.39\",\"miles\":\"0.1.1\",\"sglang\":\"0.5.21.dev69+g4e4148f\",\"sglang-router\":\"0.3.2\",\"megatron-core\":\"0.19.0+a84b10547\",\"transformers\":\"5.12.1\",\"ray\":\"2.58.0\",\"flashinfer-python\":\"0.6.18\",\"torch\":\"2.13.0+cu130\",\"torch_cuda\":\"13.0\"}" repos="{\"sglang\":\"4e4148f1b4fe9f05973da0d1e5cfe237512d5155\",\"miles\":\"c35702eefcf2862cee155e46870e6ad30568d2c6\",\"megatron-core\":\"a84b105473eca9c9c6e49dbf1c8aa460c5cf796e\"}" num_packages=425 num_env_vars=130 packages_probed=true stored=false -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) 2026-10-06 11:16:25 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_prepare_h_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) 2026-10-06 11:16:25 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_prepare_h_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) 2026-10-06 11:16:25 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_prepare_h_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) 2026-10-06 11:16:25 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_prepare_h_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) 2026-10-06 11:16:27 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_fused_chunk_gdr_bwd_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) 2026-10-06 11:16:27 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_fused_chunk_gdr_bwd_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [11:16:27] : Warning: Buffer read before initialization: 1 buffer(s) read before anything writes them -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [1] `dk_fragment` is read before anything writes it -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) --> /opt/sglang/lib/python3.12/site-packages/flash_qla/ops/gated_delta_rule/chunk/hopper/fused_bwd.py:466:1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) Initialize the buffer before its first read (for example with `T.clear`, `T.fill`, or a `T.copy` into it). If you believe this is a false positive, disable the check by setting `PassConfigKey.TL_DISABLE_BUFFER_INIT_CHECK` in the pass config. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) 2026-10-06 11:16:27 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_fused_chunk_gdr_bwd_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [11:16:27] : Warning: Buffer read before initialization: 1 buffer(s) read before anything writes them -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [1] `dk_fragment` is read before anything writes it -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) --> /opt/sglang/lib/python3.12/site-packages/flash_qla/ops/gated_delta_rule/chunk/hopper/fused_bwd.py:466:1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) Initialize the buffer before its first read (for example with `T.clear`, `T.fill`, or a `T.copy` into it). If you believe this is a false positive, disable the check by setting `PassConfigKey.TL_DISABLE_BUFFER_INIT_CHECK` in the pass config. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) 2026-10-06 11:16:27 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_fused_chunk_gdr_bwd_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [11:16:27] : Warning: Buffer read before initialization: 1 buffer(s) read before anything writes them -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [1] `dk_fragment` is read before anything writes it -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) --> /opt/sglang/lib/python3.12/site-packages/flash_qla/ops/gated_delta_rule/chunk/hopper/fused_bwd.py:466:1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) Initialize the buffer before its first read (for example with `T.clear`, `T.fill`, or a `T.copy` into it). If you believe this is a false positive, disable the check by setting `PassConfigKey.TL_DISABLE_BUFFER_INIT_CHECK` in the pass config. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [11:16:27] : Warning: Buffer read before initialization: 1 buffer(s) read before anything writes them -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [1] `dk_fragment` is read before anything writes it -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) --> /opt/sglang/lib/python3.12/site-packages/flash_qla/ops/gated_delta_rule/chunk/hopper/fused_bwd.py:466:1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) Initialize the buffer before its first read (for example with `T.clear`, `T.fill`, or a `T.copy` into it). If you believe this is a false positive, disable the check by setting `PassConfigKey.TL_DISABLE_BUFFER_INIT_CHECK` in the pass config. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:16:29] INFO: 10.0.0.8:38496 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:16:29] INFO: 10.0.0.8:37466 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":0,"rounds_completed":0,"seq":9,"time_unix":1791285392.761656,"trained_version":null,"uptime_s":270.005,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [11:16:41] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [11:16:41] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [11:16:41] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [11:16:41] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [11:16:41] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [11:16:41] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [11:16:41] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [11:16:41] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [11:16:42] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [11:16:42] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [11:16:42] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [11:16:42] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [11:16:42] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [11:16:42] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [11:16:42] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [11:16:42] : Warning: T.vectorized loop over `j_t_vec` with extent 2 is lowered as a serial loop because TileLang could not find a valid vectorization plan. Scalar accumulator updates inside the loop are a common cause; move reductions to T.unroll or T.serial if this is intended. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:16:44] INFO: 10.0.0.8:49576 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:16:44] INFO: 10.0.0.8:59338 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) 2026-10-06 11:16:54 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_fused_chunk_gdr_bwd_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) 2026-10-06 11:16:54 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_fused_chunk_gdr_bwd_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) 2026-10-06 11:16:55 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_fused_chunk_gdr_bwd_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) 2026-10-06 11:16:55 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_fused_chunk_gdr_bwd_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) 2026-10-06 11:16:56 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_chunk_local_cumsum_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) 2026-10-06 11:16:56 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_chunk_local_cumsum_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) 2026-10-06 11:16:57 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_chunk_local_cumsum_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) 2026-10-06 11:16:57 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:137): TileLang begins to compile kernel `tilelang_chunk_local_cumsum_kernel` with `out_idx=None` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) 2026-10-06 11:16:58 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_chunk_local_cumsum_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) 2026-10-06 11:16:58 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_chunk_local_cumsum_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) 2026-10-06 11:16:59 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_chunk_local_cumsum_kernel` -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) 2026-10-06 11:16:59 [TileLang:tilelang.jit.kernel:INFO] (kernel.py:145): TileLang completes to compile kernel `tilelang_chunk_local_cumsum_kernel` -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:16:59] INFO: 10.0.0.8:34464 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:16:59] INFO: 10.0.0.8:40172 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":0,"rounds_completed":0,"seq":10,"time_unix":1791285422.7621913,"trained_version":null,"uptime_s":300.006,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"available":true,"config_epoch":0,"cpu_rss_bytes":3337641984,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":18152.625,"power_w":119.296,"util_pct":0,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":18394.625,"power_w":128.382,"util_pct":0,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":18390.625,"power_w":125.811,"util_pct":0,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":17430.625,"power_w":120.564,"util_pct":0,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":20042.625,"power_w":187.439,"util_pct":0,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":19978.625,"power_w":189.163,"util_pct":31,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":20234.625,"power_w":188.889,"util_pct":83,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":19036.625,"power_w":184.099,"util_pct":0,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1265.402538667,"time_unix":1791285422.9146152,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:14] INFO: 10.0.0.8:38338 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:14] INFO: 10.0.0.8:54594 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:28.094 actor_cell00000_rank00000] _rank_utils.py:89 - Implicit initialization of Rerun State Machine! -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:28.094 actor_cell00000_rank00000] _rank_utils.py:89 - RerunStateMachine initialized in mode RerunMode.DISABLED -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[1e-05],"clip_fraction":null,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"masked_fraction":null,"nonzero_advantages":null,"resp_len_mean":1000.828125,"resp_len_p95":1024.0,"reward_p10":0.0,"reward_p50":0.0,"reward_p90":0.0,"rollout_id":0,"step_seconds":150.3467451429999,"submitted_groups":null,"time_unix":1791285449.74938,"tok_per_s":426.0351625110142,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0,"pg_clipfrac":0.0,"pg_loss":0.0,"ppo_kl":-1.000444171950221e-09,"train_rollout_kl":0.0018599902978166938,"train_rollout_logprob_abs_diff":0.03606637567281723},"train_step":1,"trained_groups":8,"trained_sample_ids_sha256":"d82d6a02dabaac03a6d9f9876c2587054e271f74ce351cdcafbd26939ae9af43","trained_samples":64,"truncated_frac":0.953125} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_round_labels","execution_mode":"colocated-serial","island_id":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/aborted_groups":0,"rl/carried_over_groups":null,"rl/clip_fraction":null,"rl/ess_ratio":1.0,"rl/filtered_groups":0,"rl/groups":8,"rl/masked_fraction":null,"rl/mean_kl":-1.000444171950221e-09,"rollout_id":0,"t":1292.237757907,"time_unix":1791285449.7497633,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1292.237971355,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":0,"start":1141.890336426,"task":"train","time_unix":1791285449.749942,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1791285449.7500346} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:29.734 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=train ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:29.735 trainer_controller_actor] group.py:291 - ft op=check errored= discarded= normal=0 decision=no_retry reason="survivors normal, gradients valid" -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:29.738 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:29.740 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:29.742 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:29.744 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:29.746 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:29.748 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:29.751 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:29.729 actor_cell00000_rank00001] model.py:646 - train op=train_step rollout=0 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:29.729 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:29.731 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:29.729 actor_cell00000_rank00000] model.py:646 - train op=train_step rollout=0 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:29.729 actor_cell00000_rank00000] timer.py:42 - Timer actor_train end (elapsed: 87.5s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:29.729 actor_cell00000_rank00000] timer.py:42 - Timer train end (elapsed: 150.3s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:29.729 actor_cell00000_rank00000] timer.py:31 - Timer train_wait start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:29.729 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:29.731 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:29.729 actor_cell00000_rank00003] model.py:646 - train op=train_step rollout=0 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:29.729 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:29.731 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:29.729 actor_cell00000_rank00002] model.py:646 - train op=train_step rollout=0 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:29.729 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:29.731 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:29.729 actor_cell00000_rank00004] model.py:646 - train op=train_step rollout=0 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:29.730 actor_cell00000_rank00004] log_utils.py:554 - step 0: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': -1.000444171950221e-09, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.03606637567281723, 'train/train_rollout_kl': 0.0018599902978166938, 'train/grad_norm': 0.0, 'train/lr-pg_0': 8.333333333333334e-06, 'train/lr-pg_1': 8.333333333333334e-06, 'train/step': 0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:29.731 actor_cell00000_rank00004] model.py:856 - step 0: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': -1.000444171950221e-09, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.03606637567281723, 'train/train_rollout_kl': 0.0018599902978166938, 'train/grad_norm': 0.0, 'train/lr-pg_0': 8.333333333333334e-06, 'train/lr-pg_1': 8.333333333333334e-06, 'train/step': 0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:29.731 actor_cell00000_rank00004] train_metric_utils.py:56 - perf 0: {'perf/sleep_time': 8.42179012298584, 'perf/wake_up_time': 3.780303955078125, 'perf/update_weights_implementation_time': 4.68604850769043, 'perf/finalize_and_resume_engines_time': 1.061631679534912, 'perf/update_weights_time': 6.861553430557251, 'perf/data_preprocess_time': 0.012430429458618164, 'perf/train_wait_time': 177.06410193443298, 'perf/log_probs_time': 62.92991280555725, 'perf/actor_train_time': 87.37146878242493, 'perf/train_time': 150.31131553649902, 'perf/log_probs_tflops': 0.27409928276011786, 'perf/actor_train_tflops': 0.5922657889767456, 'perf/actor_train_tok_per_s': 822.2936045508242, 'perf/mfu_peak_tflops': 989.0, 'perf/actor_train_mfu': 0.0005988531738895305, 'perf/step_time': 327.375417470932, 'perf/wait_time_ratio': 0.5408594918406012} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:29.731 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:29.733 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:29.729 actor_cell00000_rank00006] model.py:646 - train op=train_step rollout=0 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:29.731 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:29.732 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:29.729 actor_cell00000_rank00005] model.py:646 - train op=train_step rollout=0 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:29.731 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:29.733 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:29.729 actor_cell00000_rank00007] model.py:646 - train op=train_step rollout=0 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:29.731 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:29.733 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=150.3 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:29] INFO: 10.0.0.8:55398 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:29] INFO: 10.0.0.8:38040 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:32.043 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=2.3 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"sync","pid":18336,"policy_version":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":0,"rounds_completed":0,"seq":11,"time_unix":1791285452.811284,"trained_version":1,"uptime_s":330.055,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"action_tokens":64053,"active_groups":8,"applied_lr":1e-05,"applied_lrs":[1e-05],"base_policy_version":0,"cancelled_groups":0,"clip_fraction":null,"completed_groups":8,"completed_trajectories":64,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":8,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":0.0,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":1,"loss":0.0,"lr":1e-05,"mean_kl":-1.000444171950221e-09,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":0.0,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":18.796108502000152,"time_unix":1791285454.5753045,"tool_wait_seconds":0.0,"train_seconds":150.3467451429999,"train_step":1,"zero_variance_group_ratio":1.0} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1297.9451991,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":0,"start":1292.238123853,"task":"outer_sync","time_unix":1791285455.4573538,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1791285455.4575727} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:36.334 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:38.791 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=2.5 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:41] INFO: 10.0.0.8:33596 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:41] INFO: 10.0.0.8:33598 - "POST /pause_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:41] INFO: 10.0.0.8:33598 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:41] INFO: 10.0.0.8:33598 - "POST /begin_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:41 TP0 EP0] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:41 TP2 EP2] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:41 TP1 EP1] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:41 TP3 EP3] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:41] INFO: 10.0.0.8:45630 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:41] INFO: 10.0.0.8:45644 - "POST /pause_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:41] INFO: 10.0.0.8:45644 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:41] INFO: 10.0.0.8:45644 - "POST /begin_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:41 TP0 EP0] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:41 TP2 EP2] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:41 TP3 EP3] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:41 TP1 EP1] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:41.960 trainer_controller_actor] group.py:393 - ft op=update_weights phase=start rollout= -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:41.960 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=update_weights n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:41.962 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:41.963 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 53.3, 'used_GB': 25.88, 'allocated_GB': 3.83, 'reserved_GB': 4.79} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:41.962 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:41.962 actor_cell00000_rank00000] timer.py:31 - Timer update_weights start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:41.963 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 53.58, 'used_GB': 25.59, 'allocated_GB': 3.83, 'reserved_GB': 4.79} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:41.976 actor_cell00000_rank00000] timer.py:31 - Timer update_weights_implementation start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) Update weights: 0%| | 0/24 [00:00 63580.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:45.010 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after update_weights: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 46.7, 'used_GB': 32.48, 'allocated_GB': 3.72, 'reserved_GB': 11.24} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [miles-colocate] Update weights: 23it [00:03, 7.57it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:45.015 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after update_weights: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 48.22, 'used_GB': 30.96, 'allocated_GB': 3.72, 'reserved_GB': 10.05} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP1 EP1] [continue_generation] torch.cuda.empty_cache() called: reserved 64028.0 MB -> 63580.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP2 EP2] [continue_generation] torch.cuda.empty_cache() called: reserved 64028.0 MB -> 63580.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP3 EP3] [continue_generation] torch.cuda.empty_cache() called: reserved 64028.0 MB -> 63580.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP2 EP2] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP0 EP0] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP3 EP3] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP1 EP1] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:45.104 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 3.1s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:45.104 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:45.107 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:45.086 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:45.120 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:45.119 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:45.121 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.2 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:45.140 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=update_weights ok=true elapsed_s=3.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:45.139 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:45.136 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.2 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45] INFO: 10.0.0.8:33596 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45] INFO: 10.0.0.8:33614 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45] INFO: 10.0.0.8:33614 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP0 EP0] Weight version changed. old_version='2' new_version='yeto:1:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP1 EP1] Weight version changed. old_version='2' new_version='yeto:1:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP2 EP2] Weight version changed. old_version='2' new_version='yeto:1:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP3 EP3] Weight version changed. old_version='2' new_version='yeto:1:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45] INFO: 10.0.0.8:45630 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45] INFO: 10.0.0.8:45660 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45] INFO: 10.0.0.8:45660 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP0 EP0] Weight version changed. old_version='2' new_version='yeto:1:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP1 EP1] Weight version changed. old_version='2' new_version='yeto:1:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP2 EP2] Weight version changed. old_version='2' new_version='yeto:1:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP3 EP3] Weight version changed. old_version='2' new_version='yeto:1:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45] INFO: 10.0.0.8:33596 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP0 EP0] [WeightChecker] checksum computed for 509 tensors in 0.216s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP1 EP1] [WeightChecker] checksum computed for 509 tensors in 0.216s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP2 EP2] [WeightChecker] checksum computed for 509 tensors in 0.217s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:45 TP3 EP3] [WeightChecker] checksum computed for 509 tensors in 0.218s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45] INFO: 10.0.0.8:45630 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP0 EP0] [WeightChecker] checksum computed for 437 tensors in 0.205s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP1 EP1] [WeightChecker] checksum computed for 437 tensors in 0.205s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP3 EP3] [WeightChecker] checksum computed for 437 tensors in 0.206s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:45 TP2 EP2] [WeightChecker] checksum computed for 437 tensors in 0.206s -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_publication","island_id":0,"policy_version":1,"rl/policy_token":"yeto:1:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285465.785754} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1308.274093762,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":1297.945657347,"task":"publish","time_unix":1791285465.7861466,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":1,"time_unix":1791285465.7863526} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:45.787 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=sleep n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:45.788 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:45.788 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:45.788 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:45.788 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:45.788 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:45.788 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:45.789 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:45.789 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:45.789 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:46.548 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 0.88, 'used_GB': 78.3, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:46.568 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 1.17, 'used_GB': 78.01, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:46.600 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 1.82, 'used_GB': 77.35, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:46.541 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 0.89, 'used_GB': 78.29, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:46.593 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 1.25, 'used_GB': 77.93, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:46.587 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 1.06, 'used_GB': 78.11, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:46.582 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 2.47, 'used_GB': 76.71, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:46.636 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 1.06, 'used_GB': 78.11, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:50.314 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 11.95, 'used_GB': 67.23, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:50.314 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:50.445 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 11.24, 'used_GB': 67.93, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:50.445 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:50.616 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 11.34, 'used_GB': 67.84, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:50.616 actor_cell00000_rank00004] log_utils.py:436 - [CPU memory] after_offload_train: 147.04 GB (rollout_id=0, step=0) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:50.616 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:50.657 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 11.24, 'used_GB': 67.93, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:50.657 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:50.720 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 11.35, 'used_GB': 67.83, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:50.720 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:50.762 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 11.35, 'used_GB': 67.83, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:50.762 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:50.819 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 11.82, 'used_GB': 67.36, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:50.819 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.0 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1313.499608623,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":1,"start":1308.274435918,"task":"offload","time_unix":1791285471.0116932,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1791285471.01188} -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Rollout generation: 0%| | 0/64 [00:00system\nReasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.\n\nYou are a helpful assistant. Please put the answer within \boxed{}.<|im_end|>\n<|im_start|>user\nTina makes $18.00 an hour. If she works more than 8 hours per shift, she is eligible for overtime, which is paid by your hourly wage + 1/2 your hourly wage. If she works 10 hours eve..., label=990, reward_summary=0.0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Rollout generation: 12%|█▎ | 8/64 [00:04<00:30, 1.83it/s] Rollout generation: 100%|██████████| 64/64 [00:04<00:00, 14.61it/s] -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:17:55.398 rollout_executor] inference_rollout_train.py:161 - Finish rollout: text_preview=<|im_start|>system\nReasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.\n\nYou are a helpful assistant. Please put the answer within \boxed{}.<|im_end|>\n<|im_start|>user\nA deep-sea monster rises from the waters once every hundred years to feast on a ship and sate its hunger. Over three hundred years, it has consumed 847 people. Ships have been built la..., label=121, reward_summary=0.0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:17:55.401 rollout_executor] inference_rollout_train.py:33 - Abort request for ['http://10.0.0.8:20037', 'http://10.0.0.8:20000'] -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:17:55.411 rollout_executor] rollout_data_conversion.py:39 - Final collected 64 samples from rollout to train -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:17:55.412 rollout_executor] metrics.py:93 - perf 1: {'rollout/raw_reward_unfiltered': 0.0, 'rollout/num_training_samples': 64, 'rollout/episode_raw_reward': 0.0, 'rollout/episode_response_length/mean': 1013.78125, 'rollout/episode_response_length/median': 1024.0, 'rollout/episode_response_length/max': 1024, 'rollout/episode_response_length/min': 602, 'rollout/episode_total_response_length/mean': 1013.78125, 'rollout/response_len/mean': 1013.78125, 'rollout/response_len/median': 1024.0, 'rollout/response_len/max': 1024, 'rollout/response_len/min': 602, 'rollout/zero_std/count_0.0': 8, 'rollout/zero_std/all_zero_percentage': 1.0, 'rollout/zero_std/all_one_percentage': 0.0, 'rollout/prefix_cache_hit_rate': 0.0, 'rollout/avg_cached_tokens_per_sample': 0.0, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.96875, 'perf/rollout_time': 4.394840717315674, 'perf/tokens_per_gpu_per_sec': 1845.402489342927, 'perf/longest_sample_tokens_per_sec': 233.0004807604152, 'perf/effective_tokens_per_gpu_per_sec': 1845.402489342927, 'perf/longest_effective_sample_tokens_per_sec': 233.0004807604152} -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:17:55.414 rollout_executor] train_data_conversion.py:334 - Rollout-side DP schedule: num_samples=64, num_rollouts=[64], num_microbatches=[32] -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:55] INFO: 10.0.0.8:40762 - "POST /abort_request HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:55] INFO: 10.0.0.8:37362 - "POST /abort_request HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:55] INFO: 10.0.0.8:40766 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:55 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:55 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:55 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:55 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:55 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:55 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:55 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:55 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:55 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:55 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:55] INFO: 10.0.0.8:37378 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:55 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:55 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:55 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:55 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:55 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:55 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:55 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:55 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:55 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:55 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1318.698010377,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"rollout","rollout_id":1,"start":1313.499965289,"task":"generate","time_unix":1791285476.210133,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":1,"time_unix":1791285476.2103372} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:56] INFO: 10.0.0.8:40766 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:56] INFO: 10.0.0.8:37378 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:56.211 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=wake_up n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:56.212 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:56.213 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 70.13, 'used_GB': 9.05, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:56.212 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:56.212 actor_cell00000_rank00000] timer.py:31 - Timer wake_up start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:56.213 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 70.31, 'used_GB': 8.87, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:56.212 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:56.213 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 70.61, 'used_GB': 8.57, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:56.212 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:56.213 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 70.17, 'used_GB': 9.01, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:56.213 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:56.213 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 70.04, 'used_GB': 9.14, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:56.213 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:56.213 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 69.94, 'used_GB': 9.24, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:56.213 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:56.213 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 69.94, 'used_GB': 9.24, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:56.213 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:56.214 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 70.64, 'used_GB': 8.53, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:57.587 actor_cell00000_rank00000] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27296 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:57.589 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 66.08, 'used_GB': 13.1, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:57.589 actor_cell00000_rank00000] timer.py:42 - Timer wake_up end (elapsed: 1.4s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:57.589 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:57.564 actor_cell00000_rank00003] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27299 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:57.567 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 66.4, 'used_GB': 12.77, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:57.567 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:57.595 actor_cell00000_rank00002] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27300 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:57.597 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 65.94, 'used_GB': 13.24, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:57.598 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:57.579 actor_cell00000_rank00004] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27302 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:57.581 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 66.08, 'used_GB': 13.09, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:57.581 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:57.607 actor_cell00000_rank00006] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27304 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:57.585 actor_cell00000_rank00005] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27303 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:57.588 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 65.99, 'used_GB': 13.19, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:57.588 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:57.587 actor_cell00000_rank00007] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27305 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:57.589 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 66.69, 'used_GB': 12.48, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:57.589 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:57.609 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 65.99, 'used_GB': 13.19, 'allocated_GB': 3.72, 'reserved_GB': 4.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:57.610 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1320.136628849,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":1,"start":1318.698422893,"task":"onload","time_unix":1791285477.6487236,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1791285477.6489222} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:57.647 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=wake_up ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:57.650 trainer_controller_actor] group.py:191 - ft op=train phase=start rollout=1 attempt=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:57.651 trainer_controller_actor] group.py:540 - ft op=refresh phase=start rollout=1 alive=0 healing= all_states="(0, 'StateAllocatedAlive')" quorum=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:57.651 trainer_controller_actor] group.py:563 - ft op=refresh phase=decision rollout=1 needs_reconfigure=false reason=alive_config_unchanged,no_healing quorum=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:17:57.651 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=train n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:57.643 actor_cell00000_rank00001] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27297 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:57.646 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 65.93, 'used_GB': 13.25, 'allocated_GB': 3.83, 'reserved_GB': 4.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:57.646 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:57.652 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:57.664 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:17:57.664 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:57.652 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:57.652 actor_cell00000_rank00000] timer.py:31 - Timer data_preprocess start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:57.664 actor_cell00000_rank00000] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:57.664 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:57.664 actor_cell00000_rank00000] timer.py:42 - Timer train_wait end (elapsed: 27.9s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:57.664 actor_cell00000_rank00000] timer.py:31 - Timer train start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:57.664 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:17:57.664 actor_cell00000_rank00000] timer.py:31 - Timer log_probs start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:57.653 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:57.664 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:17:57.664 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:57.652 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:57.664 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:17:57.664 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:57.653 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:57.665 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:17:57.665 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:57.653 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:57.665 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:17:57.665 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:57.653 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:57.667 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:17:57.667 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:57.653 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:57.665 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:17:57.665 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:17:59] INFO: 10.0.0.8:40372 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:17:59] INFO: 10.0.0.8:48012 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":1,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":1,"rounds_completed":1,"seq":12,"time_unix":1791285482.8119621,"trained_version":1,"uptime_s":360.055,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:02.764 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=5.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:02.764 actor_cell00000_rank00000] timer.py:42 - Timer log_probs end (elapsed: 5.1s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:02.764 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=5.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:02.764 actor_cell00000_rank00000] timer.py:31 - Timer actor_train start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:02.763 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=5.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:02.763 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=5.1 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"available":true,"config_epoch":0,"cpu_rss_bytes":3804639232,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":18556.625,"power_w":207.945,"util_pct":54,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":18804.625,"power_w":229.677,"util_pct":37,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":18726.625,"power_w":213.555,"util_pct":66,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":17768.625,"power_w":204.834,"util_pct":37,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":20480.625,"power_w":256.025,"util_pct":53,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":20658.625,"power_w":242.575,"util_pct":36,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":20154.625,"power_w":252.484,"util_pct":46,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":18964.625,"power_w":253.466,"util_pct":53,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1325.403583624,"time_unix":1791285482.915712,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:02.865 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=5.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:02.872 actor_cell00000_rank00004] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:02.873 actor_cell00000_rank00004] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:02.874 actor_cell00000_rank00004] log_utils.py:126 - rollout 1: {'rollout/response_lengths': 1013.78125, 'rollout/rewards': 0.0, 'rollout/truncated': 0.96875, 'rollout/rollout_log_probs': -12.002046585083008, 'rollout/raw_reward': 0.0, 'rollout/total_lengths': 1150.90625, 'rollout/log_probs': -12.004202842712402, 'rollout/advantages': 0.0, 'rollout/returns': 0.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:02.863 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=5.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:02.872 actor_cell00000_rank00006] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:02.874 actor_cell00000_rank00006] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=1 success=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:02.864 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=5.2 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:02.863 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=5.2 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"aborted_in_flight_groups":0,"adv_mean":null,"adv_std":null,"applied_lrs":[8.333333333333334e-06],"clip_fraction":null,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"masked_fraction":null,"nonzero_advantages":null,"resp_len_mean":1013.78125,"resp_len_p95":1024.0,"reward_p10":0.0,"reward_p50":0.0,"reward_p90":0.0,"rollout_id":1,"step_seconds":10.55372185200008,"submitted_groups":8,"time_unix":1791285488.2030578,"tok_per_s":6147.783778071045,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0,"pg_clipfrac":0.0,"pg_loss":0.0,"ppo_kl":0.0,"train_rollout_kl":0.0019241459667682648,"train_rollout_logprob_abs_diff":0.03648190200328827},"train_step":2,"trained_groups":8,"trained_sample_ids_sha256":"5883e6fff48f762828700c935c3f3dddb9079f20a565f30b5aa34ce943e82277","trained_samples":64,"truncated_frac":0.96875} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_round_labels","execution_mode":"colocated-serial","island_id":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/aborted_groups":0,"rl/carried_over_groups":null,"rl/clip_fraction":null,"rl/ess_ratio":1.0,"rl/filtered_groups":0,"rl/groups":8,"rl/masked_fraction":null,"rl/mean_kl":0.0,"rollout_id":1,"t":1330.691430879,"time_unix":1791285488.2034228,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1330.691631206,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":1,"start":1320.136991544,"task":"train","time_unix":1791285488.20359,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1791285488.2036805} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:08.187 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=train ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:08.188 trainer_controller_actor] group.py:291 - ft op=check errored= discarded= normal=0 decision=no_retry reason="survivors normal, gradients valid" -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:08.191 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:08.193 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:08.195 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:08.197 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:08.199 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:08.201 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:08.204 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:08.183 actor_cell00000_rank00001] model.py:646 - train op=train_step rollout=1 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:08.183 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:08.184 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:08.183 actor_cell00000_rank00000] model.py:646 - train op=train_step rollout=1 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:08.183 actor_cell00000_rank00000] timer.py:42 - Timer actor_train end (elapsed: 5.4s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:08.183 actor_cell00000_rank00000] timer.py:42 - Timer train end (elapsed: 10.5s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:08.183 actor_cell00000_rank00000] timer.py:31 - Timer train_wait start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:08.183 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:08.184 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:08.183 actor_cell00000_rank00003] model.py:646 - train op=train_step rollout=1 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:08.183 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:08.184 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:08.183 actor_cell00000_rank00002] model.py:646 - train op=train_step rollout=1 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:08.184 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:08.185 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:08.183 actor_cell00000_rank00004] model.py:646 - train op=train_step rollout=1 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:08.184 actor_cell00000_rank00004] log_utils.py:554 - step 1: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.03648190200328827, 'train/train_rollout_kl': 0.0019241459667682648, 'train/grad_norm': 0.0, 'train/lr-pg_0': 6.666666666666668e-06, 'train/lr-pg_1': 6.666666666666668e-06, 'train/step': 1} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:08.184 actor_cell00000_rank00004] model.py:856 - step 1: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.03648190200328827, 'train/train_rollout_kl': 0.0019241459667682648, 'train/grad_norm': 0.0, 'train/lr-pg_0': 6.666666666666668e-06, 'train/lr-pg_1': 6.666666666666668e-06, 'train/step': 1} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:08.184 actor_cell00000_rank00004] train_metric_utils.py:56 - perf 1: {'perf/update_weights_implementation_time': 1.836961269378662, 'perf/finalize_and_resume_engines_time': 1.1489241123199463, 'perf/update_weights_time': 3.173668622970581, 'perf/sleep_time': 4.827679872512817, 'perf/wake_up_time': 1.3685288429260254, 'perf/data_preprocess_time': 0.011874675750732422, 'perf/train_wait_time': 27.934258699417114, 'perf/log_probs_time': 5.199958086013794, 'perf/actor_train_time': 5.310054063796997, 'perf/train_time': 10.518796682357788, 'perf/log_probs_tflops': 3.401762521732964, 'perf/actor_train_tflops': 9.993696289563946, 'perf/actor_train_tok_per_s': 13871.421856547022, 'perf/mfu_peak_tflops': 989.0, 'perf/actor_train_mfu': 0.010104849635555052, 'perf/step_time': 38.4530553817749, 'perf/wait_time_ratio': 0.7264509522605258} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:08.184 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:08.185 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:08.182 actor_cell00000_rank00006] model.py:646 - train op=train_step rollout=1 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:08.184 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:08.185 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:08.183 actor_cell00000_rank00005] model.py:646 - train op=train_step rollout=1 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:08.184 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:08.185 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:08.182 actor_cell00000_rank00007] model.py:646 - train op=train_step rollout=1 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:08.184 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:08.185 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.5 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:10.492 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=2.3 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"action_tokens":64882,"active_groups":8,"applied_lr":8.333333333333334e-06,"applied_lrs":[8.333333333333334e-06],"base_policy_version":1,"cancelled_groups":0,"clip_fraction":null,"completed_groups":8,"completed_trajectories":64,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":8,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":0.0,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":2,"loss":0.0,"lr":8.333333333333334e-06,"mean_kl":0.0,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":0.0,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":10.4235576760002,"time_unix":1791285493.0537043,"tool_wait_seconds":0.0,"train_seconds":10.55372185200008,"train_step":2,"zero_variance_group_ratio":1.0} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1336.426499783,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":1,"start":1330.691771297,"task":"outer_sync","time_unix":1791285493.938855,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1791285493.939187} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:14.851 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:14] INFO: 10.0.0.8:60290 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:14] INFO: 10.0.0.8:41112 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:17.118 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=2.3 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:20] INFO: 10.0.0.8:56196 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:20] INFO: 10.0.0.8:56204 - "POST /pause_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:20] INFO: 10.0.0.8:56204 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:20] INFO: 10.0.0.8:56204 - "POST /begin_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:20] INFO: 10.0.0.8:56206 - "POST /update_weights_from_tensor HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:20] INFO: 10.0.0.8:56206 - "POST /update_weights_from_tensor HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:20 TP0 EP0] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:20 TP2 EP2] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:20 TP1 EP1] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:20 TP3 EP3] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:20] INFO: 10.0.0.8:43808 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:20] INFO: 10.0.0.8:43818 - "POST /pause_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:20] INFO: 10.0.0.8:43818 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:20] INFO: 10.0.0.8:43818 - "POST /begin_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:20] INFO: 10.0.0.8:43818 - "POST /update_weights_from_tensor HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:20] INFO: 10.0.0.8:43818 - "POST /update_weights_from_tensor HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:20 TP0 EP0] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:20 TP2 EP2] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:20 TP1 EP1] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:20 TP3 EP3] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:20.125 trainer_controller_actor] group.py:393 - ft op=update_weights phase=start rollout= -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:20.125 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=update_weights n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:20.127 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:20.128 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 53.05, 'used_GB': 26.13, 'allocated_GB': 3.84, 'reserved_GB': 5.04} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:20.127 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:20.127 actor_cell00000_rank00000] timer.py:31 - Timer update_weights start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:20.128 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 53.26, 'used_GB': 25.92, 'allocated_GB': 3.83, 'reserved_GB': 5.11} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:20.139 actor_cell00000_rank00000] timer.py:31 - Timer update_weights_implementation start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) Update weights: 0%| | 0/24 [00:00 63706.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP3 EP3] [continue_generation] torch.cuda.empty_cache() called: reserved 64154.0 MB -> 63706.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP2 EP2] [continue_generation] torch.cuda.empty_cache() called: reserved 64154.0 MB -> 63706.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP0 EP0] [continue_generation] torch.cuda.empty_cache() called: reserved 64154.0 MB -> 63706.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP1 EP1] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP0 EP0] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP3 EP3] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP2 EP2] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:23.178 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=update_weights ok=true elapsed_s=3.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [miles-colocate] Update weights: 23it [00:02, 7.70it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:23.127 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 46.66, 'used_GB': 32.52, 'allocated_GB': 3.84, 'reserved_GB': 11.53} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:23.172 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [miles-colocate] Update weights: 23it [00:02, 7.69it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:23.133 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 47.63, 'used_GB': 31.55, 'allocated_GB': 3.83, 'reserved_GB': 10.84} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:23.170 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 3.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:23.170 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [miles-colocate] Update weights: 23it [00:02, 7.69it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:23.132 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after update_weights: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 47.92, 'used_GB': 31.26, 'allocated_GB': 3.84, 'reserved_GB': 11.21} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:23.173 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [miles-colocate] Update weights: 23it [00:02, 7.69it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:23.129 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after update_weights: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 46.76, 'used_GB': 32.42, 'allocated_GB': 3.84, 'reserved_GB': 11.43} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:23.162 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [miles-colocate] Update weights: 23it [00:02, 7.69it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:23.131 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after update_weights: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 48.04, 'used_GB': 31.14, 'allocated_GB': 3.72, 'reserved_GB': 10.1} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:23.175 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [miles-colocate] Update weights: 23it [00:02, 7.70it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:23.128 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after update_weights: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 46.67, 'used_GB': 32.51, 'allocated_GB': 3.73, 'reserved_GB': 11.29} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:23.176 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [miles-colocate] Update weights: 23it [00:02, 7.69it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:23.133 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after update_weights: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 46.66, 'used_GB': 32.52, 'allocated_GB': 3.73, 'reserved_GB': 11.3} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:23.177 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [miles-colocate] Update weights: 23it [00:02, 7.69it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:23.129 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after update_weights: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 48.09, 'used_GB': 31.08, 'allocated_GB': 3.73, 'reserved_GB': 11.27} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:23.177 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23] INFO: 10.0.0.8:56196 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23] INFO: 10.0.0.8:43808 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP1 EP1] Weight version changed. old_version='3' new_version='yeto:2:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP3 EP3] Weight version changed. old_version='3' new_version='yeto:2:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP0 EP0] Weight version changed. old_version='3' new_version='yeto:2:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP2 EP2] Weight version changed. old_version='3' new_version='yeto:2:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23] INFO: 10.0.0.8:43822 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23] INFO: 10.0.0.8:56220 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23] INFO: 10.0.0.8:56220 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP1 EP1] Weight version changed. old_version='3' new_version='yeto:2:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP0 EP0] Weight version changed. old_version='3' new_version='yeto:2:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP3 EP3] Weight version changed. old_version='3' new_version='yeto:2:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP2 EP2] Weight version changed. old_version='3' new_version='yeto:2:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23] INFO: 10.0.0.8:43822 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_publication","island_id":0,"policy_version":2,"rl/policy_token":"yeto:2:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285503.703498} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1346.19180621,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":1336.427271178,"task":"publish","time_unix":1791285503.7038512,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":2,"time_unix":1791285503.718487} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23] INFO: 10.0.0.8:56196 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP1 EP1] [WeightChecker] checksum computed for 509 tensors in 0.215s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP3 EP3] [WeightChecker] checksum computed for 509 tensors in 0.215s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP2 EP2] [WeightChecker] checksum computed for 509 tensors in 0.216s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:23 TP0 EP0] [WeightChecker] checksum computed for 509 tensors in 0.216s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23] INFO: 10.0.0.8:43808 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP0 EP0] [WeightChecker] checksum computed for 437 tensors in 0.206s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP1 EP1] [WeightChecker] checksum computed for 437 tensors in 0.206s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP3 EP3] [WeightChecker] checksum computed for 437 tensors in 0.207s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:23 TP2 EP2] [WeightChecker] checksum computed for 437 tensors in 0.207s -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:23.719 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=sleep n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:23.721 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:23.721 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:23.721 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:23.721 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:23.721 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:23.721 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:23.721 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:23.721 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:23.721 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:24.338 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 1.99, 'used_GB': 77.19, 'allocated_GB': 3.84, 'reserved_GB': 4.11} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:24.334 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 0.83, 'used_GB': 78.35, 'allocated_GB': 3.84, 'reserved_GB': 4.34} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:24.441 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 0.73, 'used_GB': 78.44, 'allocated_GB': 3.84, 'reserved_GB': 4.43} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:24.391 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 1.11, 'used_GB': 78.07, 'allocated_GB': 3.83, 'reserved_GB': 4.34} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:24.397 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 1.07, 'used_GB': 78.1, 'allocated_GB': 3.72, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:24.405 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 0.89, 'used_GB': 78.29, 'allocated_GB': 3.73, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:24.413 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 0.88, 'used_GB': 78.3, 'allocated_GB': 3.73, 'reserved_GB': 4.06} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:24.428 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 2.31, 'used_GB': 76.87, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:28.627 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 11.12, 'used_GB': 68.06, 'allocated_GB': 3.73, 'reserved_GB': 4.06} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:28.627 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:28.667 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 11.82, 'used_GB': 67.35, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:28.667 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:29.149 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 11.81, 'used_GB': 67.36, 'allocated_GB': 3.84, 'reserved_GB': 4.11} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:29.149 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:29.196 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 11.49, 'used_GB': 67.69, 'allocated_GB': 3.83, 'reserved_GB': 4.34} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:29.196 actor_cell00000_rank00000] timer.py:42 - Timer sleep end (elapsed: 5.5s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:29.196 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:29.202 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 11.12, 'used_GB': 68.06, 'allocated_GB': 3.73, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:29.202 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:29.325 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 11.35, 'used_GB': 67.83, 'allocated_GB': 3.84, 'reserved_GB': 4.34} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:29.326 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:29.271 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 11.22, 'used_GB': 67.96, 'allocated_GB': 3.72, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:29.271 actor_cell00000_rank00004] log_utils.py:436 - [CPU memory] after_offload_train: 150.11 GB (rollout_id=1, step=1) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:29.271 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.5 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1351.976057432,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":2,"start":1346.206538009,"task":"offload","time_unix":1791285509.4881387,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1791285509.48834} -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Rollout generation: 0%| | 0/64 [00:00system\nReasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.\n\nYou are a helpful assistant. Please put the answer within \boxed{}.<|im_end|>\n<|im_start|>user\nIt takes Roque two hours to walk to work and one hour to ride his bike to work. Roque walks to and from work three times a week and rides his bike to and from work twice a week. How ma..., label=16, reward_summary=0.0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Rollout generation: 12%|█▎ | 8/64 [00:03<00:26, 2.09it/s] Rollout generation: 100%|██████████| 64/64 [00:03<00:00, 16.69it/s] -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:18:33.327 rollout_executor] inference_rollout_train.py:161 - Finish rollout: text_preview=<|im_start|>system\nReasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.\n\nYou are a helpful assistant. Please put the answer within \boxed{}.<|im_end|>\n<|im_start|>user\nBella bought stamps at the post office. Some of the stamps had a snowflake design, some had a truck design, and some had a rose design. Bella bought 11 snowflake stamps. She bought 9 m..., label=38, reward_summary=0.0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:18:33.330 rollout_executor] inference_rollout_train.py:33 - Abort request for ['http://10.0.0.8:20037', 'http://10.0.0.8:20000'] -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:18:33.343 rollout_executor] rollout_data_conversion.py:39 - Final collected 64 samples from rollout to train -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:18:33.344 rollout_executor] metrics.py:93 - perf 2: {'rollout/raw_reward_unfiltered': 0.0, 'rollout/num_training_samples': 64, 'rollout/episode_raw_reward': 0.0, 'rollout/episode_response_length/mean': 1024.0, 'rollout/episode_response_length/median': 1024.0, 'rollout/episode_response_length/max': 1024, 'rollout/episode_response_length/min': 1024, 'rollout/episode_total_response_length/mean': 1024.0, 'rollout/response_len/mean': 1024.0, 'rollout/response_len/median': 1024.0, 'rollout/response_len/max': 1024, 'rollout/response_len/min': 1024, 'rollout/zero_std/count_0.0': 8, 'rollout/zero_std/all_zero_percentage': 1.0, 'rollout/zero_std/all_one_percentage': 0.0, 'rollout/prefix_cache_hit_rate': 0.0, 'rollout/avg_cached_tokens_per_sample': 0.0, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 1.0, 'perf/rollout_time': 3.850187063217163, 'perf/tokens_per_gpu_per_sec': 2127.688827969537, 'perf/longest_sample_tokens_per_sec': 265.96110349619215, 'perf/effective_tokens_per_gpu_per_sec': 2127.688827969537, 'perf/longest_effective_sample_tokens_per_sec': 265.96110349619215} -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:18:33.346 rollout_executor] train_data_conversion.py:334 - Rollout-side DP schedule: num_samples=64, num_rollouts=[64], num_microbatches=[32] -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:33] INFO: 10.0.0.8:54198 - "POST /abort_request HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:33] INFO: 10.0.0.8:54202 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:33 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:33 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:33 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:33 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:33 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:33 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:33 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:33 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:33 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:33 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:33] INFO: 10.0.0.8:35912 - "POST /abort_request HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:33] INFO: 10.0.0.8:35918 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:33 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:33 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:33 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:33 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:33 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:33 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:33 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:33 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:33 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:33 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:34] INFO: 10.0.0.8:54202 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1356.658371898,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"rollout","rollout_id":2,"start":1351.976419693,"task":"generate","time_unix":1791285514.1704533,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":2,"time_unix":1791285514.1706393} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:34] INFO: 10.0.0.8:35918 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:34.171 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=wake_up n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:34.173 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:34.173 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 70.17, 'used_GB': 9.01, 'allocated_GB': 3.84, 'reserved_GB': 4.43} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:34.173 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:34.173 actor_cell00000_rank00000] timer.py:31 - Timer wake_up start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:34.173 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 70.31, 'used_GB': 8.87, 'allocated_GB': 3.83, 'reserved_GB': 4.34} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:34.173 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:34.174 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 70.64, 'used_GB': 8.54, 'allocated_GB': 3.84, 'reserved_GB': 4.11} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:34.173 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:34.174 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 70.17, 'used_GB': 9.01, 'allocated_GB': 3.84, 'reserved_GB': 4.34} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:34.173 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:34.174 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 70.04, 'used_GB': 9.14, 'allocated_GB': 3.72, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:34.173 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:34.174 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 69.94, 'used_GB': 9.23, 'allocated_GB': 3.73, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:34.173 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:34.174 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 69.94, 'used_GB': 9.23, 'allocated_GB': 3.73, 'reserved_GB': 4.06} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:34.173 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:34.174 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 70.65, 'used_GB': 8.53, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:35.484 actor_cell00000_rank00003] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27299 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:35.487 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 66.57, 'used_GB': 12.61, 'allocated_GB': 3.84, 'reserved_GB': 4.11} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:35.487 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1358.050608167,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":2,"start":1356.658724345,"task":"onload","time_unix":1791285515.5626972,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1791285515.5629075} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:35.561 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=wake_up ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:35.565 trainer_controller_actor] group.py:191 - ft op=train phase=start rollout=2 attempt=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:35.565 trainer_controller_actor] group.py:540 - ft op=refresh phase=start rollout=2 alive=0 healing= all_states="(0, 'StateAllocatedAlive')" quorum=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:35.565 trainer_controller_actor] group.py:563 - ft op=refresh phase=decision rollout=2 needs_reconfigure=false reason=alive_config_unchanged,no_healing quorum=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:35.565 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=train n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:35.508 actor_cell00000_rank00001] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27297 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:35.511 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 65.78, 'used_GB': 13.4, 'allocated_GB': 3.84, 'reserved_GB': 4.43} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:35.511 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:35.566 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:35.578 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:35.578 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:35.502 actor_cell00000_rank00000] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27296 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:35.505 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 66.02, 'used_GB': 13.16, 'allocated_GB': 3.83, 'reserved_GB': 4.34} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:35.505 actor_cell00000_rank00000] timer.py:42 - Timer wake_up end (elapsed: 1.3s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:35.505 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:35.566 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:35.566 actor_cell00000_rank00000] timer.py:31 - Timer data_preprocess start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:35.578 actor_cell00000_rank00000] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:35.578 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:35.578 actor_cell00000_rank00000] timer.py:42 - Timer train_wait end (elapsed: 27.4s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:35.578 actor_cell00000_rank00000] timer.py:31 - Timer train start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:35.578 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:35.578 actor_cell00000_rank00000] timer.py:31 - Timer log_probs start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:35.567 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:35.579 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:35.579 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:35.548 actor_cell00000_rank00002] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27300 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:35.551 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 65.88, 'used_GB': 13.3, 'allocated_GB': 3.84, 'reserved_GB': 4.34} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:35.551 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:35.567 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:35.579 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:35.579 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:35.551 actor_cell00000_rank00004] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27302 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:35.554 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 66.03, 'used_GB': 13.15, 'allocated_GB': 3.72, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:35.554 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:35.567 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:35.579 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:35.579 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:35.534 actor_cell00000_rank00006] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27304 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:35.537 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 65.94, 'used_GB': 13.24, 'allocated_GB': 3.73, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:35.537 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:35.567 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:35.579 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:35.579 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:35.557 actor_cell00000_rank00005] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27303 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:35.560 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 65.93, 'used_GB': 13.25, 'allocated_GB': 3.73, 'reserved_GB': 4.06} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:35.560 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:35.567 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:35.579 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:35.579 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:35.533 actor_cell00000_rank00007] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27305 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:35.535 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 66.66, 'used_GB': 12.52, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:35.535 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:35.567 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:35.579 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:35.580 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:40.171 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:40.171 actor_cell00000_rank00000] timer.py:42 - Timer log_probs end (elapsed: 4.6s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:40.171 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:40.171 actor_cell00000_rank00000] timer.py:31 - Timer actor_train start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:40.171 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:40.171 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:40.275 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:40.291 actor_cell00000_rank00004] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:40.292 actor_cell00000_rank00004] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:40.292 actor_cell00000_rank00004] log_utils.py:126 - rollout 2: {'rollout/response_lengths': 1024.0, 'rollout/rewards': 0.0, 'rollout/truncated': 1.0, 'rollout/rollout_log_probs': -11.994685173034668, 'rollout/raw_reward': 0.0, 'rollout/total_lengths': 1153.75, 'rollout/log_probs': -11.996010303497314, 'rollout/advantages': 0.0, 'rollout/returns': 0.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:40.275 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:40.290 actor_cell00000_rank00006] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:40.292 actor_cell00000_rank00006] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=1 success=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:40.284 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:40.276 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:44] INFO: 10.0.0.8:46050 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:44] INFO: 10.0.0.8:41494 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"aborted_in_flight_groups":0,"adv_mean":null,"adv_std":null,"applied_lrs":[6.666666666666668e-06],"clip_fraction":null,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"masked_fraction":null,"nonzero_advantages":null,"resp_len_mean":1024.0,"resp_len_p95":1024.0,"reward_p10":0.0,"reward_p50":0.0,"reward_p90":0.0,"rollout_id":2,"step_seconds":10.461073258999932,"submitted_groups":8,"time_unix":1791285526.0243704,"tok_per_s":6264.749168410391,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0,"pg_clipfrac":0.0,"pg_loss":0.0,"ppo_kl":0.0,"train_rollout_kl":0.0018938753055408597,"train_rollout_logprob_abs_diff":0.036156367510557175},"train_step":3,"trained_groups":8,"trained_sample_ids_sha256":"7cb590236a092586fda2fe77d6388783755fbee91cab2e6056c152578caf1fa5","trained_samples":64,"truncated_frac":1.0} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_round_labels","execution_mode":"colocated-serial","island_id":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/aborted_groups":0,"rl/carried_over_groups":null,"rl/clip_fraction":null,"rl/ess_ratio":1.0,"rl/filtered_groups":0,"rl/groups":8,"rl/masked_fraction":null,"rl/mean_kl":0.0,"rollout_id":2,"t":1368.512745413,"time_unix":1791285526.0247386,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1368.512946339,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":2,"start":1358.050978623,"task":"train","time_unix":1791285526.0249188,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1791285526.0250077} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:46.010 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=train ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:46.010 trainer_controller_actor] group.py:291 - ft op=check errored= discarded= normal=0 decision=no_retry reason="survivors normal, gradients valid" -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:46.013 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:46.015 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:46.017 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:46.019 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:46.021 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:46.023 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:46.026 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:46.006 actor_cell00000_rank00001] model.py:646 - train op=train_step rollout=2 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:46.006 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:46.007 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:46.007 actor_cell00000_rank00000] model.py:646 - train op=train_step rollout=2 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:46.007 actor_cell00000_rank00000] timer.py:42 - Timer actor_train end (elapsed: 5.8s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:46.007 actor_cell00000_rank00000] timer.py:42 - Timer train end (elapsed: 10.4s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:46.007 actor_cell00000_rank00000] timer.py:31 - Timer train_wait start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:46.008 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:46.009 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:46.006 actor_cell00000_rank00003] model.py:646 - train op=train_step rollout=2 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:46.006 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:18:46.007 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:46.006 actor_cell00000_rank00002] model.py:646 - train op=train_step rollout=2 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:46.007 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:18:46.008 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:46.006 actor_cell00000_rank00004] model.py:646 - train op=train_step rollout=2 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:46.007 actor_cell00000_rank00004] log_utils.py:554 - step 2: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.036156367510557175, 'train/train_rollout_kl': 0.0018938753055408597, 'train/grad_norm': 0.0, 'train/lr-pg_0': 5e-06, 'train/lr-pg_1': 5e-06, 'train/step': 2} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:46.007 actor_cell00000_rank00004] model.py:856 - step 2: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.036156367510557175, 'train/train_rollout_kl': 0.0018938753055408597, 'train/grad_norm': 0.0, 'train/lr-pg_0': 5e-06, 'train/lr-pg_1': 5e-06, 'train/step': 2} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:46.007 actor_cell00000_rank00004] train_metric_utils.py:56 - perf 2: {'perf/update_weights_implementation_time': 1.7147185802459717, 'perf/finalize_and_resume_engines_time': 1.2455425262451172, 'perf/update_weights_time': 3.0473546981811523, 'perf/sleep_time': 5.54970908164978, 'perf/wake_up_time': 1.3803250789642334, 'perf/data_preprocess_time': 0.011986970901489258, 'perf/train_wait_time': 27.395413875579834, 'perf/log_probs_time': 4.696130752563477, 'perf/actor_train_time': 5.71471905708313, 'perf/train_time': 10.427717208862305, 'perf/log_probs_tflops': 3.775975484413498, 'perf/actor_train_tflops': 9.308843225443296, 'perf/actor_train_tok_per_s': 12921.02013457315, 'perf/mfu_peak_tflops': 989.0, 'perf/actor_train_mfu': 0.00941237939883043, 'perf/step_time': 37.82313108444214, 'perf/wait_time_ratio': 0.7243031734844512} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:46.007 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:18:46.009 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:46.006 actor_cell00000_rank00006] model.py:646 - train op=train_step rollout=2 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:46.007 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:18:46.008 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:46.006 actor_cell00000_rank00005] model.py:646 - train op=train_step rollout=2 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:46.007 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:18:46.008 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:46.005 actor_cell00000_rank00007] model.py:646 - train op=train_step rollout=2 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:46.007 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:18:46.008 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.4 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:48.293 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=2.3 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"action_tokens":65536,"active_groups":8,"applied_lr":6.666666666666668e-06,"applied_lrs":[6.666666666666668e-06],"base_policy_version":2,"cancelled_groups":0,"clip_fraction":null,"completed_groups":8,"completed_trajectories":64,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":8,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":0.0,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":3,"loss":0.0,"lr":6.666666666666668e-06,"mean_kl":0.0,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":0.0,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":10.451829019000115,"time_unix":1791285530.7120347,"tool_wait_seconds":0.0,"train_seconds":10.461073258999932,"train_step":3,"zero_variance_group_ratio":1.0} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1374.075931474,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":2,"start":1368.513098128,"task":"outer_sync","time_unix":1791285531.588047,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1791285531.5882518} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:52.466 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:54.746 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=2.3 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:57] INFO: 10.0.0.8:52904 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:57] INFO: 10.0.0.8:52912 - "POST /pause_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:57] INFO: 10.0.0.8:52912 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:57] INFO: 10.0.0.8:52912 - "POST /begin_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:57 TP0 EP0] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:57 TP2 EP2] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:57 TP3 EP3] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:18:57 TP1 EP1] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:57] INFO: 10.0.0.8:41530 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:57] INFO: 10.0.0.8:41540 - "POST /pause_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:57] INFO: 10.0.0.8:41540 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:57] INFO: 10.0.0.8:41540 - "POST /begin_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:57 TP0 EP0] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:57 TP2 EP2] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:57 TP3 EP3] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:18:57 TP1 EP1] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:57.762 trainer_controller_actor] group.py:393 - ft op=update_weights phase=start rollout= -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:18:57.762 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=update_weights n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:57.763 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:18:57.764 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 52.96, 'used_GB': 26.22, 'allocated_GB': 3.84, 'reserved_GB': 5.12} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:57.763 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:57.763 actor_cell00000_rank00000] timer.py:31 - Timer update_weights start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:57.764 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 53.27, 'used_GB': 25.91, 'allocated_GB': 3.84, 'reserved_GB': 5.1} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:18:57.777 actor_cell00000_rank00000] timer.py:31 - Timer update_weights_implementation start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) Update weights: 0%| | 0/24 [00:00 63704.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:00 TP3 EP3] [continue_generation] torch.cuda.empty_cache() called: reserved 64152.0 MB -> 63704.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:00 TP0 EP0] [continue_generation] torch.cuda.empty_cache() called: reserved 64152.0 MB -> 63704.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:00 TP2 EP2] [continue_generation] torch.cuda.empty_cache() called: reserved 64152.0 MB -> 63704.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:00 TP0 EP0] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:00 TP3 EP3] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:00 TP1 EP1] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:00 TP2 EP2] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:00.791 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=update_weights ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [miles-colocate] Update weights: 23it [00:02, 7.77it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:00.740 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 46.66, 'used_GB': 32.51, 'allocated_GB': 3.84, 'reserved_GB': 11.52} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:00.784 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [miles-colocate] Update weights: 23it [00:02, 7.76it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:00.742 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 47.58, 'used_GB': 31.6, 'allocated_GB': 3.84, 'reserved_GB': 10.89} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:00.772 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 3.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:00.772 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [miles-colocate] Update weights: 23it [00:02, 7.74it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:00.751 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after update_weights: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 47.92, 'used_GB': 31.26, 'allocated_GB': 3.84, 'reserved_GB': 11.21} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:00.790 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [miles-colocate] Update weights: 23it [00:02, 7.76it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:00.742 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after update_weights: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 46.72, 'used_GB': 32.46, 'allocated_GB': 3.84, 'reserved_GB': 11.46} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:00.779 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [miles-colocate] Update weights: 23it [00:02, 7.76it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:00.741 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after update_weights: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 48.07, 'used_GB': 31.11, 'allocated_GB': 3.73, 'reserved_GB': 10.08} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:00.777 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [miles-colocate] Update weights: 23it [00:02, 7.77it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:00.741 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after update_weights: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 46.67, 'used_GB': 32.5, 'allocated_GB': 3.73, 'reserved_GB': 11.29} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:00.783 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [miles-colocate] Update weights: 23it [00:02, 7.77it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:00.737 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after update_weights: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 46.64, 'used_GB': 32.54, 'allocated_GB': 3.73, 'reserved_GB': 11.31} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:00.775 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [miles-colocate] Update weights: 23it [00:02, 7.77it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:00.740 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after update_weights: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 48.09, 'used_GB': 31.09, 'allocated_GB': 3.73, 'reserved_GB': 11.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:00.785 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=3.0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01] INFO: 10.0.0.8:52904 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01] INFO: 10.0.0.8:52934 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01] INFO: 10.0.0.8:52934 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01 TP3 EP3] Weight version changed. old_version='4' new_version='yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01 TP1 EP1] Weight version changed. old_version='4' new_version='yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01 TP0 EP0] Weight version changed. old_version='4' new_version='yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01 TP2 EP2] Weight version changed. old_version='4' new_version='yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01] INFO: 10.0.0.8:41530 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01] INFO: 10.0.0.8:41560 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01] INFO: 10.0.0.8:41560 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01 TP0 EP0] Weight version changed. old_version='4' new_version='yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01 TP2 EP2] Weight version changed. old_version='4' new_version='yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01 TP1 EP1] Weight version changed. old_version='4' new_version='yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01 TP3 EP3] Weight version changed. old_version='4' new_version='yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01] INFO: 10.0.0.8:52904 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01 TP3 EP3] [WeightChecker] checksum computed for 509 tensors in 0.216s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01 TP1 EP1] [WeightChecker] checksum computed for 509 tensors in 0.216s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01 TP2 EP2] [WeightChecker] checksum computed for 509 tensors in 0.217s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:01 TP0 EP0] [WeightChecker] checksum computed for 509 tensors in 0.217s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01] INFO: 10.0.0.8:41530 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01 TP0 EP0] [WeightChecker] checksum computed for 437 tensors in 0.209s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01 TP3 EP3] [WeightChecker] checksum computed for 437 tensors in 0.210s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01 TP1 EP1] [WeightChecker] checksum computed for 437 tensors in 0.210s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:01 TP2 EP2] [WeightChecker] checksum computed for 437 tensors in 0.210s -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_publication","island_id":0,"policy_version":3,"rl/policy_token":"yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285541.456038} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1383.944375504,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":1374.076338792,"task":"publish","time_unix":1791285541.4564235,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":3,"time_unix":1791285541.4700155} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:01.471 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=sleep n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:01.472 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:01.472 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:01.472 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:01.472 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:01.472 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:01.473 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:01.473 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:01.473 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:01.473 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:02.097 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 0.74, 'used_GB': 78.44, 'allocated_GB': 3.84, 'reserved_GB': 4.42} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:02.103 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 1.06, 'used_GB': 78.12, 'allocated_GB': 3.84, 'reserved_GB': 4.39} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:02.109 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 1.99, 'used_GB': 77.19, 'allocated_GB': 3.84, 'reserved_GB': 4.11} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:02.136 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 0.8, 'used_GB': 78.38, 'allocated_GB': 3.84, 'reserved_GB': 4.37} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:02.139 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 0.86, 'used_GB': 78.32, 'allocated_GB': 3.73, 'reserved_GB': 4.08} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:02.132 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 2.3, 'used_GB': 76.88, 'allocated_GB': 3.73, 'reserved_GB': 4.04} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:02.215 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 1.1, 'used_GB': 78.08, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:02.202 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 0.89, 'used_GB': 78.29, 'allocated_GB': 3.73, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"offload","pid":18336,"policy_version":3,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":3,"rounds_completed":3,"seq":14,"time_unix":1791285542.813056,"trained_version":3,"uptime_s":420.056,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"available":true,"config_epoch":0,"cpu_rss_bytes":3852828672,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":79252.625,"power_w":120.782,"util_pct":1,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":79580.625,"power_w":128.019,"util_pct":0,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":79522.625,"power_w":125.857,"util_pct":0,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":78302.625,"power_w":120.933,"util_pct":0,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":79278.625,"power_w":124.591,"util_pct":1,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":79452.625,"power_w":121.184,"util_pct":0,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":79766.625,"power_w":124.212,"util_pct":1,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":77692.625,"power_w":121.471,"util_pct":0,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1385.413323022,"time_unix":1791285542.9253833,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:06.063 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 11.82, 'used_GB': 67.35, 'allocated_GB': 3.73, 'reserved_GB': 4.04} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:06.063 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:06.412 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 11.12, 'used_GB': 68.06, 'allocated_GB': 3.73, 'reserved_GB': 4.08} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:06.412 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:06.480 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 11.35, 'used_GB': 67.83, 'allocated_GB': 3.84, 'reserved_GB': 4.42} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:06.480 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:06.506 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 11.35, 'used_GB': 67.83, 'allocated_GB': 3.84, 'reserved_GB': 4.37} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:06.507 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:06.737 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 11.49, 'used_GB': 67.69, 'allocated_GB': 3.84, 'reserved_GB': 4.39} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:06.737 actor_cell00000_rank00000] timer.py:42 - Timer sleep end (elapsed: 5.3s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:06.737 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:06.782 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 11.81, 'used_GB': 67.36, 'allocated_GB': 3.84, 'reserved_GB': 4.11} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:06.782 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:06.795 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 11.22, 'used_GB': 67.96, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:06.796 actor_cell00000_rank00004] log_utils.py:436 - [CPU memory] after_offload_train: 151.36 GB (rollout_id=2, step=2) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:06.796 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.3 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1389.506782892,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":3,"start":1383.958063977,"task":"offload","time_unix":1791285547.0188613,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":3,"rollout_id":3,"time_unix":1791285547.0190797} -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Rollout generation: 0%| | 0/64 [00:00system\nReasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.\n\nYou are a helpful assistant. Please put the answer within \boxed{}.<|im_end|>\n<|im_start|>user\nRalph is going to practice playing tennis with a tennis ball machine that shoots out tennis balls for Ralph to hit. He loads up the machine with 175 tennis balls to start with. Out of ..., label=110, reward_summary=0.0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Rollout generation: 12%|█▎ | 8/64 [00:03<00:26, 2.13it/s] Rollout generation: 100%|██████████| 64/64 [00:03<00:00, 17.06it/s] -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:19:10.776 rollout_executor] inference_rollout_train.py:161 - Finish rollout: text_preview=<|im_start|>system\nReasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.\n\nYou are a helpful assistant. Please put the answer within \boxed{}.<|im_end|>\n<|im_start|>user\nNoah is a painter. He paints pictures and sells them at the park. He charges $60 for a large painting and $30 for a small painting. Last month he sold eight large paintings and four sm..., label=1200, reward_summary=0.0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:19:10.779 rollout_executor] inference_rollout_train.py:33 - Abort request for ['http://10.0.0.8:20037', 'http://10.0.0.8:20000'] -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:19:10.789 rollout_executor] rollout_data_conversion.py:39 - Final collected 64 samples from rollout to train -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:19:10.790 rollout_executor] metrics.py:93 - perf 3: {'rollout/raw_reward_unfiltered': 0.0, 'rollout/num_training_samples': 64, 'rollout/episode_raw_reward': 0.0, 'rollout/episode_response_length/mean': 1020.15625, 'rollout/episode_response_length/median': 1024.0, 'rollout/episode_response_length/max': 1024, 'rollout/episode_response_length/min': 778, 'rollout/episode_total_response_length/mean': 1020.15625, 'rollout/response_len/mean': 1020.15625, 'rollout/response_len/median': 1024.0, 'rollout/response_len/max': 1024, 'rollout/response_len/min': 778, 'rollout/zero_std/count_0.0': 8, 'rollout/zero_std/all_zero_percentage': 1.0, 'rollout/zero_std/all_one_percentage': 0.0, 'rollout/prefix_cache_hit_rate': 0.0, 'rollout/avg_cached_tokens_per_sample': 0.0, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.984375, 'perf/rollout_time': 3.765103816986084, 'perf/tokens_per_gpu_per_sec': 2167.602912615826, 'perf/longest_sample_tokens_per_sec': 271.9712522614313, 'perf/effective_tokens_per_gpu_per_sec': 2167.602912615826, 'perf/longest_effective_sample_tokens_per_sec': 271.9712522614313} -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:19:10.791 rollout_executor] train_data_conversion.py:334 - Rollout-side DP schedule: num_samples=64, num_rollouts=[64], num_microbatches=[32] -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:10] INFO: 10.0.0.8:50818 - "POST /abort_request HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:10 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:10 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:10 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:10 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:10] INFO: 10.0.0.8:38150 - "POST /abort_request HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:10 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:10 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:10 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:10 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:10 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:10] INFO: 10.0.0.8:38152 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:10 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:10 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:10 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:10 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:10 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:10] INFO: 10.0.0.8:50824 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:10 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:10 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:10 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:10 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:10 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:10 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:11] INFO: 10.0.0.8:50824 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1394.435067201,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"rollout","rollout_id":3,"start":1389.507162434,"task":"generate","time_unix":1791285551.9471428,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":3,"time_unix":1791285551.9473333} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:11] INFO: 10.0.0.8:38152 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:11.948 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=wake_up n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:11.949 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:11.951 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 70.17, 'used_GB': 9.01, 'allocated_GB': 3.84, 'reserved_GB': 4.42} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:11.949 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:11.949 actor_cell00000_rank00000] timer.py:31 - Timer wake_up start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:11.951 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 70.31, 'used_GB': 8.87, 'allocated_GB': 3.84, 'reserved_GB': 4.39} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:11.950 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:11.952 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 70.64, 'used_GB': 8.54, 'allocated_GB': 3.84, 'reserved_GB': 4.11} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:11.950 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:11.952 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 70.17, 'used_GB': 9.01, 'allocated_GB': 3.84, 'reserved_GB': 4.37} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:11.950 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:11.952 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 70.07, 'used_GB': 9.11, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:11.950 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:11.953 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 69.72, 'used_GB': 9.46, 'allocated_GB': 3.73, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:11.950 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:11.953 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 69.98, 'used_GB': 9.2, 'allocated_GB': 3.73, 'reserved_GB': 4.08} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:11.950 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:11.953 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 70.68, 'used_GB': 8.5, 'allocated_GB': 3.73, 'reserved_GB': 4.04} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1395.973435714,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":3,"start":1394.435419066,"task":"onload","time_unix":1791285553.485534,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":3,"time_unix":1791285553.4857311} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:13.483 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=wake_up ok=true elapsed_s=1.5 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:13.488 trainer_controller_actor] group.py:191 - ft op=train phase=start rollout=3 attempt=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:13.488 trainer_controller_actor] group.py:540 - ft op=refresh phase=start rollout=3 alive=0 healing= all_states="(0, 'StateAllocatedAlive')" quorum=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:13.488 trainer_controller_actor] group.py:563 - ft op=refresh phase=decision rollout=3 needs_reconfigure=false reason=alive_config_unchanged,no_healing quorum=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:13.488 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=train n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:13.473 actor_cell00000_rank00001] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27297 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:13.476 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 65.79, 'used_GB': 13.39, 'allocated_GB': 3.84, 'reserved_GB': 4.42} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:13.476 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:13.490 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:13.502 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:13.502 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:13.466 actor_cell00000_rank00000] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27296 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:13.468 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 65.97, 'used_GB': 13.21, 'allocated_GB': 3.84, 'reserved_GB': 4.39} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:13.468 actor_cell00000_rank00000] timer.py:42 - Timer wake_up end (elapsed: 1.5s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:13.468 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:13.490 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:13.490 actor_cell00000_rank00000] timer.py:31 - Timer data_preprocess start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:13.501 actor_cell00000_rank00000] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:13.501 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:13.502 actor_cell00000_rank00000] timer.py:42 - Timer train_wait end (elapsed: 27.5s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:13.502 actor_cell00000_rank00000] timer.py:31 - Timer train start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:13.502 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:13.502 actor_cell00000_rank00000] timer.py:31 - Timer log_probs start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:13.438 actor_cell00000_rank00003] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27299 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:13.441 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 66.57, 'used_GB': 12.61, 'allocated_GB': 3.84, 'reserved_GB': 4.11} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:13.441 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:13.490 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:13.501 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:13.502 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:13.480 actor_cell00000_rank00002] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27300 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:13.482 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 65.85, 'used_GB': 13.33, 'allocated_GB': 3.84, 'reserved_GB': 4.37} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:13.483 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:13.490 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:13.501 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:13.501 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:13.451 actor_cell00000_rank00004] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27302 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:13.453 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 66.09, 'used_GB': 13.08, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:13.453 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:13.490 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:13.502 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:13.502 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:13.453 actor_cell00000_rank00006] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27304 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:13.455 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 65.97, 'used_GB': 13.2, 'allocated_GB': 3.73, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:13.455 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:13.491 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:13.502 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:13.502 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:13.450 actor_cell00000_rank00005] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27303 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:13.453 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 65.95, 'used_GB': 13.23, 'allocated_GB': 3.73, 'reserved_GB': 4.08} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:13.453 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:13.490 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:13.502 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:13.502 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:13.434 actor_cell00000_rank00007] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27305 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:13.438 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 66.68, 'used_GB': 12.49, 'allocated_GB': 3.73, 'reserved_GB': 4.04} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:13.438 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:13.491 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:13.503 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:13.503 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:14] INFO: 10.0.0.8:50834 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:14] INFO: 10.0.0.8:38168 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:18.141 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:18.141 actor_cell00000_rank00000] timer.py:42 - Timer log_probs end (elapsed: 4.6s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:18.142 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:18.142 actor_cell00000_rank00000] timer.py:31 - Timer actor_train start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:18.141 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:18.141 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:18.244 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:18.252 actor_cell00000_rank00004] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:18.253 actor_cell00000_rank00004] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:18.254 actor_cell00000_rank00004] log_utils.py:126 - rollout 3: {'rollout/response_lengths': 1020.15625, 'rollout/rewards': 0.0, 'rollout/truncated': 0.984375, 'rollout/rollout_log_probs': -11.999644756317139, 'rollout/raw_reward': 0.0, 'rollout/total_lengths': 1155.15625, 'rollout/log_probs': -12.00189208984375, 'rollout/advantages': 0.0, 'rollout/returns': 0.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:18.245 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:18.252 actor_cell00000_rank00006] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:18.253 actor_cell00000_rank00006] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=1 success=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:18.245 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:18.245 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:23.509 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=train ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:23.510 trainer_controller_actor] group.py:291 - ft op=check errored= discarded= normal=0 decision=no_retry reason="survivors normal, gradients valid" -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:23.513 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:23.505 actor_cell00000_rank00001] model.py:646 - train op=train_step rollout=3 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:23.506 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:23.508 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:23.504 actor_cell00000_rank00000] model.py:646 - train op=train_step rollout=3 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:23.504 actor_cell00000_rank00000] timer.py:42 - Timer actor_train end (elapsed: 5.4s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:23.504 actor_cell00000_rank00000] timer.py:42 - Timer train end (elapsed: 10.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:23.504 actor_cell00000_rank00000] timer.py:31 - Timer train_wait start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:23.504 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:23.505 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:23.504 actor_cell00000_rank00003] model.py:646 - train op=train_step rollout=3 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:23.505 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:23.506 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:23.504 actor_cell00000_rank00002] model.py:646 - train op=train_step rollout=3 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:23.504 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:23.505 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:23.504 actor_cell00000_rank00004] model.py:646 - train op=train_step rollout=3 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:23.505 actor_cell00000_rank00004] log_utils.py:554 - step 3: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.03649664670228958, 'train/train_rollout_kl': 0.0019020842155441642, 'train/grad_norm': 0.0, 'train/lr-pg_0': 3.333333333333334e-06, 'train/lr-pg_1': 3.333333333333334e-06, 'train/step': 3} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:23.505 actor_cell00000_rank00004] model.py:856 - step 3: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.03649664670228958, 'train/train_rollout_kl': 0.0019020842155441642, 'train/grad_norm': 0.0, 'train/lr-pg_0': 3.333333333333334e-06, 'train/lr-pg_1': 3.333333333333334e-06, 'train/step': 3} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:23.505 actor_cell00000_rank00004] train_metric_utils.py:56 - perf 3: {'perf/update_weights_implementation_time': 1.6560392379760742, 'perf/finalize_and_resume_engines_time': 1.274815320968628, 'perf/update_weights_time': 3.0131804943084717, 'perf/sleep_time': 5.322782516479492, 'perf/wake_up_time': 1.5032298564910889, 'perf/data_preprocess_time': 0.011522054672241211, 'perf/train_wait_time': 27.495041847229004, 'perf/log_probs_time': 4.742132663726807, 'perf/actor_train_time': 5.251331806182861, 'perf/train_time': 10.003074884414673, 'perf/log_probs_tflops': 3.7440498021594046, 'perf/actor_train_tflops': 10.14301601007332, 'perf/actor_train_tok_per_s': 14078.333407337852, 'perf/mfu_peak_tflops': 989.0, 'perf/actor_train_mfu': 0.010255830141631263, 'perf/step_time': 37.49811673164368, 'perf/wait_time_ratio': 0.7332379394943496} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:23.505 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:23.507 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:23.504 actor_cell00000_rank00006] model.py:646 - train op=train_step rollout=3 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:23.505 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:23.506 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:23.504 actor_cell00000_rank00005] model.py:646 - train op=train_step rollout=3 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:23.505 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:23.507 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:23.504 actor_cell00000_rank00007] model.py:646 - train op=train_step rollout=3 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:23.505 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:23.506 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:23.515 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:23.517 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:23.521 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:23.523 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"aborted_in_flight_groups":0,"adv_mean":null,"adv_std":null,"applied_lrs":[5e-06],"clip_fraction":null,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"masked_fraction":null,"nonzero_advantages":null,"resp_len_mean":1020.15625,"resp_len_p95":1024.0,"reward_p10":0.0,"reward_p50":0.0,"reward_p90":0.0,"rollout_id":3,"step_seconds":10.04114407599991,"submitted_groups":8,"time_unix":1791285563.5272672,"tok_per_s":6502.247105093783,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0,"pg_clipfrac":0.0,"pg_loss":0.0,"ppo_kl":0.0,"train_rollout_kl":0.0019020842155441642,"train_rollout_logprob_abs_diff":0.03649664670228958},"train_step":4,"trained_groups":8,"trained_sample_ids_sha256":"7db6f0b987b89946ae44941cfccabab48ebe099810c620f74884bca9ea10ed96","trained_samples":64,"truncated_frac":0.984375} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_round_labels","execution_mode":"colocated-serial","island_id":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/aborted_groups":0,"rl/carried_over_groups":null,"rl/clip_fraction":null,"rl/ess_ratio":1.0,"rl/filtered_groups":0,"rl/groups":8,"rl/masked_fraction":null,"rl/mean_kl":0.0,"rollout_id":3,"t":1406.015629819,"time_unix":1791285563.5276225,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1406.015830443,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":3,"start":1395.973817531,"task":"train","time_unix":1791285563.527785,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":3,"time_unix":1791285563.5278757} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:23.525 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:23.528 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:25.806 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=2.3 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"action_tokens":65290,"active_groups":8,"applied_lr":5e-06,"applied_lrs":[5e-06],"base_policy_version":3,"cancelled_groups":0,"clip_fraction":null,"completed_groups":8,"completed_trajectories":64,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":8,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":0.0,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":4,"loss":0.0,"lr":5e-06,"mean_kl":0.0,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":0.0,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":10.477000067000063,"time_unix":1791285568.2051337,"tool_wait_seconds":0.0,"train_seconds":10.04114407599991,"train_step":4,"zero_variance_group_ratio":1.0} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1411.576559265,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":3,"start":1406.01596646,"task":"outer_sync","time_unix":1791285569.088938,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":4,"time_unix":1791285569.089251} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:29] INFO: 10.0.0.8:58984 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:29] INFO: 10.0.0.8:38740 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:29.973 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:32.344 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=2.4 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"publish","pid":18336,"policy_version":3,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":null,"rounds_completed":3,"seq":15,"time_unix":1791285572.8138266,"trained_version":4,"uptime_s":450.057,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:35] INFO: 10.0.0.8:58996 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:35] INFO: 10.0.0.8:59004 - "POST /pause_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:35] INFO: 10.0.0.8:59004 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:35 TP0 EP0] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:35 TP2 EP2] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:35 TP3 EP3] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:35 TP1 EP1] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:35] INFO: 10.0.0.8:38742 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:35] INFO: 10.0.0.8:38754 - "POST /pause_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:35] INFO: 10.0.0.8:38754 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:35 TP0 EP0] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:35 TP3 EP3] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:35 TP1 EP1] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:35 TP2 EP2] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:35.327 trainer_controller_actor] group.py:393 - ft op=update_weights phase=start rollout= -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:35.328 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=update_weights n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:35.329 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:35.330 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 52.97, 'used_GB': 26.21, 'allocated_GB': 3.84, 'reserved_GB': 5.12} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:35.329 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:35.329 actor_cell00000_rank00000] timer.py:31 - Timer update_weights start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:35.330 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 53.22, 'used_GB': 25.96, 'allocated_GB': 3.84, 'reserved_GB': 5.15} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:35.329 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:35.330 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before update_weights: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 53.83, 'used_GB': 25.35, 'allocated_GB': 3.84, 'reserved_GB': 5.19} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:35.329 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:35.330 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before update_weights: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 52.92, 'used_GB': 26.26, 'allocated_GB': 3.84, 'reserved_GB': 5.16} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:35.330 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:35.331 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before update_weights: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 52.07, 'used_GB': 27.11, 'allocated_GB': 3.73, 'reserved_GB': 6.02} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:35.330 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:35.331 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before update_weights: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 52.4, 'used_GB': 26.77, 'allocated_GB': 3.73, 'reserved_GB': 5.49} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:35.330 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:35.331 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before update_weights: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 51.91, 'used_GB': 27.27, 'allocated_GB': 3.73, 'reserved_GB': 5.99} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:35.330 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:35.331 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before update_weights: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 53.79, 'used_GB': 25.39, 'allocated_GB': 3.73, 'reserved_GB': 5.51} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:35] INFO: 10.0.0.8:59004 - "POST /begin_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:35] INFO: 10.0.0.8:38754 - "POST /begin_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:35.342 actor_cell00000_rank00000] timer.py:31 - Timer update_weights_implementation start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) Update weights: 0%| | 0/24 [00:00 63670.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38] INFO: 10.0.0.8:38754 - "POST /end_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38] INFO: 10.0.0.8:38754 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38] INFO: 10.0.0.8:38754 - "POST /continue_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP0 EP0] LoRA adapter loading from tensors completes: LoRARef(lora_id=7d347176e0914e4f85e83a43247f90cb, lora_name=miles_lora, lora_path=__stream__, pinned=False, reloadable=False). -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP0 EP0] Weight version changed. old_version='yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' new_version='5' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP1 EP1] Weight version changed. old_version='yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' new_version='5' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP3 EP3] Weight version changed. old_version='yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' new_version='5' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP2 EP2] Weight version changed. old_version='yeto:3:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' new_version='5' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP1 EP1] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP0 EP0] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP2 EP2] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP3 EP3] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [miles-colocate] Update weights: 23it [00:02, 8.17it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:38.129 actor_cell00000_rank00000] timer.py:42 - Timer finalize_and_resume_engines end (elapsed: 1.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [miles-colocate] Update weights: 23it [00:02, 8.17it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:38.159 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 47.56, 'used_GB': 31.62, 'allocated_GB': 3.84, 'reserved_GB': 10.91} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [miles-colocate] Update weights: 23it [00:02, 8.19it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:38.151 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after update_weights: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 46.6, 'used_GB': 32.58, 'allocated_GB': 3.73, 'reserved_GB': 11.29} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [miles-colocate] Update weights: 23it [00:02, 8.18it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:38.155 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after update_weights: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 46.53, 'used_GB': 32.65, 'allocated_GB': 3.73, 'reserved_GB': 11.35} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [miles-colocate] Update weights: 23it [00:02, 8.19it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:38.151 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after update_weights: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 48.04, 'used_GB': 31.14, 'allocated_GB': 3.73, 'reserved_GB': 11.25} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP0 EP0] [continue_generation] torch.cuda.empty_cache() called: reserved 64118.0 MB -> 63670.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP3 EP3] [continue_generation] torch.cuda.empty_cache() called: reserved 64118.0 MB -> 63670.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP1 EP1] [continue_generation] torch.cuda.empty_cache() called: reserved 64118.0 MB -> 63670.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:38.159 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 46.7, 'used_GB': 32.48, 'allocated_GB': 3.84, 'reserved_GB': 11.49} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [miles-colocate] Update weights: 23it [00:02, 8.15it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [miles-colocate] Update weights: 23it [00:02, 8.16it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:38.161 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after update_weights: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 48.1, 'used_GB': 31.07, 'allocated_GB': 3.73, 'reserved_GB': 10.08} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [miles-colocate] Update weights: 23it [00:02, 8.14it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:38.169 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after update_weights: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 47.87, 'used_GB': 31.31, 'allocated_GB': 3.84, 'reserved_GB': 11.25} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:38.166 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after update_weights: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 46.76, 'used_GB': 32.42, 'allocated_GB': 3.84, 'reserved_GB': 11.43} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:38.207 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=update_weights ok=true elapsed_s=2.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:38.198 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:38.195 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 2.9s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:38.195 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:38.206 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:38.205 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:38.201 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:38.197 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:38.194 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:38.201 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.9 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38] INFO: 10.0.0.8:58996 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38] INFO: 10.0.0.8:44938 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38] INFO: 10.0.0.8:44938 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP0 EP0] Weight version changed. old_version='5' new_version='yeto:4:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP1 EP1] Weight version changed. old_version='5' new_version='yeto:4:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP2 EP2] Weight version changed. old_version='5' new_version='yeto:4:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP3 EP3] Weight version changed. old_version='5' new_version='yeto:4:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38] INFO: 10.0.0.8:38742 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38] INFO: 10.0.0.8:38688 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38] INFO: 10.0.0.8:38688 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP0 EP0] Weight version changed. old_version='5' new_version='yeto:4:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP1 EP1] Weight version changed. old_version='5' new_version='yeto:4:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP3 EP3] Weight version changed. old_version='5' new_version='yeto:4:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP2 EP2] Weight version changed. old_version='5' new_version='yeto:4:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP0 EP0] [WeightChecker] checksum computed for 437 tensors in 0.209s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP1 EP1] [WeightChecker] checksum computed for 437 tensors in 0.210s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP3 EP3] [WeightChecker] checksum computed for 437 tensors in 0.211s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP3 EP3] [WeightChecker] checksum computed for 509 tensors in 0.213s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP1 EP1] [WeightChecker] checksum computed for 509 tensors in 0.214s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP0 EP0] [WeightChecker] checksum computed for 509 tensors in 0.215s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38 TP2 EP2] [WeightChecker] checksum computed for 509 tensors in 0.215s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38 TP2 EP2] [WeightChecker] checksum computed for 437 tensors in 0.214s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:38] INFO: 10.0.0.8:38742 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:38] INFO: 10.0.0.8:58996 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_publication","island_id":0,"policy_version":4,"rl/policy_token":"yeto:4:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285578.7419362} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1421.230272057,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":1411.577336723,"task":"publish","time_unix":1791285578.742331,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":4,"time_unix":1791285578.7562606} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:38.757 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=sleep n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:38.758 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:38.758 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:38.758 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:38.758 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:38.758 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:38.759 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:38.759 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:38.759 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:38.759 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:39.447 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 0.77, 'used_GB': 78.41, 'allocated_GB': 3.84, 'reserved_GB': 4.39} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:39.368 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 1.05, 'used_GB': 78.13, 'allocated_GB': 3.84, 'reserved_GB': 4.4} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:39.370 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 1.95, 'used_GB': 77.23, 'allocated_GB': 3.84, 'reserved_GB': 4.15} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:39.372 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 0.83, 'used_GB': 78.35, 'allocated_GB': 3.84, 'reserved_GB': 4.33} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:39.407 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 1.13, 'used_GB': 78.05, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:39.411 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 0.92, 'used_GB': 78.26, 'allocated_GB': 3.73, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:39.444 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 0.86, 'used_GB': 78.32, 'allocated_GB': 3.73, 'reserved_GB': 4.12} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:39.420 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 2.37, 'used_GB': 76.81, 'allocated_GB': 3.73, 'reserved_GB': 4.01} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:43.812 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 11.86, 'used_GB': 67.32, 'allocated_GB': 3.73, 'reserved_GB': 4.01} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:43.812 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:44.341 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 11.81, 'used_GB': 67.36, 'allocated_GB': 3.84, 'reserved_GB': 4.15} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:44.341 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:44.279 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 11.16, 'used_GB': 68.02, 'allocated_GB': 3.73, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:44.280 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:44.286 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 11.15, 'used_GB': 68.02, 'allocated_GB': 3.73, 'reserved_GB': 4.12} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:44.286 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.5 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:44] INFO: 10.0.0.8:44948 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:44] INFO: 10.0.0.8:38700 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:44.904 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 11.49, 'used_GB': 67.69, 'allocated_GB': 3.84, 'reserved_GB': 4.4} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:44.904 actor_cell00000_rank00000] timer.py:42 - Timer sleep end (elapsed: 6.1s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:44.904 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=6.1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:44.906 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 11.25, 'used_GB': 67.93, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:44.906 actor_cell00000_rank00004] log_utils.py:436 - [CPU memory] after_offload_train: 150.96 GB (rollout_id=3, step=3) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:44.907 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=6.1 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1427.538119505,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":4,"start":1421.24431026,"task":"offload","time_unix":1791285585.0502071,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":4,"rollout_id":4,"time_unix":1791285585.0503945} -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Rollout generation: 0%| | 0/64 [00:00system\nReasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.\n\nYou are a helpful assistant. Please put the answer within \boxed{}.<|im_end|>\n<|im_start|>user\nMr. Sam shared a certain amount of money between his two sons, Ken and Tony. If Ken got $1750, and Tony got twice as much as Ken, how much was the money shared?<|im_end|>\n<|im_start|>..., label=5250, reward_summary=0.0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Rollout generation: 12%|█▎ | 8/64 [00:03<00:27, 2.05it/s] -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:44958 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:44960 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:44968 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:44970 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:44996 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45008 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45016 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45018 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45026 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45032 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45034 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45040 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45062 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45076 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45080 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45082 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45096 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45118 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45104 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45126 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45136 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45144 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45148 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45162 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:48] INFO: 10.0.0.8:45166 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49 TP0 EP0] Decode batch, #running-req: 37, #full token: 41792, full token usage: 0.00, mamba num: 37, mamba usage: 0.00, cuda graph: True, gen throughput (token/s): 11839.05, #queue-req: 0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45170 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45186 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45190 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45192 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45202 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45198 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45212 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45222 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45228 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45238 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45254 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45270 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45284 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45288 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45302 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45318 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45324 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45328 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45342 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45356 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45372 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45376 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45392 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45396 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45400 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45404 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45428 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45434 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45444 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45446 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45450 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45462 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45466 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45470 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45486 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45502 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:45514 - "POST /generate HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49 TP0 EP0] Decode batch, #running-req: 37, #full token: 42688, full token usage: 0.00, mamba num: 37, mamba usage: 0.00, cuda graph: True, gen throughput (token/s): 12394.19, #queue-req: 0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Rollout generation: 38%|███▊ | 24/64 [00:04<00:05, 7.09it/s] Rollout generation: 100%|██████████| 64/64 [00:04<00:00, 15.16it/s] -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:19:49.276 rollout_executor] inference_rollout_train.py:161 - Finish rollout: text_preview=<|im_start|>system\nReasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.\n\nYou are a helpful assistant. Please put the answer within \boxed{}.<|im_end|>\n<|im_start|>user\nA car is driving through a tunnel with many turns. After a while, the car must travel through a ring that requires a total of 4 right-hand turns. After the 1st turn, it travels 5 meter..., label=10, reward_summary=0.0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:19:49.279 rollout_executor] inference_rollout_train.py:33 - Abort request for ['http://10.0.0.8:20037', 'http://10.0.0.8:20000'] -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:19:49.289 rollout_executor] rollout_data_conversion.py:39 - Final collected 64 samples from rollout to train -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:19:49.290 rollout_executor] metrics.py:93 - perf 4: {'rollout/raw_reward_unfiltered': 0.0, 'rollout/num_training_samples': 64, 'rollout/episode_raw_reward': 0.0, 'rollout/episode_response_length/mean': 999.953125, 'rollout/episode_response_length/median': 1024.0, 'rollout/episode_response_length/max': 1024, 'rollout/episode_response_length/min': 66, 'rollout/episode_total_response_length/mean': 999.953125, 'rollout/response_len/mean': 999.953125, 'rollout/response_len/median': 1024.0, 'rollout/response_len/max': 1024, 'rollout/response_len/min': 66, 'rollout/zero_std/count_0.0': 8, 'rollout/zero_std/all_zero_percentage': 1.0, 'rollout/zero_std/all_one_percentage': 0.0, 'rollout/prefix_cache_hit_rate': 0.0, 'rollout/avg_cached_tokens_per_sample': 0.0, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.96875, 'perf/rollout_time': 4.23476505279541, 'perf/tokens_per_gpu_per_sec': 1889.0363220314593, 'perf/longest_sample_tokens_per_sec': 241.80798396927534, 'perf/effective_tokens_per_gpu_per_sec': 1889.0363220314593, 'perf/longest_effective_sample_tokens_per_sec': 241.80798396927534} -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:19:49.292 rollout_executor] train_data_conversion.py:334 - Rollout-side DP schedule: num_samples=64, num_rollouts=[64], num_microbatches=[32] -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:48528 - "POST /abort_request HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:49] INFO: 10.0.0.8:58534 - "POST /abort_request HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:49] INFO: 10.0.0.8:58546 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:49 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:49 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:49 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:49 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:49 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:49 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:49 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:49 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:49 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:49 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49] INFO: 10.0.0.8:48540 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:49 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:50] INFO: 10.0.0.8:48540 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:50] INFO: 10.0.0.8:58546 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1432.560233736,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"rollout","rollout_id":4,"start":1427.538480802,"task":"generate","time_unix":1791285590.0723002,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":4,"time_unix":1791285590.072497} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:50.073 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=wake_up n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:50.075 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:50.075 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 70.17, 'used_GB': 9.01, 'allocated_GB': 3.84, 'reserved_GB': 4.39} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:50.074 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:50.075 actor_cell00000_rank00000] timer.py:31 - Timer wake_up start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:50.075 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 70.31, 'used_GB': 8.87, 'allocated_GB': 3.84, 'reserved_GB': 4.4} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:50.075 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:50.076 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 70.64, 'used_GB': 8.54, 'allocated_GB': 3.84, 'reserved_GB': 4.15} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:50.075 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:50.076 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 70.17, 'used_GB': 9.01, 'allocated_GB': 3.84, 'reserved_GB': 4.33} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:50.075 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:50.076 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 69.97, 'used_GB': 9.21, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:50.075 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:50.076 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 69.88, 'used_GB': 9.3, 'allocated_GB': 3.73, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:50.075 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:50.076 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 69.88, 'used_GB': 9.3, 'allocated_GB': 3.73, 'reserved_GB': 4.12} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:50.075 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:50.076 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 70.58, 'used_GB': 8.6, 'allocated_GB': 3.73, 'reserved_GB': 4.01} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:51.425 actor_cell00000_rank00000] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27296 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:51.427 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 65.95, 'used_GB': 13.23, 'allocated_GB': 3.84, 'reserved_GB': 4.4} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:51.427 actor_cell00000_rank00000] timer.py:42 - Timer wake_up end (elapsed: 1.4s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:51.427 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:51.414 actor_cell00000_rank00003] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27299 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:51.417 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 66.53, 'used_GB': 12.65, 'allocated_GB': 3.84, 'reserved_GB': 4.15} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:51.417 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:51.421 actor_cell00000_rank00002] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27300 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:51.423 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 65.88, 'used_GB': 13.3, 'allocated_GB': 3.84, 'reserved_GB': 4.33} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:51.424 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1433.980336106,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":4,"start":1432.56058292,"task":"onload","time_unix":1791285591.4924161,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":4,"time_unix":1791285591.4926155} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:51.490 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=wake_up ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:51.494 trainer_controller_actor] group.py:191 - ft op=train phase=start rollout=4 attempt=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:51.494 trainer_controller_actor] group.py:540 - ft op=refresh phase=start rollout=4 alive=0 healing= all_states="(0, 'StateAllocatedAlive')" quorum=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:51.494 trainer_controller_actor] group.py:563 - ft op=refresh phase=decision rollout=4 needs_reconfigure=false reason=alive_config_unchanged,no_healing quorum=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:19:51.494 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=train n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:51.471 actor_cell00000_rank00001] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27297 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:51.474 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 65.82, 'used_GB': 13.36, 'allocated_GB': 3.84, 'reserved_GB': 4.39} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:51.474 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:51.496 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:51.507 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:51.507 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:51.496 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:51.496 actor_cell00000_rank00000] timer.py:31 - Timer data_preprocess start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:51.507 actor_cell00000_rank00000] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:51.507 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:51.507 actor_cell00000_rank00000] timer.py:42 - Timer train_wait end (elapsed: 28.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:51.507 actor_cell00000_rank00000] timer.py:31 - Timer train start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:51.507 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:51.507 actor_cell00000_rank00000] timer.py:31 - Timer log_probs start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:51.496 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:51.507 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:51.508 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:51.496 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:51.508 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:51.508 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:51.443 actor_cell00000_rank00004] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27302 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:51.446 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 65.99, 'used_GB': 13.19, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:51.446 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:51.497 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:51.508 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:51.508 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:51.459 actor_cell00000_rank00006] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27304 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:51.462 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 65.87, 'used_GB': 13.31, 'allocated_GB': 3.73, 'reserved_GB': 4.05} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:51.462 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:51.497 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:51.509 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:51.509 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:51.487 actor_cell00000_rank00005] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27303 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:51.489 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 65.81, 'used_GB': 13.37, 'allocated_GB': 3.73, 'reserved_GB': 4.12} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:51.490 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:51.497 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:51.508 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:51.508 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:51.456 actor_cell00000_rank00007] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27305 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:51.459 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 66.62, 'used_GB': 12.56, 'allocated_GB': 3.73, 'reserved_GB': 4.01} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:51.459 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:51.497 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:51.508 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:51.508 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:19:55.937 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:55.937 actor_cell00000_rank00000] timer.py:42 - Timer log_probs end (elapsed: 4.4s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:55.937 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:19:55.937 actor_cell00000_rank00000] timer.py:31 - Timer actor_train start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:19:55.936 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:19:55.936 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:56.047 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:56.059 actor_cell00000_rank00004] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:56.060 actor_cell00000_rank00004] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:19:56.060 actor_cell00000_rank00004] log_utils.py:126 - rollout 4: {'rollout/response_lengths': 999.953125, 'rollout/rewards': 0.0, 'rollout/truncated': 0.96875, 'rollout/rollout_log_probs': -11.986705303192139, 'rollout/raw_reward': 0.0, 'rollout/total_lengths': 1135.328125, 'rollout/log_probs': -11.988543033599854, 'rollout/advantages': 0.0, 'rollout/returns': 0.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:56.052 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:56.059 actor_cell00000_rank00006] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:19:56.060 actor_cell00000_rank00006] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=1 success=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:19:56.049 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.5 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:19:56.047 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.5 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:19:59] INFO: 10.0.0.8:48700 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:19:59] INFO: 10.0.0.8:54552 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"aborted_in_flight_groups":0,"adv_mean":null,"adv_std":null,"applied_lrs":[3.333333333333334e-06],"clip_fraction":null,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"masked_fraction":null,"nonzero_advantages":null,"resp_len_mean":999.953125,"resp_len_p95":1024.0,"reward_p10":0.0,"reward_p50":0.0,"reward_p90":0.0,"rollout_id":4,"step_seconds":9.901529057999824,"submitted_groups":8,"time_unix":1791285601.3945644,"tok_per_s":6463.345168723651,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0,"pg_clipfrac":0.0,"pg_loss":0.0,"ppo_kl":0.0,"train_rollout_kl":0.0019019710598513484,"train_rollout_logprob_abs_diff":0.036261189728975296},"train_step":5,"trained_groups":8,"trained_sample_ids_sha256":"5662988e66c78ab4f31c26050379f5f0c1c6341ce00255741b05527f544c6708","trained_samples":64,"truncated_frac":0.96875} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_round_labels","execution_mode":"colocated-serial","island_id":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/aborted_groups":0,"rl/carried_over_groups":null,"rl/clip_fraction":null,"rl/ess_ratio":1.0,"rl/filtered_groups":0,"rl/groups":8,"rl/masked_fraction":null,"rl/mean_kl":0.0,"rollout_id":4,"t":1443.882962364,"time_unix":1791285601.3949869,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1443.883215266,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":4,"start":1433.980700018,"task":"train","time_unix":1791285601.3951726,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":4,"time_unix":1791285601.3952582} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:01.380 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=train ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:01.380 trainer_controller_actor] group.py:291 - ft op=check errored= discarded= normal=0 decision=no_retry reason="survivors normal, gradients valid" -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:01.383 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:01.385 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:01.387 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:01.389 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:01.391 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:01.393 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:01.396 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:01.376 actor_cell00000_rank00001] model.py:646 - train op=train_step rollout=4 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:01.376 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:01.377 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:01.376 actor_cell00000_rank00000] model.py:646 - train op=train_step rollout=4 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:01.376 actor_cell00000_rank00000] timer.py:42 - Timer actor_train end (elapsed: 5.4s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:01.376 actor_cell00000_rank00000] timer.py:42 - Timer train end (elapsed: 9.9s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:01.376 actor_cell00000_rank00000] timer.py:31 - Timer train_wait start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:01.376 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:01.377 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:01.376 actor_cell00000_rank00003] model.py:646 - train op=train_step rollout=4 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:01.376 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:01.377 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:01.376 actor_cell00000_rank00002] model.py:646 - train op=train_step rollout=4 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:01.376 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:01.378 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:01.376 actor_cell00000_rank00004] model.py:646 - train op=train_step rollout=4 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:01.377 actor_cell00000_rank00004] log_utils.py:554 - step 4: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.036261189728975296, 'train/train_rollout_kl': 0.0019019710598513484, 'train/grad_norm': 0.0, 'train/lr-pg_0': 1.6666666666666665e-06, 'train/lr-pg_1': 1.6666666666666665e-06, 'train/step': 4} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:01.377 actor_cell00000_rank00004] model.py:856 - step 4: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.036261189728975296, 'train/train_rollout_kl': 0.0019019710598513484, 'train/grad_norm': 0.0, 'train/lr-pg_0': 1.6666666666666665e-06, 'train/lr-pg_1': 1.6666666666666665e-06, 'train/step': 4} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:01.377 actor_cell00000_rank00004] train_metric_utils.py:56 - perf 4: {'perf/update_weights_implementation_time': 1.818197250366211, 'perf/finalize_and_resume_engines_time': 0.9687867164611816, 'perf/update_weights_time': 2.8709990978240967, 'perf/sleep_time': 6.147735118865967, 'perf/wake_up_time': 1.3707523345947266, 'perf/data_preprocess_time': 0.01134800910949707, 'perf/train_wait_time': 28.00313448905945, 'perf/log_probs_time': 4.538585186004639, 'perf/actor_train_time': 5.316507577896118, 'perf/train_time': 9.868537187576294, 'perf/log_probs_tflops': 3.8446059095981386, 'perf/actor_train_tflops': 9.846165648220362, 'perf/actor_train_tok_per_s': 13667.0547225579, 'perf/mfu_peak_tflops': 989.0, 'perf/actor_train_mfu': 0.009955678107401781, 'perf/step_time': 37.87167167663574, 'perf/wait_time_ratio': 0.7394216639857355} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:01.377 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:01.378 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:01.375 actor_cell00000_rank00006] model.py:646 - train op=train_step rollout=4 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:01.377 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:01.378 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:01.376 actor_cell00000_rank00005] model.py:646 - train op=train_step rollout=4 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:01.377 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:01.378 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:01.375 actor_cell00000_rank00007] model.py:646 - train op=train_step rollout=4 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:01.377 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:01.378 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=9.9 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"sync","pid":18336,"policy_version":4,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":4,"rounds_completed":4,"seq":16,"time_unix":1791285602.814457,"trained_version":5,"uptime_s":480.058,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"available":true,"config_epoch":0,"cpu_rss_bytes":3852902400,"event":"rl_resource_sample","execution_mode":"colocated-serial","gpus":[{"index":0,"mem_total_mb":81559.0,"mem_used_mb":21032.625,"power_w":120.757,"util_pct":0,"uuid":"GPU-b083b8ab-9c49-9cf7-15b1-236f2ab0f918"},{"index":1,"mem_total_mb":81559.0,"mem_used_mb":21250.625,"power_w":128.811,"util_pct":0,"uuid":"GPU-08972e89-2cc0-c995-0997-f3fd6940c1f2"},{"index":2,"mem_total_mb":81559.0,"mem_used_mb":21242.625,"power_w":126.728,"util_pct":0,"uuid":"GPU-ed102080-5f98-398b-6317-4f90ce3e98b2"},{"index":3,"mem_total_mb":81559.0,"mem_used_mb":20406.625,"power_w":121.819,"util_pct":0,"uuid":"GPU-bf89177a-1800-a4e0-c3c1-02771d48d2fe"},{"index":4,"mem_total_mb":81559.0,"mem_used_mb":22374.625,"power_w":125.05,"util_pct":0,"uuid":"GPU-900ca427-fbc4-d281-a2ad-d30e0254bced"},{"index":5,"mem_total_mb":81559.0,"mem_used_mb":22404.625,"power_w":123.251,"util_pct":0,"uuid":"GPU-8d3613eb-49dd-7293-bbd5-fbf69d58a45d"},{"index":6,"mem_total_mb":81559.0,"mem_used_mb":22082.625,"power_w":124.423,"util_pct":0,"uuid":"GPU-7292f600-c28f-e52f-e353-6dfc4a4500a8"},{"index":7,"mem_total_mb":81559.0,"mem_used_mb":20568.625,"power_w":121.369,"util_pct":0,"uuid":"GPU-a1304a7d-5213-784e-8b47-cbe64f06dedf"}],"island_id":0,"pid":18336,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","t":1445.414239647,"time_unix":1791285602.9263175,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:03.643 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=2.2 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"action_tokens":63997,"active_groups":8,"applied_lr":3.333333333333334e-06,"applied_lrs":[3.333333333333334e-06],"base_policy_version":4,"cancelled_groups":0,"clip_fraction":null,"completed_groups":8,"completed_trajectories":64,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":8,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":0.0,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":5,"loss":0.0,"lr":3.333333333333334e-06,"mean_kl":0.0,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":0.0,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":11.315920646999984,"time_unix":1791285606.0933363,"tool_wait_seconds":0.0,"train_seconds":9.901529057999824,"train_step":5,"zero_variance_group_ratio":1.0} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1449.462540842,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":4,"start":1443.883346481,"task":"outer_sync","time_unix":1791285606.9749565,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":5,"time_unix":1791285606.9752853} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:07.856 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:10.134 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=2.3 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:13] INFO: 10.0.0.8:45394 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:13] INFO: 10.0.0.8:59132 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:13.155 trainer_controller_actor] group.py:393 - ft op=update_weights phase=start rollout= -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:13.155 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=update_weights n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:13.157 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:13.158 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 52.99, 'used_GB': 26.18, 'allocated_GB': 3.84, 'reserved_GB': 5.09} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:13.156 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:13.157 actor_cell00000_rank00000] timer.py:31 - Timer update_weights start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:13.158 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 53.21, 'used_GB': 25.97, 'allocated_GB': 3.84, 'reserved_GB': 5.16} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:13.157 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:13.158 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before update_weights: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 53.82, 'used_GB': 25.36, 'allocated_GB': 3.84, 'reserved_GB': 5.2} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:13.157 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:13.158 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before update_weights: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 53.0, 'used_GB': 26.18, 'allocated_GB': 3.84, 'reserved_GB': 5.08} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:13.157 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:13.158 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before update_weights: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 51.9, 'used_GB': 27.28, 'allocated_GB': 3.73, 'reserved_GB': 6.08} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:13.157 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:13.159 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before update_weights: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 52.18, 'used_GB': 27.0, 'allocated_GB': 3.73, 'reserved_GB': 5.61} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:13.157 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:13.158 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before update_weights: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 51.87, 'used_GB': 27.31, 'allocated_GB': 3.73, 'reserved_GB': 5.93} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:13.158 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:13.159 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before update_weights: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 53.66, 'used_GB': 25.52, 'allocated_GB': 3.73, 'reserved_GB': 5.54} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:13] INFO: 10.0.0.8:59134 - "POST /pause_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:13] INFO: 10.0.0.8:59134 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:13 TP0 EP0] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:13 TP1 EP1] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:13 TP3 EP3] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:13 TP2 EP2] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:13] INFO: 10.0.0.8:45398 - "POST /pause_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:13] INFO: 10.0.0.8:45398 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:13 TP2 EP2] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:13 TP0 EP0] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:13 TP3 EP3] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:13 TP1 EP1] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:13] INFO: 10.0.0.8:59134 - "POST /begin_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:13] INFO: 10.0.0.8:45398 - "POST /begin_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:13.169 actor_cell00000_rank00000] timer.py:31 - Timer update_weights_implementation start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) Update weights: 0%| | 0/24 [00:00 63770.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:15 TP1 EP1] [continue_generation] torch.cuda.empty_cache() called: reserved 64218.0 MB -> 63770.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:15 TP0 EP0] [continue_generation] torch.cuda.empty_cache() called: reserved 64218.0 MB -> 63770.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:15 TP2 EP2] [continue_generation] torch.cuda.empty_cache() called: reserved 64218.0 MB -> 63770.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:15 TP0 EP0] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:15 TP2 EP2] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:15 TP1 EP1] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:15 TP3 EP3] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [miles-colocate] Update weights: 23it [00:02, 8.86it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:15.768 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after update_weights: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 46.68, 'used_GB': 32.5, 'allocated_GB': 3.84, 'reserved_GB': 11.5} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [miles-colocate] Update weights: 23it [00:02, 8.85it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:15.767 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after update_weights: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 46.59, 'used_GB': 32.59, 'allocated_GB': 3.73, 'reserved_GB': 11.31} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [miles-colocate] Update weights: 23it [00:02, 8.85it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [miles-colocate] Update weights: 23it [00:02, 8.84it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:15.771 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after update_weights: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 48.0, 'used_GB': 31.18, 'allocated_GB': 3.73, 'reserved_GB': 10.08} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [miles-colocate] Update weights: 23it [00:02, 8.84it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:15.773 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after update_weights: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 46.62, 'used_GB': 32.56, 'allocated_GB': 3.73, 'reserved_GB': 11.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:15.770 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after update_weights: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 48.0, 'used_GB': 31.17, 'allocated_GB': 3.73, 'reserved_GB': 11.3} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:15.773 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 46.64, 'used_GB': 32.54, 'allocated_GB': 3.84, 'reserved_GB': 11.54} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [miles-colocate] Update weights: 23it [00:02, 8.81it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:15.781 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after update_weights: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 47.89, 'used_GB': 31.28, 'allocated_GB': 3.84, 'reserved_GB': 11.23} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:15.821 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=update_weights ok=true elapsed_s=2.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:15.818 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:15.807 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 2.7s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:15.807 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:15.820 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:15.801 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:15.810 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:15.817 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:15.793 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:15.803 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.6 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16] INFO: 10.0.0.8:59132 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16] INFO: 10.0.0.8:59162 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16] INFO: 10.0.0.8:59162 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16 TP0 EP0] Weight version changed. old_version='6' new_version='yeto:5:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16 TP2 EP2] Weight version changed. old_version='6' new_version='yeto:5:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16 TP1 EP1] Weight version changed. old_version='6' new_version='yeto:5:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16 TP3 EP3] Weight version changed. old_version='6' new_version='yeto:5:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16] INFO: 10.0.0.8:45394 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16] INFO: 10.0.0.8:45418 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16] INFO: 10.0.0.8:45418 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16 TP0 EP0] Weight version changed. old_version='6' new_version='yeto:5:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16 TP1 EP1] Weight version changed. old_version='6' new_version='yeto:5:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16 TP3 EP3] Weight version changed. old_version='6' new_version='yeto:5:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16 TP2 EP2] Weight version changed. old_version='6' new_version='yeto:5:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16 TP0 EP0] [WeightChecker] checksum computed for 437 tensors in 0.208s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16 TP3 EP3] [WeightChecker] checksum computed for 437 tensors in 0.208s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16 TP2 EP2] [WeightChecker] checksum computed for 437 tensors in 0.208s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16 TP1 EP1] [WeightChecker] checksum computed for 437 tensors in 0.208s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:16] INFO: 10.0.0.8:45394 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16 TP3 EP3] [WeightChecker] checksum computed for 509 tensors in 0.216s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16 TP1 EP1] [WeightChecker] checksum computed for 509 tensors in 0.216s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16 TP2 EP2] [WeightChecker] checksum computed for 509 tensors in 0.217s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16 TP0 EP0] [WeightChecker] checksum computed for 509 tensors in 0.217s -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_publication","island_id":0,"policy_version":5,"rl/policy_token":"yeto:5:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285616.3555238} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1458.843831943,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":1449.463370069,"task":"publish","time_unix":1791285616.3558664,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"offload","rollout_id":5,"time_unix":1791285616.3702774} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:16] INFO: 10.0.0.8:59132 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:16.371 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=sleep n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:16.373 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:16.373 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:16.373 actor_cell00000_rank00000] timer.py:31 - Timer sleep start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:16.373 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:16.373 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:16.373 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:16.373 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:16.373 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:16.373 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=sleep phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:17.018 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 0.71, 'used_GB': 78.47, 'allocated_GB': 3.84, 'reserved_GB': 4.45} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:16.984 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 1.07, 'used_GB': 78.11, 'allocated_GB': 3.84, 'reserved_GB': 4.38} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:16.976 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 1.97, 'used_GB': 77.21, 'allocated_GB': 3.84, 'reserved_GB': 4.13} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:17.021 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 0.75, 'used_GB': 78.42, 'allocated_GB': 3.84, 'reserved_GB': 4.41} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:17.036 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before offload model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 1.03, 'used_GB': 78.15, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:17.052 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 0.8, 'used_GB': 78.38, 'allocated_GB': 3.73, 'reserved_GB': 4.07} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:17.063 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 0.83, 'used_GB': 78.34, 'allocated_GB': 3.73, 'reserved_GB': 4.04} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:17.079 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 2.22, 'used_GB': 76.96, 'allocated_GB': 3.73, 'reserved_GB': 4.06} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:20.755 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after offload model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 11.81, 'used_GB': 67.36, 'allocated_GB': 3.84, 'reserved_GB': 4.13} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:20.755 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:21.108 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after offload model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 11.06, 'used_GB': 68.12, 'allocated_GB': 3.73, 'reserved_GB': 4.04} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:21.108 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:21.184 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after offload model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 11.06, 'used_GB': 68.12, 'allocated_GB': 3.73, 'reserved_GB': 4.07} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:21.184 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:21.213 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after offload model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 11.76, 'used_GB': 67.42, 'allocated_GB': 3.73, 'reserved_GB': 4.06} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:21.214 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:21.318 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after offload model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 11.34, 'used_GB': 67.83, 'allocated_GB': 3.84, 'reserved_GB': 4.41} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:21.319 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=4.9 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:21.336 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after offload model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 11.34, 'used_GB': 67.83, 'allocated_GB': 3.84, 'reserved_GB': 4.45} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:21.337 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:21.483 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after offload model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 11.49, 'used_GB': 67.69, 'allocated_GB': 3.84, 'reserved_GB': 4.38} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:21.483 actor_cell00000_rank00000] timer.py:42 - Timer sleep end (elapsed: 5.1s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:21.483 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=sleep phase=end ok=true elapsed_s=5.1 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1464.16865433,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":5,"start":1458.858330573,"task":"offload","time_unix":1791285621.6807451,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":5,"rollout_id":5,"time_unix":1791285621.680951} -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Rollout generation: 0%| | 0/64 [00:00system\nReasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.\n\nYou are a helpful assistant. Please put the answer within \boxed{}.<|im_end|>\n<|im_start|>user\nAn earthquake caused four buildings to collapse. Experts predicted that each following earthquake would have double the number of collapsing buildings as the previous one, since each o..., label=60, reward_summary=0.0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) Rollout generation: 12%|█▎ | 8/64 [00:03<00:27, 2.02it/s] Rollout generation: 100%|██████████| 64/64 [00:03<00:00, 16.14it/s] -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:20:25.651 rollout_executor] inference_rollout_train.py:161 - Finish rollout: text_preview=<|im_start|>system\nReasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.\n\nYou are a helpful assistant. Please put the answer within \boxed{}.<|im_end|>\n<|im_start|>user\nA concert ticket costs $40. Mr. Benson bought 12 tickets and received a 5% discount for every ticket bought that exceeds 10. How much did Mr. Benson pay in all?<|im_end|>\n<|im_start|>..., label=476, reward_summary=0.0 -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:20:25.654 rollout_executor] inference_rollout_train.py:33 - Abort request for ['http://10.0.0.8:20037', 'http://10.0.0.8:20000'] -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:20:25.666 rollout_executor] rollout_data_conversion.py:39 - Final collected 64 samples from rollout to train -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:20:25.667 rollout_executor] metrics.py:93 - perf 5: {'rollout/raw_reward_unfiltered': 0.0, 'rollout/num_training_samples': 64, 'rollout/episode_raw_reward': 0.0, 'rollout/episode_response_length/mean': 999.03125, 'rollout/episode_response_length/median': 1024.0, 'rollout/episode_response_length/max': 1024, 'rollout/episode_response_length/min': 88, 'rollout/episode_total_response_length/mean': 999.03125, 'rollout/response_len/mean': 999.03125, 'rollout/response_len/median': 1024.0, 'rollout/response_len/max': 1024, 'rollout/response_len/min': 88, 'rollout/zero_std/count_0.0': 8, 'rollout/zero_std/all_zero_percentage': 1.0, 'rollout/zero_std/all_one_percentage': 0.0, 'rollout/prefix_cache_hit_rate': 0.0, 'rollout/avg_cached_tokens_per_sample': 0.0, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.96875, 'perf/rollout_time': 3.980242967605591, 'perf/tokens_per_gpu_per_sec': 2007.9804336185855, 'perf/longest_sample_tokens_per_sec': 257.2707265194947, 'perf/effective_tokens_per_gpu_per_sec': 2007.9804336185855, 'perf/longest_effective_sample_tokens_per_sec': 257.2707265194947} -(yeto-rl-island-0, pid=4051) (RolloutExecutor pid=26746) [2026-10-06 11:20:25.668 rollout_executor] train_data_conversion.py:334 - Rollout-side DP schedule: num_samples=64, num_rollouts=[64], num_microbatches=[32] -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:25] INFO: 10.0.0.8:50996 - "POST /abort_request HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:25] INFO: 10.0.0.8:49038 - "POST /abort_request HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:25 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:25 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:25 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:25 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:25 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:25 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:25 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:25 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:25 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:25] INFO: 10.0.0.8:49050 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:25 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:25 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:25 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:25 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:25 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:25] INFO: 10.0.0.8:50998 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:25 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:25 TP0 EP0] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:25 TP2 EP2] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:25 TP1 EP1] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:25 TP3 EP3] Reset HybridReqToTokenPool -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:25 TP0 EP0] Cache flushed successfully! -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1468.909227733,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"rollout","rollout_id":5,"start":1464.169036524,"task":"generate","time_unix":1791285626.4212992,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"onload","rollout_id":5,"time_unix":1791285626.4214826} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:26] INFO: 10.0.0.8:50998 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:26] INFO: 10.0.0.8:49050 - "POST /release_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:26.422 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=wake_up n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:26.423 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:26.423 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:26.423 actor_cell00000_rank00000] timer.py:31 - Timer wake_up start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:26.424 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:26.424 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:26.424 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:26.425 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage before wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 69.97, 'used_GB': 9.21, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:26.424 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:26.425 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage before wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 69.88, 'used_GB': 9.3, 'allocated_GB': 3.73, 'reserved_GB': 4.04} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:26.424 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:26.425 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage before wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 69.72, 'used_GB': 9.46, 'allocated_GB': 3.73, 'reserved_GB': 4.07} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:26.424 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=wake_up phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:26.425 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage before wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 70.47, 'used_GB': 8.7, 'allocated_GB': 3.73, 'reserved_GB': 4.06} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:26.425 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 70.17, 'used_GB': 9.01, 'allocated_GB': 3.84, 'reserved_GB': 4.45} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:26.424 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 70.31, 'used_GB': 8.87, 'allocated_GB': 3.84, 'reserved_GB': 4.38} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:26.425 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage before wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 70.64, 'used_GB': 8.54, 'allocated_GB': 3.84, 'reserved_GB': 4.13} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:26.425 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage before wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 70.17, 'used_GB': 9.01, 'allocated_GB': 3.84, 'reserved_GB': 4.41} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:27.757 actor_cell00000_rank00000] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27296 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:27.759 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after wake_up model: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 65.98, 'used_GB': 13.2, 'allocated_GB': 3.84, 'reserved_GB': 4.38} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:27.759 actor_cell00000_rank00000] timer.py:42 - Timer wake_up end (elapsed: 1.3s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:27.759 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:27.752 actor_cell00000_rank00003] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27299 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:27.754 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after wake_up model: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 66.55, 'used_GB': 12.63, 'allocated_GB': 3.84, 'reserved_GB': 4.13} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:27.755 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.3 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:27.782 actor_cell00000_rank00004] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27302 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:27.785 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after wake_up model: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 65.99, 'used_GB': 13.19, 'allocated_GB': 3.73, 'reserved_GB': 4.03} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:27.785 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:27.775 actor_cell00000_rank00006] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27304 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:27.778 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after wake_up model: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 65.89, 'used_GB': 13.29, 'allocated_GB': 3.73, 'reserved_GB': 4.04} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:27.778 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1470.331685289,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":5,"start":1468.909568983,"task":"onload","time_unix":1791285627.8437645,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":5,"time_unix":1791285627.8439708} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:27.842 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=wake_up ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:27.846 trainer_controller_actor] group.py:191 - ft op=train phase=start rollout=5 attempt=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:27.846 trainer_controller_actor] group.py:540 - ft op=refresh phase=start rollout=5 alive=0 healing= all_states="(0, 'StateAllocatedAlive')" quorum=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:27.846 trainer_controller_actor] group.py:563 - ft op=refresh phase=decision rollout=5 needs_reconfigure=false reason=alive_config_unchanged,no_healing quorum=0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:27.846 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=train n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:27.825 actor_cell00000_rank00001] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27297 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:27.828 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after wake_up model: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 65.76, 'used_GB': 13.41, 'allocated_GB': 3.84, 'reserved_GB': 4.45} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:27.828 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:27.847 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:27.859 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:27.859 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:27.847 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:27.848 actor_cell00000_rank00000] timer.py:31 - Timer data_preprocess start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:27.859 actor_cell00000_rank00000] timer.py:42 - Timer data_preprocess end (elapsed: 0.0s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:27.859 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:27.859 actor_cell00000_rank00000] timer.py:42 - Timer train_wait end (elapsed: 26.5s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:27.859 actor_cell00000_rank00000] timer.py:31 - Timer train start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:27.859 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:27.859 actor_cell00000_rank00000] timer.py:31 - Timer log_probs start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:27.848 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:27.862 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:27.862 actor_cell00000_rank00003] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:27.790 actor_cell00000_rank00002] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27300 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:27.792 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after wake_up model: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 65.81, 'used_GB': 13.37, 'allocated_GB': 3.84, 'reserved_GB': 4.41} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:27.792 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:27.848 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:27.859 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:27.859 actor_cell00000_rank00002] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:27.848 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:27.860 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:27.860 actor_cell00000_rank00004] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:27.848 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:27.860 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:27.860 actor_cell00000_rank00006] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:27.838 actor_cell00000_rank00005] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27303 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:27.841 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after wake_up model: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 65.86, 'used_GB': 13.32, 'allocated_GB': 3.73, 'reserved_GB': 4.07} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:27.841 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:27.848 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:27.860 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:27.860 actor_cell00000_rank00005] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:27.829 actor_cell00000_rank00007] reloadable_process_group.py:151 - Reloading 36 process groups in pid 27305 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:27.832 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after wake_up model: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 66.57, 'used_GB': 12.61, 'allocated_GB': 3.73, 'reserved_GB': 4.06} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:27.832 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=wake_up phase=end ok=true elapsed_s=1.4 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:27.848 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=train phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:27.860 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_train_actor phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:27.860 actor_cell00000_rank00007] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=start -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:29] INFO: 10.0.0.8:49202 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:29] INFO: 10.0.0.8:40492 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:32.514 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:32.514 actor_cell00000_rank00000] timer.py:42 - Timer log_probs end (elapsed: 4.7s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:32.514 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:32.514 actor_cell00000_rank00000] timer.py:31 - Timer actor_train start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:32.513 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:32.513 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:32.617 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:32.629 actor_cell00000_rank00004] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=0 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:32.630 actor_cell00000_rank00004] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=0 success=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:32.630 actor_cell00000_rank00004] log_utils.py:126 - rollout 5: {'rollout/response_lengths': 999.03125, 'rollout/rewards': 0.0, 'rollout/truncated': 0.96875, 'rollout/rollout_log_probs': -11.993880271911621, 'rollout/raw_reward': 0.0, 'rollout/total_lengths': 1126.28125, 'rollout/log_probs': -11.99585247039795, 'rollout/advantages': 0.0, 'rollout/returns': 0.0} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:32.617 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:32.629 actor_cell00000_rank00006] log_utils.py:100 - ft op=cross_cell phase=start kind=log_gather rank=1 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:32.630 actor_cell00000_rank00006] log_utils.py:106 - ft op=cross_cell phase=end kind=log_gather rank=1 success=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:32.618 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:32.622 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_compute_log_prob phase=end ok=true elapsed_s=4.8 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_heartbeat","execution_mode":"colocated-serial","interval_s":30.0,"island_id":0,"phase":"train","pid":18336,"policy_version":5,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rollout_id":5,"rounds_completed":5,"seq":17,"time_unix":1791285632.8150492,"trained_version":5,"uptime_s":510.058,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"aborted_in_flight_groups":0,"adv_mean":null,"adv_std":null,"applied_lrs":[1.6666666666666665e-06],"clip_fraction":null,"dynamic_filter_source":{"dropped_groups":"all_samples_hook_not_trained","generated_groups":"all_samples_hook_completed_groups","replacement_attempts":"proxy_filtered"},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"masked_fraction":null,"nonzero_advantages":null,"resp_len_mean":999.03125,"resp_len_p95":1024.0,"reward_p10":0.0,"reward_p50":0.0,"reward_p90":0.0,"rollout_id":5,"step_seconds":10.804810546999988,"submitted_groups":8,"time_unix":1791285638.6492066,"tok_per_s":5917.549384311298,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0,"pg_clipfrac":0.0,"pg_loss":0.0,"ppo_kl":0.0,"train_rollout_kl":0.0018683435628190637,"train_rollout_logprob_abs_diff":0.0359635204076767},"train_step":6,"trained_groups":8,"trained_sample_ids_sha256":"4f9af3ed01dbaa4e2e7788efbcbea5c6ce3ec4901d040d08a2037abcd607687d","trained_samples":64,"truncated_frac":0.96875} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"config_epoch":0,"event":"rl_round_labels","execution_mode":"colocated-serial","island_id":0,"profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","rl/aborted_groups":0,"rl/carried_over_groups":null,"rl/clip_fraction":null,"rl/ess_ratio":1.0,"rl/filtered_groups":0,"rl/groups":8,"rl/masked_fraction":null,"rl/mean_kl":0.0,"rollout_id":5,"t":1481.137571521,"time_unix":1791285638.6495662,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1481.137779501,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":5,"start":1470.332056308,"task":"train","time_unix":1791285638.6497386,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":5,"time_unix":1791285638.6498363} -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:38.629 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=train ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:38.630 trainer_controller_actor] group.py:291 - ft op=check errored= discarded= normal=0 decision=no_retry reason="survivors normal, gradients valid" -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:38.633 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:38.635 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:38.637 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:38.640 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:38.644 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:38.647 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=0.0 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:38.650 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:38.625 actor_cell00000_rank00001] model.py:646 - train op=train_step rollout=5 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:38.626 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:38.627 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:38.625 actor_cell00000_rank00000] model.py:646 - train op=train_step rollout=5 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:38.625 actor_cell00000_rank00000] timer.py:42 - Timer actor_train end (elapsed: 6.1s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:38.625 actor_cell00000_rank00000] timer.py:42 - Timer train end (elapsed: 10.8s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:38.626 actor_cell00000_rank00000] timer.py:31 - Timer train_wait start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:38.626 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:38.627 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:38.625 actor_cell00000_rank00003] model.py:646 - train op=train_step rollout=5 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:38.626 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:38.627 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:38.625 actor_cell00000_rank00002] model.py:646 - train op=train_step rollout=5 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:38.626 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:38.627 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:38.625 actor_cell00000_rank00004] model.py:646 - train op=train_step rollout=5 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:38.626 actor_cell00000_rank00004] log_utils.py:554 - step 5: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.0359635204076767, 'train/train_rollout_kl': 0.0018683435628190637, 'train/grad_norm': 0.0, 'train/lr-pg_0': 0.0, 'train/lr-pg_1': 0.0, 'train/step': 5} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:38.626 actor_cell00000_rank00004] model.py:856 - step 5: {'train/loss': 0.0, 'train/pg_loss': 0.0, 'train/entropy_loss': 0.0, 'train/pg_clipfrac': 0.0, 'train/ppo_kl': 0.0, 'train/ess_ratio': 1.0, 'train/train_rollout_logprob_abs_diff': 0.0359635204076767, 'train/train_rollout_kl': 0.0018683435628190637, 'train/grad_norm': 0.0, 'train/lr-pg_0': 0.0, 'train/lr-pg_1': 0.0, 'train/step': 5} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:38.626 actor_cell00000_rank00004] train_metric_utils.py:56 - perf 5: {'perf/update_weights_implementation_time': 1.6525883674621582, 'perf/finalize_and_resume_engines_time': 0.9163856506347656, 'perf/update_weights_time': 2.6523282527923584, 'perf/sleep_time': 5.304517984390259, 'perf/wake_up_time': 1.3604962825775146, 'perf/data_preprocess_time': 0.01159524917602539, 'perf/train_wait_time': 26.483285427093506, 'perf/log_probs_time': 4.757153272628784, 'perf/actor_train_time': 5.9959352016448975, 'perf/train_time': 10.766058206558228, 'perf/log_probs_tflops': 3.638272721394105, 'perf/actor_train_tflops': 8.659777199667461, 'perf/actor_train_tok_per_s': 12021.811039623202, 'perf/mfu_peak_tflops': 989.0, 'perf/actor_train_mfu': 0.00875609423626639, 'perf/step_time': 37.24934363365173, 'perf/wait_time_ratio': 0.7109732103619679} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:38.627 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:38.628 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:38.625 actor_cell00000_rank00006] model.py:646 - train op=train_step rollout=5 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:38.626 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:38.627 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:38.625 actor_cell00000_rank00005] model.py:646 - train op=train_step rollout=5 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:38.626 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:38.627 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:38.625 actor_cell00000_rank00007] model.py:646 - train op=train_step rollout=5 step=0 attempt=0 outcome=NORMAL valid_step=true -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:38.626 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=_train_actor phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:38.627 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=train phase=end ok=true elapsed_s=10.8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:40.955 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=2.3 -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"action_tokens":63938,"active_groups":8,"applied_lr":1.6666666666666665e-06,"applied_lrs":[1.6666666666666665e-06],"base_policy_version":5,"cancelled_groups":0,"clip_fraction":null,"completed_groups":8,"completed_trajectories":64,"delta_l2_norm":0.0,"dynamic_filter_dropped_groups":0,"dynamic_filter_generated_groups":8,"dynamic_filter_replacement_attempts":0,"ess_ratio":1.0,"event":"rl_local_round","grad_norm":0.0,"group_p50_seconds":0.0,"group_p95_seconds":0.0,"group_p99_seconds":0.0,"island_id":0,"local_round_id":6,"loss":0.0,"lr":1.6666666666666665e-06,"mean_kl":0.0,"nonzero_advantage_count":null,"pass_rate":null,"pg_loss":0.0,"reward_mean":0.0,"reward_std":0.0,"rollout_seconds":10.050893723999934,"time_unix":1791285643.5051532,"tool_wait_seconds":0.0,"train_seconds":10.804810546999988,"train_step":6,"zero_variance_group_ratio":1.0} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1486.881580093,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"compute","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer","rollout_id":5,"start":1481.137926151,"task":"outer_sync","time_unix":1791285644.3938332,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":6,"time_unix":1791285644.3941019} -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:44] INFO: 10.0.0.8:52950 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:44] INFO: 10.0.0.8:50678 - "GET /health HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:45.288 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=run_plugin n_actors=8 -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:47.593 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=run_plugin ok=true elapsed_s=2.3 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:50] INFO: 10.0.0.8:48326 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:50] INFO: 10.0.0.8:48332 - "POST /pause_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:50] INFO: 10.0.0.8:48332 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:50] INFO: 10.0.0.8:48332 - "POST /begin_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:50 TP0 EP0] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:50 TP2 EP2] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:50 TP3 EP3] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:50 TP1 EP1] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:50] INFO: 10.0.0.8:32978 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:50] INFO: 10.0.0.8:32994 - "POST /pause_generation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:50] INFO: 10.0.0.8:32994 - "GET /flush_cache HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:50] INFO: 10.0.0.8:32994 - "POST /begin_weight_update HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:50 TP0 EP0] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:50 TP2 EP2] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:50 TP1 EP1] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:50 TP3 EP3] Cache flush skipped: KV cache memory is released. -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:50.623 trainer_controller_actor] group.py:393 - ft op=update_weights phase=start rollout= -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:50.624 trainer_controller_actor] cell.py:231 - ft op=execute phase=start cell=trainer-engine-actor-00000 fn=update_weights n_actors=8 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:50.625 actor_cell00000_rank00001] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:50.626 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage before update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 53.02, 'used_GB': 26.16, 'allocated_GB': 3.84, 'reserved_GB': 5.06} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:50.625 actor_cell00000_rank00000] structured_log.py:24 - actor cls=MegatronTrainRayActor fn=update_weights phase=start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:50.625 actor_cell00000_rank00000] timer.py:31 - Timer update_weights start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:50.626 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage before update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 53.29, 'used_GB': 25.89, 'allocated_GB': 3.84, 'reserved_GB': 5.07} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:50.637 actor_cell00000_rank00000] timer.py:31 - Timer update_weights_implementation start -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [miles-colocate] Update weights: 0it [00:00, ?it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) Update weights: 0%| | 0/24 [00:00 63770.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP2 EP2] [continue_generation] torch.cuda.empty_cache() called: reserved 64218.0 MB -> 63770.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP0 EP0] [continue_generation] torch.cuda.empty_cache() called: reserved 64218.0 MB -> 63770.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP1 EP1] [continue_generation] torch.cuda.empty_cache() called: reserved 64218.0 MB -> 63770.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP1 EP1] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP2 EP2] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP3 EP3] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP0 EP0] [continue_generation] torch.cuda.empty_cache() called: reserved 63928.0 MB -> 63480.0 MB (freed 448.0 MB) -(yeto-rl-island-0, pid=4051) (TrainerController pid=26752) [2026-10-06 11:20:53.286 trainer_controller_actor] cell.py:239 - ft op=execute phase=end cell=trainer-engine-actor-00000 fn=update_weights ok=true elapsed_s=2.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [miles-colocate] Update weights: 23it [00:02, 8.85it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:53.236 actor_cell00000_rank00001] memory_utils.py:41 - [Rank 1] Memory-Usage after update_weights: {'gpu': '1', 'total_GB': 79.18, 'free_GB': 46.58, 'used_GB': 32.6, 'allocated_GB': 3.84, 'reserved_GB': 11.6} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27297) [2026-10-06 11:20:53.270 actor_cell00000_rank00001] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [miles-colocate] Update weights: 23it [00:02, 8.83it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:53.243 actor_cell00000_rank00000] memory_utils.py:41 - [Rank 0] Memory-Usage after update_weights: {'gpu': '0', 'total_GB': 79.18, 'free_GB': 47.59, 'used_GB': 31.59, 'allocated_GB': 3.84, 'reserved_GB': 10.88} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:53.283 actor_cell00000_rank00000] timer.py:42 - Timer update_weights end (elapsed: 2.7s) -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27296) [2026-10-06 11:20:53.283 actor_cell00000_rank00000] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [miles-colocate] Update weights: 23it [00:02, 8.81it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:53.251 actor_cell00000_rank00003] memory_utils.py:41 - [Rank 3] Memory-Usage after update_weights: {'gpu': '3', 'total_GB': 79.18, 'free_GB': 47.88, 'used_GB': 31.3, 'allocated_GB': 3.84, 'reserved_GB': 11.24} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27299) [2026-10-06 11:20:53.285 actor_cell00000_rank00003] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [miles-colocate] Update weights: 23it [00:02, 8.83it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:53.244 actor_cell00000_rank00002] memory_utils.py:41 - [Rank 2] Memory-Usage after update_weights: {'gpu': '2', 'total_GB': 79.18, 'free_GB': 46.71, 'used_GB': 32.47, 'allocated_GB': 3.84, 'reserved_GB': 11.48} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27300) [2026-10-06 11:20:53.281 actor_cell00000_rank00002] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [miles-colocate] Update weights: 23it [00:02, 8.84it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:53.242 actor_cell00000_rank00004] memory_utils.py:41 - [Rank 4] Memory-Usage after update_weights: {'gpu': '4', 'total_GB': 79.18, 'free_GB': 48.0, 'used_GB': 31.18, 'allocated_GB': 3.73, 'reserved_GB': 10.08} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27302) [2026-10-06 11:20:53.284 actor_cell00000_rank00004] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.7 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [miles-colocate] Update weights: 23it [00:02, 8.87it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:53.232 actor_cell00000_rank00006] memory_utils.py:41 - [Rank 6] Memory-Usage after update_weights: {'gpu': '6', 'total_GB': 79.18, 'free_GB': 46.61, 'used_GB': 32.57, 'allocated_GB': 3.73, 'reserved_GB': 11.28} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27304) [2026-10-06 11:20:53.263 actor_cell00000_rank00006] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [miles-colocate] Update weights: 23it [00:02, 8.85it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:53.239 actor_cell00000_rank00005] memory_utils.py:41 - [Rank 5] Memory-Usage after update_weights: {'gpu': '5', 'total_GB': 79.18, 'free_GB': 46.59, 'used_GB': 32.59, 'allocated_GB': 3.73, 'reserved_GB': 11.3} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27303) [2026-10-06 11:20:53.274 actor_cell00000_rank00005] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.6 -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [miles-colocate] Update weights: 23it [00:02, 8.85it/s] -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:53.236 actor_cell00000_rank00007] memory_utils.py:41 - [Rank 7] Memory-Usage after update_weights: {'gpu': '7', 'total_GB': 79.18, 'free_GB': 48.04, 'used_GB': 31.14, 'allocated_GB': 3.73, 'reserved_GB': 11.27} -(yeto-rl-island-0, pid=4051) (MegatronTrainRayActor pid=27305) [2026-10-06 11:20:53.265 actor_cell00000_rank00007] structured_log.py:28 - actor cls=MegatronTrainRayActor fn=update_weights phase=end ok=true elapsed_s=2.6 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53] INFO: 10.0.0.8:48326 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53] INFO: 10.0.0.8:32978 - "POST /resume_memory_occupation HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53] INFO: 10.0.0.8:48344 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53] INFO: 10.0.0.8:48344 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP0 EP0] Weight version changed. old_version='7' new_version='yeto:6:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP1 EP1] Weight version changed. old_version='7' new_version='yeto:6:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP3 EP3] Weight version changed. old_version='7' new_version='yeto:6:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP2 EP2] Weight version changed. old_version='7' new_version='yeto:6:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53] INFO: 10.0.0.8:32996 - "POST /update_weight_version HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53] INFO: 10.0.0.8:32996 - "GET /model_info HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP0 EP0] Weight version changed. old_version='7' new_version='yeto:6:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP1 EP1] Weight version changed. old_version='7' new_version='yeto:6:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP3 EP3] Weight version changed. old_version='7' new_version='yeto:6:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP2 EP2] Weight version changed. old_version='7' new_version='yeto:6:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117' num_recorded=0 -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP0 EP0] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP2 EP2] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP1 EP1] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP3 EP3] [WeightChecker] handle action=checksum allow_quant_error=False skip_tensor_list=None -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53] INFO: 10.0.0.8:32978 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP3 EP3] [WeightChecker] checksum computed for 437 tensors in 0.205s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP1 EP1] [WeightChecker] checksum computed for 437 tensors in 0.205s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP0 EP0] [WeightChecker] checksum computed for 437 tensors in 0.206s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53 TP2 EP2] [WeightChecker] checksum computed for 437 tensors in 0.206s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP2 EP2] [WeightChecker] checksum computed for 509 tensors in 0.215s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP0 EP0] [WeightChecker] checksum computed for 509 tensors in 0.215s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP3 EP3] [WeightChecker] checksum computed for 509 tensors in 0.216s -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53 TP1 EP1] [WeightChecker] checksum computed for 509 tensors in 0.218s -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_publication","island_id":0,"policy_version":6,"rl/policy_token":"yeto:6:cd9466cc8ea0fd63764eef10afd5b10c2566e5104f83ea17e9b1f606797d8117","sync/publication_members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"sync/publication_payload_bytes":466769920,"sync/publication_payload_hash":"d10f320b55baf2080e43158ae6d94cde6d85d0a6fb3e5e1b045c4fd9be8809ce","time_unix":1791285653.8974712} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"end":1496.385882499,"epoch":0,"event":"rl_timeline_span","island_id":0,"kind":"transfer","profile_hash":"sha256:d092c18462e56335bc44664da540599511cc82c39ea2660bc1ec662efa3c044f","role":"trainer+rollout","rollout_id":null,"start":1486.882185187,"task":"publish","time_unix":1791285653.8980012,"weight_transport":"cuda-ipc"} -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":6,"time_unix":1791285653.9136086} -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:53.941 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='rollout-executor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f6a74429440>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.rollout_executor.RolloutExecutor', ctor_kwargs=. at 0x7f6a744294e0>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=0, spec=ServeWorkerSpec(name='rollout-executor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f6a74429440>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.rollout_executor.RolloutExecutor', ctor_kwargs=. at 0x7f6a744294e0>, concurrency_groups=None), actor_handle=Actor(RolloutExecutor, da3ad76a26235c04893541f701000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20072, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:53.942 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='inference-controller', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f6a74429620>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.inference_controller.InferenceController', ctor_kwargs=. at 0x7f69f5a8a3e0>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=0, spec=ServeWorkerSpec(name='inference-controller', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f6a74429620>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.rollout.inference_controller.InferenceController', ctor_kwargs=. at 0x7f69f5a8a3e0>, concurrency_groups=None), actor_handle=Actor(InferenceController, 4e83849c78443e8987e1920c01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20073, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:53.943 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-controller-actor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8aa20>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.train.group.TrainerController', ctor_kwargs=. at 0x7f69f5a8ab60>, concurrency_groups=None), actors=[...], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-controller-actor', category=None, port_infos=[PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8aa20>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=1.0, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=None, deploy_component=, platform_access=, worker_class='miles.ray.train.group.TrainerController', ctor_kwargs=. at 0x7f69f5a8ab60>, concurrency_groups=None), actor_handle=Actor(TrainerController, 9464bc970d3025ff9ffb53f501000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20074, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:53.943 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actors=[..., _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=1, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 22f55169b6328d3ea952b73301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20077, external_host=None)}, local_gpu_ids=[1], gpu_slot_index=1), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=2, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 6014634ca786d194f3b8636501000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20083, external_host=None)}, local_gpu_ids=[2], gpu_slot_index=2), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=3, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, cb8ca3c4e1a6652340249e0d01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20079, external_host=None)}, local_gpu_ids=[3], gpu_slot_index=3), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=4, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, daa357fc73acd2064050b0f401000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20080, external_host=None)}, local_gpu_ids=[4], gpu_slot_index=4), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=5, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 34c2ca3a4c7670df3a0b472101000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20081, external_host=None)}, local_gpu_ids=[5], gpu_slot_index=5), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=6, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, c7f035550931cd81fc90217a01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20082, external_host=None)}, local_gpu_ids=[6], gpu_slot_index=6), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=7, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 8bbca804f97f2f49aef7dad301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20078, external_host=None)}, local_gpu_ids=[7], gpu_slot_index=7)], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 1ebf23d1f7a8ad5257607d9a01000000), self_addrs={'master': HostAndPort(host='10.0.0.8', port=20075, external_host=None), 'rpc': HostAndPort(host='10.0.0.8', port=20076, external_host=None)}, local_gpu_ids=[0], gpu_slot_index=0) -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:53.948 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actors=[_ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 1ebf23d1f7a8ad5257607d9a01000000), self_addrs={'master': HostAndPort(host='10.0.0.8', port=20075, external_host=None), 'rpc': HostAndPort(host='10.0.0.8', port=20076, external_host=None)}, local_gpu_ids=[0], gpu_slot_index=0), ..., _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=2, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 6014634ca786d194f3b8636501000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20083, external_host=None)}, local_gpu_ids=[2], gpu_slot_index=2), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=3, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, cb8ca3c4e1a6652340249e0d01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20079, external_host=None)}, local_gpu_ids=[3], gpu_slot_index=3), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=4, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, daa357fc73acd2064050b0f401000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20080, external_host=None)}, local_gpu_ids=[4], gpu_slot_index=4), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=5, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 34c2ca3a4c7670df3a0b472101000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20081, external_host=None)}, local_gpu_ids=[5], gpu_slot_index=5), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=6, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, c7f035550931cd81fc90217a01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20082, external_host=None)}, local_gpu_ids=[6], gpu_slot_index=6), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=7, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 8bbca804f97f2f49aef7dad301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20078, external_host=None)}, local_gpu_ids=[7], gpu_slot_index=7)], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=1, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 22f55169b6328d3ea952b73301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20077, external_host=None)}, local_gpu_ids=[1], gpu_slot_index=1) -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:53.949 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actors=[_ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 1ebf23d1f7a8ad5257607d9a01000000), self_addrs={'master': HostAndPort(host='10.0.0.8', port=20075, external_host=None), 'rpc': HostAndPort(host='10.0.0.8', port=20076, external_host=None)}, local_gpu_ids=[0], gpu_slot_index=0), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=1, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 22f55169b6328d3ea952b73301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20077, external_host=None)}, local_gpu_ids=[1], gpu_slot_index=1), ..., _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=3, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, cb8ca3c4e1a6652340249e0d01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20079, external_host=None)}, local_gpu_ids=[3], gpu_slot_index=3), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=4, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, daa357fc73acd2064050b0f401000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20080, external_host=None)}, local_gpu_ids=[4], gpu_slot_index=4), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=5, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 34c2ca3a4c7670df3a0b472101000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20081, external_host=None)}, local_gpu_ids=[5], gpu_slot_index=5), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=6, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, c7f035550931cd81fc90217a01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20082, external_host=None)}, local_gpu_ids=[6], gpu_slot_index=6), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=7, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 8bbca804f97f2f49aef7dad301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20078, external_host=None)}, local_gpu_ids=[7], gpu_slot_index=7)], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=2, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 6014634ca786d194f3b8636501000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20083, external_host=None)}, local_gpu_ids=[2], gpu_slot_index=2) -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:53.950 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actors=[_ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 1ebf23d1f7a8ad5257607d9a01000000), self_addrs={'master': HostAndPort(host='10.0.0.8', port=20075, external_host=None), 'rpc': HostAndPort(host='10.0.0.8', port=20076, external_host=None)}, local_gpu_ids=[0], gpu_slot_index=0), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=1, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 22f55169b6328d3ea952b73301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20077, external_host=None)}, local_gpu_ids=[1], gpu_slot_index=1), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=2, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 6014634ca786d194f3b8636501000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20083, external_host=None)}, local_gpu_ids=[2], gpu_slot_index=2), ..., _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=4, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, daa357fc73acd2064050b0f401000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20080, external_host=None)}, local_gpu_ids=[4], gpu_slot_index=4), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=5, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 34c2ca3a4c7670df3a0b472101000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20081, external_host=None)}, local_gpu_ids=[5], gpu_slot_index=5), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=6, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, c7f035550931cd81fc90217a01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20082, external_host=None)}, local_gpu_ids=[6], gpu_slot_index=6), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=7, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 8bbca804f97f2f49aef7dad301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20078, external_host=None)}, local_gpu_ids=[7], gpu_slot_index=7)], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=3, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, cb8ca3c4e1a6652340249e0d01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20079, external_host=None)}, local_gpu_ids=[3], gpu_slot_index=3) -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:53.950 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actors=[_ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 1ebf23d1f7a8ad5257607d9a01000000), self_addrs={'master': HostAndPort(host='10.0.0.8', port=20075, external_host=None), 'rpc': HostAndPort(host='10.0.0.8', port=20076, external_host=None)}, local_gpu_ids=[0], gpu_slot_index=0), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=1, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 22f55169b6328d3ea952b73301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20077, external_host=None)}, local_gpu_ids=[1], gpu_slot_index=1), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=2, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 6014634ca786d194f3b8636501000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20083, external_host=None)}, local_gpu_ids=[2], gpu_slot_index=2), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=3, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, cb8ca3c4e1a6652340249e0d01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20079, external_host=None)}, local_gpu_ids=[3], gpu_slot_index=3), ..., _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=5, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 34c2ca3a4c7670df3a0b472101000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20081, external_host=None)}, local_gpu_ids=[5], gpu_slot_index=5), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=6, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, c7f035550931cd81fc90217a01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20082, external_host=None)}, local_gpu_ids=[6], gpu_slot_index=6), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=7, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 8bbca804f97f2f49aef7dad301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20078, external_host=None)}, local_gpu_ids=[7], gpu_slot_index=7)], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=4, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, daa357fc73acd2064050b0f401000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20080, external_host=None)}, local_gpu_ids=[4], gpu_slot_index=4) -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:53.951 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actors=[_ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 1ebf23d1f7a8ad5257607d9a01000000), self_addrs={'master': HostAndPort(host='10.0.0.8', port=20075, external_host=None), 'rpc': HostAndPort(host='10.0.0.8', port=20076, external_host=None)}, local_gpu_ids=[0], gpu_slot_index=0), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=1, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 22f55169b6328d3ea952b73301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20077, external_host=None)}, local_gpu_ids=[1], gpu_slot_index=1), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=2, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 6014634ca786d194f3b8636501000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20083, external_host=None)}, local_gpu_ids=[2], gpu_slot_index=2), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=3, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, cb8ca3c4e1a6652340249e0d01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20079, external_host=None)}, local_gpu_ids=[3], gpu_slot_index=3), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=4, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, daa357fc73acd2064050b0f401000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20080, external_host=None)}, local_gpu_ids=[4], gpu_slot_index=4), ..., _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=6, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, c7f035550931cd81fc90217a01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20082, external_host=None)}, local_gpu_ids=[6], gpu_slot_index=6), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=7, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 8bbca804f97f2f49aef7dad301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20078, external_host=None)}, local_gpu_ids=[7], gpu_slot_index=7)], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=5, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 34c2ca3a4c7670df3a0b472101000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20081, external_host=None)}, local_gpu_ids=[5], gpu_slot_index=5) -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:53.951 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actors=[_ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 1ebf23d1f7a8ad5257607d9a01000000), self_addrs={'master': HostAndPort(host='10.0.0.8', port=20075, external_host=None), 'rpc': HostAndPort(host='10.0.0.8', port=20076, external_host=None)}, local_gpu_ids=[0], gpu_slot_index=0), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=1, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 22f55169b6328d3ea952b73301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20077, external_host=None)}, local_gpu_ids=[1], gpu_slot_index=1), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=2, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 6014634ca786d194f3b8636501000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20083, external_host=None)}, local_gpu_ids=[2], gpu_slot_index=2), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=3, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, cb8ca3c4e1a6652340249e0d01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20079, external_host=None)}, local_gpu_ids=[3], gpu_slot_index=3), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=4, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, daa357fc73acd2064050b0f401000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20080, external_host=None)}, local_gpu_ids=[4], gpu_slot_index=4), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=5, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 34c2ca3a4c7670df3a0b472101000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20081, external_host=None)}, local_gpu_ids=[5], gpu_slot_index=5), ..., _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=7, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 8bbca804f97f2f49aef7dad301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20078, external_host=None)}, local_gpu_ids=[7], gpu_slot_index=7)], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=6, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, c7f035550931cd81fc90217a01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20082, external_host=None)}, local_gpu_ids=[6], gpu_slot_index=6) -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:53.953 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_ServeActorRayCommManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actors=[_ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=0, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 1ebf23d1f7a8ad5257607d9a01000000), self_addrs={'master': HostAndPort(host='10.0.0.8', port=20075, external_host=None), 'rpc': HostAndPort(host='10.0.0.8', port=20076, external_host=None)}, local_gpu_ids=[0], gpu_slot_index=0), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=1, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 22f55169b6328d3ea952b73301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20077, external_host=None)}, local_gpu_ids=[1], gpu_slot_index=1), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=2, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 6014634ca786d194f3b8636501000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20083, external_host=None)}, local_gpu_ids=[2], gpu_slot_index=2), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=3, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, cb8ca3c4e1a6652340249e0d01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20079, external_host=None)}, local_gpu_ids=[3], gpu_slot_index=3), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=4, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, daa357fc73acd2064050b0f401000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20080, external_host=None)}, local_gpu_ids=[4], gpu_slot_index=4), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=5, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 34c2ca3a4c7670df3a0b472101000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20081, external_host=None)}, local_gpu_ids=[5], gpu_slot_index=5), _ServeActorRayCommManager(manager=, parent=..., worker_in_cell_index=6, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, c7f035550931cd81fc90217a01000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20082, external_host=None)}, local_gpu_ids=[6], gpu_slot_index=6), ...], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=7, spec=ServeWorkerSpec(name='trainer-engine-actor', category='trainer_engine', port_infos=[PortInfo(name='master', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='rpc', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8ac00>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=8, num_gpus_per_worker=0.4, num_cpus_per_worker=0.4, num_gpu_slots_per_worker=1, num_gpus_per_node=8, pg_name='actor', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5aac0e0>, deploy_component=, platform_access=, worker_class='miles.backends.megatron_utils.actor.MegatronTrainRayActor', ctor_kwargs=. at 0x7f69f5aaccc0>, concurrency_groups=None), actor_handle=Actor(MegatronTrainRayActor, 8bbca804f97f2f49aef7dad301000000), self_addrs={'rpc': HostAndPort(host='10.0.0.8', port=20078, external_host=None)}, local_gpu_ids=[7], gpu_slot_index=7) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53] INFO: 10.0.0.8:48326 - "POST /weights_checker HTTP/1.1" 200 OK -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:53] SIGTERM received. signum=None frame=None. Draining requests and shutting down... -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:20:53.933 inference_controller] rollout_server.py:132 - Killing server cell_id='inference-engine-all-0-0-00000'... -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:20:53.937 inference_controller] server_cell.py:315 - Cell inference-engine-all-0-0-00000 dispose start old=addr_info=CellAddrInfo(server_url='http://10.0.0.8:20000', bootstrap_port=None, gate_url='http://10.0.0.8:20034') -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:20:53.937 inference_controller] server_cell.py:318 - Cell inference-engine-all-0-0-00000 dispose end new= -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:20:53.937 inference_controller] rollout_server.py:132 - Killing server cell_id='inference-engine-all-0-0-00001'... -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:20:53.939 inference_controller] server_cell.py:315 - Cell inference-engine-all-0-0-00001 dispose start old=addr_info=CellAddrInfo(server_url='http://10.0.0.8:20037', bootstrap_port=None, gate_url='http://10.0.0.8:20071') -(yeto-rl-island-0, pid=4051) (InferenceController pid=26748) [2026-10-06 11:20:53.939 inference_controller] server_cell.py:318 - Cell inference-engine-all-0-0-00001 dispose end new= -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:53] SIGTERM received. signum=None frame=None. Draining requests and shutting down... -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:54.063 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_CommandActorManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=CommandWorkerSpec(name='inference-router-0', category=None, port_infos=[PortInfo(name='primary', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='prometheus', static_port=9000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8a480>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=None, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x7f69f5a8a520>), actors=[...], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=0, spec=CommandWorkerSpec(name='inference-router-0', category=None, port_infos=[PortInfo(name='primary', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='prometheus', static_port=9000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8a480>, scheduling=SchedulingSpec(num_cells=1, num_workers_per_cell=1, num_gpus_per_worker=0.0, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=0, num_gpus_per_node=0, pg_name=None, pg_slot_offset=0, pin_to_head=True, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=None, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x7f69f5a8a520>), actor_handle=Actor(CommandActor, 157af7221c1b56116e6ede3001000000), self_addrs={'primary': HostAndPort(host='10.0.0.8', port=20035, external_host=None), 'prometheus': HostAndPort(host='10.0.0.8', port=20036, external_host=None)}, local_gpu_ids=[], gpu_slot_index=None) -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:54] Subprocess detokenizer (pid=30297) crashed with exit code -15. Triggering SIGQUIT for cleanup... -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:54] SIGQUIT received. signum=None, frame=None. It usually means one child failed. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) [2026-10-06 11:20:54] Sleeping 5 seconds before crash diagnostics to let GPU activity settle. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:54] Subprocess detokenizer (pid=30295) crashed with exit code -15. Triggering SIGQUIT for cleanup... -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:54] SIGQUIT received. signum=None, frame=None. It usually means one child failed. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) [2026-10-06 11:20:54] Sleeping 5 seconds before crash diagnostics to let GPU activity settle. -(yeto-rl-island-0, pid=4051) (CommandActor pid=26751) WARNING:miles.utils.workers.process_utils:Process 28485 did not exit within 5.0s after SIGTERM; escalating to SIGKILL -(yeto-rl-island-0, pid=4051) (CommandActor pid=26750) WARNING:miles.utils.workers.process_utils:Process 28483 did not exit within 5.0s after SIGTERM; escalating to SIGKILL -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:59.040 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_CommandActorManager(manager=, parent=_CellManager(manager=, cell_index=1, spec=CommandWorkerSpec(name='inference-engine-all-0-0', category='inference_engine', port_infos=[PortInfo(name='primary', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='dist_init', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=31, offset_by_cell=False), PortInfo(name='nccl', static_port=10000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='engine_info_bootstrap', static_port=12000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='gate', static_port=13000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8a700>, scheduling=SchedulingSpec(num_cells=2, num_workers_per_cell=1, num_gpus_per_worker=0.2, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=4, num_gpus_per_node=8, pg_name='rollout', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5a8a840>, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x7f69f5a8a980>), actors=[...], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=0, spec=CommandWorkerSpec(name='inference-engine-all-0-0', category='inference_engine', port_infos=[PortInfo(name='primary', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='dist_init', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=31, offset_by_cell=False), PortInfo(name='nccl', static_port=10000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='engine_info_bootstrap', static_port=12000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='gate', static_port=13000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8a700>, scheduling=SchedulingSpec(num_cells=2, num_workers_per_cell=1, num_gpus_per_worker=0.2, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=4, num_gpus_per_node=8, pg_name='rollout', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5a8a840>, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x7f69f5a8a980>), actor_handle=Actor(CommandActor, 4b1d2cec8386d65b88b62d4301000000), self_addrs={'primary': HostAndPort(host='10.0.0.8', port=20037, external_host=None), 'dist_init': HostAndPort(host='10.0.0.8', port=20038, external_host=None), 'nccl': HostAndPort(host='10.0.0.8', port=20069, external_host=None), 'engine_info_bootstrap': HostAndPort(host='10.0.0.8', port=20070, external_host=None), 'gate': HostAndPort(host='10.0.0.8', port=20071, external_host=None)}, local_gpu_ids=[4, 5, 6, 7], gpu_slot_index=4) -(yeto-rl-island-0, pid=4051) YETO_RL_EVENT {"event":"rl_learner_finalized","island_id":0,"time_unix":1791285659.0665615} -(yeto-rl-island-0, pid=4051) [rl] learner 0 finalized (rl_engine=ports) -(yeto-rl-island-0, pid=4051) (RayWorkerManager pid=26006) [2026-10-06 11:20:59.064 worker_manager] ray_worker_manager.py:704 - Killed actor at self=_CommandActorManager(manager=, parent=_CellManager(manager=, cell_index=0, spec=CommandWorkerSpec(name='inference-engine-all-0-0', category='inference_engine', port_infos=[PortInfo(name='primary', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='dist_init', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=31, offset_by_cell=False), PortInfo(name='nccl', static_port=10000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='engine_info_bootstrap', static_port=12000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='gate', static_port=13000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8a700>, scheduling=SchedulingSpec(num_cells=2, num_workers_per_cell=1, num_gpus_per_worker=0.2, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=4, num_gpus_per_node=8, pg_name='rollout', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5a8a840>, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x7f69f5a8a980>), actors=[...], generation=1, liveness_scan_task= wait_for=>, binding=None, deferred=False, lost_worker_names=None), worker_in_cell_index=0, spec=CommandWorkerSpec(name='inference-engine-all-0-0', category='inference_engine', port_infos=[PortInfo(name='primary', static_port=8000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='dist_init', static_port=9000, mode='master', allow_dynamic=True, num_consecutive=31, offset_by_cell=False), PortInfo(name='nccl', static_port=10000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='engine_info_bootstrap', static_port=12000, mode='per_worker', allow_dynamic=True, num_consecutive=1, offset_by_cell=False), PortInfo(name='gate', static_port=13000, mode='master', allow_dynamic=True, num_consecutive=1, offset_by_cell=False)], env_var=. at 0x7f69f5a8a700>, scheduling=SchedulingSpec(num_cells=2, num_workers_per_cell=1, num_gpus_per_worker=0.2, num_cpus_per_worker=0.2, num_gpu_slots_per_worker=4, num_gpus_per_node=8, pg_name='rollout', pg_slot_offset=0, pin_to_head=False, num_deferred_cells=0, cell_aliases=(), initial_bindings=()), meta=. at 0x7f69f5a8a840>, deploy_component=, platform_access=, launch_command=._compute_launch_command at 0x7f69f5a8a980>), actor_handle=Actor(CommandActor, 803253774402999a0097899701000000), self_addrs={'primary': HostAndPort(host='10.0.0.8', port=20000, external_host=None), 'dist_init': HostAndPort(host='10.0.0.8', port=20001, external_host=None), 'nccl': HostAndPort(host='10.0.0.8', port=20032, external_host=None), 'engine_info_bootstrap': HostAndPort(host='10.0.0.8', port=20033, external_host=None), 'gate': HostAndPort(host='10.0.0.8', port=20034, external_host=None)}, local_gpu_ids=[0, 1, 2, 3], gpu_slot_index=0) diff --git a/openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fnprep/fnprep-remote.log b/openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fnprep/fnprep-remote.log deleted file mode 100644 index bef54e403..000000000 --- a/openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fnprep/fnprep-remote.log +++ /dev/null @@ -1,16 +0,0 @@ -Warning: Permanently added '[localhost]:10022' (ED25519) to the list of known hosts. -RUNTIME py=/opt/sglang/bin/python python3=/opt/sglang/bin/python3 torchrun=/opt/sglang/bin/torchrun miles=/root/miles megatron=/root/Megatron-LM -PREFLIGHT_OK -DF0 1099511627776 399152648192 700358979584 -FSTYPE virtiofs filesystem-skypilot-1 -FS_SIZE_G0 1024 -SNAPSHOT_BYTES 30520078752 -POPULATE already complete -POPULATE_S 0 -DF1 1099511627776 399152648192 700358979584 -CONVERT already release -CONVERT_S 0 -DF2 1099511627776 399152648192 700358979584 -TRACKER release -TORCHDIST_BYTES 23390890947 /mnt/yeto-models/torch_dist/qwen3.8-flash-next-4layer_torch_dist -FNPREP_OK diff --git a/openspec/changes/rl-infra-spec/evidence/image/attempt2/imports_cpu.log b/openspec/changes/rl-infra-spec/evidence/image/attempt2/imports_cpu.log deleted file mode 100644 index 05d10ff55..000000000 --- a/openspec/changes/rl-infra-spec/evidence/image/attempt2/imports_cpu.log +++ /dev/null @@ -1,8 +0,0 @@ -W0929 16:29:27.326000 16 torch/utils/cpp_extension.py:178] No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' -miles /root/miles/miles/__init__.py 0.1.1 -sglang /sgl-workspace/sglang/python/sglang/__init__.py 0.5.21.dev67+g9f29303 0.5.21.dev67+g9f29303 -sglang dists: ['0.5.21.dev67+g9f29303'] -PASS imports_point_at_forks - -rc 0 -terminated sb-88pkNZTN7G3r9UF63pup52 diff --git a/openspec/changes/rl-infra-spec/evidence/image/attempt2/ns_probe.log b/openspec/changes/rl-infra-spec/evidence/image/attempt2/ns_probe.log deleted file mode 100644 index 33cb6eb51..000000000 --- a/openspec/changes/rl-infra-spec/evidence/image/attempt2/ns_probe.log +++ /dev/null @@ -1,22 +0,0 @@ -== ports cwd=/ -ModuleSpec(name='miles', loader=<_frozen_importlib_external.NamespaceLoader object at 0x2b36f3c8bfe0>, submodule_search_locations=_NamespacePath(['/root/miles'])) -['/root/miles'] -['', '/opt/sglang/lib/python3.12/site-packages/nvidia_cutlass_dsl/dsl_packages', '/opt/sglang/lib/python3.12/site-packages/nvidia_cutlass_dsl/dsl_packages', '/pkg', '/root', '/usr/lib/python312.zip', '/usr/lib/python3.12', '/usr/lib/python3.12/lib-dynload', '/opt/sglang/lib/python3.12/site-packages', '__editable__.megatron_core-0.19.0+a84b10547.finder.__path_hook__', '__editable__.sglang-0.5.21.dev67+g9f29303.finder.__path_hook__'] - -== ports cwd=/root -ModuleSpec(name='miles', loader=<_frozen_importlib_external.NamespaceLoader object at 0x2b08ecec14c0>, submodule_search_locations=_NamespacePath(['/root/miles', '/root/miles'])) -['/root/miles', '/root/miles'] -['', '/opt/sglang/lib/python3.12/site-packages/nvidia_cutlass_dsl/dsl_packages', '/opt/sglang/lib/python3.12/site-packages/nvidia_cutlass_dsl/dsl_packages', '/pkg', '/root', '/usr/lib/python312.zip', '/usr/lib/python3.12', '/usr/lib/python3.12/lib-dynload', '/opt/sglang/lib/python3.12/site-packages', '__editable__.megatron_core-0.19.0+a84b10547.finder.__path_hook__', '__editable__.sglang-0.5.21.dev67+g9f29303.finder.__path_hook__'] - -terminated ports sb-Yo0Q0xLJGQsVmaYH7zmwXP -== base cwd=/ -ModuleSpec(name='miles', loader=<_frozen_importlib_external.NamespaceLoader object at 0x2aac47369580>, submodule_search_locations=_NamespacePath(['/root/miles'])) -['/root/miles'] -['', '/opt/sglang/lib/python3.12/site-packages/nvidia_cutlass_dsl/dsl_packages', '/opt/sglang/lib/python3.12/site-packages/nvidia_cutlass_dsl/dsl_packages', '/pkg', '/root', '/usr/lib/python312.zip', '/usr/lib/python3.12', '/usr/lib/python3.12/lib-dynload', '/opt/sglang/lib/python3.12/site-packages', '__editable__.megatron_core-0.19.0+a84b10547.finder.__path_hook__', '__editable__.sglang-0.5.21.dev64+ge5bba1f.finder.__path_hook__'] - -== base cwd=/root -ModuleSpec(name='miles', loader=<_frozen_importlib_external.NamespaceLoader object at 0x2ad92e7e14f0>, submodule_search_locations=_NamespacePath(['/root/miles', '/root/miles'])) -['/root/miles', '/root/miles'] -['', '/opt/sglang/lib/python3.12/site-packages/nvidia_cutlass_dsl/dsl_packages', '/opt/sglang/lib/python3.12/site-packages/nvidia_cutlass_dsl/dsl_packages', '/pkg', '/root', '/usr/lib/python312.zip', '/usr/lib/python3.12', '/usr/lib/python3.12/lib-dynload', '/opt/sglang/lib/python3.12/site-packages', '__editable__.megatron_core-0.19.0+a84b10547.finder.__path_hook__', '__editable__.sglang-0.5.21.dev64+ge5bba1f.finder.__path_hook__'] - -terminated base sb-2YA7SK8NJWdQC02w8d4THs diff --git a/openspec/changes/rl-infra-spec/evidence/infra-a/2.2-2.3/round2-e0a/tape.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-a/2.2-2.3/round2-e0a/tape.jsonl deleted file mode 100644 index 5132fcc1a..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-a/2.2-2.3/round2-e0a/tape.jsonl +++ /dev/null @@ -1,42 +0,0 @@ -{"event": "rl_driver_start", "execution_mode": "colocated-serial", "island_id": 0, "placement": "colocated", "rl/algorithm_spec_sha256": "27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea", "runtime_fingerprint": "sha256:2293c72aa23a368cf66d263f53f4f23259a076f3c628fb4b24a66008fc83610e", "time_unix": 1790704528.498677} -{"event": "rl_driver_phase", "island_id": 0, "optimizer": "reset", "phase": "apply", "policy_version": 0, "time_unix": 1790704542.0236206} -{"event": "rl_policy_apply", "island_id": 0, "local_step": 0, "optimizer": "reset", "partial_fragment_apply": false, "policy_version": 0, "sync/apply_seconds": 121.57577682699997, "sync/global_policy_hash": "a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4", "time_unix": 1790704663.6001942} -{"event": "rl_driver_phase", "island_id": 0, "phase": "publish", "policy_version": 0, "time_unix": 1790704663.668085} -{"event": "rl_publication", "island_id": 0, "policy_version": 0, "rl/policy_token": "yeto:0:a34cd30f37ee5ffaad784b92defd77b3b6ce68095faf29288d5ea460912617c4", "sync/publication_members": ["engine:inference-engine-all-0-0-00000"], "sync/publication_payload_bytes": 40370176, "sync/publication_payload_hash": "373abf70cb687c7dfe3b8391c678d27be175a19d0e847349c362ed042f7ba43f", "time_unix": 1790704767.7674854} -{"event": "rl_driver_phase", "island_id": 0, "phase": "offload", "rollout_id": 0, "time_unix": 1790704767.7679694} -{"event": "rl_driver_phase", "island_id": 0, "phase": "generate", "policy_version": 0, "rollout_id": 0, "time_unix": 1790704767.7743087} -{"event": "rl_driver_phase", "island_id": 0, "phase": "onload", "rollout_id": 0, "time_unix": 1790704781.2186983} -{"event": "rl_driver_phase", "island_id": 0, "phase": "train", "rollout_id": 0, "time_unix": 1790704782.0743556} -{"clip_fraction": null, "event": "rl_round_trained", "island_id": 0, "masked_fraction": null, "nonzero_advantages": null, "rollout_id": 0, "time_unix": 1790704815.562172, "trained_groups": 4, "trained_sample_ids_sha256": "6ac54d62824c59fa3a8f59264bca872f865403d905926ce326c231e62752ac7d", "trained_samples": 32} -{"event": "rl_driver_phase", "island_id": 0, "phase": "sync", "rollout_id": 0, "time_unix": 1790704815.5625854} -{"event": "rl_driver_phase", "island_id": 0, "phase": "export_push", "rollout_id": 0, "time_unix": 1790704815.5628083} -{"event": "rl_driver_phase", "island_id": 0, "phase": "wait_global", "policy_version": 1, "time_unix": 1790704816.3949282} -{"event": "rl_driver_phase", "island_id": 0, "optimizer": "reset", "phase": "apply", "policy_version": 1, "time_unix": 1790704833.1697624} -{"event": "rl_policy_apply", "island_id": 0, "local_step": 1, "optimizer": "reset", "partial_fragment_apply": false, "policy_version": 1, "sync/apply_seconds": 76.89053808799997, "sync/global_policy_hash": "3660f9f99dfafaf1f63ff494b54c3edb5e4bc73d5bd8359163f0e60248fbea42", "time_unix": 1790704910.0609124} -{"event": "rl_driver_phase", "island_id": 0, "phase": "publish", "policy_version": 1, "time_unix": 1790704910.1317556} -{"event": "rl_publication", "island_id": 0, "policy_version": 1, "rl/policy_token": "yeto:1:3660f9f99dfafaf1f63ff494b54c3edb5e4bc73d5bd8359163f0e60248fbea42", "sync/publication_members": ["engine:inference-engine-all-0-0-00000"], "sync/publication_payload_bytes": 40370176, "sync/publication_payload_hash": "5024bb7ae491408587e157ac502dd118c6b34362203e904a7b2471e79e78c6f9", "time_unix": 1790704911.210658} -{"event": "rl_driver_phase", "island_id": 0, "phase": "offload", "rollout_id": 1, "time_unix": 1790704911.2110636} -{"event": "rl_driver_phase", "island_id": 0, "phase": "generate", "policy_version": 1, "rollout_id": 1, "time_unix": 1790704912.629549} -{"event": "rl_driver_phase", "island_id": 0, "phase": "onload", "rollout_id": 1, "time_unix": 1790704920.000759} -{"event": "rl_driver_phase", "island_id": 0, "phase": "train", "rollout_id": 1, "time_unix": 1790704921.0192537} -{"clip_fraction": null, "event": "rl_round_trained", "island_id": 0, "masked_fraction": null, "nonzero_advantages": null, "rollout_id": 1, "time_unix": 1790704936.6229029, "trained_groups": 4, "trained_sample_ids_sha256": "2db44ca8e1db6f0b44c22e6134a0eb33edb7e03f9323974771a6138c41bc9a04", "trained_samples": 32} -{"event": "rl_driver_phase", "island_id": 0, "phase": "sync", "rollout_id": 1, "time_unix": 1790704936.6233435} -{"event": "rl_driver_phase", "island_id": 0, "phase": "export_push", "rollout_id": 1, "time_unix": 1790704936.6235397} -{"event": "rl_driver_phase", "island_id": 0, "phase": "wait_global", "policy_version": 2, "time_unix": 1790704937.341068} -{"event": "rl_driver_phase", "island_id": 0, "optimizer": "reset", "phase": "apply", "policy_version": 2, "time_unix": 1790704948.6503818} -{"event": "rl_policy_apply", "island_id": 0, "local_step": 2, "optimizer": "reset", "partial_fragment_apply": false, "policy_version": 2, "sync/apply_seconds": 57.586905409999986, "sync/global_policy_hash": "fdbd6e1642916e544678a43ba58abc29034e50c96e8534f00301e9e86ac28c69", "time_unix": 1790705006.237992} -{"event": "rl_driver_phase", "island_id": 0, "phase": "publish", "policy_version": 2, "time_unix": 1790705006.3086848} -{"event": "rl_publication", "island_id": 0, "policy_version": 2, "rl/policy_token": "yeto:2:fdbd6e1642916e544678a43ba58abc29034e50c96e8534f00301e9e86ac28c69", "sync/publication_members": ["engine:inference-engine-all-0-0-00000"], "sync/publication_payload_bytes": 40370176, "sync/publication_payload_hash": "f8b038b3af34ab949ae9845aab971b5adf5e74f6d70f1a8347359a67c7c49a1a", "time_unix": 1790705007.3068175} -{"event": "rl_driver_phase", "island_id": 0, "phase": "offload", "rollout_id": 2, "time_unix": 1790705007.3075945} -{"event": "rl_driver_phase", "island_id": 0, "phase": "generate", "policy_version": 2, "rollout_id": 2, "time_unix": 1790705008.7238533} -{"event": "rl_driver_phase", "island_id": 0, "phase": "onload", "rollout_id": 2, "time_unix": 1790705013.9620929} -{"event": "rl_driver_phase", "island_id": 0, "phase": "train", "rollout_id": 2, "time_unix": 1790705015.1601632} -{"clip_fraction": null, "event": "rl_round_trained", "island_id": 0, "masked_fraction": null, "nonzero_advantages": null, "rollout_id": 2, "time_unix": 1790705031.4668138, "trained_groups": 4, "trained_sample_ids_sha256": "69be7217a18d7585a50c20b2a7b0af4ddc13146706816ed0c22cf668eda0aaef", "trained_samples": 32} -{"event": "rl_driver_phase", "island_id": 0, "phase": "sync", "rollout_id": 2, "time_unix": 1790705031.4672177} -{"event": "rl_driver_phase", "island_id": 0, "phase": "export_push", "rollout_id": 2, "time_unix": 1790705031.4674268} -{"event": "rl_driver_phase", "island_id": 0, "phase": "wait_global", "policy_version": 3, "time_unix": 1790705032.1778858} -{"event": "rl_driver_phase", "island_id": 0, "optimizer": "reset", "phase": "apply", "policy_version": 3, "time_unix": 1790705041.878685} -{"event": "rl_policy_apply", "island_id": 0, "local_step": 3, "optimizer": "reset", "partial_fragment_apply": false, "policy_version": 3, "sync/apply_seconds": 58.35841030500001, "sync/global_policy_hash": "aa83be67e10e5146d3ca51ec00821aa232893eda3ed588101a19e24396fbcb95", "time_unix": 1790705100.2378254} -{"event": "rl_driver_phase", "island_id": 0, "phase": "publish", "policy_version": 3, "time_unix": 1790705100.3092992} -{"event": "rl_publication", "island_id": 0, "policy_version": 3, "rl/policy_token": "yeto:3:aa83be67e10e5146d3ca51ec00821aa232893eda3ed588101a19e24396fbcb95", "sync/publication_members": ["engine:inference-engine-all-0-0-00000"], "sync/publication_payload_bytes": 40370176, "sync/publication_payload_hash": "0869a9ed55a43ed0efd241134d50d142ef704b5689609cf90abe7f7441307120", "time_unix": 1790705101.2882462} -{"event": "rl_driver_phase", "island_id": 0, "phase": "finish", "rollout_id": 3, "time_unix": 1790705101.2890892} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-b1/state/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-b1/state/elastic-state/ledger/journal.jsonl deleted file mode 100644 index 339284122..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-b1/state/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,30 +0,0 @@ -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","group_ids":["g0","g1"],"kind":"prepared","monotonic":527.051790076,"policy_token":"yeto:0:fe39114a65ec59a3d3093455c5999c3c7cd3f0c08ad8f2b4ad16ab0a51ebc1b5","rollout_id":0,"samples":16,"seq":1,"wall_time":1790771670.0358078} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":527.052298886,"rollout_id":0,"seq":2,"wall_time":1790771670.036317} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":527.052691898,"rollout_id":0,"seq":3,"wall_time":1790771670.0367105} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","input_batch_hash":"22b4712b530f69cb591a4d62209f33563c77ee9f825874712bc6bec05ee07df0","kind":"optimizer_applied","monotonic":550.969922409,"rollout_id":0,"seq":4,"wall_time":1790771693.953941} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","kind":"outer_recorded","monotonic":551.837566061,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790771694.8215842} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","group_ids":["g2","g3"],"kind":"prepared","monotonic":554.88977611,"policy_token":"yeto:1:f85f3d066cac8a02ad3e4d902ee57c696ea32fc5950a23044913eb541511e07d","rollout_id":1,"samples":16,"seq":6,"wall_time":1790771697.8737946} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":554.890397332,"rollout_id":1,"seq":7,"wall_time":1790771697.8744156} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":554.890825692,"rollout_id":1,"seq":8,"wall_time":1790771697.8748438} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","input_batch_hash":"e0d9e9dd21cb517faed3b0af39da30424c59cbe1d8ddd673eb2b2a3547a3a999","kind":"optimizer_applied","monotonic":562.814651169,"rollout_id":1,"seq":9,"wall_time":1790771705.7986696} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","kind":"outer_recorded","monotonic":563.603193672,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790771706.587212} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","group_ids":["g4","g5"],"kind":"prepared","monotonic":567.254932422,"policy_token":"yeto:2:7684720b1187c35800c8ab3c4692de6a43e9b49419aa9338e3e253900c1cd6b1","rollout_id":2,"samples":16,"seq":11,"wall_time":1790771710.238951} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":567.255424287,"rollout_id":2,"seq":12,"wall_time":1790771710.2394426} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":567.255864078,"rollout_id":2,"seq":13,"wall_time":1790771710.2398822} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","input_batch_hash":"a550eded31c86fa298d6cfc5595f1dc13f322e9b22fb35211b7e5140d41c698b","kind":"optimizer_applied","monotonic":571.896519613,"rollout_id":2,"seq":14,"wall_time":1790771714.8805382} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","kind":"outer_recorded","monotonic":572.55779619,"next_policy_version":3,"rollout_id":2,"seq":15,"wall_time":1790771715.5418146} -{"attempt":0,"batch_hash":"3e3baa054dd7ccce7ad84ed24c1796cd081ace596844984617e226de1fa2c75c","group_ids":["g6","g7"],"kind":"prepared","monotonic":576.011203274,"policy_token":"yeto:3:09f5875758115d1fd88c332c17baaed9e41866e11581172d055562f1182c7ec6","rollout_id":3,"samples":16,"seq":16,"wall_time":1790771718.9952219} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":576.011784021,"rollout_id":3,"seq":17,"wall_time":1790771718.9958024} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":576.012192709,"rollout_id":3,"seq":18,"wall_time":1790771718.996211} -{"attempt":0,"batch_hash":"3e3baa054dd7ccce7ad84ed24c1796cd081ace596844984617e226de1fa2c75c","input_batch_hash":"51338f10759818830eb56bd5919e4abdccfc5afdbf17dc3ae30d60ed8e090b71","kind":"optimizer_applied","monotonic":579.973345498,"rollout_id":3,"seq":19,"wall_time":1790771722.957364} -{"attempt":0,"batch_hash":"3e3baa054dd7ccce7ad84ed24c1796cd081ace596844984617e226de1fa2c75c","kind":"outer_recorded","monotonic":580.700062101,"next_policy_version":4,"rollout_id":3,"seq":20,"wall_time":1790771723.6840804} -{"attempt":0,"batch_hash":"1823b89734bb73c95d36bde489b891dd9570224297a3c310531601655326f78a","group_ids":["g8","g9"],"kind":"prepared","monotonic":584.767464316,"policy_token":"yeto:4:863ae98bb148e39f2d17f5fb6e846bc291d3a71449cc36aa47a735f0cb221dde","rollout_id":4,"samples":16,"seq":21,"wall_time":1790771727.751483} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":584.768055261,"rollout_id":4,"seq":22,"wall_time":1790771727.7520735} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":584.768400309,"rollout_id":4,"seq":23,"wall_time":1790771727.7524185} -{"attempt":0,"batch_hash":"1823b89734bb73c95d36bde489b891dd9570224297a3c310531601655326f78a","input_batch_hash":"922cdb2b327254e70cfeac286d6d3b6a0b1bb1b9d8070bdc681c3e917bd5fc07","kind":"optimizer_applied","monotonic":589.246326154,"rollout_id":4,"seq":24,"wall_time":1790771732.2303448} -{"attempt":0,"batch_hash":"1823b89734bb73c95d36bde489b891dd9570224297a3c310531601655326f78a","kind":"outer_recorded","monotonic":589.890099015,"next_policy_version":5,"rollout_id":4,"seq":25,"wall_time":1790771732.8741174} -{"attempt":0,"batch_hash":"061ffbed993026ad2c26256dce4b3fdab686c8722d8738dddbc6c3572e63357d","group_ids":["g10","g11"],"kind":"prepared","monotonic":593.927728486,"policy_token":"yeto:5:46f3c5f021a72ab6129426f0891fbe01ffd62bb4203aac6ca6fa3efadb9ed4da","rollout_id":5,"samples":16,"seq":26,"wall_time":1790771736.911747} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":593.928359663,"rollout_id":5,"seq":27,"wall_time":1790771736.912378} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":593.928754203,"rollout_id":5,"seq":28,"wall_time":1790771736.9127727} -{"attempt":0,"batch_hash":"061ffbed993026ad2c26256dce4b3fdab686c8722d8738dddbc6c3572e63357d","input_batch_hash":"bc52106f68f3c53ab7c2d20c76efc3ae315b4e33054cf822f55be7600d6a1f5d","kind":"optimizer_applied","monotonic":598.297918914,"rollout_id":5,"seq":29,"wall_time":1790771741.2819374} -{"attempt":0,"batch_hash":"061ffbed993026ad2c26256dce4b3fdab686c8722d8738dddbc6c3572e63357d","kind":"outer_recorded","monotonic":598.896057136,"next_policy_version":6,"rollout_id":5,"seq":30,"wall_time":1790771741.8800752} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-b1/state/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-b1/state/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index 9d8d6fd07..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-b1/state/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"kind":"finalization","monotonic":600.04131975,"rollout_id":6,"seq":1,"wall_time":1790771743.025338} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f1/state/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f1/state/elastic-state/ledger/journal.jsonl deleted file mode 100644 index 0cc7add28..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f1/state/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,15 +0,0 @@ -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","group_ids":["g0","g1"],"kind":"prepared","monotonic":437.668027664,"policy_token":"yeto:0:fe39114a65ec59a3d3093455c5999c3c7cd3f0c08ad8f2b4ad16ab0a51ebc1b5","rollout_id":0,"samples":16,"seq":1,"wall_time":1790774029.0942557} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":437.668427338,"rollout_id":0,"seq":2,"wall_time":1790774029.0946555} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":437.668802027,"rollout_id":0,"seq":3,"wall_time":1790774029.0950303} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","input_batch_hash":"22b4712b530f69cb591a4d62209f33563c77ee9f825874712bc6bec05ee07df0","kind":"optimizer_applied","monotonic":457.086128227,"rollout_id":0,"seq":4,"wall_time":1790774048.5123568} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","kind":"outer_recorded","monotonic":457.820281576,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790774049.2465098} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","group_ids":["g2","g3"],"kind":"prepared","monotonic":460.401755579,"policy_token":"yeto:1:f85f3d066cac8a02ad3e4d902ee57c696ea32fc5950a23044913eb541511e07d","rollout_id":1,"samples":16,"seq":6,"wall_time":1790774051.827984} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":460.402168335,"rollout_id":1,"seq":7,"wall_time":1790774051.8283966} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":460.402484951,"rollout_id":1,"seq":8,"wall_time":1790774051.8287134} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","input_batch_hash":"e0d9e9dd21cb517faed3b0af39da30424c59cbe1d8ddd673eb2b2a3547a3a999","kind":"optimizer_applied","monotonic":466.550778525,"rollout_id":1,"seq":9,"wall_time":1790774057.9770072} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","kind":"outer_recorded","monotonic":467.259781718,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790774058.68601} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","group_ids":["g4","g5"],"kind":"prepared","monotonic":470.80060808,"policy_token":"yeto:2:7684720b1187c35800c8ab3c4692de6a43e9b49419aa9338e3e253900c1cd6b1","rollout_id":2,"samples":16,"seq":11,"wall_time":1790774062.2268367} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":470.801094446,"rollout_id":2,"seq":12,"wall_time":1790774062.2273228} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":470.801421011,"rollout_id":2,"seq":13,"wall_time":1790774062.2276495} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","input_batch_hash":"a550eded31c86fa298d6cfc5595f1dc13f322e9b22fb35211b7e5140d41c698b","kind":"optimizer_applied","monotonic":474.587027746,"rollout_id":2,"seq":14,"wall_time":1790774066.0132565} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","kind":"outer_recorded","monotonic":475.3282791,"next_policy_version":3,"rollout_id":2,"seq":15,"wall_time":1790774066.7545075} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f1/state/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f1/state/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index 3074ac5f5..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f1/state/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -{"body":{"deadline_s":900.0,"expected_config_epoch":0,"kind":"trainer-rebuild"},"body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","deadline_wall":1790774968.015837,"kind":"request","monotonic":476.589622548,"plan":{"expected_config_epoch":0,"expected_pause_s":900.0,"kind":"trainer-rebuild","pause_budget_s":null,"profile_hash":"sha256:c6ed5d2db4cd81cf67ecf7031a921044cfce6426cddfaaba6107f6b10e3eb7d4","request_body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","source":"T2R1S0","source_engines":1,"target":"T2R1S0","target_engines":1,"trainer":null},"request_id":"rb1","seq":1,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790774068.0158513} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":476.590000008,"phase":"VALIDATING","plan":{"expected_config_epoch":0,"expected_pause_s":900.0,"kind":"trainer-rebuild","pause_budget_s":null,"profile_hash":"sha256:c6ed5d2db4cd81cf67ecf7031a921044cfce6426cddfaaba6107f6b10e3eb7d4","request_body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","source":"T2R1S0","source_engines":1,"target":"T2R1S0","target_engines":1,"trainer":null},"request_id":"rb1","seq":2,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790774068.0162284} -{"allowed":true,"budget_s":null,"expected_pause_s":900.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":476.592323761,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":3,"seq":3,"stalls_peers":false,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790774068.0185523} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":476.592604174,"phase":"WAIT_SAFE","request_id":"rb1","safe_point_rollout_id":3,"seq":4,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790774068.0188327} -{"config_epoch":0,"cut_id":"rb-0-cf5ecb1130c6","fork_epoch":0,"kind":"phase","monotonic":476.701453592,"optimizer_step":3,"phase":"REBUILDING_TRAINER","policy_hash":"09f5875758115d1fd88c332c17baaed9e41866e11581172d055562f1182c7ec6","published_version":3,"request_id":"rb1","rollout_id":3,"seq":5,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790774068.1276817} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f2/state/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f2/state/elastic-state/ledger/journal.jsonl deleted file mode 100644 index 7e346680a..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f2/state/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,15 +0,0 @@ -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","group_ids":["g0","g1"],"kind":"prepared","monotonic":429.675304692,"policy_token":"yeto:0:fe39114a65ec59a3d3093455c5999c3c7cd3f0c08ad8f2b4ad16ab0a51ebc1b5","rollout_id":0,"samples":16,"seq":1,"wall_time":1790774783.1564498} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":429.675837495,"rollout_id":0,"seq":2,"wall_time":1790774783.1569822} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":429.676166941,"rollout_id":0,"seq":3,"wall_time":1790774783.1573122} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","input_batch_hash":"22b4712b530f69cb591a4d62209f33563c77ee9f825874712bc6bec05ee07df0","kind":"optimizer_applied","monotonic":450.26123589,"rollout_id":0,"seq":4,"wall_time":1790774803.7423813} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","kind":"outer_recorded","monotonic":451.048905321,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790774804.5300503} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","group_ids":["g2","g3"],"kind":"prepared","monotonic":453.818628411,"policy_token":"yeto:1:f85f3d066cac8a02ad3e4d902ee57c696ea32fc5950a23044913eb541511e07d","rollout_id":1,"samples":16,"seq":6,"wall_time":1790774807.299774} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":453.819172968,"rollout_id":1,"seq":7,"wall_time":1790774807.3003182} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":453.819501513,"rollout_id":1,"seq":8,"wall_time":1790774807.3006465} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","input_batch_hash":"e0d9e9dd21cb517faed3b0af39da30424c59cbe1d8ddd673eb2b2a3547a3a999","kind":"optimizer_applied","monotonic":460.252985233,"rollout_id":1,"seq":9,"wall_time":1790774813.7341306} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","kind":"outer_recorded","monotonic":461.360040334,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790774814.8411856} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","group_ids":["g4","g5"],"kind":"prepared","monotonic":465.47223111,"policy_token":"yeto:2:7684720b1187c35800c8ab3c4692de6a43e9b49419aa9338e3e253900c1cd6b1","rollout_id":2,"samples":16,"seq":11,"wall_time":1790774818.9533765} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":465.472724953,"rollout_id":2,"seq":12,"wall_time":1790774818.95387} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":465.473045895,"rollout_id":2,"seq":13,"wall_time":1790774818.9541914} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","input_batch_hash":"a550eded31c86fa298d6cfc5595f1dc13f322e9b22fb35211b7e5140d41c698b","kind":"optimizer_applied","monotonic":469.357919785,"rollout_id":2,"seq":14,"wall_time":1790774822.8390653} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","kind":"outer_recorded","monotonic":470.143429644,"next_policy_version":3,"rollout_id":2,"seq":15,"wall_time":1790774823.6245747} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f2/state/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f2/state/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index 4ffe62b98..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f2/state/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -{"body":{"deadline_s":900.0,"expected_config_epoch":0,"kind":"trainer-rebuild"},"body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","deadline_wall":1790775724.9441817,"kind":"request","monotonic":471.463050297,"plan":{"expected_config_epoch":0,"expected_pause_s":900.0,"kind":"trainer-rebuild","pause_budget_s":null,"profile_hash":"sha256:c6ed5d2db4cd81cf67ecf7031a921044cfce6426cddfaaba6107f6b10e3eb7d4","request_body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","source":"T2R1S0","source_engines":1,"target":"T2R1S0","target_engines":1,"trainer":null},"request_id":"rb1","seq":1,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790774824.9441957} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":471.463395263,"phase":"VALIDATING","plan":{"expected_config_epoch":0,"expected_pause_s":900.0,"kind":"trainer-rebuild","pause_budget_s":null,"profile_hash":"sha256:c6ed5d2db4cd81cf67ecf7031a921044cfce6426cddfaaba6107f6b10e3eb7d4","request_body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","source":"T2R1S0","source_engines":1,"target":"T2R1S0","target_engines":1,"trainer":null},"request_id":"rb1","seq":2,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790774824.9445407} -{"allowed":true,"budget_s":null,"expected_pause_s":900.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":471.465460537,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":3,"seq":3,"stalls_peers":false,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790774824.946606} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":471.465732739,"phase":"WAIT_SAFE","request_id":"rb1","safe_point_rollout_id":3,"seq":4,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790774824.946878} -{"config_epoch":0,"cut_id":"rb-0-cf5ecb1130c6","fork_epoch":0,"kind":"phase","monotonic":471.619331889,"optimizer_step":3,"phase":"REBUILDING_TRAINER","policy_hash":"09f5875758115d1fd88c332c17baaed9e41866e11581172d055562f1182c7ec6","published_version":3,"request_id":"rb1","rollout_id":3,"seq":5,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790774825.100477} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f3/state/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f3/state/elastic-state/ledger/journal.jsonl deleted file mode 100644 index 53dea0590..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f3/state/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,15 +0,0 @@ -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","group_ids":["g0","g1"],"kind":"prepared","monotonic":434.655405413,"policy_token":"yeto:0:fe39114a65ec59a3d3093455c5999c3c7cd3f0c08ad8f2b4ad16ab0a51ebc1b5","rollout_id":0,"samples":16,"seq":1,"wall_time":1790775940.1169467} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":434.65586191,"rollout_id":0,"seq":2,"wall_time":1790775940.117403} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":434.656175903,"rollout_id":0,"seq":3,"wall_time":1790775940.1177175} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","input_batch_hash":"22b4712b530f69cb591a4d62209f33563c77ee9f825874712bc6bec05ee07df0","kind":"optimizer_applied","monotonic":454.262105566,"rollout_id":0,"seq":4,"wall_time":1790775959.7236476} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","kind":"outer_recorded","monotonic":455.008511939,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790775960.4700534} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","group_ids":["g2","g3"],"kind":"prepared","monotonic":457.761565601,"policy_token":"yeto:1:f85f3d066cac8a02ad3e4d902ee57c696ea32fc5950a23044913eb541511e07d","rollout_id":1,"samples":16,"seq":6,"wall_time":1790775963.2231073} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":457.762070249,"rollout_id":1,"seq":7,"wall_time":1790775963.2236116} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":457.762425921,"rollout_id":1,"seq":8,"wall_time":1790775963.2239676} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","input_batch_hash":"e0d9e9dd21cb517faed3b0af39da30424c59cbe1d8ddd673eb2b2a3547a3a999","kind":"optimizer_applied","monotonic":463.829451795,"rollout_id":1,"seq":9,"wall_time":1790775969.2909937} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","kind":"outer_recorded","monotonic":464.588025989,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790775970.0495675} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","group_ids":["g4","g5"],"kind":"prepared","monotonic":468.207560942,"policy_token":"yeto:2:7684720b1187c35800c8ab3c4692de6a43e9b49419aa9338e3e253900c1cd6b1","rollout_id":2,"samples":16,"seq":11,"wall_time":1790775973.669103} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":468.208004231,"rollout_id":2,"seq":12,"wall_time":1790775973.6695457} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":468.20827139,"rollout_id":2,"seq":13,"wall_time":1790775973.669813} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","input_batch_hash":"a550eded31c86fa298d6cfc5595f1dc13f322e9b22fb35211b7e5140d41c698b","kind":"optimizer_applied","monotonic":471.978824353,"rollout_id":2,"seq":14,"wall_time":1790775977.440366} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","kind":"outer_recorded","monotonic":472.605562771,"next_policy_version":3,"rollout_id":2,"seq":15,"wall_time":1790775978.0671043} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f3/state/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f3/state/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index 374c6c61a..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f3/state/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1,4 +0,0 @@ -{"body":{"deadline_s":900.0,"expected_config_epoch":0,"kind":"trainer-rebuild"},"body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","deadline_wall":1790776879.221076,"kind":"request","monotonic":473.759549557,"plan":{"expected_config_epoch":0,"expected_pause_s":900.0,"kind":"trainer-rebuild","pause_budget_s":null,"profile_hash":"sha256:c6ed5d2db4cd81cf67ecf7031a921044cfce6426cddfaaba6107f6b10e3eb7d4","request_body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","source":"T2R1S0","source_engines":1,"target":"T2R1S0","target_engines":1,"trainer":null},"request_id":"rb1","seq":1,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790775979.2210913} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":473.759912302,"phase":"VALIDATING","plan":{"expected_config_epoch":0,"expected_pause_s":900.0,"kind":"trainer-rebuild","pause_budget_s":null,"profile_hash":"sha256:c6ed5d2db4cd81cf67ecf7031a921044cfce6426cddfaaba6107f6b10e3eb7d4","request_body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","source":"T2R1S0","source_engines":1,"target":"T2R1S0","target_engines":1,"trainer":null},"request_id":"rb1","seq":2,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790775979.221454} -{"allowed":true,"budget_s":null,"expected_pause_s":900.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":473.76208198,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":3,"seq":3,"stalls_peers":false,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790775979.2236238} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":473.76231655,"phase":"WAIT_SAFE","request_id":"rb1","safe_point_rollout_id":3,"seq":4,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790775979.223858} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5-e2r/state/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5-e2r/state/elastic-state/ledger/journal.jsonl deleted file mode 100644 index 01d1248c7..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5-e2r/state/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,13 +0,0 @@ -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","group_ids":["g0","g1"],"kind":"prepared","monotonic":546.953349599,"policy_token":"yeto:0:fe39114a65ec59a3d3093455c5999c3c7cd3f0c08ad8f2b4ad16ab0a51ebc1b5","rollout_id":0,"samples":16,"seq":1,"wall_time":1790776733.5175087} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":546.953949195,"rollout_id":0,"seq":2,"wall_time":1790776733.518109} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":546.954388756,"rollout_id":0,"seq":3,"wall_time":1790776733.5185487} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","input_batch_hash":"22b4712b530f69cb591a4d62209f33563c77ee9f825874712bc6bec05ee07df0","kind":"optimizer_applied","monotonic":572.470136237,"rollout_id":0,"seq":4,"wall_time":1790776759.0342965} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","kind":"outer_recorded","monotonic":573.405169506,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790776759.9693294} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","group_ids":["g2","g3"],"kind":"prepared","monotonic":576.309730483,"policy_token":"yeto:1:f85f3d066cac8a02ad3e4d902ee57c696ea32fc5950a23044913eb541511e07d","rollout_id":1,"samples":16,"seq":6,"wall_time":1790776762.8738909} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":576.310439821,"rollout_id":1,"seq":7,"wall_time":1790776762.8746} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":576.310863387,"rollout_id":1,"seq":8,"wall_time":1790776762.8750236} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","input_batch_hash":"e0d9e9dd21cb517faed3b0af39da30424c59cbe1d8ddd673eb2b2a3547a3a999","kind":"optimizer_applied","monotonic":583.912451027,"rollout_id":1,"seq":9,"wall_time":1790776770.4766111} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","kind":"outer_recorded","monotonic":584.710584908,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790776771.2747447} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","group_ids":["g4","g5"],"kind":"prepared","monotonic":588.683291724,"policy_token":"yeto:2:7684720b1187c35800c8ab3c4692de6a43e9b49419aa9338e3e253900c1cd6b1","rollout_id":2,"samples":16,"seq":11,"wall_time":1790776775.247452} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":588.683859405,"rollout_id":2,"seq":12,"wall_time":1790776775.2480195} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":588.684255863,"rollout_id":2,"seq":13,"wall_time":1790776775.248416} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5-e2r/state/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5-e2r/state/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index e69de29bb..000000000 diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5/state/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5/state/elastic-state/ledger/journal.jsonl deleted file mode 100644 index 01d1248c7..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5/state/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,13 +0,0 @@ -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","group_ids":["g0","g1"],"kind":"prepared","monotonic":546.953349599,"policy_token":"yeto:0:fe39114a65ec59a3d3093455c5999c3c7cd3f0c08ad8f2b4ad16ab0a51ebc1b5","rollout_id":0,"samples":16,"seq":1,"wall_time":1790776733.5175087} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":546.953949195,"rollout_id":0,"seq":2,"wall_time":1790776733.518109} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":546.954388756,"rollout_id":0,"seq":3,"wall_time":1790776733.5185487} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","input_batch_hash":"22b4712b530f69cb591a4d62209f33563c77ee9f825874712bc6bec05ee07df0","kind":"optimizer_applied","monotonic":572.470136237,"rollout_id":0,"seq":4,"wall_time":1790776759.0342965} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","kind":"outer_recorded","monotonic":573.405169506,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790776759.9693294} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","group_ids":["g2","g3"],"kind":"prepared","monotonic":576.309730483,"policy_token":"yeto:1:f85f3d066cac8a02ad3e4d902ee57c696ea32fc5950a23044913eb541511e07d","rollout_id":1,"samples":16,"seq":6,"wall_time":1790776762.8738909} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":576.310439821,"rollout_id":1,"seq":7,"wall_time":1790776762.8746} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":576.310863387,"rollout_id":1,"seq":8,"wall_time":1790776762.8750236} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","input_batch_hash":"e0d9e9dd21cb517faed3b0af39da30424c59cbe1d8ddd673eb2b2a3547a3a999","kind":"optimizer_applied","monotonic":583.912451027,"rollout_id":1,"seq":9,"wall_time":1790776770.4766111} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","kind":"outer_recorded","monotonic":584.710584908,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790776771.2747447} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","group_ids":["g4","g5"],"kind":"prepared","monotonic":588.683291724,"policy_token":"yeto:2:7684720b1187c35800c8ab3c4692de6a43e9b49419aa9338e3e253900c1cd6b1","rollout_id":2,"samples":16,"seq":11,"wall_time":1790776775.247452} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":588.683859405,"rollout_id":2,"seq":12,"wall_time":1790776775.2480195} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":588.684255863,"rollout_id":2,"seq":13,"wall_time":1790776775.248416} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5/state/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f5/state/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index e69de29bb..000000000 diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f6a/state/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f6a/state/elastic-state/ledger/journal.jsonl deleted file mode 100644 index 4d7d1b581..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f6a/state/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,15 +0,0 @@ -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","group_ids":["g0","g1"],"kind":"prepared","monotonic":517.737171693,"policy_token":"yeto:0:fe39114a65ec59a3d3093455c5999c3c7cd3f0c08ad8f2b4ad16ab0a51ebc1b5","rollout_id":0,"samples":16,"seq":1,"wall_time":1790777595.9476204} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":517.737802405,"rollout_id":0,"seq":2,"wall_time":1790777595.9482512} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":517.738250469,"rollout_id":0,"seq":3,"wall_time":1790777595.948699} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","input_batch_hash":"22b4712b530f69cb591a4d62209f33563c77ee9f825874712bc6bec05ee07df0","kind":"optimizer_applied","monotonic":541.827731196,"rollout_id":0,"seq":4,"wall_time":1790777620.0381804} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","kind":"outer_recorded","monotonic":542.716176093,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790777620.926625} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","group_ids":["g2","g3"],"kind":"prepared","monotonic":545.562038237,"policy_token":"yeto:1:f85f3d066cac8a02ad3e4d902ee57c696ea32fc5950a23044913eb541511e07d","rollout_id":1,"samples":16,"seq":6,"wall_time":1790777623.7724874} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":545.562696815,"rollout_id":1,"seq":7,"wall_time":1790777623.7731457} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":545.563155715,"rollout_id":1,"seq":8,"wall_time":1790777623.7736049} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","input_batch_hash":"e0d9e9dd21cb517faed3b0af39da30424c59cbe1d8ddd673eb2b2a3547a3a999","kind":"optimizer_applied","monotonic":553.239439765,"rollout_id":1,"seq":9,"wall_time":1790777631.449889} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","kind":"outer_recorded","monotonic":554.094337104,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790777632.3047857} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","group_ids":["g4","g5"],"kind":"prepared","monotonic":557.865601773,"policy_token":"yeto:2:7684720b1187c35800c8ab3c4692de6a43e9b49419aa9338e3e253900c1cd6b1","rollout_id":2,"samples":16,"seq":11,"wall_time":1790777636.076051} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":557.866250777,"rollout_id":2,"seq":12,"wall_time":1790777636.0766997} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":557.866678891,"rollout_id":2,"seq":13,"wall_time":1790777636.0771277} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","input_batch_hash":"a550eded31c86fa298d6cfc5595f1dc13f322e9b22fb35211b7e5140d41c698b","kind":"optimizer_applied","monotonic":562.296249108,"rollout_id":2,"seq":14,"wall_time":1790777640.5066981} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","kind":"outer_recorded","monotonic":563.096301389,"next_policy_version":3,"rollout_id":2,"seq":15,"wall_time":1790777641.3067503} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f6a/state/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f6a/state/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index 601346175..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-f6a/state/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"body":{"deadline_s":900.0,"expected_config_epoch":0,"kind":"trainer-rebuild"},"body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","deadline_wall":1790778542.7491665,"kind":"request","monotonic":564.538733388,"plan":{"expected_config_epoch":0,"expected_pause_s":900.0,"kind":"trainer-rebuild","pause_budget_s":null,"profile_hash":"sha256:c6ed5d2db4cd81cf67ecf7031a921044cfce6426cddfaaba6107f6b10e3eb7d4","request_body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","source":"T2R1S0","source_engines":1,"target":"T2R1S0","target_engines":1,"trainer":null},"request_id":"rb1","seq":1,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790777642.7491825} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":564.539240638,"phase":"VALIDATING","plan":{"expected_config_epoch":0,"expected_pause_s":900.0,"kind":"trainer-rebuild","pause_budget_s":null,"profile_hash":"sha256:c6ed5d2db4cd81cf67ecf7031a921044cfce6426cddfaaba6107f6b10e3eb7d4","request_body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","source":"T2R1S0","source_engines":1,"target":"T2R1S0","target_engines":1,"trainer":null},"request_id":"rb1","seq":2,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790777642.7496896} -{"allowed":true,"budget_s":null,"expected_pause_s":900.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":564.542145214,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":3,"seq":3,"stalls_peers":false,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790777642.752594} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":564.542605937,"phase":"WAIT_SAFE","request_id":"rb1","safe_point_rollout_id":3,"seq":4,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790777642.7530546} -{"config_epoch":0,"cut_id":"rb-0-cf5ecb1130c6","fork_epoch":0,"kind":"phase","monotonic":564.665024319,"optimizer_step":3,"phase":"REBUILDING_TRAINER","policy_hash":"09f5875758115d1fd88c332c17baaed9e41866e11581172d055562f1182c7ec6","published_version":3,"request_id":"rb1","rollout_id":3,"seq":5,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790777642.8754728} -{"config_epoch":0,"error":"learner restarted after release, before commit","fork_epoch":0,"kind":"phase","monotonic":944.591668443,"phase":"RECOVERY_REQUIRED","request_id":null,"seq":6,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790778022.802117} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rb/state/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rb/state/elastic-state/ledger/journal.jsonl deleted file mode 100644 index 6cd65e0b9..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rb/state/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","group_ids":["g0","g1"],"kind":"prepared","monotonic":502.072952369,"policy_token":"yeto:0:fe39114a65ec59a3d3093455c5999c3c7cd3f0c08ad8f2b4ad16ab0a51ebc1b5","rollout_id":0,"samples":16,"seq":1,"wall_time":1790778776.2400537} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":502.073564358,"rollout_id":0,"seq":2,"wall_time":1790778776.240666} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":502.073949512,"rollout_id":0,"seq":3,"wall_time":1790778776.2410517} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","input_batch_hash":"22b4712b530f69cb591a4d62209f33563c77ee9f825874712bc6bec05ee07df0","kind":"optimizer_applied","monotonic":524.864327637,"rollout_id":0,"seq":4,"wall_time":1790778799.03143} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","kind":"outer_recorded","monotonic":525.672872943,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790778799.8399746} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rb/state/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rb/state/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index e69de29bb..000000000 diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rbold/state/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rbold/state/elastic-state/ledger/journal.jsonl deleted file mode 100644 index 7618fc6ad..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rbold/state/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,28 +0,0 @@ -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","group_ids":["g0","g1"],"kind":"prepared","monotonic":417.743480009,"policy_token":"yeto:0:fe39114a65ec59a3d3093455c5999c3c7cd3f0c08ad8f2b4ad16ab0a51ebc1b5","rollout_id":0,"samples":16,"seq":1,"wall_time":1790773199.9221768} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":417.744012473,"rollout_id":0,"seq":2,"wall_time":1790773199.9227095} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":417.744430682,"rollout_id":0,"seq":3,"wall_time":1790773199.9231281} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","input_batch_hash":"22b4712b530f69cb591a4d62209f33563c77ee9f825874712bc6bec05ee07df0","kind":"optimizer_applied","monotonic":435.002806814,"rollout_id":0,"seq":4,"wall_time":1790773217.1815042} -{"attempt":0,"batch_hash":"010e29cccdc82b02db0e8b58a6b6d460f07ad3fd9a6bae166bd31c68ce89d586","kind":"outer_recorded","monotonic":435.683088014,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790773217.8617852} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","group_ids":["g2","g3"],"kind":"prepared","monotonic":438.133466754,"policy_token":"yeto:1:f85f3d066cac8a02ad3e4d902ee57c696ea32fc5950a23044913eb541511e07d","rollout_id":1,"samples":16,"seq":6,"wall_time":1790773220.312164} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":438.134083678,"rollout_id":1,"seq":7,"wall_time":1790773220.312781} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":438.134499808,"rollout_id":1,"seq":8,"wall_time":1790773220.3131971} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","input_batch_hash":"e0d9e9dd21cb517faed3b0af39da30424c59cbe1d8ddd673eb2b2a3547a3a999","kind":"optimizer_applied","monotonic":443.843473293,"rollout_id":1,"seq":9,"wall_time":1790773226.0221705} -{"attempt":0,"batch_hash":"30cfd3163c8a7f5d0ef9c4957195de3250d6c9a59646ba3c39798100a8c917db","kind":"outer_recorded","monotonic":444.419507839,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790773226.5982049} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","group_ids":["g4","g5"],"kind":"prepared","monotonic":447.876924767,"policy_token":"yeto:2:7684720b1187c35800c8ab3c4692de6a43e9b49419aa9338e3e253900c1cd6b1","rollout_id":2,"samples":16,"seq":11,"wall_time":1790773230.055622} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":447.877454888,"rollout_id":2,"seq":12,"wall_time":1790773230.056152} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":447.877797425,"rollout_id":2,"seq":13,"wall_time":1790773230.056495} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","input_batch_hash":"a550eded31c86fa298d6cfc5595f1dc13f322e9b22fb35211b7e5140d41c698b","kind":"optimizer_applied","monotonic":451.369004524,"rollout_id":2,"seq":14,"wall_time":1790773233.547702} -{"attempt":0,"batch_hash":"e51f95e9f3e04d796df8ddf24c68856f7365dab5ea927fececd1bf360150591e","kind":"outer_recorded","monotonic":451.899970822,"next_policy_version":3,"rollout_id":2,"seq":15,"wall_time":1790773234.078668} -{"attempt":0,"batch_hash":"3e3baa054dd7ccce7ad84ed24c1796cd081ace596844984617e226de1fa2c75c","group_ids":["g6","g7"],"kind":"prepared","monotonic":574.196783302,"policy_token":"yeto:3:09f5875758115d1fd88c332c17baaed9e41866e11581172d055562f1182c7ec6","rollout_id":3,"samples":16,"seq":16,"wall_time":1790773356.375481} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":574.197290409,"rollout_id":3,"seq":17,"wall_time":1790773356.3759878} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":574.197622251,"rollout_id":3,"seq":18,"wall_time":1790773356.3763194} -{"attempt":0,"batch_hash":"3e3baa054dd7ccce7ad84ed24c1796cd081ace596844984617e226de1fa2c75c","input_batch_hash":"51338f10759818830eb56bd5919e4abdccfc5afdbf17dc3ae30d60ed8e090b71","kind":"optimizer_applied","monotonic":585.834771931,"rollout_id":3,"seq":19,"wall_time":1790773368.0134695} -{"attempt":0,"batch_hash":"3e3baa054dd7ccce7ad84ed24c1796cd081ace596844984617e226de1fa2c75c","kind":"outer_recorded","monotonic":586.441275678,"next_policy_version":4,"rollout_id":3,"seq":20,"wall_time":1790773368.619973} -{"attempt":0,"batch_hash":"1823b89734bb73c95d36bde489b891dd9570224297a3c310531601655326f78a","group_ids":["g8","g9"],"kind":"prepared","monotonic":590.165188953,"policy_token":"yeto:4:863ae98bb148e39f2d17f5fb6e846bc291d3a71449cc36aa47a735f0cb221dde","rollout_id":4,"samples":16,"seq":21,"wall_time":1790773372.3438864} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":590.165747197,"rollout_id":4,"seq":22,"wall_time":1790773372.3444445} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":590.166073812,"rollout_id":4,"seq":23,"wall_time":1790773372.3447711} -{"attempt":0,"batch_hash":"1823b89734bb73c95d36bde489b891dd9570224297a3c310531601655326f78a","input_batch_hash":"922cdb2b327254e70cfeac286d6d3b6a0b1bb1b9d8070bdc681c3e917bd5fc07","kind":"optimizer_applied","monotonic":593.912881636,"rollout_id":4,"seq":24,"wall_time":1790773376.0915792} -{"attempt":0,"batch_hash":"1823b89734bb73c95d36bde489b891dd9570224297a3c310531601655326f78a","kind":"outer_recorded","monotonic":594.493190043,"next_policy_version":5,"rollout_id":4,"seq":25,"wall_time":1790773376.6718872} -{"attempt":0,"batch_hash":"061ffbed993026ad2c26256dce4b3fdab686c8722d8738dddbc6c3572e63357d","group_ids":["g10","g11"],"kind":"prepared","monotonic":598.335755322,"policy_token":"yeto:5:46f3c5f021a72ab6129426f0891fbe01ffd62bb4203aac6ca6fa3efadb9ed4da","rollout_id":5,"samples":16,"seq":26,"wall_time":1790773380.5144527} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":598.336219965,"rollout_id":5,"seq":27,"wall_time":1790773380.5149174} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":598.33648376,"rollout_id":5,"seq":28,"wall_time":1790773380.515181} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rbold/state/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rbold/state/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index 2457e5673..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-e2/4.2-4.5/gpu-v6-c3-rbold/state/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"body":{"deadline_s":900.0,"expected_config_epoch":0,"kind":"trainer-rebuild"},"body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","deadline_wall":1790774135.0542266,"kind":"request","monotonic":452.875540512,"plan":{"expected_config_epoch":0,"expected_pause_s":900.0,"kind":"trainer-rebuild","pause_budget_s":null,"profile_hash":"sha256:c6ed5d2db4cd81cf67ecf7031a921044cfce6426cddfaaba6107f6b10e3eb7d4","request_body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","source":"T2R1S0","source_engines":1,"target":"T2R1S0","target_engines":1,"trainer":null},"request_id":"rb1","seq":1,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790773235.0542378} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":452.875954137,"phase":"VALIDATING","plan":{"expected_config_epoch":0,"expected_pause_s":900.0,"kind":"trainer-rebuild","pause_budget_s":null,"profile_hash":"sha256:c6ed5d2db4cd81cf67ecf7031a921044cfce6426cddfaaba6107f6b10e3eb7d4","request_body_hash":"cf5ecb1130c6f172619f31be70fa55bd022f6ea94e62907f7a9834a35d42d101","source":"T2R1S0","source_engines":1,"target":"T2R1S0","target_engines":1,"trainer":null},"request_id":"rb1","seq":2,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790773235.0546515} -{"allowed":true,"budget_s":null,"expected_pause_s":900.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":452.878171504,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":3,"seq":3,"stalls_peers":false,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790773235.0568688} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":452.878475318,"phase":"WAIT_SAFE","request_id":"rb1","safe_point_rollout_id":3,"seq":4,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790773235.0571728} -{"config_epoch":0,"cut_id":"rb-0-cf5ecb1130c6","fork_epoch":0,"kind":"phase","monotonic":452.965578566,"optimizer_step":3,"phase":"REBUILDING_TRAINER","policy_hash":"09f5875758115d1fd88c332c17baaed9e41866e11581172d055562f1182c7ec6","published_version":3,"request_id":"rb1","rollout_id":3,"seq":5,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790773235.1442757} -{"config_epoch":0,"cut_id":"rb-0-cf5ecb1130c6","fork_epoch":0,"kind":"phase","monotonic":572.467728766,"phase":"SUCCEEDED","rebuild":{"attempts":[{"attempt":0,"cleanup_error":null,"error":"RuntimeError('yeto test injection: create_training_models failed')","stage":"create_training_models"},{"attempt":1,"generation":1,"stage":"done"}],"cut_id":"rb-0-cf5ecb1130c6","generation":1,"local_step":3,"outcome":"REBUILD_OLD","policy_hash":"09f5875758115d1fd88c332c17baaed9e41866e11581172d055562f1182c7ec6"},"request_id":"rb1","seq":6,"tx_id":"tx-0-cf5ecb1130c6-rb1","wall_time":1790773354.6464262} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/base/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/base/elastic-state/ledger/journal.jsonl deleted file mode 100644 index b1d939760..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/base/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,60 +0,0 @@ -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","group_ids":["g0","g1","g2","g3"],"kind":"prepared","monotonic":977.112215242,"policy_token":"yeto:0:c65b6a4d0c9dc0ff99b65278e9410b13a5b953f175f2dbde702c3df255b79213","rollout_id":0,"samples":32,"seq":1,"wall_time":1790775226.4140375} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":977.115793925,"rollout_id":0,"seq":2,"wall_time":1790775226.4176166} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":977.117368903,"rollout_id":0,"seq":3,"wall_time":1790775226.4191914} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","input_batch_hash":"09c86025cbe7393dd354c7055d163d9068ab0812f6dc200ea8cbcf941f795f07","kind":"optimizer_applied","monotonic":991.550891741,"rollout_id":0,"seq":4,"wall_time":1790775240.8527145} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","kind":"outer_recorded","monotonic":992.003698843,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790775241.3055212} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","group_ids":["g4","g5","g6","g7"],"kind":"prepared","monotonic":994.834102283,"policy_token":"yeto:1:53e62a6ce5566341ca2a1c0ef2909a511eb667d8612d96a5d1609f9fced71831","rollout_id":1,"samples":32,"seq":6,"wall_time":1790775244.135925} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":994.836814834,"rollout_id":1,"seq":7,"wall_time":1790775244.1386375} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":994.83792639,"rollout_id":1,"seq":8,"wall_time":1790775244.139749} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","input_batch_hash":"dc8543c1c3582ce670b65cf904f5fa88486821dc9e1e0983d12dce5baaed4202","kind":"optimizer_applied","monotonic":998.559310483,"rollout_id":1,"seq":9,"wall_time":1790775247.861133} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","kind":"outer_recorded","monotonic":999.002246427,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790775248.3040688} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","group_ids":["g10","g11","g8","g9"],"kind":"prepared","monotonic":1001.960677782,"policy_token":"yeto:2:f5b71dd19389abfe8e2621e5b743e552b76fcddb017e50308db2713bae77502e","rollout_id":2,"samples":32,"seq":11,"wall_time":1790775251.2625003} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1001.96383799,"rollout_id":2,"seq":12,"wall_time":1790775251.2656608} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1001.96528017,"rollout_id":2,"seq":13,"wall_time":1790775251.2671027} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","input_batch_hash":"6fb5375d0a27c2b61baa2731e66a71c5a857057f331c449300bf7e7ca72b667d","kind":"optimizer_applied","monotonic":1005.92766823,"rollout_id":2,"seq":14,"wall_time":1790775255.229491} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","kind":"outer_recorded","monotonic":1006.345035033,"next_policy_version":3,"rollout_id":2,"seq":15,"wall_time":1790775255.6468575} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","group_ids":["g12","g13","g14","g15"],"kind":"prepared","monotonic":1009.10638364,"policy_token":"yeto:3:b39309fecb7d3c6bc6f57746151617cdc81fbe90ba296e5f2c5edcb195600f12","rollout_id":3,"samples":32,"seq":16,"wall_time":1790775258.4082062} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1009.109394217,"rollout_id":3,"seq":17,"wall_time":1790775258.411217} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1009.110666448,"rollout_id":3,"seq":18,"wall_time":1790775258.4124892} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","input_batch_hash":"be6add3c86bac216aac2b7be868d115d5d36b9fc112a9f1593ea562b752d5151","kind":"optimizer_applied","monotonic":1012.821941227,"rollout_id":3,"seq":19,"wall_time":1790775262.123764} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","kind":"outer_recorded","monotonic":1013.243475284,"next_policy_version":4,"rollout_id":3,"seq":20,"wall_time":1790775262.5452976} -{"attempt":0,"batch_hash":"7328d450bd7fdc43c18fca9149e109342e83a89da53a2ea7d497e99fc4236ca2","group_ids":["g16","g17","g18","g19"],"kind":"prepared","monotonic":1016.008793707,"policy_token":"yeto:4:8d92c23d3d88517f7e76be1f3742f61e1f5b030cc83c75a85188bbd5e3432c5f","rollout_id":4,"samples":32,"seq":21,"wall_time":1790775265.3106163} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1016.010733349,"rollout_id":4,"seq":22,"wall_time":1790775265.312556} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1016.011997325,"rollout_id":4,"seq":23,"wall_time":1790775265.31382} -{"attempt":0,"batch_hash":"7328d450bd7fdc43c18fca9149e109342e83a89da53a2ea7d497e99fc4236ca2","input_batch_hash":"043dff4d12ad624febcabef7da9868271e15c618ceb9c531eccf974c523c7ad0","kind":"optimizer_applied","monotonic":1019.817811223,"rollout_id":4,"seq":24,"wall_time":1790775269.119634} -{"attempt":0,"batch_hash":"7328d450bd7fdc43c18fca9149e109342e83a89da53a2ea7d497e99fc4236ca2","kind":"outer_recorded","monotonic":1020.241068594,"next_policy_version":5,"rollout_id":4,"seq":25,"wall_time":1790775269.542891} -{"attempt":0,"batch_hash":"a50a36b23943193b6ed09930c9f46ad4cfa19cb56cc4281a8d6f210cd4230b4b","group_ids":["g20","g21","g22","g23"],"kind":"prepared","monotonic":1023.017319795,"policy_token":"yeto:5:c482ce266a856d7a827dbcd2cfab61d1c62175038aebe6738e9b4684c4371e5d","rollout_id":5,"samples":32,"seq":26,"wall_time":1790775272.3191423} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1023.020156726,"rollout_id":5,"seq":27,"wall_time":1790775272.3219793} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1023.021449196,"rollout_id":5,"seq":28,"wall_time":1790775272.323272} -{"attempt":0,"batch_hash":"a50a36b23943193b6ed09930c9f46ad4cfa19cb56cc4281a8d6f210cd4230b4b","input_batch_hash":"366991d09f5a3f506e3bc3375f7f029d15d0d3071bc0cef7b8cea8a9708ab3f6","kind":"optimizer_applied","monotonic":1026.891397359,"rollout_id":5,"seq":29,"wall_time":1790775276.1932201} -{"attempt":0,"batch_hash":"a50a36b23943193b6ed09930c9f46ad4cfa19cb56cc4281a8d6f210cd4230b4b","kind":"outer_recorded","monotonic":1027.319188755,"next_policy_version":6,"rollout_id":5,"seq":30,"wall_time":1790775276.6210113} -{"attempt":0,"batch_hash":"15d40faf583d63d9ec1c5145495d64b529d30a9efe3f4e063e3be83735f51b15","group_ids":["g24","g25","g26","g27"],"kind":"prepared","monotonic":1029.646021526,"policy_token":"yeto:6:4971872ea4ddd4f58890af451087b1cc043779833c27ab21f6fc4004691da293","rollout_id":6,"samples":32,"seq":31,"wall_time":1790775278.947844} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1029.649604247,"rollout_id":6,"seq":32,"wall_time":1790775278.9514267} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1029.650879487,"rollout_id":6,"seq":33,"wall_time":1790775278.9527023} -{"attempt":0,"batch_hash":"15d40faf583d63d9ec1c5145495d64b529d30a9efe3f4e063e3be83735f51b15","input_batch_hash":"91a7358ee020e4865a401a82ee980907ee85167f2e322b19e64b44ed3910c753","kind":"optimizer_applied","monotonic":1034.187267505,"rollout_id":6,"seq":34,"wall_time":1790775283.48909} -{"attempt":0,"batch_hash":"15d40faf583d63d9ec1c5145495d64b529d30a9efe3f4e063e3be83735f51b15","kind":"outer_recorded","monotonic":1034.614236664,"next_policy_version":7,"rollout_id":6,"seq":35,"wall_time":1790775283.916059} -{"attempt":0,"batch_hash":"ffd63fa458e4755fc9b5371a07295f8c893f3728afeed5f497bca25e1992d1ac","group_ids":["g28","g29","g30","g31"],"kind":"prepared","monotonic":1037.147921452,"policy_token":"yeto:7:e74b67b648bc5a015dbcb1249803114c12825474823c4f14930d40cbdab01dcb","rollout_id":7,"samples":32,"seq":36,"wall_time":1790775286.4497442} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1037.150539969,"rollout_id":7,"seq":37,"wall_time":1790775286.4523625} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1037.151848743,"rollout_id":7,"seq":38,"wall_time":1790775286.4536715} -{"attempt":0,"batch_hash":"ffd63fa458e4755fc9b5371a07295f8c893f3728afeed5f497bca25e1992d1ac","input_batch_hash":"c5ddfb48d9e4855e1de9692f5cf8facf4bb8ccd9ac88df243b6df3a3dae6e300","kind":"optimizer_applied","monotonic":1041.774171421,"rollout_id":7,"seq":39,"wall_time":1790775291.075994} -{"attempt":0,"batch_hash":"ffd63fa458e4755fc9b5371a07295f8c893f3728afeed5f497bca25e1992d1ac","kind":"outer_recorded","monotonic":1042.205066035,"next_policy_version":8,"rollout_id":7,"seq":40,"wall_time":1790775291.5068884} -{"attempt":0,"batch_hash":"739ca7c1d0a47584bf0a8253aae932c0e23f0589eecde6a4b02020bee9b18f6c","group_ids":["g32","g33","g34","g35"],"kind":"prepared","monotonic":1044.079681116,"policy_token":"yeto:8:cba6b584968e4bf8c33b787cf9212395cc9af74c4f0f682bfef9127626c6ffc0","rollout_id":8,"samples":32,"seq":41,"wall_time":1790775293.3815038} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1044.082640375,"rollout_id":8,"seq":42,"wall_time":1790775293.384463} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1044.084052004,"rollout_id":8,"seq":43,"wall_time":1790775293.3858747} -{"attempt":0,"batch_hash":"739ca7c1d0a47584bf0a8253aae932c0e23f0589eecde6a4b02020bee9b18f6c","input_batch_hash":"f4d2e4c913eab5eadcbf5e019dc6a9b23acebf265d59a634537c64c6e383c20f","kind":"optimizer_applied","monotonic":1047.779684756,"rollout_id":8,"seq":44,"wall_time":1790775297.0815074} -{"attempt":0,"batch_hash":"739ca7c1d0a47584bf0a8253aae932c0e23f0589eecde6a4b02020bee9b18f6c","kind":"outer_recorded","monotonic":1048.209937411,"next_policy_version":9,"rollout_id":8,"seq":45,"wall_time":1790775297.5117598} -{"attempt":0,"batch_hash":"4184c7ee9bac678381d11410779bb9a297f44d08b1e4a28915cabca03e934973","group_ids":["g36","g37","g38","g39"],"kind":"prepared","monotonic":1050.438132729,"policy_token":"yeto:9:535598b3f485c8fe16c83382d146b43659f388888da9bfec257d7807ce3674b3","rollout_id":9,"samples":32,"seq":46,"wall_time":1790775299.7399552} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1050.441491612,"rollout_id":9,"seq":47,"wall_time":1790775299.7433143} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1050.442696464,"rollout_id":9,"seq":48,"wall_time":1790775299.7445192} -{"attempt":0,"batch_hash":"4184c7ee9bac678381d11410779bb9a297f44d08b1e4a28915cabca03e934973","input_batch_hash":"602df94a0c1b822aa73137d9c20e0a322397cddfa160679e7256802cfd7c5e54","kind":"optimizer_applied","monotonic":1054.17693858,"rollout_id":9,"seq":49,"wall_time":1790775303.4787612} -{"attempt":0,"batch_hash":"4184c7ee9bac678381d11410779bb9a297f44d08b1e4a28915cabca03e934973","kind":"outer_recorded","monotonic":1054.61209558,"next_policy_version":10,"rollout_id":9,"seq":50,"wall_time":1790775303.913918} -{"attempt":0,"batch_hash":"10e4da1b11192667eeba01635d7d0c9ae69f2d568fd0f4f99cca12f4f2b5f141","group_ids":["g40","g41","g42","g43"],"kind":"prepared","monotonic":1056.910419949,"policy_token":"yeto:10:56bd64e9fa1dfac6c1ba8f9d4f3d07ddcaae1d0de073fb9b106b6d99e6c72c56","rollout_id":10,"samples":32,"seq":51,"wall_time":1790775306.2122426} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1056.91329593,"rollout_id":10,"seq":52,"wall_time":1790775306.2151186} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1056.914613418,"rollout_id":10,"seq":53,"wall_time":1790775306.216436} -{"attempt":0,"batch_hash":"10e4da1b11192667eeba01635d7d0c9ae69f2d568fd0f4f99cca12f4f2b5f141","input_batch_hash":"a9b315199a797b5d7ee030db759f00512a87a59d7e5449e735efba78735e9c0a","kind":"optimizer_applied","monotonic":1060.663539081,"rollout_id":10,"seq":54,"wall_time":1790775309.9653616} -{"attempt":0,"batch_hash":"10e4da1b11192667eeba01635d7d0c9ae69f2d568fd0f4f99cca12f4f2b5f141","kind":"outer_recorded","monotonic":1061.149224898,"next_policy_version":11,"rollout_id":10,"seq":55,"wall_time":1790775310.4510472} -{"attempt":0,"batch_hash":"5a4cd1c2d6a6a4607ba54b49dd304bd3121aaf4455535d393f54dfbbb4cc48a9","group_ids":["g44","g45","g46","g47"],"kind":"prepared","monotonic":1063.421267744,"policy_token":"yeto:11:6fe1cbd21d550c76a6487c50c246a18c6ebb76301704d90cc28651974b9168a6","rollout_id":11,"samples":32,"seq":56,"wall_time":1790775312.7230902} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1063.426528035,"rollout_id":11,"seq":57,"wall_time":1790775312.7283506} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1063.427781009,"rollout_id":11,"seq":58,"wall_time":1790775312.7296035} -{"attempt":0,"batch_hash":"5a4cd1c2d6a6a4607ba54b49dd304bd3121aaf4455535d393f54dfbbb4cc48a9","input_batch_hash":"845c9ce088802a1283c621ab531f085f7fe3046db3f5d47ef5e93286a1276096","kind":"optimizer_applied","monotonic":1067.153866028,"rollout_id":11,"seq":59,"wall_time":1790775316.4556887} -{"attempt":0,"batch_hash":"5a4cd1c2d6a6a4607ba54b49dd304bd3121aaf4455535d393f54dfbbb4cc48a9","kind":"outer_recorded","monotonic":1067.610294402,"next_policy_version":12,"rollout_id":11,"seq":60,"wall_time":1790775316.9121165} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/base/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/base/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index 9ae8deb6e..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/base/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"kind":"finalization","monotonic":1068.379834877,"rollout_id":12,"seq":1,"wall_time":1790775317.6816576} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/e1b/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/e1b/elastic-state/ledger/journal.jsonl deleted file mode 100644 index 1ef8018ff..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/e1b/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,60 +0,0 @@ -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","group_ids":["g0","g1","g2","g3"],"kind":"prepared","monotonic":958.513961808,"policy_token":"yeto:0:c65b6a4d0c9dc0ff99b65278e9410b13a5b953f175f2dbde702c3df255b79213","rollout_id":0,"samples":32,"seq":1,"wall_time":1790778448.5300958} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":958.517242991,"rollout_id":0,"seq":2,"wall_time":1790778448.5333772} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":958.518736969,"rollout_id":0,"seq":3,"wall_time":1790778448.5348716} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","input_batch_hash":"09c86025cbe7393dd354c7055d163d9068ab0812f6dc200ea8cbcf941f795f07","kind":"optimizer_applied","monotonic":973.040081659,"rollout_id":0,"seq":4,"wall_time":1790778463.056216} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","kind":"outer_recorded","monotonic":973.485981354,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790778463.5021155} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","group_ids":["g4","g5","g6","g7"],"kind":"prepared","monotonic":976.282811834,"policy_token":"yeto:1:a6fa0de7fface8438eb79c3cee8dea33d39879a4e11c71755a3c269b63ffa243","rollout_id":1,"samples":32,"seq":6,"wall_time":1790778466.2989464} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":976.286008913,"rollout_id":1,"seq":7,"wall_time":1790778466.3021433} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":976.287852957,"rollout_id":1,"seq":8,"wall_time":1790778466.3039875} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","input_batch_hash":"70fae16c8ced99e9b8f8ca4d522c2d3478d05b2037d36f468d72702c75589739","kind":"optimizer_applied","monotonic":980.112674665,"rollout_id":1,"seq":9,"wall_time":1790778470.1288092} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","kind":"outer_recorded","monotonic":980.568573087,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790778470.584707} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","group_ids":["g10","g11","g8","g9"],"kind":"prepared","monotonic":1134.466769429,"policy_token":"yeto:2:5502008ae0c0083ed4b95ef17395311a8518227a4b63995c24f2be64b45b86a6","rollout_id":2,"samples":32,"seq":11,"wall_time":1790778624.4829037} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1134.469860857,"rollout_id":2,"seq":12,"wall_time":1790778624.4859953} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1134.471173768,"rollout_id":2,"seq":13,"wall_time":1790778624.4873083} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","input_batch_hash":"f50b3444ba22c288a62a1b89a380704ebdf27440f57bebf04cac4ae7d4bf0344","kind":"optimizer_applied","monotonic":1138.726164595,"rollout_id":2,"seq":14,"wall_time":1790778628.7422988} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","kind":"outer_recorded","monotonic":1139.166621198,"next_policy_version":3,"rollout_id":2,"seq":15,"wall_time":1790778629.1827552} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","group_ids":["g12","g13","g14","g15"],"kind":"prepared","monotonic":1142.522363296,"policy_token":"yeto:3:950109f338f627598a2a3c31cc645b88c3696cd85918ae8e786c676c16440429","rollout_id":3,"samples":32,"seq":16,"wall_time":1790778632.5384977} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1142.52588235,"rollout_id":3,"seq":17,"wall_time":1790778632.5420167} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1142.527205773,"rollout_id":3,"seq":18,"wall_time":1790778632.5433402} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","input_batch_hash":"e76218bc1da05c4ab5aca481b234c0ab0daa6fa18b91ac054849273b5c90b25d","kind":"optimizer_applied","monotonic":1146.355287159,"rollout_id":3,"seq":19,"wall_time":1790778636.3714218} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","kind":"outer_recorded","monotonic":1146.80710064,"next_policy_version":4,"rollout_id":3,"seq":20,"wall_time":1790778636.8232348} -{"attempt":0,"batch_hash":"7328d450bd7fdc43c18fca9149e109342e83a89da53a2ea7d497e99fc4236ca2","group_ids":["g16","g17","g18","g19"],"kind":"prepared","monotonic":1149.680405276,"policy_token":"yeto:4:6212af24ff8d3fb1c74279a307b292c2a8758c51b9660647cdd9f6bb77097752","rollout_id":4,"samples":32,"seq":21,"wall_time":1790778639.6965396} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1149.685430222,"rollout_id":4,"seq":22,"wall_time":1790778639.7015648} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1149.686919576,"rollout_id":4,"seq":23,"wall_time":1790778639.7030542} -{"attempt":0,"batch_hash":"7328d450bd7fdc43c18fca9149e109342e83a89da53a2ea7d497e99fc4236ca2","input_batch_hash":"97d7b34554745aa3ed7a719d52a7c2300bd86776cd831f8f1c30b00b88fdb4ce","kind":"optimizer_applied","monotonic":1153.539665279,"rollout_id":4,"seq":24,"wall_time":1790778643.5557997} -{"attempt":0,"batch_hash":"7328d450bd7fdc43c18fca9149e109342e83a89da53a2ea7d497e99fc4236ca2","kind":"outer_recorded","monotonic":1153.973202946,"next_policy_version":5,"rollout_id":4,"seq":25,"wall_time":1790778643.9893372} -{"attempt":0,"batch_hash":"a50a36b23943193b6ed09930c9f46ad4cfa19cb56cc4281a8d6f210cd4230b4b","group_ids":["g20","g21","g22","g23"],"kind":"prepared","monotonic":1156.747550502,"policy_token":"yeto:5:a061b5d593a06b2faeda17d077c81adf708bc7b8a21740f82a5867a7b5bdecc0","rollout_id":5,"samples":32,"seq":26,"wall_time":1790778646.7636847} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1156.750092083,"rollout_id":5,"seq":27,"wall_time":1790778646.7662265} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1156.75130935,"rollout_id":5,"seq":28,"wall_time":1790778646.7674437} -{"attempt":0,"batch_hash":"a50a36b23943193b6ed09930c9f46ad4cfa19cb56cc4281a8d6f210cd4230b4b","input_batch_hash":"fa848d3e765e0f2c54580d56bc8e11f798feff60030599ed90fbe6810c65e111","kind":"optimizer_applied","monotonic":1160.609480656,"rollout_id":5,"seq":29,"wall_time":1790778650.6256151} -{"attempt":0,"batch_hash":"a50a36b23943193b6ed09930c9f46ad4cfa19cb56cc4281a8d6f210cd4230b4b","kind":"outer_recorded","monotonic":1161.063092369,"next_policy_version":6,"rollout_id":5,"seq":30,"wall_time":1790778651.0792263} -{"attempt":0,"batch_hash":"15d40faf583d63d9ec1c5145495d64b529d30a9efe3f4e063e3be83735f51b15","group_ids":["g24","g25","g26","g27"],"kind":"prepared","monotonic":1163.386978266,"policy_token":"yeto:6:dd4559d67f40ca827a8d9dfcd4bc2863c1ec3c6b6e09d6f2e472474d02b244db","rollout_id":6,"samples":32,"seq":31,"wall_time":1790778653.4031126} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1163.389550058,"rollout_id":6,"seq":32,"wall_time":1790778653.4056842} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1163.390749702,"rollout_id":6,"seq":33,"wall_time":1790778653.4068842} -{"attempt":0,"batch_hash":"15d40faf583d63d9ec1c5145495d64b529d30a9efe3f4e063e3be83735f51b15","input_batch_hash":"176eaa44cc9b5358023890013a522a5096a303caa5ca4dbb2b2770fe798547ad","kind":"optimizer_applied","monotonic":1168.053960463,"rollout_id":6,"seq":34,"wall_time":1790778658.0700948} -{"attempt":0,"batch_hash":"15d40faf583d63d9ec1c5145495d64b529d30a9efe3f4e063e3be83735f51b15","kind":"outer_recorded","monotonic":1168.511251431,"next_policy_version":7,"rollout_id":6,"seq":35,"wall_time":1790778658.5273852} -{"attempt":0,"batch_hash":"ffd63fa458e4755fc9b5371a07295f8c893f3728afeed5f497bca25e1992d1ac","group_ids":["g28","g29","g30","g31"],"kind":"prepared","monotonic":1171.29525427,"policy_token":"yeto:7:e5cacce30d0df23266d94abce6b99d6023268782186505454fd209fff1cd8c49","rollout_id":7,"samples":32,"seq":36,"wall_time":1790778661.3113887} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1171.298134325,"rollout_id":7,"seq":37,"wall_time":1790778661.3142686} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1171.299345052,"rollout_id":7,"seq":38,"wall_time":1790778661.3154795} -{"attempt":0,"batch_hash":"ffd63fa458e4755fc9b5371a07295f8c893f3728afeed5f497bca25e1992d1ac","input_batch_hash":"58a9bed8493a3ba479a7f3e9a5beb17800fe0f5fb0f138e4fb0c716bdd77347e","kind":"optimizer_applied","monotonic":1175.841141292,"rollout_id":7,"seq":39,"wall_time":1790778665.8572757} -{"attempt":0,"batch_hash":"ffd63fa458e4755fc9b5371a07295f8c893f3728afeed5f497bca25e1992d1ac","kind":"outer_recorded","monotonic":1176.293793047,"next_policy_version":8,"rollout_id":7,"seq":40,"wall_time":1790778666.3099272} -{"attempt":0,"batch_hash":"739ca7c1d0a47584bf0a8253aae932c0e23f0589eecde6a4b02020bee9b18f6c","group_ids":["g32","g33","g34","g35"],"kind":"prepared","monotonic":1178.403112236,"policy_token":"yeto:8:60fffc6504efea55065cb1c36ade9d5c1e266a592a70fc7e3667029fd8076da9","rollout_id":8,"samples":32,"seq":41,"wall_time":1790778668.4192467} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1178.406108222,"rollout_id":8,"seq":42,"wall_time":1790778668.4222426} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1178.407230493,"rollout_id":8,"seq":43,"wall_time":1790778668.4233649} -{"attempt":0,"batch_hash":"739ca7c1d0a47584bf0a8253aae932c0e23f0589eecde6a4b02020bee9b18f6c","input_batch_hash":"c0cec13bfc576d568bb7585d304359f2077a77d92cce01a97256fc1b6d12af51","kind":"optimizer_applied","monotonic":1182.240532857,"rollout_id":8,"seq":44,"wall_time":1790778672.2566674} -{"attempt":0,"batch_hash":"739ca7c1d0a47584bf0a8253aae932c0e23f0589eecde6a4b02020bee9b18f6c","kind":"outer_recorded","monotonic":1182.662544611,"next_policy_version":9,"rollout_id":8,"seq":45,"wall_time":1790778672.6786788} -{"attempt":0,"batch_hash":"4184c7ee9bac678381d11410779bb9a297f44d08b1e4a28915cabca03e934973","group_ids":["g36","g37","g38","g39"],"kind":"prepared","monotonic":1184.571609341,"policy_token":"yeto:9:67004ea000bcf8ac5825be466e9b169d872d8d0bd083b39fd807746f7d29ba75","rollout_id":9,"samples":32,"seq":46,"wall_time":1790778674.5877438} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1184.574387396,"rollout_id":9,"seq":47,"wall_time":1790778674.5905218} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1184.575694375,"rollout_id":9,"seq":48,"wall_time":1790778674.5918288} -{"attempt":0,"batch_hash":"4184c7ee9bac678381d11410779bb9a297f44d08b1e4a28915cabca03e934973","input_batch_hash":"4ba1d59a854c86a3b26c5cba90e9d90650b30c3e6536fcba59ceffbd8d091255","kind":"optimizer_applied","monotonic":1188.423778073,"rollout_id":9,"seq":49,"wall_time":1790778678.4399126} -{"attempt":0,"batch_hash":"4184c7ee9bac678381d11410779bb9a297f44d08b1e4a28915cabca03e934973","kind":"outer_recorded","monotonic":1188.86578433,"next_policy_version":10,"rollout_id":9,"seq":50,"wall_time":1790778678.8819184} -{"attempt":0,"batch_hash":"10e4da1b11192667eeba01635d7d0c9ae69f2d568fd0f4f99cca12f4f2b5f141","group_ids":["g40","g41","g42","g43"],"kind":"prepared","monotonic":1191.104976401,"policy_token":"yeto:10:49346369c6c28c7ce9569299a92b37a43b0a8095f273d7760349a4d0b7a6edbe","rollout_id":10,"samples":32,"seq":51,"wall_time":1790778681.1211107} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1191.10855615,"rollout_id":10,"seq":52,"wall_time":1790778681.1246908} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1191.109864182,"rollout_id":10,"seq":53,"wall_time":1790778681.1259985} -{"attempt":0,"batch_hash":"10e4da1b11192667eeba01635d7d0c9ae69f2d568fd0f4f99cca12f4f2b5f141","input_batch_hash":"2beb3f3ec330f4608bc216f040cea16554ad149ba15d56797bf29103cb708deb","kind":"optimizer_applied","monotonic":1194.868101418,"rollout_id":10,"seq":54,"wall_time":1790778684.8842359} -{"attempt":0,"batch_hash":"10e4da1b11192667eeba01635d7d0c9ae69f2d568fd0f4f99cca12f4f2b5f141","kind":"outer_recorded","monotonic":1195.322119392,"next_policy_version":11,"rollout_id":10,"seq":55,"wall_time":1790778685.3382535} -{"attempt":0,"batch_hash":"5a4cd1c2d6a6a4607ba54b49dd304bd3121aaf4455535d393f54dfbbb4cc48a9","group_ids":["g44","g45","g46","g47"],"kind":"prepared","monotonic":1197.601472577,"policy_token":"yeto:11:53a27f4d8f3ae7c5c262f66ced8990fa660209508fc15400376d2e4ea2277ec2","rollout_id":11,"samples":32,"seq":56,"wall_time":1790778687.6176069} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1197.604171049,"rollout_id":11,"seq":57,"wall_time":1790778687.6203053} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1197.605379488,"rollout_id":11,"seq":58,"wall_time":1790778687.621514} -{"attempt":0,"batch_hash":"5a4cd1c2d6a6a4607ba54b49dd304bd3121aaf4455535d393f54dfbbb4cc48a9","input_batch_hash":"983d55191b3cc14c5a071ddfdd300599a52231542327d7f18b8d9eb078da6a06","kind":"optimizer_applied","monotonic":1201.401069122,"rollout_id":11,"seq":59,"wall_time":1790778691.4172034} -{"attempt":0,"batch_hash":"5a4cd1c2d6a6a4607ba54b49dd304bd3121aaf4455535d393f54dfbbb4cc48a9","kind":"outer_recorded","monotonic":1201.851034272,"next_policy_version":12,"rollout_id":11,"seq":60,"wall_time":1790778691.8671682} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/e1b/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/e1b/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index 02899c344..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/e1b/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1,15 +0,0 @@ -{"body":{"deadline_s":600.0,"expected_config_epoch":0,"target":"T4R4S0"},"body_hash":"d13c83e0da4b99bc12d568249c1b63d50b6500b2bda3074c19b8c3bc78ef7286","deadline_wall":1790779071.4265888,"kind":"request","monotonic":981.410468671,"plan":{"expected_config_epoch":0,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"d13c83e0da4b99bc12d568249c1b63d50b6500b2bda3074c19b8c3bc78ef7286","source":"T4R2S2","source_engines":2,"target":"T4R4S0","target_engines":4,"trainer":null},"request_id":"b-up","seq":1,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778471.4266033} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":981.412860316,"phase":"VALIDATING","plan":{"expected_config_epoch":0,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"d13c83e0da4b99bc12d568249c1b63d50b6500b2bda3074c19b8c3bc78ef7286","source":"T4R2S2","source_engines":2,"target":"T4R4S0","target_engines":4,"trainer":null},"request_id":"b-up","seq":2,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778471.428995} -{"allowed":true,"budget_s":null,"expected_pause_s":600.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":981.414809821,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":2,"seq":3,"stalls_peers":false,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778471.4309442} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":981.416148957,"phase":"WAIT_SAFE","request_id":"b-up","safe_point_rollout_id":2,"seq":4,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778471.4322834} -{"config_epoch":0,"cut":{"optimizer_step":2,"policy_hash":"5502008ae0c0083ed4b95ef17395311a8518227a4b63995c24f2be64b45b86a6","policy_token":"yeto:2:5502008ae0c0083ed4b95ef17395311a8518227a4b63995c24f2be64b45b86a6","published_version":2,"rollout_id":2,"unconsumed_batches":[]},"fork_epoch":0,"kind":"phase","monotonic":981.486359164,"phase":"QUIESCING","remove":[],"request_id":"b-up","seq":5,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778471.5024934} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":981.487926882,"phase":"INITIALIZING","request_id":"b-up","seq":6,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778471.5040612} -{"kind":"add_intent","members":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"monotonic":981.490708398,"seq":7,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778471.5068429} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":0,"kind":"fork_op","monotonic":981.492081188,"op":"start","seq":8,"status":"issued","tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778471.508216} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":0,"kind":"fork_op","monotonic":1128.307290888,"op":"start","result_fork_epoch":1,"seq":9,"status":"done","tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778618.3234253} -{"config_epoch":0,"fork_epoch":1,"kind":"phase","monotonic":1128.310061232,"phase":"VERIFYING","request_id":"b-up","seq":10,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778618.3261957} -{"kind":"weight_admission","manifest_hash":"93d11b307f1198f81a301edaa2169ef67e60974b50656efd197fa0945e45d74b","members":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"monotonic":1130.624351603,"policy_token":"yeto:2:5502008ae0c0083ed4b95ef17395311a8518227a4b63995c24f2be64b45b86a6","seq":11,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778620.6404862} -{"config_epoch":1,"fork_epoch":1,"kind":"phase","members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001","engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"monotonic":1130.69965652,"phase":"COMMITTED","request_id":"b-up","seq":12,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778620.7157907} -{"config_epoch":1,"fork_epoch":1,"kind":"phase","monotonic":1130.701055915,"phase":"RESUMING","request_id":"b-up","seq":13,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778620.7171905} -{"config_epoch":1,"config_epoch_to":1,"fork_epoch":1,"kind":"phase","monotonic":1130.702302127,"phase":"SUCCEEDED","request_id":"b-up","seq":14,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790778620.7184367} -{"kind":"finalization","monotonic":1202.660696466,"rollout_id":12,"seq":15,"wall_time":1790778692.676831} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/x2/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/x2/elastic-state/ledger/journal.jsonl deleted file mode 100644 index 1f786d65b..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/x2/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,60 +0,0 @@ -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","group_ids":["g0","g1","g2","g3"],"kind":"prepared","monotonic":950.285882906,"policy_token":"yeto:0:c65b6a4d0c9dc0ff99b65278e9410b13a5b953f175f2dbde702c3df255b79213","rollout_id":0,"samples":32,"seq":1,"wall_time":1790776635.8487968} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":950.289713053,"rollout_id":0,"seq":2,"wall_time":1790776635.8526275} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":950.291031994,"rollout_id":0,"seq":3,"wall_time":1790776635.8539464} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","input_batch_hash":"09c86025cbe7393dd354c7055d163d9068ab0812f6dc200ea8cbcf941f795f07","kind":"optimizer_applied","monotonic":964.453521059,"rollout_id":0,"seq":4,"wall_time":1790776650.0164354} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","kind":"outer_recorded","monotonic":964.873166226,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790776650.4360805} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","group_ids":["g4","g5","g6","g7"],"kind":"prepared","monotonic":967.760510643,"policy_token":"yeto:1:6ef6f00bb9dbe9d555fcbd383e22da306f92a9dc6964cae3ab0661f269f35cb0","rollout_id":1,"samples":32,"seq":6,"wall_time":1790776653.323425} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":967.763631825,"rollout_id":1,"seq":7,"wall_time":1790776653.3265462} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":967.764767484,"rollout_id":1,"seq":8,"wall_time":1790776653.3276818} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","input_batch_hash":"d1d547c5c4984c3c8407643f6d983a1cf5b86e57bc4514310ee603a4b9c4315a","kind":"optimizer_applied","monotonic":971.429436276,"rollout_id":1,"seq":9,"wall_time":1790776656.9923508} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","kind":"outer_recorded","monotonic":971.834080236,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790776657.3969944} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","group_ids":["g10","g11","g8","g9"],"kind":"prepared","monotonic":1124.813315271,"policy_token":"yeto:2:0aa91476e6ff0509ba898730c888b05746fc21065cdf70cefe7347e4b5e16f8b","rollout_id":2,"samples":32,"seq":11,"wall_time":1790776810.3762295} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1124.816786852,"rollout_id":2,"seq":12,"wall_time":1790776810.3797011} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1124.818221342,"rollout_id":2,"seq":13,"wall_time":1790776810.3811357} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","input_batch_hash":"8db006d4ff44b64bce16c08a956d89f3cd1857864a847ab5c34ca3dd0469d841","kind":"optimizer_applied","monotonic":1128.744592156,"rollout_id":2,"seq":14,"wall_time":1790776814.3075063} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","kind":"outer_recorded","monotonic":1129.169294761,"next_policy_version":3,"rollout_id":2,"seq":15,"wall_time":1790776814.732209} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","group_ids":["g12","g13","g14","g15"],"kind":"prepared","monotonic":1132.503344009,"policy_token":"yeto:3:0fcf608b0c54b1d472a2b6b819884191f6296db1e913485b59813b8c47272d1b","rollout_id":3,"samples":32,"seq":16,"wall_time":1790776818.0662582} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1132.50603644,"rollout_id":3,"seq":17,"wall_time":1790776818.068951} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1132.507392223,"rollout_id":3,"seq":18,"wall_time":1790776818.0703065} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","input_batch_hash":"5fc7e15b5ecac397a9d82de4443227ae5d55f202ea197c3948f031cf585f686c","kind":"optimizer_applied","monotonic":1136.258531562,"rollout_id":3,"seq":19,"wall_time":1790776821.821446} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","kind":"outer_recorded","monotonic":1136.682533356,"next_policy_version":4,"rollout_id":3,"seq":20,"wall_time":1790776822.2454474} -{"attempt":0,"batch_hash":"7328d450bd7fdc43c18fca9149e109342e83a89da53a2ea7d497e99fc4236ca2","group_ids":["g16","g17","g18","g19"],"kind":"prepared","monotonic":1139.517563716,"policy_token":"yeto:4:cb84f3353d3c6f1efd1c38daaa80f2345891b5185c9ad3c0dfaafe500be2d2f7","rollout_id":4,"samples":32,"seq":21,"wall_time":1790776825.080478} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1139.520894606,"rollout_id":4,"seq":22,"wall_time":1790776825.0838087} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1139.522285004,"rollout_id":4,"seq":23,"wall_time":1790776825.0851994} -{"attempt":0,"batch_hash":"7328d450bd7fdc43c18fca9149e109342e83a89da53a2ea7d497e99fc4236ca2","input_batch_hash":"01c4e660769498f40118caa7d5174075dbdee0508518b457360a44f603bb638e","kind":"optimizer_applied","monotonic":1143.387170005,"rollout_id":4,"seq":24,"wall_time":1790776828.9500844} -{"attempt":0,"batch_hash":"7328d450bd7fdc43c18fca9149e109342e83a89da53a2ea7d497e99fc4236ca2","kind":"outer_recorded","monotonic":1143.798897554,"next_policy_version":5,"rollout_id":4,"seq":25,"wall_time":1790776829.3618114} -{"attempt":0,"batch_hash":"a50a36b23943193b6ed09930c9f46ad4cfa19cb56cc4281a8d6f210cd4230b4b","group_ids":["g20","g21","g22","g23"],"kind":"prepared","monotonic":1146.554401135,"policy_token":"yeto:5:1095c07f668ee2bfc5828cfb99a963d9b1db63807ed1c87cf19dbec1a6312a6b","rollout_id":5,"samples":32,"seq":26,"wall_time":1790776832.1173153} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1146.557121915,"rollout_id":5,"seq":27,"wall_time":1790776832.1200364} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1146.558343649,"rollout_id":5,"seq":28,"wall_time":1790776832.121258} -{"attempt":0,"batch_hash":"a50a36b23943193b6ed09930c9f46ad4cfa19cb56cc4281a8d6f210cd4230b4b","input_batch_hash":"152fb0a5dbdc33f7276ae20739a49bb0def22dad61d367899f6524f922cc0198","kind":"optimizer_applied","monotonic":1150.333831084,"rollout_id":5,"seq":29,"wall_time":1790776835.8967454} -{"attempt":0,"batch_hash":"a50a36b23943193b6ed09930c9f46ad4cfa19cb56cc4281a8d6f210cd4230b4b","kind":"outer_recorded","monotonic":1150.755659825,"next_policy_version":6,"rollout_id":5,"seq":30,"wall_time":1790776836.3185742} -{"attempt":0,"batch_hash":"15d40faf583d63d9ec1c5145495d64b529d30a9efe3f4e063e3be83735f51b15","group_ids":["g24","g25","g26","g27"],"kind":"prepared","monotonic":1153.055190545,"policy_token":"yeto:6:1a9f4e3d09c9151a62506926bac74126003fa648197de15eb72ecf3020c60141","rollout_id":6,"samples":32,"seq":31,"wall_time":1790776838.618105} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1153.058266917,"rollout_id":6,"seq":32,"wall_time":1790776838.6211812} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1153.059648747,"rollout_id":6,"seq":33,"wall_time":1790776838.6225631} -{"attempt":0,"batch_hash":"15d40faf583d63d9ec1c5145495d64b529d30a9efe3f4e063e3be83735f51b15","input_batch_hash":"fa1af6fe91c2699a7a4fdccb11df89191c13beb51d00bdeb024a5414afdc4408","kind":"optimizer_applied","monotonic":1157.42328052,"rollout_id":6,"seq":34,"wall_time":1790776842.9861948} -{"attempt":0,"batch_hash":"15d40faf583d63d9ec1c5145495d64b529d30a9efe3f4e063e3be83735f51b15","kind":"outer_recorded","monotonic":1157.840860636,"next_policy_version":7,"rollout_id":6,"seq":35,"wall_time":1790776843.4037745} -{"attempt":0,"batch_hash":"ffd63fa458e4755fc9b5371a07295f8c893f3728afeed5f497bca25e1992d1ac","group_ids":["g28","g29","g30","g31"],"kind":"prepared","monotonic":1165.940560733,"policy_token":"yeto:7:2eb8fe4b60823494cc3ebbb871e15710458d02ff8205870eef6af254df176887","rollout_id":7,"samples":32,"seq":36,"wall_time":1790776851.503475} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1165.943271127,"rollout_id":7,"seq":37,"wall_time":1790776851.5061855} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1165.944463536,"rollout_id":7,"seq":38,"wall_time":1790776851.5073779} -{"attempt":0,"batch_hash":"ffd63fa458e4755fc9b5371a07295f8c893f3728afeed5f497bca25e1992d1ac","input_batch_hash":"056e0a11a609b97e85d7a3337c94224303c99c7c747b04489733ff3ccd6be7f5","kind":"optimizer_applied","monotonic":1170.534360326,"rollout_id":7,"seq":39,"wall_time":1790776856.0972745} -{"attempt":0,"batch_hash":"ffd63fa458e4755fc9b5371a07295f8c893f3728afeed5f497bca25e1992d1ac","kind":"outer_recorded","monotonic":1170.963686853,"next_policy_version":8,"rollout_id":7,"seq":40,"wall_time":1790776856.526601} -{"attempt":0,"batch_hash":"739ca7c1d0a47584bf0a8253aae932c0e23f0589eecde6a4b02020bee9b18f6c","group_ids":["g32","g33","g34","g35"],"kind":"prepared","monotonic":1173.644907546,"policy_token":"yeto:8:b78df62120e4b5465eaf694ba9f80d2783d58e1aa7494c1b546c0348db865ae2","rollout_id":8,"samples":32,"seq":41,"wall_time":1790776859.2078218} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1173.648053138,"rollout_id":8,"seq":42,"wall_time":1790776859.2109675} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1173.649262966,"rollout_id":8,"seq":43,"wall_time":1790776859.2121775} -{"attempt":0,"batch_hash":"739ca7c1d0a47584bf0a8253aae932c0e23f0589eecde6a4b02020bee9b18f6c","input_batch_hash":"c6d701b057c37251566285b1373f837e58e33e9ea44a052a21243d47b9838db3","kind":"optimizer_applied","monotonic":1177.362046004,"rollout_id":8,"seq":44,"wall_time":1790776862.9249604} -{"attempt":0,"batch_hash":"739ca7c1d0a47584bf0a8253aae932c0e23f0589eecde6a4b02020bee9b18f6c","kind":"outer_recorded","monotonic":1177.784256898,"next_policy_version":9,"rollout_id":8,"seq":45,"wall_time":1790776863.347171} -{"attempt":0,"batch_hash":"4184c7ee9bac678381d11410779bb9a297f44d08b1e4a28915cabca03e934973","group_ids":["g36","g37","g38","g39"],"kind":"prepared","monotonic":1180.187169428,"policy_token":"yeto:9:59e0e66ad0f668695b1adb8dc86f2fada5311de758044323a03b513e73ea1944","rollout_id":9,"samples":32,"seq":46,"wall_time":1790776865.750084} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1180.191483091,"rollout_id":9,"seq":47,"wall_time":1790776865.7543974} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1180.192861965,"rollout_id":9,"seq":48,"wall_time":1790776865.7557762} -{"attempt":0,"batch_hash":"4184c7ee9bac678381d11410779bb9a297f44d08b1e4a28915cabca03e934973","input_batch_hash":"d52c8bf0ed115de5d2083129eda98f8448f75a45e1c24298c73a3ab827a9a6fe","kind":"optimizer_applied","monotonic":1183.848301661,"rollout_id":9,"seq":49,"wall_time":1790776869.4112158} -{"attempt":0,"batch_hash":"4184c7ee9bac678381d11410779bb9a297f44d08b1e4a28915cabca03e934973","kind":"outer_recorded","monotonic":1184.274294838,"next_policy_version":10,"rollout_id":9,"seq":50,"wall_time":1790776869.8372087} -{"attempt":0,"batch_hash":"10e4da1b11192667eeba01635d7d0c9ae69f2d568fd0f4f99cca12f4f2b5f141","group_ids":["g40","g41","g42","g43"],"kind":"prepared","monotonic":1186.630554007,"policy_token":"yeto:10:60093f9b42a560b601d0ad59fd0e5ceb4c351316ead20d91773a319495c380b3","rollout_id":10,"samples":32,"seq":51,"wall_time":1790776872.193468} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1186.633542604,"rollout_id":10,"seq":52,"wall_time":1790776872.196457} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1186.634924266,"rollout_id":10,"seq":53,"wall_time":1790776872.1978385} -{"attempt":0,"batch_hash":"10e4da1b11192667eeba01635d7d0c9ae69f2d568fd0f4f99cca12f4f2b5f141","input_batch_hash":"da6e2fbd24154fe70ed23a9aacf792582e328f376e49e528a647fbd92d0adf20","kind":"optimizer_applied","monotonic":1190.399480382,"rollout_id":10,"seq":54,"wall_time":1790776875.9623947} -{"attempt":0,"batch_hash":"10e4da1b11192667eeba01635d7d0c9ae69f2d568fd0f4f99cca12f4f2b5f141","kind":"outer_recorded","monotonic":1190.821598732,"next_policy_version":11,"rollout_id":10,"seq":55,"wall_time":1790776876.3845127} -{"attempt":0,"batch_hash":"5a4cd1c2d6a6a4607ba54b49dd304bd3121aaf4455535d393f54dfbbb4cc48a9","group_ids":["g44","g45","g46","g47"],"kind":"prepared","monotonic":1192.928106878,"policy_token":"yeto:11:ad75252a1663b96d75255c80e1232b9fdd0542987cc3beee787905595d181ef9","rollout_id":11,"samples":32,"seq":56,"wall_time":1790776878.4910214} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1192.930813365,"rollout_id":11,"seq":57,"wall_time":1790776878.4937277} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1192.932112199,"rollout_id":11,"seq":58,"wall_time":1790776878.4950264} -{"attempt":0,"batch_hash":"5a4cd1c2d6a6a4607ba54b49dd304bd3121aaf4455535d393f54dfbbb4cc48a9","input_batch_hash":"92f14d0f955d2110d6756e28d2026e496c8e985596c8bdb44984fde3545c9d99","kind":"optimizer_applied","monotonic":1196.613082141,"rollout_id":11,"seq":59,"wall_time":1790776882.1759963} -{"attempt":0,"batch_hash":"5a4cd1c2d6a6a4607ba54b49dd304bd3121aaf4455535d393f54dfbbb4cc48a9","kind":"outer_recorded","monotonic":1197.041442523,"next_policy_version":12,"rollout_id":11,"seq":60,"wall_time":1790776882.6043565} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/x2/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/x2/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index d30c835c3..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4/x2/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1,27 +0,0 @@ -{"body":{"deadline_s":600.0,"expected_config_epoch":0,"target":"T4R4S0"},"body_hash":"d13c83e0da4b99bc12d568249c1b63d50b6500b2bda3074c19b8c3bc78ef7286","deadline_wall":1790777258.17151,"kind":"request","monotonic":972.608608753,"plan":{"expected_config_epoch":0,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"d13c83e0da4b99bc12d568249c1b63d50b6500b2bda3074c19b8c3bc78ef7286","source":"T4R2S2","source_engines":2,"target":"T4R4S0","target_engines":4,"trainer":null},"request_id":"a4up","seq":1,"tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776658.171523} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":972.612405054,"phase":"VALIDATING","plan":{"expected_config_epoch":0,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"d13c83e0da4b99bc12d568249c1b63d50b6500b2bda3074c19b8c3bc78ef7286","source":"T4R2S2","source_engines":2,"target":"T4R4S0","target_engines":4,"trainer":null},"request_id":"a4up","seq":2,"tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776658.1753192} -{"allowed":true,"budget_s":null,"expected_pause_s":600.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":972.614462883,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":2,"seq":3,"stalls_peers":false,"tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776658.1773772} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":972.615676344,"phase":"WAIT_SAFE","request_id":"a4up","safe_point_rollout_id":2,"seq":4,"tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776658.1785908} -{"config_epoch":0,"cut":{"optimizer_step":2,"policy_hash":"0aa91476e6ff0509ba898730c888b05746fc21065cdf70cefe7347e4b5e16f8b","policy_token":"yeto:2:0aa91476e6ff0509ba898730c888b05746fc21065cdf70cefe7347e4b5e16f8b","published_version":2,"rollout_id":2,"unconsumed_batches":[]},"fork_epoch":0,"kind":"phase","monotonic":972.681385885,"phase":"QUIESCING","remove":[],"request_id":"a4up","seq":5,"tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776658.2443} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":972.683325845,"phase":"INITIALIZING","request_id":"a4up","seq":6,"tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776658.24624} -{"kind":"add_intent","members":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"monotonic":972.686278857,"seq":7,"tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776658.2491932} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":0,"kind":"fork_op","monotonic":972.687455699,"op":"start","seq":8,"status":"issued","tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776658.25037} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":0,"kind":"fork_op","monotonic":1119.36307297,"op":"start","result_fork_epoch":1,"seq":9,"status":"done","tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776804.9259872} -{"config_epoch":0,"fork_epoch":1,"kind":"phase","monotonic":1119.368446541,"phase":"VERIFYING","request_id":"a4up","seq":10,"tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776804.9313607} -{"kind":"weight_admission","manifest_hash":"e86e92bf3add3c4194d7f239869d80c922e3acf2d0d2f1d334748efc04abbc61","members":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"monotonic":1120.988383667,"policy_token":"yeto:2:0aa91476e6ff0509ba898730c888b05746fc21065cdf70cefe7347e4b5e16f8b","seq":11,"tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776806.5512981} -{"config_epoch":1,"fork_epoch":1,"kind":"phase","members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001","engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"monotonic":1121.063394476,"phase":"COMMITTED","request_id":"a4up","seq":12,"tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776806.6263087} -{"config_epoch":1,"fork_epoch":1,"kind":"phase","monotonic":1121.064612436,"phase":"RESUMING","request_id":"a4up","seq":13,"tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776806.627527} -{"config_epoch":1,"config_epoch_to":1,"fork_epoch":1,"kind":"phase","monotonic":1121.065753519,"phase":"SUCCEEDED","request_id":"a4up","seq":14,"tx_id":"tx-0-d13c83e0da4b-a4up","wall_time":1790776806.6286678} -{"body":{"deadline_s":600.0,"expected_config_epoch":1,"target":"T4R2S2"},"body_hash":"80e0a696a247a840248f7ad0fca30ea8ea2ba6f817f86f659727d2e3aaeaad15","deadline_wall":1790777444.2094896,"kind":"request","monotonic":1158.646590006,"plan":{"expected_config_epoch":1,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"80e0a696a247a840248f7ad0fca30ea8ea2ba6f817f86f659727d2e3aaeaad15","source":"T4R4S0","source_engines":4,"target":"T4R2S2","target_engines":2,"trainer":null},"request_id":"a4down","seq":15,"tx_id":"tx-1-80e0a696a247-a4down","wall_time":1790776844.2095044} -{"config_epoch":1,"fork_epoch":1,"kind":"phase","monotonic":1158.649788692,"phase":"VALIDATING","plan":{"expected_config_epoch":1,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"80e0a696a247a840248f7ad0fca30ea8ea2ba6f817f86f659727d2e3aaeaad15","source":"T4R4S0","source_engines":4,"target":"T4R2S2","target_engines":2,"trainer":null},"request_id":"a4down","seq":16,"tx_id":"tx-1-80e0a696a247-a4down","wall_time":1790776844.212703} -{"allowed":true,"budget_s":null,"expected_pause_s":600.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":1158.652041105,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":7,"seq":17,"stalls_peers":false,"tx_id":"tx-1-80e0a696a247-a4down","wall_time":1790776844.2149556} -{"config_epoch":1,"fork_epoch":1,"kind":"phase","monotonic":1158.653658038,"phase":"WAIT_SAFE","request_id":"a4down","safe_point_rollout_id":7,"seq":18,"tx_id":"tx-1-80e0a696a247-a4down","wall_time":1790776844.2165725} -{"config_epoch":1,"cut":{"optimizer_step":7,"policy_hash":"2eb8fe4b60823494cc3ebbb871e15710458d02ff8205870eef6af254df176887","policy_token":"yeto:7:2eb8fe4b60823494cc3ebbb871e15710458d02ff8205870eef6af254df176887","published_version":7,"rollout_id":7,"unconsumed_batches":[]},"fork_epoch":1,"kind":"phase","monotonic":1158.718707507,"phase":"QUIESCING","remove":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"request_id":"a4down","seq":19,"tx_id":"tx-1-80e0a696a247-a4down","wall_time":1790776844.2816215} -{"config_epoch":1,"fork_epoch":1,"kind":"phase","monotonic":1158.729208544,"phase":"TRANSFERRING","release":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"request_id":"a4down","rollback_boundary":"release","seq":20,"tx_id":"tx-1-80e0a696a247-a4down","wall_time":1790776844.2921228} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":1,"kind":"fork_op","monotonic":1158.730567642,"op":"stop","seq":21,"status":"issued","tx_id":"tx-1-80e0a696a247-a4down","wall_time":1790776844.293482} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":1,"kind":"fork_op","monotonic":1163.855232241,"op":"stop","result_fork_epoch":2,"seq":22,"status":"done","tx_id":"tx-1-80e0a696a247-a4down","wall_time":1790776849.4181466} -{"config_epoch":1,"fork_epoch":2,"kind":"phase","monotonic":1163.858275039,"phase":"VERIFYING","request_id":"a4down","seq":23,"tx_id":"tx-1-80e0a696a247-a4down","wall_time":1790776849.4211893} -{"config_epoch":2,"fork_epoch":2,"kind":"phase","members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"monotonic":1163.932527134,"phase":"COMMITTED","request_id":"a4down","seq":24,"tx_id":"tx-1-80e0a696a247-a4down","wall_time":1790776849.4954412} -{"config_epoch":2,"fork_epoch":2,"kind":"phase","monotonic":1163.933972048,"phase":"RESUMING","request_id":"a4down","seq":25,"tx_id":"tx-1-80e0a696a247-a4down","wall_time":1790776849.496886} -{"config_epoch":2,"config_epoch_to":2,"fork_epoch":2,"kind":"phase","monotonic":1163.935410341,"phase":"SUCCEEDED","request_id":"a4down","seq":26,"tx_id":"tx-1-80e0a696a247-a4down","wall_time":1790776849.4983244} -{"kind":"finalization","monotonic":1197.818940383,"rollout_id":12,"seq":27,"wall_time":1790776883.3818548} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/e1b/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/e1b/elastic-state/ledger/journal.jsonl deleted file mode 100644 index b9bd57b18..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/e1b/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,14 +0,0 @@ -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","group_ids":["g0","g1","g2","g3"],"kind":"prepared","monotonic":946.05248051,"policy_token":"yeto:0:c65b6a4d0c9dc0ff99b65278e9410b13a5b953f175f2dbde702c3df255b79213","rollout_id":0,"samples":32,"seq":1,"wall_time":1790790433.3998628} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":946.056128597,"rollout_id":0,"seq":2,"wall_time":1790790433.403511} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":946.057749926,"rollout_id":0,"seq":3,"wall_time":1790790433.4051325} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","input_batch_hash":"09c86025cbe7393dd354c7055d163d9068ab0812f6dc200ea8cbcf941f795f07","kind":"optimizer_applied","monotonic":960.296701166,"rollout_id":0,"seq":4,"wall_time":1790790447.6440837} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","kind":"outer_recorded","monotonic":960.727599853,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790790448.0749822} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","group_ids":["g4","g5","g6","g7"],"kind":"prepared","monotonic":963.65772739,"policy_token":"yeto:1:6ef6f00bb9dbe9d555fcbd383e22da306f92a9dc6964cae3ab0661f269f35cb0","rollout_id":1,"samples":32,"seq":6,"wall_time":1790790451.00511} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":963.661018522,"rollout_id":1,"seq":7,"wall_time":1790790451.0084012} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":963.662585821,"rollout_id":1,"seq":8,"wall_time":1790790451.0099683} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","input_batch_hash":"d1d547c5c4984c3c8407643f6d983a1cf5b86e57bc4514310ee603a4b9c4315a","kind":"optimizer_applied","monotonic":967.428251259,"rollout_id":1,"seq":9,"wall_time":1790790454.7756338} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","kind":"outer_recorded","monotonic":967.86640172,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790790455.213784} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","group_ids":["g10","g11","g8","g9"],"kind":"prepared","monotonic":1123.832835314,"policy_token":"yeto:2:0aa91476e6ff0509ba898730c888b05746fc21065cdf70cefe7347e4b5e16f8b","rollout_id":2,"samples":32,"seq":11,"wall_time":1790790611.1802177} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1123.83586994,"rollout_id":2,"seq":12,"wall_time":1790790611.1832526} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1123.837390219,"rollout_id":2,"seq":13,"wall_time":1790790611.184773} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","input_batch_hash":"8db006d4ff44b64bce16c08a956d89f3cd1857864a847ab5c34ca3dd0469d841","kind":"optimizer_applied","monotonic":1127.831103685,"rollout_id":2,"seq":14,"wall_time":1790790615.1784863} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/e1b/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/e1b/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index 612cfaa16..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/e1b/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1,14 +0,0 @@ -{"body":{"deadline_s":600.0,"expected_config_epoch":0,"target":"T4R4S0"},"body_hash":"d13c83e0da4b99bc12d568249c1b63d50b6500b2bda3074c19b8c3bc78ef7286","deadline_wall":1790791055.9952173,"kind":"request","monotonic":968.647850764,"plan":{"expected_config_epoch":0,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"d13c83e0da4b99bc12d568249c1b63d50b6500b2bda3074c19b8c3bc78ef7286","source":"T4R2S2","source_engines":2,"target":"T4R4S0","target_engines":4,"trainer":null},"request_id":"b-up","seq":1,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790455.9952333} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":968.649737863,"phase":"VALIDATING","plan":{"expected_config_epoch":0,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"d13c83e0da4b99bc12d568249c1b63d50b6500b2bda3074c19b8c3bc78ef7286","source":"T4R2S2","source_engines":2,"target":"T4R4S0","target_engines":4,"trainer":null},"request_id":"b-up","seq":2,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790455.9971206} -{"allowed":true,"budget_s":null,"expected_pause_s":600.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":968.651451648,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":2,"seq":3,"stalls_peers":false,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790455.9988344} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":968.652644037,"phase":"WAIT_SAFE","request_id":"b-up","safe_point_rollout_id":2,"seq":4,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790456.0000267} -{"config_epoch":0,"cut":{"optimizer_step":2,"policy_hash":"0aa91476e6ff0509ba898730c888b05746fc21065cdf70cefe7347e4b5e16f8b","policy_token":"yeto:2:0aa91476e6ff0509ba898730c888b05746fc21065cdf70cefe7347e4b5e16f8b","published_version":2,"rollout_id":2,"unconsumed_batches":[]},"fork_epoch":0,"kind":"phase","monotonic":968.717519297,"phase":"QUIESCING","remove":[],"request_id":"b-up","seq":5,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790456.0649016} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":968.719073672,"phase":"INITIALIZING","request_id":"b-up","seq":6,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790456.0664563} -{"kind":"add_intent","members":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"monotonic":968.721760155,"seq":7,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790456.0691428} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":0,"kind":"fork_op","monotonic":968.723016433,"op":"start","seq":8,"status":"issued","tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790456.0703993} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":0,"kind":"fork_op","monotonic":1116.059891626,"op":"start","result_fork_epoch":1,"seq":9,"status":"done","tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790603.4072742} -{"config_epoch":0,"fork_epoch":1,"kind":"phase","monotonic":1116.062456121,"phase":"VERIFYING","request_id":"b-up","seq":10,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790603.409839} -{"config_epoch":0,"error":"PublicationError: member update_weights failed: This event loop is already running","fork_epoch":1,"kind":"phase","monotonic":1116.632046224,"phase":"REBUILD_OLD","request_id":"b-up","seq":11,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790603.9794288} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":1,"kind":"fork_op","monotonic":1116.633669178,"op":"stop","seq":12,"status":"issued","tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790603.981052} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":1,"kind":"fork_op","monotonic":1121.755707574,"op":"stop","result_fork_epoch":2,"seq":13,"status":"done","tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790609.1030903} -{"config_epoch":0,"error":"PublicationError: member update_weights failed: This event loop is already running","fork_epoch":2,"kind":"phase","monotonic":1121.76466561,"phase":"REBUILT_OLD","request_id":"b-up","seq":14,"tx_id":"tx-0-d13c83e0da4b-b-up","wall_time":1790790609.1120481} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/elastic-state/ledger/journal.jsonl deleted file mode 100644 index 52d72e517..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,20 +0,0 @@ -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","group_ids":["g0","g1","g2","g3"],"kind":"prepared","monotonic":941.333162001,"policy_token":"yeto:0:c65b6a4d0c9dc0ff99b65278e9410b13a5b953f175f2dbde702c3df255b79213","rollout_id":0,"samples":32,"seq":1,"wall_time":1790788909.0933027} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":941.33618999,"rollout_id":0,"seq":2,"wall_time":1790788909.0963311} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":941.337660014,"rollout_id":0,"seq":3,"wall_time":1790788909.0978012} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","input_batch_hash":"09c86025cbe7393dd354c7055d163d9068ab0812f6dc200ea8cbcf941f795f07","kind":"optimizer_applied","monotonic":955.580968943,"rollout_id":0,"seq":4,"wall_time":1790788923.34111} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","kind":"outer_recorded","monotonic":956.015567506,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790788923.7757084} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","group_ids":["g4","g5","g6","g7"],"kind":"prepared","monotonic":958.894029052,"policy_token":"yeto:1:ca1f75f8a797641d52f2bc8813c21e5d550e3bde9b0d77f5597f2f15fa6a3927","rollout_id":1,"samples":32,"seq":6,"wall_time":1790788926.65417} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":958.897210619,"rollout_id":1,"seq":7,"wall_time":1790788926.6573517} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":958.898862152,"rollout_id":1,"seq":8,"wall_time":1790788926.6590035} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","input_batch_hash":"f8498d1c3d8e7109d208e7d8e3c2175f3a43365518a79d76f0fb6b9c808e2486","kind":"optimizer_applied","monotonic":962.596014745,"rollout_id":1,"seq":9,"wall_time":1790788930.3561559} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","kind":"outer_recorded","monotonic":963.027474052,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790788930.787615} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","group_ids":["g10","g11","g8","g9"],"kind":"prepared","monotonic":1113.69941419,"policy_token":"yeto:2:3444bfed46ac106bab67c4f008e9377bd5ea5dd9d214b87477e96324c7cccf73","rollout_id":2,"samples":32,"seq":11,"wall_time":1790789081.4595554} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1113.702946418,"rollout_id":2,"seq":12,"wall_time":1790789081.4630876} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1113.704582085,"rollout_id":2,"seq":13,"wall_time":1790789081.464723} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","input_batch_hash":"0914d3dab703be9ad94f68c75d22acb6c4a8765daefe284f144c78ba6bf42d97","kind":"optimizer_applied","monotonic":1117.548827395,"rollout_id":2,"seq":14,"wall_time":1790789085.3089685} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","kind":"outer_recorded","monotonic":1117.975932432,"next_policy_version":3,"rollout_id":2,"seq":15,"wall_time":1790789085.7360733} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","group_ids":["g12","g13","g14","g15"],"kind":"prepared","monotonic":1120.761859035,"policy_token":"yeto:3:5750b367a07931bb7133fcd85199f9d6d2bd69631d02404430acde78b08fcf85","rollout_id":3,"samples":32,"seq":16,"wall_time":1790789088.522} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":1120.764714636,"rollout_id":3,"seq":17,"wall_time":1790789088.5248559} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":1120.766107368,"rollout_id":3,"seq":18,"wall_time":1790789088.5262485} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","input_batch_hash":"aab991ac83e1e71d270f2a68b847644526ac3723fb6c1812b04077de47e5d9af","kind":"optimizer_applied","monotonic":1124.44331642,"rollout_id":3,"seq":19,"wall_time":1790789092.2034576} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","kind":"outer_recorded","monotonic":1124.858487263,"next_policy_version":4,"rollout_id":3,"seq":20,"wall_time":1790789092.618628} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index d1267987c..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1,16 +0,0 @@ -{"body":{"deadline_s":120.0,"expected_config_epoch":0,"target":"T4R4S0"},"body_hash":"b5d74b5bf6b7e03c0d3775cf4796f8a8a5dc3b4056e2e56a3f4017150258283a","deadline_wall":1790789051.5769036,"kind":"request","monotonic":963.816778041,"plan":{"expected_config_epoch":0,"expected_pause_s":120.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"b5d74b5bf6b7e03c0d3775cf4796f8a8a5dc3b4056e2e56a3f4017150258283a","source":"T4R2S2","source_engines":2,"target":"T4R4S0","target_engines":4,"trainer":null},"request_id":"wd-up","seq":1,"tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790788931.576919} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":963.819112217,"phase":"VALIDATING","plan":{"expected_config_epoch":0,"expected_pause_s":120.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"b5d74b5bf6b7e03c0d3775cf4796f8a8a5dc3b4056e2e56a3f4017150258283a","source":"T4R2S2","source_engines":2,"target":"T4R4S0","target_engines":4,"trainer":null},"request_id":"wd-up","seq":2,"tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790788931.5792534} -{"allowed":true,"budget_s":null,"expected_pause_s":120.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":963.821079187,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":2,"seq":3,"stalls_peers":false,"tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790788931.5812204} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":963.82355817,"phase":"WAIT_SAFE","request_id":"wd-up","safe_point_rollout_id":2,"seq":4,"tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790788931.5836995} -{"config_epoch":0,"cut":{"optimizer_step":2,"policy_hash":"3444bfed46ac106bab67c4f008e9377bd5ea5dd9d214b87477e96324c7cccf73","policy_token":"yeto:2:3444bfed46ac106bab67c4f008e9377bd5ea5dd9d214b87477e96324c7cccf73","published_version":2,"rollout_id":2,"unconsumed_batches":[]},"fork_epoch":0,"kind":"phase","monotonic":963.888066139,"phase":"QUIESCING","remove":[],"request_id":"wd-up","seq":5,"tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790788931.6482072} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":963.889954291,"phase":"INITIALIZING","request_id":"wd-up","seq":6,"tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790788931.6500955} -{"kind":"add_intent","members":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"monotonic":963.892850952,"seq":7,"tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790788931.652992} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":0,"kind":"fork_op","monotonic":963.894115474,"op":"start","seq":8,"status":"issued","tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790788931.6542566} -{"kind":"watchdog","monotonic":1083.817085359,"note":"absolute transaction deadline passed while a step was running","phase":"INITIALIZING","seq":9,"target_cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790789051.5772262} -{"errors":[{"cell":"engine:inference-engine-all-0-0-00002","error":"RayTaskError(AssertionError)(AssertionError(\"cell_id='engine:inference-engine-all-0-0-00002' matches=[]\"))"},{"cell":"engine:inference-engine-all-0-0-00003","error":"RayTaskError(AssertionError)(AssertionError(\"cell_id='engine:inference-engine-all-0-0-00003' matches=[]\"))"}],"killed":[],"kind":"watchdog_action","monotonic":1083.826458067,"phase":"INITIALIZING","seq":10,"tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790789051.5865989} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":0,"kind":"fork_op","monotonic":1106.504279015,"op":"start","result_fork_epoch":1,"seq":11,"status":"done","tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790789074.26442} -{"config_epoch":0,"error":"INITIALIZING: transaction deadline passed during engine start","fork_epoch":1,"kind":"phase","monotonic":1106.507194524,"phase":"REBUILD_OLD","request_id":"wd-up","seq":12,"tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790789074.2673357} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":1,"kind":"fork_op","monotonic":1106.508584231,"op":"stop","seq":13,"status":"issued","tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790789074.2687254} -{"cells":["engine:inference-engine-all-0-0-00002","engine:inference-engine-all-0-0-00003"],"expected_fork_epoch":1,"kind":"fork_op","monotonic":1111.628586375,"op":"stop","result_fork_epoch":2,"seq":14,"status":"done","tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790789079.3887277} -{"config_epoch":0,"error":"INITIALIZING: transaction deadline passed during engine start","fork_epoch":2,"kind":"phase","monotonic":1111.638487114,"phase":"REBUILT_OLD","request_id":"wd-up","seq":15,"tx_id":"tx-0-b5d74b5bf6b7-wd-up","wall_time":1790789079.398628} -{"kind":"finalization","monotonic":1125.639866782,"rollout_id":4,"seq":16,"wall_time":1790789093.400008} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/watchdog.out b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/watchdog.out deleted file mode 100644 index daa1e3b92..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/a4s3/watchdog/watchdog.out +++ /dev/null @@ -1,2 +0,0 @@ -Cluster(s) not found: infra-v2-b1-a4wd-20260930-3-l0-eu-north1. -Cluster(s) not found (tip: see `sky status`). diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1/elastic-state/ledger/journal.jsonl deleted file mode 100644 index fa92f0552..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,23 +0,0 @@ -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","group_ids":["g0","g1","g2","g3"],"kind":"prepared","monotonic":418.036770645,"policy_token":"yeto:0:c1573d237d410918ad6f36e0e0692faa04e5f8d1e6ec95e3b3411d46aa917797","rollout_id":0,"samples":32,"seq":1,"wall_time":1790742232.1719716} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":418.037208087,"rollout_id":0,"seq":2,"wall_time":1790742232.1724088} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":418.037490436,"rollout_id":0,"seq":3,"wall_time":1790742232.1726918} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","input_batch_hash":"68d333551d6449f25b8433187bba5a353808cdec8ce099897a9023a46f9bc6a6","kind":"optimizer_applied","monotonic":455.134680065,"rollout_id":0,"seq":4,"wall_time":1790742269.2698815} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","kind":"outer_recorded","monotonic":455.776013334,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790742269.9112146} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","group_ids":["g4","g5","g6","g7"],"kind":"prepared","monotonic":461.226240106,"policy_token":"yeto:1:9cac4a12070727f8867b1811066b3046b3f6c8ad5842327c229ad65d9b5a8541","rollout_id":1,"samples":32,"seq":6,"wall_time":1790742275.3614416} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":461.2267199,"rollout_id":1,"seq":7,"wall_time":1790742275.3619213} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":461.22706467,"rollout_id":1,"seq":8,"wall_time":1790742275.362266} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","input_batch_hash":"2d99e30929ed8b6c11918427e04a2471075d07f9d7bed774aa9d2c743909dfec","kind":"optimizer_applied","monotonic":477.536421989,"rollout_id":1,"seq":9,"wall_time":1790742291.6716235} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","kind":"outer_recorded","monotonic":478.158190874,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790742292.2933922} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","group_ids":["g10","g11","g8","g9"],"kind":"prepared","monotonic":483.69591047,"policy_token":"yeto:2:b5c461c1f9736fc3156f34998baeda92a77ab7707a57cf6689487ec55e8c6512","rollout_id":2,"samples":32,"seq":11,"wall_time":1790742297.831112} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":483.696437185,"rollout_id":2,"seq":12,"wall_time":1790742297.8316383} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":483.696792475,"rollout_id":2,"seq":13,"wall_time":1790742297.8319936} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","input_batch_hash":"01672fb4d0d19f62363612ab5d396f171b800f0558586956b3b4e08ca153c598","kind":"optimizer_applied","monotonic":499.837041816,"rollout_id":2,"seq":14,"wall_time":1790742313.9722433} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","kind":"outer_recorded","monotonic":500.398581571,"next_policy_version":3,"rollout_id":2,"seq":15,"wall_time":1790742314.5337827} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","group_ids":["g12","g13","g14","g15"],"kind":"prepared","monotonic":505.501746126,"policy_token":"yeto:3:df036c1262fbbc189b2c23920c8d1e3a193e346a8e1ff80e01ad57130d9a8c9a","rollout_id":3,"samples":32,"seq":16,"wall_time":1790742319.6369474} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":505.502300152,"rollout_id":3,"seq":17,"wall_time":1790742319.6375015} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":505.502630142,"rollout_id":3,"seq":18,"wall_time":1790742319.6378314} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","input_batch_hash":"c8103f35470b33ed0a6bb34696273c0bf6038e72d79042326d2a609370461a4b","kind":"optimizer_applied","monotonic":521.30799933,"rollout_id":3,"seq":19,"wall_time":1790742335.4432008} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","kind":"outer_recorded","monotonic":521.867924983,"next_policy_version":4,"rollout_id":3,"seq":20,"wall_time":1790742336.0031264} -{"attempt":0,"batch_hash":"7328d450bd7fdc43c18fca9149e109342e83a89da53a2ea7d497e99fc4236ca2","group_ids":["g16","g17","g18","g19"],"kind":"prepared","monotonic":527.283936829,"policy_token":"yeto:4:d63ba64ac8581628139ac25c01ff35230e94162ee46f58f5d5611e7ffdfdd315","rollout_id":4,"samples":32,"seq":21,"wall_time":1790742341.4191382} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":527.284445844,"rollout_id":4,"seq":22,"wall_time":1790742341.419647} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":527.284793714,"rollout_id":4,"seq":23,"wall_time":1790742341.4199948} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index 77434b74b..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"body":{"deadline_s":600.0,"expected_config_epoch":0,"target":"T1R2S0"},"body_hash":"a863ab500960bb593d339883ae0aed04f0470a6d79df0148c53cda764dafeee6","deadline_wall":1790742893.3280137,"kind":"request","monotonic":479.192823888,"plan":{"expected_config_epoch":0,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"a863ab500960bb593d339883ae0aed04f0470a6d79df0148c53cda764dafeee6","source":"T1R1S1","source_engines":1,"target":"T1R2S0","target_engines":2,"trainer":null},"request_id":"fe1-up","seq":1,"tx_id":"tx-0-a863ab500960-fe1-up","wall_time":1790742293.3280253} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":479.193150158,"phase":"VALIDATING","plan":{"expected_config_epoch":0,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"a863ab500960bb593d339883ae0aed04f0470a6d79df0148c53cda764dafeee6","source":"T1R1S1","source_engines":1,"target":"T1R2S0","target_engines":2,"trainer":null},"request_id":"fe1-up","seq":2,"tx_id":"tx-0-a863ab500960-fe1-up","wall_time":1790742293.3283515} -{"allowed":true,"budget_s":null,"expected_pause_s":600.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":479.195499847,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":2,"seq":3,"stalls_peers":false,"tx_id":"tx-0-a863ab500960-fe1-up","wall_time":1790742293.3307014} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":479.195765175,"phase":"WAIT_SAFE","request_id":"fe1-up","safe_point_rollout_id":2,"seq":4,"tx_id":"tx-0-a863ab500960-fe1-up","wall_time":1790742293.3309665} -{"config_epoch":0,"cut":{"optimizer_step":2,"policy_hash":"b5c461c1f9736fc3156f34998baeda92a77ab7707a57cf6689487ec55e8c6512","policy_token":"yeto:2:b5c461c1f9736fc3156f34998baeda92a77ab7707a57cf6689487ec55e8c6512","published_version":2,"rollout_id":2,"unconsumed_batches":[]},"fork_epoch":0,"kind":"phase","monotonic":479.265790816,"phase":"QUIESCING","remove":[],"request_id":"fe1-up","seq":5,"tx_id":"tx-0-a863ab500960-fe1-up","wall_time":1790742293.400992} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":479.266336492,"phase":"INITIALIZING","request_id":"fe1-up","seq":6,"tx_id":"tx-0-a863ab500960-fe1-up","wall_time":1790742293.4015374} -{"kind":"add_intent","members":["engine:c0"],"monotonic":479.269726922,"seq":7,"tx_id":"tx-0-a863ab500960-fe1-up","wall_time":1790742293.4049282} -{"cells":["engine:c0"],"expected_fork_epoch":0,"kind":"fork_op","monotonic":479.270080342,"op":"start","seq":8,"status":"issued","tx_id":"tx-0-a863ab500960-fe1-up","wall_time":1790742293.4052818} -{"cells":["engine:c0"],"error":"\u001b[36mray::InferenceController.start_cells()\u001b[39m (pid=2911, ip=172.20.3.100, actor_id=8e55ec011e374f7e12bac52d01000000, repr=)\n File \"/usr/lib/python3.12/concurrent/futures/_base.py\", line 456, in result\n return self.__get_result()\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.12/concurrent/futures/_base.py\", line 401, in __get_result\n raise self._exception\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/utils/context_lock.py\", line 155, in wrapper\n return await fn(self, *args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/ray/rollout/inference_controller.py\", line 253, in start_cells\n epoch = await self._change_membership(\"start\", cell_ids, expected_epoch=expected_epoch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/utils/context_lock.py\", line 205, in wrapper\n return await fn(self, *args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/ray/rollout/inference_controller.py\", line 289, in _change_membership\n raise KeyError(f\"cells {unknown} were not declared at startup; declared cells are {sorted(declared)}\")\nKeyError: \"cells ['c0'] were not declared at startup; declared cells are ['inference-engine-all-0-0-00000']\"","expected_fork_epoch":0,"fork_epoch":0,"kind":"fork_op","monotonic":479.281751814,"op":"start","seq":9,"status":"failed","tx_id":"tx-0-a863ab500960-fe1-up","wall_time":1790742293.4169533} -{"config_epoch":0,"error":"INITIALIZING: start failed: \u001b[36mray::InferenceController.start_cells()\u001b[39m (pid=2911, ip=172.20.3.100, actor_id=8e55ec011e374f7e12bac52d01000000, repr=)\n File \"/usr/lib/python3.12/concurrent/futures/_base.py\", line 456, in result\n return self.__get_result()\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.12/concurrent/futures/_base.py\", line 401, in __get_result\n raise self._exception\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/utils/context_lock.py\", line 155, in wrapper\n return await fn(self, *args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/ray/rollout/inference_controller.py\", line 253, in start_cells\n epoch = await self._change_membership(\"start\", cell_ids, expected_epoch=expected_epoch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/utils/context_lock.py\", line 205, in wrapper\n return await fn(self, *args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/ray/rollout/inference_controller.py\", line 289, in _change_membership\n raise KeyError(f\"cells {unknown} were not declared at startup; declared cells are {sorted(declared)}\")\nKeyError: \"cells ['c0'] were not declared at startup; declared cells are ['inference-engine-all-0-0-00000']\"","fork_epoch":0,"kind":"phase","monotonic":479.282249999,"phase":"REBUILD_OLD","request_id":"fe1-up","seq":10,"tx_id":"tx-0-a863ab500960-fe1-up","wall_time":1790742293.4174514} -{"config_epoch":0,"error":"INITIALIZING: start failed: \u001b[36mray::InferenceController.start_cells()\u001b[39m (pid=2911, ip=172.20.3.100, actor_id=8e55ec011e374f7e12bac52d01000000, repr=)\n File \"/usr/lib/python3.12/concurrent/futures/_base.py\", line 456, in result\n return self.__get_result()\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.12/concurrent/futures/_base.py\", line 401, in __get_result\n raise self._exception\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/utils/context_lock.py\", line 155, in wrapper\n return await fn(self, *args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/ray/rollout/inference_controller.py\", line 253, in start_cells\n epoch = await self._change_membership(\"start\", cell_ids, expected_epoch=expected_epoch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/utils/context_lock.py\", line 205, in wrapper\n return await fn(self, *args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/ray/rollout/inference_controller.py\", line 289, in _change_membership\n raise KeyError(f\"cells {unknown} were not declared at startup; declared cells are {sorted(declared)}\")\nKeyError: \"cells ['c0'] were not declared at startup; declared cells are ['inference-engine-all-0-0-00000']\"","fork_epoch":0,"kind":"phase","monotonic":479.288843662,"phase":"REBUILT_OLD","request_id":"fe1-up","seq":11,"tx_id":"tx-0-a863ab500960-fe1-up","wall_time":1790742293.424045} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r/fe1r/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r/fe1r/elastic-state/ledger/journal.jsonl deleted file mode 100644 index 942b3683a..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r/fe1r/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,8 +0,0 @@ -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","group_ids":["g0","g1","g2","g3"],"kind":"prepared","monotonic":424.362172434,"policy_token":"yeto:0:c1573d237d410918ad6f36e0e0692faa04e5f8d1e6ec95e3b3411d46aa917797","rollout_id":0,"samples":32,"seq":1,"wall_time":1790751602.4063828} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":424.362646544,"rollout_id":0,"seq":2,"wall_time":1790751602.406857} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":424.362937291,"rollout_id":0,"seq":3,"wall_time":1790751602.4071484} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","input_batch_hash":"68d333551d6449f25b8433187bba5a353808cdec8ce099897a9023a46f9bc6a6","kind":"optimizer_applied","monotonic":461.284497012,"rollout_id":0,"seq":4,"wall_time":1790751639.3287082} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","kind":"outer_recorded","monotonic":461.950750559,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790751639.9949613} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","group_ids":["g4","g5","g6","g7"],"kind":"prepared","monotonic":601.959839509,"policy_token":"yeto:1:d2ef4fe978a3b6ad24bed7d374d95dd14416ea3b0c838ea1533092ce3262798d","rollout_id":1,"samples":32,"seq":6,"wall_time":1790751780.0040505} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":601.96028552,"rollout_id":1,"seq":7,"wall_time":1790751780.0044963} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":601.960632219,"rollout_id":1,"seq":8,"wall_time":1790751780.004843} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r/fe1r/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r/fe1r/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index ef8542909..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r/fe1r/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1,14 +0,0 @@ -{"body":{"deadline_s":600.0,"expected_config_epoch":0,"target":"T1R2S0"},"body_hash":"a863ab500960bb593d339883ae0aed04f0470a6d79df0148c53cda764dafeee6","deadline_wall":1790752241.1075616,"kind":"request","monotonic":463.063361828,"plan":{"expected_config_epoch":0,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"a863ab500960bb593d339883ae0aed04f0470a6d79df0148c53cda764dafeee6","source":"T1R1S1","source_engines":1,"target":"T1R2S0","target_engines":2,"trainer":null},"request_id":"up1","seq":1,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790751641.107573} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":463.063741167,"phase":"VALIDATING","plan":{"expected_config_epoch":0,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"a863ab500960bb593d339883ae0aed04f0470a6d79df0148c53cda764dafeee6","source":"T1R1S1","source_engines":1,"target":"T1R2S0","target_engines":2,"trainer":null},"request_id":"up1","seq":2,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790751641.107952} -{"allowed":true,"budget_s":null,"expected_pause_s":600.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":463.065887189,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":1,"seq":3,"stalls_peers":false,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790751641.110098} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":463.066142925,"phase":"WAIT_SAFE","request_id":"up1","safe_point_rollout_id":1,"seq":4,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790751641.110354} -{"config_epoch":0,"cut":{"optimizer_step":1,"policy_hash":"d2ef4fe978a3b6ad24bed7d374d95dd14416ea3b0c838ea1533092ce3262798d","policy_token":"yeto:1:d2ef4fe978a3b6ad24bed7d374d95dd14416ea3b0c838ea1533092ce3262798d","published_version":1,"rollout_id":1,"unconsumed_batches":[]},"fork_epoch":0,"kind":"phase","monotonic":463.137348335,"phase":"QUIESCING","remove":[],"request_id":"up1","seq":5,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790751641.1815588} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":463.137811096,"phase":"INITIALIZING","request_id":"up1","seq":6,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790751641.182022} -{"kind":"add_intent","members":["engine:inference-engine-all-0-0-00001"],"monotonic":463.142042569,"seq":7,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790751641.1862535} -{"cells":["engine:inference-engine-all-0-0-00001"],"expected_fork_epoch":0,"kind":"fork_op","monotonic":463.142406978,"op":"start","seq":8,"status":"issued","tx_id":"tx-0-a863ab500960-up1","wall_time":1790751641.1866179} -{"cells":["engine:inference-engine-all-0-0-00001"],"expected_fork_epoch":0,"kind":"fork_op","monotonic":591.708133038,"op":"start","result_fork_epoch":1,"seq":9,"status":"done","tx_id":"tx-0-a863ab500960-up1","wall_time":1790751769.7523441} -{"config_epoch":0,"fork_epoch":1,"kind":"phase","monotonic":591.7086124,"phase":"VERIFYING","request_id":"up1","seq":10,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790751769.7528236} -{"config_epoch":0,"error":"PublicationError: member update_weights failed: \u001b[36mray::InferenceController.start_update_weights()\u001b[39m (pid=2980, ip=172.20.0.95, actor_id=e97f98d84924396e451109cd01000000, repr=)\n File \"/usr/lib/python3.12/concurrent/futures/_base.py\", line 449, in result\n return self.__get_result()\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.12/concurrent/futures/_base.py\", line 401, in __get_result\n raise self._exception\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/utils/context_lock.py\", line 169, in wrapper\n result = await fn(self, *args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/ray/rollout/inference_controller.py\", line 621, in start_update_weights\n raise ValueError(\"admit_cordoned needs the Miles router (--use-miles-router)\")\nValueError: admit_cordoned needs the Miles router (--use-miles-router)","fork_epoch":1,"kind":"phase","monotonic":592.399368302,"phase":"REBUILD_OLD","request_id":"up1","seq":11,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790751770.4435794} -{"cells":["engine:inference-engine-all-0-0-00001"],"expected_fork_epoch":1,"kind":"fork_op","monotonic":592.399916176,"op":"stop","seq":12,"status":"issued","tx_id":"tx-0-a863ab500960-up1","wall_time":1790751770.444127} -{"cells":["engine:inference-engine-all-0-0-00001"],"expected_fork_epoch":1,"kind":"fork_op","monotonic":597.501764099,"op":"stop","result_fork_epoch":2,"seq":13,"status":"done","tx_id":"tx-0-a863ab500960-up1","wall_time":1790751775.545975} -{"config_epoch":0,"error":"PublicationError: member update_weights failed: \u001b[36mray::InferenceController.start_update_weights()\u001b[39m (pid=2980, ip=172.20.0.95, actor_id=e97f98d84924396e451109cd01000000, repr=)\n File \"/usr/lib/python3.12/concurrent/futures/_base.py\", line 449, in result\n return self.__get_result()\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.12/concurrent/futures/_base.py\", line 401, in __get_result\n raise self._exception\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/utils/context_lock.py\", line 169, in wrapper\n result = await fn(self, *args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/root/miles/miles/ray/rollout/inference_controller.py\", line 621, in start_update_weights\n raise ValueError(\"admit_cordoned needs the Miles router (--use-miles-router)\")\nValueError: admit_cordoned needs the Miles router (--use-miles-router)","fork_epoch":2,"kind":"phase","monotonic":597.510364635,"phase":"REBUILT_OLD","request_id":"up1","seq":14,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790751775.5545757} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r4/fe1r5/elastic-state/ledger/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r4/fe1r5/elastic-state/ledger/journal.jsonl deleted file mode 100644 index fec274af0..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r4/fe1r5/elastic-state/ledger/journal.jsonl +++ /dev/null @@ -1,23 +0,0 @@ -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","group_ids":["g0","g1","g2","g3"],"kind":"prepared","monotonic":333.358013917,"policy_token":"yeto:0:499b6e4a5c2d088c4683cf76edf6a3ae5cb55701ad75ee7618b04b28e74e8090","rollout_id":0,"samples":32,"seq":1,"wall_time":1790760750.5430722} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":333.358374007,"rollout_id":0,"seq":2,"wall_time":1790760750.5434325} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":333.358601593,"rollout_id":0,"seq":3,"wall_time":1790760750.5436604} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","input_batch_hash":"9ff157ea04f5f7e6175feb4e25957284aa1f003295e64c41525f5b73489fe576","kind":"optimizer_applied","monotonic":364.236166545,"rollout_id":0,"seq":4,"wall_time":1790760781.4212253} -{"attempt":0,"batch_hash":"566c157023d0d11a57fddb6526ac7666ea23e7e4944df2cd50de500e84d02630","kind":"outer_recorded","monotonic":364.769479256,"next_policy_version":1,"rollout_id":0,"seq":5,"wall_time":1790760781.9545376} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","group_ids":["g4","g5","g6","g7"],"kind":"prepared","monotonic":467.137062589,"policy_token":"yeto:1:32e42708d3cfb263a01fb5276e2a2cf74ddb39ec5d31c3c21ba807bdc0a074fb","rollout_id":1,"samples":32,"seq":6,"wall_time":1790760884.322121} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":467.137481291,"rollout_id":1,"seq":7,"wall_time":1790760884.3225398} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":467.137716848,"rollout_id":1,"seq":8,"wall_time":1790760884.3227756} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","input_batch_hash":"dacc4e8b7bc1c689a23bdde2a04f59815210d6f72c53c34c5a996a935b78819e","kind":"optimizer_applied","monotonic":483.069155848,"rollout_id":1,"seq":9,"wall_time":1790760900.2542148} -{"attempt":0,"batch_hash":"8c0aafe41f3dae4a8d6692cac383821700da8d94a45d8fe8d879e81d604c26da","kind":"outer_recorded","monotonic":484.252992585,"next_policy_version":2,"rollout_id":1,"seq":10,"wall_time":1790760901.4380507} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","group_ids":["g10","g11","g8","g9"],"kind":"prepared","monotonic":493.349978307,"policy_token":"yeto:2:fdece0609309b0ae1de7069abab18de620554a6e6eb8aeedff5c8748d18d5955","rollout_id":2,"samples":32,"seq":11,"wall_time":1790760910.535037} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":493.350438144,"rollout_id":2,"seq":12,"wall_time":1790760910.5354967} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":493.350697349,"rollout_id":2,"seq":13,"wall_time":1790760910.5357559} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","input_batch_hash":"c9d182cbe34ac326d40b3d1eef5cd00e777ff7e51c10b86251b9cbf98d67f0a5","kind":"optimizer_applied","monotonic":512.328414738,"rollout_id":2,"seq":14,"wall_time":1790760929.5134735} -{"attempt":0,"batch_hash":"bb660c0fbe7634ecb3d0897efa0d1aa9825f247151d4589b5c44ace9f81979b1","kind":"outer_recorded","monotonic":512.786609717,"next_policy_version":3,"rollout_id":2,"seq":15,"wall_time":1790760929.971668} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","group_ids":["g12","g13","g14","g15"],"kind":"prepared","monotonic":526.198850355,"policy_token":"yeto:3:9065fc2b9d18002d3423ebd34e183aed48db002b1c692ab7725c7b55bd047295","rollout_id":3,"samples":32,"seq":16,"wall_time":1790760943.3839092} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":526.199326886,"rollout_id":3,"seq":17,"wall_time":1790760943.3843856} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":526.199617736,"rollout_id":3,"seq":18,"wall_time":1790760943.384676} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","input_batch_hash":"6bd4ac0715627efc1494e7cf13175d87cb2a2e767b01b0d3642d98c282de315c","kind":"optimizer_applied","monotonic":542.768361371,"rollout_id":3,"seq":19,"wall_time":1790760959.95342} -{"attempt":0,"batch_hash":"15b25426f12aaf738d0761ee55320b2cd0d34993883682e2beff23d16254ce12","kind":"outer_recorded","monotonic":543.242506794,"next_policy_version":4,"rollout_id":3,"seq":20,"wall_time":1790760960.427565} -{"attempt":0,"batch_hash":"7328d450bd7fdc43c18fca9149e109342e83a89da53a2ea7d497e99fc4236ca2","group_ids":["g16","g17","g18","g19"],"kind":"prepared","monotonic":551.738687209,"policy_token":"yeto:4:ce8cee2f6738d272d95784c284809b9f6f5cb2bcaf3e00019e035d139f00e954","rollout_id":4,"samples":32,"seq":21,"wall_time":1790760968.9237459} -{"attempt":0,"detail":{"groups":0,"mechanism":"rollout_meta_hook.record_trained_groups","reason":"completed groups not trained this round: dynamic-filter drops and over-sampling surplus (not returned to the buffer)"},"kind":"filtered","monotonic":551.739086422,"rollout_id":4,"seq":22,"wall_time":1790760968.924145} -{"attempt":0,"buffer_length":0,"carried_over":0,"carried_over_reported":true,"kind":"carried_over_report","monotonic":551.739386339,"rollout_id":4,"seq":23,"wall_time":1790760968.924445} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r4/fe1r5/elastic-state/reconfig/journal.jsonl b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r4/fe1r5/elastic-state/reconfig/journal.jsonl deleted file mode 100644 index c3b762b11..000000000 --- a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/fe1r4/fe1r5/elastic-state/reconfig/journal.jsonl +++ /dev/null @@ -1,26 +0,0 @@ -{"body":{"deadline_s":600.0,"expected_config_epoch":0,"target":"T1R2S0"},"body_hash":"a863ab500960bb593d339883ae0aed04f0470a6d79df0148c53cda764dafeee6","deadline_wall":1790761382.870328,"kind":"request","monotonic":365.685281849,"plan":{"expected_config_epoch":0,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"a863ab500960bb593d339883ae0aed04f0470a6d79df0148c53cda764dafeee6","source":"T1R1S1","source_engines":1,"target":"T1R2S0","target_engines":2,"trainer":null},"request_id":"up1","seq":1,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790760782.8703406} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":365.685596614,"phase":"VALIDATING","plan":{"expected_config_epoch":0,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"a863ab500960bb593d339883ae0aed04f0470a6d79df0148c53cda764dafeee6","source":"T1R1S1","source_engines":1,"target":"T1R2S0","target_engines":2,"trainer":null},"request_id":"up1","seq":2,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790760782.870655} -{"allowed":true,"budget_s":null,"expected_pause_s":600.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":365.687238531,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":1,"seq":3,"stalls_peers":false,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790760782.8722973} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":365.6874356,"phase":"WAIT_SAFE","request_id":"up1","safe_point_rollout_id":1,"seq":4,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790760782.872494} -{"config_epoch":0,"cut":{"optimizer_step":1,"policy_hash":"32e42708d3cfb263a01fb5276e2a2cf74ddb39ec5d31c3c21ba807bdc0a074fb","policy_token":"yeto:1:32e42708d3cfb263a01fb5276e2a2cf74ddb39ec5d31c3c21ba807bdc0a074fb","published_version":1,"rollout_id":1,"unconsumed_batches":[]},"fork_epoch":0,"kind":"phase","monotonic":365.759097132,"phase":"QUIESCING","remove":[],"request_id":"up1","seq":5,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790760782.9441555} -{"config_epoch":0,"fork_epoch":0,"kind":"phase","monotonic":365.759606184,"phase":"INITIALIZING","request_id":"up1","seq":6,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790760782.9446647} -{"kind":"add_intent","members":["engine:inference-engine-all-0-0-00001"],"monotonic":365.761529193,"seq":7,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790760782.9465878} -{"cells":["engine:inference-engine-all-0-0-00001"],"expected_fork_epoch":0,"kind":"fork_op","monotonic":365.761997341,"op":"start","seq":8,"status":"issued","tx_id":"tx-0-a863ab500960-up1","wall_time":1790760782.9470558} -{"cells":["engine:inference-engine-all-0-0-00001"],"expected_fork_epoch":0,"kind":"fork_op","monotonic":457.749504439,"op":"start","result_fork_epoch":1,"seq":9,"status":"done","tx_id":"tx-0-a863ab500960-up1","wall_time":1790760874.9345632} -{"config_epoch":0,"fork_epoch":1,"kind":"phase","monotonic":457.749918574,"phase":"VERIFYING","request_id":"up1","seq":10,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790760874.9349773} -{"kind":"weight_admission","manifest_hash":"3539eb6605538a341664c4e9458be30e04b3f4b01c13ba909dc1c0c93dccb639","members":["engine:inference-engine-all-0-0-00001"],"monotonic":459.021404728,"policy_token":"yeto:1:32e42708d3cfb263a01fb5276e2a2cf74ddb39ec5d31c3c21ba807bdc0a074fb","seq":11,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790760876.2064636} -{"config_epoch":1,"fork_epoch":1,"kind":"phase","members":["engine:inference-engine-all-0-0-00000","engine:inference-engine-all-0-0-00001"],"monotonic":459.100960673,"phase":"COMMITTED","request_id":"up1","seq":12,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790760876.2860188} -{"config_epoch":1,"fork_epoch":1,"kind":"phase","monotonic":459.101251621,"phase":"RESUMING","request_id":"up1","seq":13,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790760876.2863102} -{"config_epoch":1,"config_epoch_to":1,"fork_epoch":1,"kind":"phase","monotonic":459.101578179,"phase":"SUCCEEDED","request_id":"up1","seq":14,"tx_id":"tx-0-a863ab500960-up1","wall_time":1790760876.2866368} -{"body":{"deadline_s":600.0,"expected_config_epoch":1,"target":"T1R1S1"},"body_hash":"1a22d1c53ef961ad4ef9d6463aa923f2cb87a7675c59fb7c9e10fc9ba8bbd79e","deadline_wall":1790761530.8933477,"kind":"request","monotonic":513.708300421,"plan":{"expected_config_epoch":1,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"1a22d1c53ef961ad4ef9d6463aa923f2cb87a7675c59fb7c9e10fc9ba8bbd79e","source":"T1R2S0","source_engines":2,"target":"T1R1S1","target_engines":1,"trainer":null},"request_id":"down1","seq":15,"tx_id":"tx-1-1a22d1c53ef9-down1","wall_time":1790760930.8933592} -{"config_epoch":1,"fork_epoch":1,"kind":"phase","monotonic":513.708619426,"phase":"VALIDATING","plan":{"expected_config_epoch":1,"expected_pause_s":600.0,"kind":"rollout-only","pause_budget_s":null,"profile_hash":"sha256:6039d75ff8514328f4edc2e2d341f2e20f230b299b905a69c6a463ea499e12f3","request_body_hash":"1a22d1c53ef961ad4ef9d6463aa923f2cb87a7675c59fb7c9e10fc9ba8bbd79e","source":"T1R2S0","source_engines":2,"target":"T1R1S1","target_engines":1,"trainer":null},"request_id":"down1","seq":16,"tx_id":"tx-1-1a22d1c53ef9-down1","wall_time":1790760930.8936782} -{"allowed":true,"budget_s":null,"expected_pause_s":600.0,"idle_flow_timeout_s":null,"kind":"pause_decision","margin":0.5,"monotonic":513.710557208,"outer_phase":"round-boundary-published","quorum_timeout_s":900.0,"reason":"LocalOnlySync: no syncer connection; only in-island collectives must be quiescent","rollout_id":3,"seq":17,"stalls_peers":false,"tx_id":"tx-1-1a22d1c53ef9-down1","wall_time":1790760930.8956158} -{"config_epoch":1,"fork_epoch":1,"kind":"phase","monotonic":513.71079737,"phase":"WAIT_SAFE","request_id":"down1","safe_point_rollout_id":3,"seq":18,"tx_id":"tx-1-1a22d1c53ef9-down1","wall_time":1790760930.8958561} -{"config_epoch":1,"cut":{"optimizer_step":3,"policy_hash":"9065fc2b9d18002d3423ebd34e183aed48db002b1c692ab7725c7b55bd047295","policy_token":"yeto:3:9065fc2b9d18002d3423ebd34e183aed48db002b1c692ab7725c7b55bd047295","published_version":3,"rollout_id":3,"unconsumed_batches":[]},"fork_epoch":1,"kind":"phase","monotonic":513.785627709,"phase":"QUIESCING","remove":["engine:inference-engine-all-0-0-00001"],"request_id":"down1","seq":19,"tx_id":"tx-1-1a22d1c53ef9-down1","wall_time":1790760930.970686} -{"config_epoch":1,"fork_epoch":1,"kind":"phase","monotonic":513.793471198,"phase":"TRANSFERRING","release":["engine:inference-engine-all-0-0-00001"],"request_id":"down1","rollback_boundary":"release","seq":20,"tx_id":"tx-1-1a22d1c53ef9-down1","wall_time":1790760930.97853} -{"cells":["engine:inference-engine-all-0-0-00001"],"expected_fork_epoch":1,"kind":"fork_op","monotonic":513.7941796,"op":"stop","seq":21,"status":"issued","tx_id":"tx-1-1a22d1c53ef9-down1","wall_time":1790760930.979238} -{"cells":["engine:inference-engine-all-0-0-00001"],"expected_fork_epoch":1,"kind":"fork_op","monotonic":518.885472097,"op":"stop","result_fork_epoch":2,"seq":22,"status":"done","tx_id":"tx-1-1a22d1c53ef9-down1","wall_time":1790760936.070531} -{"config_epoch":1,"fork_epoch":2,"kind":"phase","monotonic":518.885888047,"phase":"VERIFYING","request_id":"down1","seq":23,"tx_id":"tx-1-1a22d1c53ef9-down1","wall_time":1790760936.0709465} -{"config_epoch":2,"fork_epoch":2,"kind":"phase","members":["engine:inference-engine-all-0-0-00000"],"monotonic":518.968512161,"phase":"COMMITTED","request_id":"down1","seq":24,"tx_id":"tx-1-1a22d1c53ef9-down1","wall_time":1790760936.1535707} -{"config_epoch":2,"fork_epoch":2,"kind":"phase","monotonic":518.968748342,"phase":"RESUMING","request_id":"down1","seq":25,"tx_id":"tx-1-1a22d1c53ef9-down1","wall_time":1790760936.153807} -{"config_epoch":2,"config_epoch_to":2,"fork_epoch":2,"kind":"phase","monotonic":518.969018726,"phase":"SUCCEEDED","request_id":"down1","seq":26,"tx_id":"tx-1-1a22d1c53ef9-down1","wall_time":1790760936.1540775} diff --git a/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/nsmoke/launcher.log b/openspec/changes/rl-infra-spec/evidence/infra-v2-b1/nsmoke/launcher.log deleted file mode 100644 index e69de29bb..000000000 diff --git a/openspec/changes/rl-infra-spec/evidence/ports-image/2026-10-02-m3a27/check-old.log b/openspec/changes/rl-infra-spec/evidence/ports-image/2026-10-02-m3a27/check-old.log deleted file mode 100644 index 5165f52d6..000000000 --- a/openspec/changes/rl-infra-spec/evidence/ports-image/2026-10-02-m3a27/check-old.log +++ /dev/null @@ -1,15 +0,0 @@ -PASS miles_head: e3a11ab38cbb7fd911b23fdd62a4eb6dfbb1c841 -PASS sglang_head: a1240c530d406b5d0f252511f8e0a9a93b6a079d -PASS sglang_version: 0.5.21.dev68+ga1240c5 -PASS sglang_dist_info: 1 -PASS manifest_miles: e3a11ab38cbb7fd911b23fdd62a4eb6dfbb1c841 -PASS manifest_sglang: a1240c530d406b5d0f252511f8e0a9a93b6a079d -PASS miles_origin: https://github.com/michaellchung/miles -PASS sglang_origin: https://github.com/michaellchung/sglang -FAIL m3_lora_plugin_present: got want yes -FAIL qwen4exp_lora_hooks: got no want yes -FAIL a27_workers_lost: got no want yes -megatron_head: a84b105473eca9c9c6e49dbf1c8aa460c5cf796e -PASS d2_hc_head_contraction: yes -out=/tmp/img-m3a27/check-old -exit=1 diff --git a/tests/test_fn_learning_signal.py b/tests/test_fn_learning_signal.py index 7ef152221..3a7cf412a 100644 --- a/tests/test_fn_learning_signal.py +++ b/tests/test_fn_learning_signal.py @@ -15,7 +15,9 @@ REPO = Path(__file__).resolve().parents[1] MG = REPO / "tests" / "multinode_gpu" -TRY27 = (REPO / "openspec/changes/rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fna/pulled" +# Run logs are kept outside the repository (rl-code-quality-cleanup 0.3). +EVIDENCE = Path(os.environ.get("YETO_EVIDENCE_DIR", "/home/michael/work/infra-drafts/evidence")) +TRY27 = (EVIDENCE / "rl-infra-spec/evidence/fn/try27-s12-h100-20261006a/fna/pulled" / "rl-island-0.jsonl") sys.path.insert(0, str(REPO / "scripts")) import judge_fn_learning_signal as jfs # noqa: E402 @@ -50,6 +52,7 @@ def test_length_reward_spec_is_a_valid_callable_in_repo(): # ------------------------------------------------------------------ judge +@pytest.mark.skipif(not TRY27.exists(), reason="out-of-repo evidence log not present") def test_signal_judge_fails_try27(): r = subprocess.run([sys.executable, str(REPO / "scripts/judge_fn_learning_signal.py"), str(TRY27)], capture_output=True, text=True) diff --git a/tests/test_no_tracked_logs.py b/tests/test_no_tracked_logs.py new file mode 100644 index 000000000..0bc9e9df3 --- /dev/null +++ b/tests/test_no_tracked_logs.py @@ -0,0 +1,49 @@ +"""Run logs never enter the repository (rl-code-quality-cleanup, spec +"运行日志不入库"). + +Evidence logs live outside the repository under +/home/michael/work/infra-drafts/evidence// ; each old directory keeps +an EVIDENCE.md pointer. Test fixtures under tests/ and data/ are inputs, not +run logs, and are out of scope. +""" +from __future__ import annotations + +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[1] +LOG_PATTERN = re.compile( + r"(^|/)run-logs/(?!EVIDENCE\.md$)|\.log$|\.stderr$|\.stdout$|metrics[^/]*\.json$|\.jsonl$|\.out$|\.err$") +FIXTURE_ROOTS = ("tests/", "data/") + + +def _git(*args: str) -> str: + if shutil.which("git") is None or not (REPO / ".git").exists(): + pytest.skip("not a git checkout") + return subprocess.run(["git", *args], cwd=REPO, check=True, capture_output=True, text=True).stdout + + +def test_no_tracked_run_logs(): + tracked = _git("ls-files").splitlines() + hits = [f for f in tracked if LOG_PATTERN.search(f) and not f.startswith(FIXTURE_ROOTS)] + assert hits == [], f"{len(hits)} tracked run-log files: {hits[:20]}" + + +@pytest.mark.parametrize("name", ["openspec/changes/x/run-logs/a.log", + "openspec/changes/x/run-logs/b.txt", + "openspec/changes/x/evidence/c/events.jsonl", + "openspec/changes/x/evidence/c/run.stderr", + "openspec/changes/x/evidence/c/train_metrics.json"]) +def test_gitignore_blocks_new_logs(name): + out = _git("check-ignore", "--no-index", "-v", name) + assert out.strip() and not out.split("\t")[0].rsplit(":", 1)[-1].startswith("!"), out + + +def test_gitignore_has_no_log_whitelist(): + for line in (REPO / ".gitignore").read_text().splitlines(): + if line.startswith("!") and line != "!**/run-logs/EVIDENCE.md": + assert not LOG_PATTERN.search(line[1:]), line From 3700cfb5e551f7577c499c626e2dc8f397ae9465 Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:15:33 +0000 Subject: [PATCH 2/2] Skip the committed-evidence equivalence check when its run logs are absent The run logs this test reads were moved out of the repository, so in a clean checkout the arm discovery raises FileNotFoundError instead of skipping. Co-Authored-By: Claude Opus 5.5 --- tests/test_rl_engine_equivalence.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_rl_engine_equivalence.py b/tests/test_rl_engine_equivalence.py index 376e567cb..f975c08a3 100644 --- a/tests/test_rl_engine_equivalence.py +++ b/tests/test_rl_engine_equivalence.py @@ -136,7 +136,10 @@ def _committed_arm_or_skip(run_dir: Path) -> Path: ``[truncated: ...]`` marker, and the parquet shards were removed. round1_check needs the full per-step metrics, so skip when the arm is missing or trimmed rather than fail on evidence this tree does not carry.""" - arms = eq.discover_arms(run_dir) if run_dir.is_dir() else {} + try: + arms = eq.discover_arms(run_dir) if run_dir.is_dir() else {} + except FileNotFoundError: # events.jsonl moved out of the repo (no run logs in git) + arms = {} if not arms: pytest.skip(f"committed evidence missing under {run_dir} (trimmed by #77)") arm = next(iter(arms.values()))