From 1bcb985f13d05fbb07f600063e8aedd786fb87d1 Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:31:35 +0000 Subject: [PATCH] =?UTF-8?q?rlf-codex-rl:=205.2/5.8=20pass=20=E2=80=94=20GP?= =?UTF-8?q?U=20RL=20gate=20verdict=3DTrue=20(rlf-gate-20261011-9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five criteria on the ports engine tape (32 rl_trajectory_reward records, 2 rounds x 16 samples): reward_sum=6.0, reward_std=0.390, success_count=6, advantage_abs_sum=7.0, policy_update payload hashes differ (v0/v1), collector=miles_island. Decision log: -6/-7/-8 root causes (cuDNN fused refusal, ports empty completed-group queue, all-samples hook occupied by extract_rollout_metadata). Co-authored-by: Cursor --- evidence/rlf/acceptance-status.json | 28 +- evidence/rlf/decision-log.md | 8 + evidence/rlf/gate-20261011-9.json | 351 +++++++++++++++++++++++++ openspec/changes/rlf-codex-rl/tasks.md | 4 +- 4 files changed, 375 insertions(+), 16 deletions(-) create mode 100644 evidence/rlf/gate-20261011-9.json diff --git a/evidence/rlf/acceptance-status.json b/evidence/rlf/acceptance-status.json index c858cd9c..608fc8c6 100644 --- a/evidence/rlf/acceptance-status.json +++ b/evidence/rlf/acceptance-status.json @@ -1,21 +1,21 @@ { - "package_admission": "pass", - "modal_vm_dockerd": "pass", - "ctf_image_build": "pass", - "target_health": "pass", - "trusted_reference_flag": "pass", "codex_trajectory_metadata": "pass_cpu_tests", + "compaction_evidence": "run18 (s1-runs/.../run18-out): 12K window/4K reserve, compaction fired at 10320 tokens, summary_ok, context 8210->2176, episode continued to 12 turns + submit; run14: qwen3_cyber 24K profile active with correct no-fire gating", + "compaction_real_firing": "pass", "cpu_reward_gae_gate": "pass", - "modal_gpu_rl_gate": "attempted_failed_trainer_cutlass", - "qwen3_baseline": "pass", + "ctf_image_build": "pass", + "ctf_lease_evidence": "evidence/rlf/modal-ctf-lease-e2e.log (vm sb-01M4MEKE66M30GA31EZBGEBRR0, destroyed, describe=gone)", + "ctf_lease_vm_e2e": "pass", "failure_class": "resolved_transport_single_recv", - "swe": "deferred", + "modal_gpu_rl_gate": "pass", + "modal_gpu_rl_gate_evidence": "evidence/rlf/gate-20261011-9.json (rlf-gate-20261011-9, Modal H200 ports+Qwen3.5-4B rev 851bf6e8 LoRA r16 unfused, 2轮×4组×4样本: reward_sum=6.0 std=0.390 success_count=6 adv_abs_sum=7.0 policy_update=True, collector=miles_island; tape run9-tape, head logs run9-head-logs.tgz, island+head torn down)", + "modal_vm_dockerd": "pass", "online_harness": "codex_only", - "trusted_flag_evidence": "evidence/rlf/modal-ctf-trusted-recvfix2.log (sandbox sb-01M4MB62RAE1FR0Y3HDGWZJRX7, terminated)", - "ctf_lease_vm_e2e": "pass", - "ctf_lease_evidence": "evidence/rlf/modal-ctf-lease-e2e.log (vm sb-01M4MEKE66M30GA31EZBGEBRR0, destroyed, describe=gone)", + "package_admission": "pass", + "qwen3_baseline": "pass", "qwen3_baseline_evidence": "run10 (s1-runs/s21-rlf-baseline-20261011a/run10-out): ezshell success reward=1.0 (2 turns), ret2win failed reward=0.0 (9 turns, wrong win addr); success_count=1, reward mean=0.5 std>0; run8: ret2win 36-turn 31K-token max_seq_len truncation (5.7 trigger evidence)", - "compaction_real_firing": "pass", - "compaction_evidence": "run18 (s1-runs/.../run18-out): 12K window/4K reserve, compaction fired at 10320 tokens, summary_ok, context 8210->2176, episode continued to 12 turns + submit; run14: qwen3_cyber 24K profile active with correct no-fire gating", - "modal_gpu_rl_gate_evidence": "s1-runs/s21-rlf-baseline-20261011a/gate-launch7.log (rlf-gate-20261011-3): rollout+RLF reward path live; Megatron train step cutlass crd2idx compile error; artifacts lost with island (pull-before-stop lesson recorded)" + "swe": "deferred", + "target_health": "pass", + "trusted_flag_evidence": "evidence/rlf/modal-ctf-trusted-recvfix2.log (sandbox sb-01M4MB62RAE1FR0Y3HDGWZJRX7, terminated)", + "trusted_reference_flag": "pass" } \ No newline at end of file diff --git a/evidence/rlf/decision-log.md b/evidence/rlf/decision-log.md index 13e2814d..418fdeec 100644 --- a/evidence/rlf/decision-log.md +++ b/evidence/rlf/decision-log.md @@ -70,3 +70,11 @@ 3. 其余不变（LoRA r16 attention、seq 8192、GRPO 2×4×4、worker 8）。预算预登记 ≤$6。 第五跑（rlf-gate-20261011-7）head 形态（代拍板）：本地 preflight 线程守卫把内核线程计入 root（~1000），真实用户线程仅 ~1841 仍卡 2800 线；按用户指示改用 `--controller head`（Nebius eu-north1 无卡 head）+ `--preflight-threads off`——理由：guard 的目的是保护跑 driver 的机器，head 模式下 driver 在 Nebius VM，本地只剩瘦客户端，本地计数是不适用的机器。Modal 令牌按 s19-critic-g3.sh 模式走环境变量进 head（不落盘）。跑完先从 head 拉回产物再 sky down。 + +## 2026-10-11 5.8 gate 第六至八跑（-7/-8/-9）与最终通过 + +- **-6（fused/L40S/0.8B）**：训练步 cuDNN fused 报 "No dot product attention backend is available"——flash(cute JIT 拒)、fused(cuDNN 拒)、unfused(OOM) 三后端在 L40S/sm89 全灭。代拍板：换 H200×1(141G)+unfused+Qwen3.5-4B（rev 851bf6e8 M1 钉版；0.8B 连续两跑 reward 全零过不了 success_count）。 +- **-7（H200+4B）**：训练 SUCCEEDED（raw_reward 0.125→0.375）但判定卡证据缺口：ports 引擎 completed_groups 恒空，样本 outcome 随岛销毁。修 054c20d5（legacy hook 写磁带）。 +- **-8**：训练 SUCCEEDED（0.5→0.1875）但仍零 outcome——ports 启动配置里 --rollout-all-samples-process-path 被 rollout_meta_hook.extract_rollout_metadata 占用（config.py:834），054c20d5 的 queue_completed_groups 只在 legacy 非 ports 路径注册，修复打在了不执行的位置。根因教训：**改代码前先验证目标函数在真实配置下被调用**（日志 2164 行一锤定音）。 +- **-9（通过）**：judge 改收 driver 侧现成的 rl_trajectory_reward 磁带事件（rl-fn-codex-rollout 1.0 管线，--rl-observe-timeline 打开，ec656e4e）。磁带 32 条轨迹事件（16样本×2轮），五判据全过：reward_sum=6.0、std=0.390、success_count=6、advantage_abs_sum=7.0、policy_update=True（v0/v1 payload hash 相异），collector=miles_island，verdict=True。**rlf-codex-rl 5.2/5.8 通过。** +- 奖励曲线（4B，ezshell+ret2win 2×2）：-7: 0.125→0.375；-8: 0.5→0.1875；-9: 0.25→0.125。ret2win 全程零（符合设计，30B 也解不出）；轮间波动为采样噪声，gate 不考趋势。 diff --git a/evidence/rlf/gate-20261011-9.json b/evidence/rlf/gate-20261011-9.json new file mode 100644 index 00000000..0cf0f991 --- /dev/null +++ b/evidence/rlf/gate-20261011-9.json @@ -0,0 +1,351 @@ +{ + "collected_at": 1791725338.4261763, + "collector": "miles_island", + "gate": { + "passed": true, + "result": { + "advantage_abs_sum": 7.0, + "passed": true, + "policy_update": true, + "reward_std": 0.3903123748998999, + "reward_sum": 6.0, + "success_count": 6 + } + }, + "policy_update_evidence": { + "after_digest": "54546bee1d72ccdd11e832bfc8fd1e2a87bc1598c13cd2bdc8d91a4bb4f9cd6f", + "before_digest": "fb37744257923fd8159a3eaa81b99025ddddecc362f8f12b4810b3076dd4f3f0", + "changed": [ + "policy_payload" + ], + "step": 2, + "updated": true + }, + "policy_update_source": "event_tape", + "rows": [ + { + "advantage": 0.5, + "metadata": { + "group_index": 0, + "success": true, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 1.0 + }, + { + "advantage": -0.5, + "metadata": { + "group_index": 0, + "success": false, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.5, + "metadata": { + "group_index": 0, + "success": true, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 1.0 + }, + { + "advantage": -0.5, + "metadata": { + "group_index": 0, + "success": false, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.5, + "metadata": { + "group_index": 1, + "success": true, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 1.0 + }, + { + "advantage": -0.5, + "metadata": { + "group_index": 1, + "success": false, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.5, + "metadata": { + "group_index": 1, + "success": true, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 1.0 + }, + { + "advantage": -0.5, + "metadata": { + "group_index": 1, + "success": false, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 2, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 2, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 2, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 2, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 3, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 3, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 3, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 3, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 4, + "success": false, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 4, + "success": false, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 4, + "success": false, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 4, + "success": false, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": -0.25, + "metadata": { + "group_index": 5, + "success": false, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": -0.25, + "metadata": { + "group_index": 5, + "success": false, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.75, + "metadata": { + "group_index": 5, + "success": true, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 1.0 + }, + { + "advantage": -0.25, + "metadata": { + "group_index": 5, + "success": false, + "task_id": "fixture-ctf-ezshell" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 6, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 6, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 6, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.0, + "metadata": { + "group_index": 6, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": -0.25, + "metadata": { + "group_index": 7, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": -0.25, + "metadata": { + "group_index": 7, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + }, + { + "advantage": 0.75, + "metadata": { + "group_index": 7, + "success": true, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 1.0 + }, + { + "advantage": -0.25, + "metadata": { + "group_index": 7, + "success": false, + "task_id": "fixture-ctf-ret2win" + }, + "policy_update": true, + "reward": 0.0 + } + ], + "sample_source": "event_tape", + "schema": "yeto.rlf.rl_gate.evidence/v1", + "verdict": true, + "verdict_note": "gate criteria evaluated on real Miles island artifacts" +} diff --git a/openspec/changes/rlf-codex-rl/tasks.md b/openspec/changes/rlf-codex-rl/tasks.md index 9a322a66..f1343e56 100644 --- a/openspec/changes/rlf-codex-rl/tasks.md +++ b/openspec/changes/rlf-codex-rl/tasks.md @@ -28,13 +28,13 @@ ## 5. End-to-end gates - [x] 5.1 增加 CPU fake-provider 到 learner 的端到端测试。 -- [ ] 5.2 在 Modal GPU 上运行小规模 Codex-only RL gate，核对 rollout、trusted reward、success metadata 和 policy update。 +- [x] 5.2 在 Modal GPU 上运行小规模 Codex-only RL gate，核对 rollout、trusted reward、success metadata 和 policy update。 - [x] 5.3 记录资源清理、失败分类和可复现 manifest；确认 SWE/其他 harness 未成为前置依赖。 - [x] 5.4 更新运行文档与 handoff，说明旧 `rl-agentic-reward-env` change 保持搁置及后续 apply 顺序。 - [x] 5.5 明确并验证 AgentFlow 仅用于离线任务生产/验收，在线 Yeto Codex RL rollout 不依赖 AgentFlow。 - [x] 5.6 用 Qwen3-30B-A3B thinking mode 在固定 cyber task 集上先跑 32K 单窗口能力 baseline，分析成功率、tool trace、提交 turn、`max_seq_len_hit` 和 reward 分布。 - [x] 5.7 若 baseline 在提交 flag 前截断，显式注册 Qwen compaction budget，启用 24K window、4K–8K reserve 和最多 2–3 次 compaction，并验证 segment identity/reward 合并。 -- [ ] 5.8 在 Modal GPU RL gate 中硬性验证 reward sum/std、success count、`advantage_abs_sum > 0` 和 policy update；全零或恒定 reward 必须失败。 +- [x] 5.8 在 Modal GPU RL gate 中硬性验证 reward sum/std、success count、`advantage_abs_sum > 0` 和 policy update；全零或恒定 reward 必须失败。