diff --git a/tests/test_rlf_env_lease.py b/tests/test_rlf_env_lease.py index 29571911..58ae9009 100644 --- a/tests/test_rlf_env_lease.py +++ b/tests/test_rlf_env_lease.py @@ -416,3 +416,34 @@ def test_finish_trusted_tb2_fallback_byte_unchanged(): task_id="tb2-task", sample_id="s-1", key=HMAC_KEY)) assert md_timeout[tbench_outcome.OUTCOME_KEY]["verifier"] == tbench_outcome.TIMEOUT_VERIFIER assert not any(k.startswith("rlf_") for k in md_timeout) + + +def test_acquire_vm_idle_timeout_follows_ttl(monkeypatch): + """Sandbox idle kill must not reclaim a VM mid-generation (S21 gate-5).""" + import sys + created: dict[str, Any] = {} + + class _Sb: + object_id = "sb-x" + + def poll(self): + return None + + def terminate(self): + pass + + fake_modal = SimpleNamespace( + App=SimpleNamespace(lookup=lambda *a, **k: object()), + Image=SimpleNamespace(from_registry=lambda *a, **k: object()), + Sandbox=SimpleNamespace(create=lambda *a, **k: (created.update(k) or _Sb())), + ) + monkeypatch.setitem(sys.modules, "modal", fake_modal) + + rlf_modal.acquire_vm("t", "ep") + assert created["timeout"] == 1800 + assert created["idle_timeout"] == 1800 # not the old min(ttl, 300) + + created.clear() + cfg = rlf_modal.RlfModalVmConfig(ttl_s=600, idle_timeout_s=900) + rlf_modal.acquire_vm("t", "ep", config=cfg) + assert created["timeout"] == 600 and created["idle_timeout"] == 900 diff --git a/yeto/launcher.py b/yeto/launcher.py index 00ca4369..ddf73ed7 100644 --- a/yeto/launcher.py +++ b/yeto/launcher.py @@ -3491,7 +3491,7 @@ def checkout( "OPENENV_RUN_ID", "SECRLENV_MAX_TURNS", ) -HARNESS_PASSTHROUGH_ENV_PREFIXES = ("YETO_HARNESS_TB2_",) +HARNESS_PASSTHROUGH_ENV_PREFIXES = ("YETO_HARNESS_TB2_", "YETO_RLF_") # tb2_provider's Modal Sandbox backend runs inside the island (rollout # workers create the task sandboxes), so the island python needs the Modal # client. The ports image does not ship it; install it in setup without @@ -3500,7 +3500,8 @@ def checkout( MODAL_SANDBOX_PROVIDER = "yeto.rl.harness.codex.tb2_provider:modal_provider" # rl-agentic-reward-env: same TB2 provider, sandboxes from prebaked images. PREBAKED_MODAL_SANDBOX_PROVIDER = "yeto.cloud.modal_reward_env:modal_provider" -MODAL_SANDBOX_PROVIDERS = frozenset({MODAL_SANDBOX_PROVIDER, PREBAKED_MODAL_SANDBOX_PROVIDER}) +RLF_MODAL_SANDBOX_PROVIDER = "yeto.cloud.modal_reward_env:rlf_modal_provider" +MODAL_SANDBOX_PROVIDERS = frozenset({MODAL_SANDBOX_PROVIDER, PREBAKED_MODAL_SANDBOX_PROVIDER, RLF_MODAL_SANDBOX_PROVIDER}) SANDBOX_MODAL_TOKEN_ENVS = ("YETO_SANDBOX_MODAL_TOKEN_ID", "YETO_SANDBOX_MODAL_TOKEN_SECRET") diff --git a/yeto/rl/rlf/modal.py b/yeto/rl/rlf/modal.py index 84a35f8f..07124bb3 100644 --- a/yeto/rl/rlf/modal.py +++ b/yeto/rl/rlf/modal.py @@ -55,6 +55,11 @@ class RlfModalVmConfig: cpu: float = 2.0 memory_mb: int = 4096 ttl_s: int = 1800 + # Idle kill must not fire below ttl_s: agent turns can spend minutes in model + # generation with no exec running; a shorter idle timeout reclaims the VM + # mid-episode (Modal NotFoundError) and the fail-closed preflight then drops + # the whole group. Default: follow ttl_s (hard cap still bounds leaks). + idle_timeout_s: int | None = None image: str = 'debian:bookworm-slim' runtime: str = 'vm' image_run_commands: tuple = () @@ -106,8 +111,9 @@ def acquire_vm(task_id: str, episode_id: str, *, config: RlfModalVmConfig | None image = modal.Image.from_registry(cfg.image) if cfg.image_run_commands: image = image.run_commands(*cfg.image_run_commands) + idle_timeout_s = cfg.idle_timeout_s if cfg.idle_timeout_s is not None else cfg.ttl_s sb = modal.Sandbox.create('sleep', 'infinity', app=app, image=image, runtime='vm', cpu=cfg.cpu, - memory=cfg.memory_mb, timeout=cfg.ttl_s, idle_timeout=min(cfg.ttl_s, 300), + memory=cfg.memory_mb, timeout=cfg.ttl_s, idle_timeout=idle_timeout_s, tags={'yeto-purpose': 'rlf-environment', 'yeto-task': task_id[:63], 'yeto-episode': episode_id[:63]}) return RlfVmLease(sb, task_id=task_id, episode_id=episode_id, trajectory_id=trajectory_id)