|
| 1 | +"""Integration test: verify provenance tracking in W&B run config via ServerlessBackend.""" |
| 2 | + |
| 3 | +import asyncio |
| 4 | +from datetime import datetime |
| 5 | + |
| 6 | +from dotenv import load_dotenv |
| 7 | + |
| 8 | +import art |
| 9 | +from art.serverless.backend import ServerlessBackend |
| 10 | + |
| 11 | +load_dotenv() |
| 12 | + |
| 13 | + |
| 14 | +async def simple_rollout(model: art.TrainableModel) -> art.Trajectory: |
| 15 | + """Minimal rollout that produces a single turn with a reward.""" |
| 16 | + traj = art.Trajectory( |
| 17 | + messages_and_choices=[ |
| 18 | + {"role": "system", "content": "Reply with exactly 'hello'."}, |
| 19 | + ], |
| 20 | + reward=0.0, |
| 21 | + ) |
| 22 | + |
| 23 | + choice = ( |
| 24 | + await model.openai_client().chat.completions.create( |
| 25 | + model=model.get_inference_name(), |
| 26 | + messages=traj.messages(), |
| 27 | + max_completion_tokens=16, |
| 28 | + timeout=30, |
| 29 | + ) |
| 30 | + ).choices[0] |
| 31 | + |
| 32 | + traj.messages_and_choices.append(choice) |
| 33 | + traj.reward = ( |
| 34 | + 1.0 if (choice.message.content or "").strip().lower() == "hello" else 0.0 |
| 35 | + ) |
| 36 | + return traj |
| 37 | + |
| 38 | + |
| 39 | +async def make_group(model: art.TrainableModel) -> art.TrajectoryGroup: |
| 40 | + return art.TrajectoryGroup(simple_rollout(model) for _ in range(4)) |
| 41 | + |
| 42 | + |
| 43 | +async def main() -> None: |
| 44 | + backend = ServerlessBackend() |
| 45 | + |
| 46 | + model = art.TrainableModel( |
| 47 | + name=f"provenance-test-{datetime.now().strftime('%Y%m%d-%H%M%S')}", |
| 48 | + project="provenance-test", |
| 49 | + base_model="OpenPipe/Qwen3-14B-Instruct", |
| 50 | + ) |
| 51 | + await model.register(backend) |
| 52 | + |
| 53 | + # --- Step 1: first training call --- |
| 54 | + groups = await art.gather_trajectory_groups(make_group(model) for _ in range(1)) |
| 55 | + result = await backend.train(model, groups) |
| 56 | + await model.log(groups, metrics=result.metrics, step=result.step, split="train") |
| 57 | + |
| 58 | + # Check provenance after first train call |
| 59 | + run = model._get_wandb_run() |
| 60 | + assert run is not None, "W&B run should exist" |
| 61 | + provenance = run.config.get("provenance") |
| 62 | + print(f"After step 1: provenance = {provenance}") |
| 63 | + assert provenance == ["serverless-rl"], ( |
| 64 | + f"Expected ['serverless-rl'], got {provenance}" |
| 65 | + ) |
| 66 | + |
| 67 | + # --- Step 2: second training call (same technique, should NOT duplicate) --- |
| 68 | + # Provenance is recorded at the start of train(), before the remote call, |
| 69 | + # so we can verify deduplication even if the server-side training fails. |
| 70 | + groups2 = await art.gather_trajectory_groups(make_group(model) for _ in range(1)) |
| 71 | + try: |
| 72 | + result2 = await backend.train(model, groups2) |
| 73 | + await model.log( |
| 74 | + groups2, metrics=result2.metrics, step=result2.step, split="train" |
| 75 | + ) |
| 76 | + except RuntimeError as e: |
| 77 | + print(f"Step 2 training failed (transient server error, OK for this test): {e}") |
| 78 | + |
| 79 | + provenance = run.config.get("provenance") |
| 80 | + print(f"After step 2: provenance = {provenance}") |
| 81 | + assert provenance == ["serverless-rl"], ( |
| 82 | + f"Expected ['serverless-rl'] (no duplicate), got {provenance}" |
| 83 | + ) |
| 84 | + |
| 85 | + print("\nAll provenance checks passed!") |
| 86 | + |
| 87 | + await backend.close() |
| 88 | + |
| 89 | + |
| 90 | +if __name__ == "__main__": |
| 91 | + asyncio.run(main()) |
0 commit comments