From 3e47c90d12f36b47fd8abeb70fdeae3c2bd4876c Mon Sep 17 00:00:00 2001 From: Sasan Sotoodehfar Date: Thu, 22 Oct 2026 15:42:45 -0400 Subject: [PATCH 1/2] feat: vendor Codex packages from plugin releases --- .github/workflows/sync-codex-package.yml | 94 + README.md | 11 +- packages/codex/expected-integrity.json | 43 +- .../codex/mlx-agent/.codex-plugin/plugin.json | 4 +- packages/codex/mlx-agent/provenance.json | 18 +- .../mlx-agent/requirements-lab-inference.txt | 10 + packages/codex/mlx-agent/requirements-lab.txt | 9 + .../{skills/mlx-adopt => }/scripts/mlx-agent | 6 +- .../codex/mlx-agent/skills/mlx-adopt/SKILL.md | 6 +- .../mlx-adopt/src/mlx_agent/__init__.py | 6 - .../skills/mlx-adopt/src/mlx_agent/bench.py | 431 -- .../skills/mlx-adopt/src/mlx_agent/cli.py | 1924 ----- .../skills/mlx-adopt/src/mlx_agent/convert.py | 230 - .../skills/mlx-adopt/src/mlx_agent/fleet.py | 218 - .../mlx-adopt/src/mlx_agent/gemini_args.py | 340 - .../src/mlx_agent/gemini_executor.py | 140 - .../src/mlx_agent/gemini_transport.py | 128 - .../mlx-adopt/src/mlx_agent/huggingface.py | 457 -- .../skills/mlx-adopt/src/mlx_agent/lora.py | 326 - .../mlx-adopt/src/mlx_agent/model_doctor.py | 495 -- .../mlx-adopt/src/mlx_agent/providers.py | 436 -- .../skills/mlx-adopt/src/mlx_agent/serve.py | 528 -- .../mlx-adopt/src/mlx_agent/transactions.py | 1343 ---- .../mlx-adopt/src/mlx_agent/verification.py | 1163 --- .../skills/mlx-adopt/src/mlx_agent/watch.py | 205 - .../codex/mlx-agent/skills/mlx-bench/SKILL.md | 4 +- .../skills/mlx-bench/scripts/mlx-agent | 16 - .../mlx-bench/src/mlx_agent/__main__.py | 9 - .../mlx-bench/src/mlx_agent/adoption.py | 870 --- .../skills/mlx-bench/src/mlx_agent/bench.py | 431 -- .../mlx-bench/src/mlx_agent/blueprint.py | 128 - .../src/mlx_agent/command_executor.py | 74 - .../mlx-bench/src/mlx_agent/contextfit.py | 87 - .../mlx-bench/src/mlx_agent/contracts.py | 98 - .../skills/mlx-bench/src/mlx_agent/convert.py | 230 - .../mlx-bench/src/mlx_agent/discovery.py | 528 -- .../skills/mlx-bench/src/mlx_agent/fleet.py | 218 - .../skills/mlx-bench/src/mlx_agent/fuse.py | 213 - .../mlx-bench/src/mlx_agent/gemini_args.py | 340 - .../src/mlx_agent/gemini_executor.py | 140 - .../src/mlx_agent/gemini_transport.py | 128 - .../skills/mlx-bench/src/mlx_agent/host.py | 54 - .../mlx-bench/src/mlx_agent/installer.py | 696 -- .../mlx-bench/src/mlx_agent/interview.py | 302 - .../skills/mlx-bench/src/mlx_agent/lora.py | 326 - .../mlx-bench/src/mlx_agent/modality.py | 283 - .../mlx-bench/src/mlx_agent/model_doctor.py | 495 -- .../skills/mlx-bench/src/mlx_agent/models.py | 184 - .../mlx-bench/src/mlx_agent/probe_assets.py | 116 - .../src/mlx_agent/project_blueprint.py | 257 - .../mlx-bench/src/mlx_agent/research.py | 502 -- .../mlx_agent/resources/adapter-runtime.json | 4 - .../mlx_agent/resources/bench-community.json | 4 - .../resources/references/model-families.md | 35 - .../resources/references/quantization.md | 25 - .../resources/references/troubleshooting.md | 33 - .../mlx_agent/resources/serve-recipes.json | 21 - .../src/mlx_agent/runtime_preference.py | 142 - .../skills/mlx-bench/src/mlx_agent/scoring.py | 194 - .../skills/mlx-bench/src/mlx_agent/serve.py | 528 -- .../mlx-bench/src/mlx_agent/transactions.py | 1343 ---- .../skills/mlx-bench/src/mlx_agent/wiring.py | 353 - .../mlx-agent/skills/mlx-doctor/SKILL.md | 18 + .../codex/mlx-agent/skills/mlx-fleet/SKILL.md | 26 + .../codex/mlx-agent/skills/mlx-scout/SKILL.md | 6 +- .../skills/mlx-scout/scripts/mlx-agent | 16 - .../mlx-scout/src/mlx_agent/__init__.py | 6 - .../mlx-scout/src/mlx_agent/__main__.py | 9 - .../mlx-scout/src/mlx_agent/adoption.py | 870 --- .../mlx-scout/src/mlx_agent/blueprint.py | 128 - .../skills/mlx-scout/src/mlx_agent/cli.py | 1924 ----- .../src/mlx_agent/command_executor.py | 74 - .../mlx-scout/src/mlx_agent/contextfit.py | 87 - .../mlx-scout/src/mlx_agent/contracts.py | 98 - .../skills/mlx-scout/src/mlx_agent/convert.py | 230 - .../mlx-scout/src/mlx_agent/discovery.py | 528 -- .../skills/mlx-scout/src/mlx_agent/fuse.py | 213 - .../mlx-scout/src/mlx_agent/gemini_args.py | 340 - .../src/mlx_agent/gemini_executor.py | 140 - .../src/mlx_agent/gemini_transport.py | 128 - .../skills/mlx-scout/src/mlx_agent/host.py | 54 - .../mlx-scout/src/mlx_agent/huggingface.py | 457 -- .../mlx-scout/src/mlx_agent/installer.py | 696 -- .../mlx-scout/src/mlx_agent/interview.py | 302 - .../skills/mlx-scout/src/mlx_agent/lora.py | 326 - .../mlx-scout/src/mlx_agent/modality.py | 283 - .../skills/mlx-scout/src/mlx_agent/models.py | 184 - .../mlx-scout/src/mlx_agent/probe_assets.py | 116 - .../src/mlx_agent/project_blueprint.py | 257 - .../mlx-scout/src/mlx_agent/providers.py | 436 -- .../mlx-scout/src/mlx_agent/research.py | 502 -- .../mlx_agent/resources/adapter-runtime.json | 4 - .../mlx_agent/resources/bench-community.json | 4 - .../resources/references/model-families.md | 35 - .../resources/references/quantization.md | 25 - .../resources/references/troubleshooting.md | 33 - .../mlx_agent/resources/serve-recipes.json | 21 - .../src/mlx_agent/runtime_preference.py | 142 - .../skills/mlx-scout/src/mlx_agent/scoring.py | 194 - .../skills/mlx-scout/src/mlx_agent/serve.py | 528 -- .../mlx-scout/src/mlx_agent/verification.py | 1163 --- .../skills/mlx-scout/src/mlx_agent/watch.py | 205 - .../skills/mlx-scout/src/mlx_agent/wiring.py | 353 - .../codex/mlx-agent/skills/mlx-watch/SKILL.md | 19 + .../codex/mlx-agent/skills/mlx-wire/SKILL.md | 8 +- .../skills/mlx-wire/scripts/mlx-agent | 16 - .../skills/mlx-wire/src/mlx_agent/__init__.py | 6 - .../skills/mlx-wire/src/mlx_agent/__main__.py | 9 - .../skills/mlx-wire/src/mlx_agent/adoption.py | 870 --- .../skills/mlx-wire/src/mlx_agent/bench.py | 431 -- .../mlx-wire/src/mlx_agent/blueprint.py | 128 - .../skills/mlx-wire/src/mlx_agent/cli.py | 1924 ----- .../src/mlx_agent/command_executor.py | 74 - .../mlx-wire/src/mlx_agent/contextfit.py | 87 - .../mlx-wire/src/mlx_agent/contracts.py | 98 - .../skills/mlx-wire/src/mlx_agent/convert.py | 230 - .../mlx-wire/src/mlx_agent/discovery.py | 528 -- .../skills/mlx-wire/src/mlx_agent/fleet.py | 218 - .../skills/mlx-wire/src/mlx_agent/fuse.py | 213 - .../mlx-wire/src/mlx_agent/gemini_args.py | 340 - .../mlx-wire/src/mlx_agent/gemini_executor.py | 140 - .../src/mlx_agent/gemini_transport.py | 128 - .../skills/mlx-wire/src/mlx_agent/host.py | 54 - .../mlx-wire/src/mlx_agent/huggingface.py | 457 -- .../mlx-wire/src/mlx_agent/installer.py | 696 -- .../mlx-wire/src/mlx_agent/interview.py | 302 - .../skills/mlx-wire/src/mlx_agent/lora.py | 326 - .../skills/mlx-wire/src/mlx_agent/modality.py | 283 - .../mlx-wire/src/mlx_agent/model_doctor.py | 495 -- .../skills/mlx-wire/src/mlx_agent/models.py | 184 - .../mlx-wire/src/mlx_agent/probe_assets.py | 116 - .../src/mlx_agent/project_blueprint.py | 257 - .../mlx-wire/src/mlx_agent/providers.py | 436 -- .../skills/mlx-wire/src/mlx_agent/research.py | 502 -- .../mlx_agent/resources/adapter-runtime.json | 4 - .../mlx_agent/resources/bench-community.json | 4 - .../resources/references/model-families.md | 35 - .../resources/references/quantization.md | 25 - .../resources/references/troubleshooting.md | 33 - .../mlx_agent/resources/serve-recipes.json | 21 - .../src/mlx_agent/runtime_preference.py | 142 - .../skills/mlx-wire/src/mlx_agent/scoring.py | 194 - .../skills/mlx-wire/src/mlx_agent/serve.py | 528 -- .../mlx-wire/src/mlx_agent/transactions.py | 1343 ---- .../mlx-wire/src/mlx_agent/verification.py | 1163 --- .../skills/mlx-wire/src/mlx_agent/watch.py | 205 - .../skills/mlx-wire/src/mlx_agent/wiring.py | 353 - .../mlx-bench => }/src/mlx_agent/__init__.py | 2 +- .../mlx-adopt => }/src/mlx_agent/__main__.py | 0 .../src/mlx_agent/_lab/video_lab/__init__.py | 24 + .../_lab/video_lab/adapter_protocol.py | 152 + .../_lab/video_lab/adapter_registry.py | 300 + .../_lab/video_lab/adapters/__init__.py | 1 + .../_lab/video_lab/adapters/animegen_mps.py | 1242 ++++ .../mlx_agent/_lab/video_lab/adapters/fake.py | 164 + .../_lab/video_lab/adapters/wan_mps.py | 523 ++ .../_lab/video_lab/adaptive_study_models.py | 415 ++ .../video_lab/adaptive_study_repository.py | 1069 +++ .../_lab/video_lab/adaptive_study_service.py | 810 +++ .../adaptive_web_creation_repository.py | 250 + .../_lab/video_lab/animegen_feasibility.py | 596 ++ .../_lab/video_lab/animegen_pair_contract.py | 6 + .../src/mlx_agent/_lab/video_lab/artifacts.py | 988 +++ .../mlx_agent/_lab/video_lab/capabilities.py | 82 + .../video_lab/character_benchmark_analysis.py | 283 + .../video_lab/character_benchmark_models.py | 264 + .../video_lab/character_benchmark_protocol.py | 304 + .../character_benchmark_repository.py | 919 +++ .../video_lab/character_benchmark_service.py | 1062 +++ .../_lab/video_lab/character_images.py | 62 + .../_lab/video_lab/character_models.py | 110 + .../_lab/video_lab/character_pack.py | 177 + .../_lab/video_lab/character_repository.py | 691 ++ .../_lab/video_lab/character_service.py | 528 ++ .../character_web_grant_repository.py | 174 + .../src/mlx_agent/_lab/video_lab/cli.py | 2801 ++++++++ .../mlx_agent/_lab/video_lab/comparison.py | 70 + .../_lab/video_lab/compilers/__init__.py | 5 + .../_lab/video_lab/compilers/wan_curated.py | 473 ++ .../mlx_agent/_lab/video_lab/curated_ids.py | 16 + .../_lab/video_lab/curated_metrics.py | 86 + .../_lab/video_lab/curated_models.py | 269 + .../_lab/video_lab/curated_source.py | 468 ++ .../_lab/video_lab/curated_splits.py | 279 + .../_lab/video_lab/curated_validation.py | 557 ++ .../src/mlx_agent/_lab/video_lab/database.py | 6351 +++++++++++++++++ .../mlx_agent/_lab/video_lab/dataset_cli.py | 1324 ++++ .../_lab/video_lab/dataset_compiler.py | 565 ++ .../_lab/video_lab/dataset_receipts.py | 547 ++ .../src/mlx_agent/_lab/video_lab/domain.py | 249 + .../_lab/video_lab/evidence_service.py | 525 ++ .../mlx_agent/_lab/video_lab/feasibility.py | 689 ++ .../_lab/video_lab/feasibility_models.py | 300 + .../_lab/video_lab/feasibility_projection.py | 531 ++ .../_lab/video_lab/feasibility_repository.py | 900 +++ .../_lab/video_lab/feasibility_telemetry.py | 128 + .../_lab/video_lab/host_canaries/__init__.py | 29 + .../video_lab/host_canaries/wan_curated.py | 1046 +++ .../host_canaries/wan_qualification.py | 1555 ++++ .../src/mlx_agent/_lab/video_lab/launcher.py | 13 + .../_lab/video_lab/local_authority.py | 689 ++ .../_lab/video_lab/normal_generation.py | 161 + .../mlx_agent/_lab/video_lab/operations.py | 118 + .../_lab/video_lab/preference_optimizer.py | 560 ++ .../src/mlx_agent/_lab/video_lab/py.typed | 0 .../mlx_agent/_lab/video_lab/queue_service.py | 198 + .../mlx_agent/_lab/video_lab/repositories.py | 1293 ++++ .../_lab/video_lab/review_service.py | 545 ++ .../src/mlx_agent/_lab/video_lab/reviews.py | 452 ++ .../_lab/video_lab/runtime_identity.py | 87 + .../src/mlx_agent/_lab/video_lab/services.py | 137 + .../_lab/video_lab/source_inventory.py | 556 ++ .../src/mlx_agent/_lab/video_lab/states.py | 26 + .../src/mlx_agent/_lab/video_lab/telemetry.py | 54 + .../_lab/video_lab/trainer_protocol.py | 855 +++ .../_lab/video_lab/trainers/__init__.py | 106 + .../mlx_agent/_lab/video_lab/trainers/fake.py | 434 ++ .../_lab/video_lab/trainers/wan_batch.py | 314 + .../video_lab/trainers/wan_branch_loader.py | 623 ++ .../video_lab/trainers/wan_branch_runtime.py | 916 +++ .../_lab/video_lab/trainers/wan_checkpoint.py | 1642 +++++ .../_lab/video_lab/trainers/wan_continuity.py | 147 + .../trainers/wan_diffusers_runtime.py | 1289 ++++ .../_lab/video_lab/trainers/wan_factory.py | 1069 +++ .../_lab/video_lab/trainers/wan_mps.py | 1442 ++++ .../trainers/wan_production_models.py | 735 ++ .../mlx_agent/_lab/video_lab/training_data.py | 181 + .../_lab/video_lab/training_data_cli.py | 66 + .../_lab/video_lab/training_dataset.py | 206 + .../_lab/video_lab/training_media.py | 586 ++ .../_lab/video_lab/training_models.py | 1041 +++ .../_lab/video_lab/training_repository.py | 1822 +++++ .../_lab/video_lab/training_service.py | 2759 +++++++ .../_lab/video_lab/validation_analysis.py | 154 + .../_lab/video_lab/validation_models.py | 263 + .../_lab/video_lab/validation_panel.py | 151 + .../_lab/video_lab/validation_repository.py | 641 ++ .../_lab/video_lab/validation_service.py | 818 +++ .../mlx_agent/_lab/video_lab/wan_pending.py | 174 + .../src/mlx_agent/_lab/video_lab/worker.py | 525 ++ .../_lab/video_lab/worker_control.py | 21 + .../_lab/video_lab/worker_supervisor.py | 327 + .../mlx-adopt => }/src/mlx_agent/adoption.py | 0 .../codex/mlx-agent/src/mlx_agent/atomicio.py | 11 + .../src/mlx_agent/backend_install.py | 263 + .../mlx-agent/src/mlx_agent/backend_runner.py | 73 + .../codex/mlx-agent/src/mlx_agent/backends.py | 676 ++ .../mlx-scout => }/src/mlx_agent/bench.py | 18 +- .../mlx-adopt => }/src/mlx_agent/blueprint.py | 0 .../src/mlx_agent/capture_platform.py | 197 + .../mlx-agent/src/mlx_agent/capture_volume.py | 285 + .../mlx-bench => }/src/mlx_agent/cli.py | 909 ++- .../mlx-agent/src/mlx_agent/coding_capture.py | 140 + .../src/mlx_agent/coding_capture_client.py | 128 + .../src/mlx_agent/coding_capture_contracts.py | 244 + .../src/mlx_agent/coding_capture_plugins.py | 254 + .../src/mlx_agent/coding_capture_policy.py | 121 + .../src/mlx_agent/coding_capture_process.py | 163 + .../src/mlx_agent/coding_capture_storage.py | 443 ++ .../mlx-agent/src/mlx_agent/command_args.py | 508 ++ .../src/mlx_agent/command_executor.py | 8 +- .../codex/mlx-agent/src/mlx_agent/compute.py | 174 + .../src/mlx_agent/contextfit.py | 0 .../mlx-adopt => }/src/mlx_agent/contracts.py | 0 .../codex/mlx-agent/src/mlx_agent/convert.py | 549 ++ .../mlx-agent/src/mlx_agent/convert_queue.py | 532 ++ .../codex/mlx-agent/src/mlx_agent/decide.py | 117 + .../mlx-agent/src/mlx_agent/decide_runner.py | 31 + .../codex/mlx-agent/src/mlx_agent/describe.py | 147 + .../src/mlx_agent/describe_runner.py | 125 + .../mlx-adopt => }/src/mlx_agent/discovery.py | 16 +- .../mlx-agent/src/mlx_agent/fetch_runner.py | 64 + .../mlx-scout => }/src/mlx_agent/fleet.py | 75 +- .../mlx-adopt => }/src/mlx_agent/fuse.py | 0 .../codex/mlx-agent/src/mlx_agent/generate.py | 95 + .../src/mlx_agent/generate_runner.py | 36 + .../codex/mlx-agent/src/mlx_agent/gguf.py | 1038 +++ .../mlx-agent/src/mlx_agent/gguf_runner.py | 242 + .../mlx-adopt => }/src/mlx_agent/host.py | 0 .../src/mlx_agent/huggingface.py | 242 +- .../src/mlx_agent/inference_runner.py | 90 + .../mlx-adopt => }/src/mlx_agent/installer.py | 199 +- .../codex/mlx-agent/src/mlx_agent/intake.py | 387 + .../mlx-agent/src/mlx_agent/intake_fetch.py | 179 + .../mlx-agent/src/mlx_agent/intake_source.py | 106 + .../mlx-adopt => }/src/mlx_agent/interview.py | 0 .../mlx-agent/src/mlx_agent/jit_serve.py | 564 ++ .../mlx-agent/src/mlx_agent/kokoro_runtime.py | 41 + packages/codex/mlx-agent/src/mlx_agent/lab.py | 146 + .../mlx-agent/src/mlx_agent/lab_adoption.py | 193 + .../mlx-agent/src/mlx_agent/lab_studies.py | 52 + .../mlx-agent/src/mlx_agent/lab_workspace.py | 246 + .../codex/mlx-agent/src/mlx_agent/lora.py | 568 ++ .../mlx-adopt => }/src/mlx_agent/modality.py | 0 .../src/mlx_agent/model_doctor.py | 14 +- .../mlx-adopt => }/src/mlx_agent/models.py | 0 .../codex/mlx-agent/src/mlx_agent/music.py | 83 + .../src/mlx_agent/music_requantize_runner.py | 71 + .../mlx-agent/src/mlx_agent/music_runner.py | 89 + .../mlx-agent/src/mlx_agent/port_analysis.py | 181 + .../mlx-agent/src/mlx_agent/port_plan.py | 113 + .../src/mlx_agent/probe_assets.py | 0 .../src/mlx_agent/project_blueprint.py | 0 .../mlx-bench => }/src/mlx_agent/providers.py | 289 +- .../mlx-agent/src/mlx_agent/queue_bridge.py | 139 + .../mlx-agent/src/mlx_agent/queue_cli.py | 190 + .../mlx-adopt => }/src/mlx_agent/research.py | 0 .../mlx_agent/resources/adapter-runtime.json | 0 .../mlx_agent/resources/backends/mflux.json | 39 + .../mlx_agent/resources/backends/mflux.lock | 1447 ++++ .../resources/backends/mlx-audio-build.lock | 9 + .../resources/backends/mlx-audio.json | 176 + .../resources/backends/mlx-audio.lock | 2262 ++++++ .../resources/backends/mlx-embeddings.json | 50 + .../resources/backends/mlx-embeddings.lock | 1576 ++++ .../mlx_agent/resources/backends/mlx-lm.json | 123 + .../resources/backends/mlx-video.json | 27 + .../resources/backends/mlx-video.lock | 1635 +++++ .../mlx_agent/resources/backends/mlx-vlm.json | 356 + .../mlx_agent/resources/backends/mlx-vlm.lock | 1495 ++++ .../mlx_agent/resources/bench-community.json | 0 .../resources/coding_capture/freeze.mjs | 99 + .../resources/coding_capture/legacy.mjs | 77 + .../resources/coding_capture/proxy.mjs | 47 + .../resources/coding_capture/rpc.mjs | 54 + .../resources/coding_capture/worker.mjs | 36 + .../ports/mflux/qwen_image_21/__init__.py | 5 + .../ports/mflux/qwen_image_21/convert.py | 81 + .../mflux/qwen_image_21/qwen_image_21.py | 73 + .../mlx-audio/audio8_asr_infinite/__init__.py | 4 + .../mlx-audio/audio8_asr_infinite/audio.py | 96 + .../audio8_asr_infinite.py | 494 ++ .../mlx-audio/audio8_asr_infinite/config.py | 106 + .../ports/mlx-embeddings/laya/__init__.py | 5 + .../ports/mlx-embeddings/laya/convert.py | 94 + .../ports/mlx-embeddings/laya/laya.py | 337 + .../mlx-lm/deepseek_v4_dspark/__init__.py | 1 + .../mlx-lm/deepseek_v4_dspark/convert.py | 311 + .../resources/ports/mlx-video/t2v/__init__.py | 6 + .../resources/ports/mlx-video/t2v/convert.py | 87 + .../resources/ports/mlx-video/t2v/t2v.py | 145 + .../resources/ports/mlx-video/t2v/training.py | 292 + .../ports/mlx-video/t2v/training_inputs.py | 253 + .../resources/references/model-families.md | 0 .../resources/references/quantization.md | 0 .../resources/references/troubleshooting.md | 0 .../mlx_agent/resources/serve-recipes.json | 11 +- .../src/mlx_agent/runtime_preference.py | 0 .../mlx-adopt => }/src/mlx_agent/scoring.py | 0 .../codex/mlx-agent/src/mlx_agent/serve.py | 972 +++ .../codex/mlx-agent/src/mlx_agent/speak.py | 133 + .../mlx-agent/src/mlx_agent/speak_runner.py | 86 + .../mlx-agent/src/mlx_agent/strata_runtime.py | 277 + .../mlx-agent/src/mlx_agent/study_recipes.py | 206 + .../codex/mlx-agent/src/mlx_agent/task_cli.py | 92 + .../mlx-agent/src/mlx_agent/task_contracts.py | 206 + .../mlx-agent/src/mlx_agent/task_identity.py | 60 + .../mlx-agent/src/mlx_agent/task_kinds.py | 306 + .../src/mlx_agent/task_mp4_validator.py | 46 + .../src/mlx_agent/task_png_validator.py | 25 + .../mlx-agent/src/mlx_agent/task_runtime.py | 375 + .../mlx-agent/src/mlx_agent/task_store.py | 481 ++ .../mlx-agent/src/mlx_agent/task_worker.py | 19 + .../codex/mlx-agent/src/mlx_agent/taxonomy.py | 266 + .../src/mlx_agent/training_artifacts.py | 43 + .../src/mlx_agent/training_eval_runtime.py | 31 + .../src/mlx_agent/training_evaluation.py | 235 + .../src/mlx_agent/training_runner.py | 30 + .../src/mlx_agent/transactions.py | 241 +- .../mlx-agent/src/mlx_agent/transcribe.py | 78 + .../src/mlx_agent/transcribe_runner.py | 66 + .../src/mlx_agent/verification.py | 54 +- .../codex/mlx-agent/src/mlx_agent/video.py | 148 + .../mlx-agent/src/mlx_agent/video_runner.py | 38 + .../mlx-bench => }/src/mlx_agent/watch.py | 3 +- .../mlx-adopt => }/src/mlx_agent/wiring.py | 31 +- scripts/sync-codex-package.mjs | 73 + scripts/sync-codex-package.test.mjs | 80 + scripts/validate-catalog.mjs | 50 +- scripts/validate-catalog.test.mjs | 28 +- 380 files changed, 83576 insertions(+), 42901 deletions(-) create mode 100644 .github/workflows/sync-codex-package.yml create mode 100644 packages/codex/mlx-agent/requirements-lab-inference.txt create mode 100644 packages/codex/mlx-agent/requirements-lab.txt rename packages/codex/mlx-agent/{skills/mlx-adopt => }/scripts/mlx-agent (54%) delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/__init__.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/bench.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/cli.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/convert.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/fleet.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/gemini_args.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/gemini_executor.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/gemini_transport.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/huggingface.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/lora.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/model_doctor.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/providers.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/serve.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/transactions.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/verification.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/watch.py delete mode 100755 packages/codex/mlx-agent/skills/mlx-bench/scripts/mlx-agent delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/__main__.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/adoption.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/bench.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/blueprint.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/command_executor.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/contextfit.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/contracts.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/convert.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/discovery.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/fleet.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/fuse.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/gemini_args.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/gemini_executor.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/gemini_transport.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/host.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/installer.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/interview.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/lora.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/modality.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/model_doctor.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/models.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/probe_assets.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/project_blueprint.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/research.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/adapter-runtime.json delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/bench-community.json delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/references/model-families.md delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/references/quantization.md delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/references/troubleshooting.md delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/serve-recipes.json delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/runtime_preference.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/scoring.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/serve.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/transactions.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/wiring.py create mode 100644 packages/codex/mlx-agent/skills/mlx-doctor/SKILL.md create mode 100644 packages/codex/mlx-agent/skills/mlx-fleet/SKILL.md delete mode 100755 packages/codex/mlx-agent/skills/mlx-scout/scripts/mlx-agent delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/__init__.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/__main__.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/adoption.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/blueprint.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/cli.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/command_executor.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/contextfit.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/contracts.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/convert.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/discovery.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/fuse.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/gemini_args.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/gemini_executor.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/gemini_transport.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/host.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/huggingface.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/installer.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/interview.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/lora.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/modality.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/models.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/probe_assets.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/project_blueprint.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/providers.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/research.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/adapter-runtime.json delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/bench-community.json delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/references/model-families.md delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/references/quantization.md delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/references/troubleshooting.md delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/serve-recipes.json delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/runtime_preference.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/scoring.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/serve.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/verification.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/watch.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/wiring.py create mode 100644 packages/codex/mlx-agent/skills/mlx-watch/SKILL.md delete mode 100755 packages/codex/mlx-agent/skills/mlx-wire/scripts/mlx-agent delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/__init__.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/__main__.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/adoption.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/bench.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/blueprint.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/cli.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/command_executor.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/contextfit.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/contracts.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/convert.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/discovery.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/fleet.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/fuse.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/gemini_args.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/gemini_executor.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/gemini_transport.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/host.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/huggingface.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/installer.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/interview.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/lora.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/modality.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/model_doctor.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/models.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/probe_assets.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/project_blueprint.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/providers.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/research.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/adapter-runtime.json delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/bench-community.json delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/references/model-families.md delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/references/quantization.md delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/references/troubleshooting.md delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/serve-recipes.json delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/runtime_preference.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/scoring.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/serve.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/transactions.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/verification.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/watch.py delete mode 100644 packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/wiring.py rename packages/codex/mlx-agent/{skills/mlx-bench => }/src/mlx_agent/__init__.py (84%) rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/__main__.py (100%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/__init__.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapter_protocol.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapter_registry.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/__init__.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/animegen_mps.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/fake.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/wan_mps.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_study_models.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_study_repository.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_study_service.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_web_creation_repository.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/animegen_feasibility.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/animegen_pair_contract.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/artifacts.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/capabilities.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_analysis.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_models.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_protocol.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_repository.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_service.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_images.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_models.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_pack.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_repository.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_service.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_web_grant_repository.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/cli.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/comparison.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/compilers/__init__.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/compilers/wan_curated.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_ids.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_metrics.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_models.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_source.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_splits.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_validation.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/database.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/dataset_cli.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/dataset_compiler.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/dataset_receipts.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/domain.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/evidence_service.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_models.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_projection.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_repository.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_telemetry.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/host_canaries/__init__.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/host_canaries/wan_curated.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/host_canaries/wan_qualification.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/launcher.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/local_authority.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/normal_generation.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/operations.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/preference_optimizer.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/py.typed create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/queue_service.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/repositories.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/review_service.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/reviews.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/runtime_identity.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/services.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/source_inventory.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/states.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/telemetry.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainer_protocol.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/__init__.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/fake.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_batch.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_branch_loader.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_branch_runtime.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_checkpoint.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_continuity.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_diffusers_runtime.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_factory.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_mps.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_production_models.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_data.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_data_cli.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_dataset.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_media.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_models.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_repository.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_service.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_analysis.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_models.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_panel.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_repository.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_service.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/wan_pending.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/worker.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/worker_control.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/worker_supervisor.py rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/adoption.py (100%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/atomicio.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/backend_install.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/backend_runner.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/backends.py rename packages/codex/mlx-agent/{skills/mlx-scout => }/src/mlx_agent/bench.py (95%) rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/blueprint.py (100%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/capture_platform.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/capture_volume.py rename packages/codex/mlx-agent/{skills/mlx-bench => }/src/mlx_agent/cli.py (64%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/coding_capture.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/coding_capture_client.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/coding_capture_contracts.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/coding_capture_plugins.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/coding_capture_policy.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/coding_capture_process.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/coding_capture_storage.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/command_args.py rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/command_executor.py (89%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/compute.py rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/contextfit.py (100%) rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/contracts.py (100%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/convert.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/convert_queue.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/decide.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/decide_runner.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/describe.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/describe_runner.py rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/discovery.py (96%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/fetch_runner.py rename packages/codex/mlx-agent/{skills/mlx-scout => }/src/mlx_agent/fleet.py (75%) rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/fuse.py (100%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/generate.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/generate_runner.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/gguf.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/gguf_runner.py rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/host.py (100%) rename packages/codex/mlx-agent/{skills/mlx-bench => }/src/mlx_agent/huggingface.py (60%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/inference_runner.py rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/installer.py (81%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/intake.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/intake_fetch.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/intake_source.py rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/interview.py (100%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/jit_serve.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/kokoro_runtime.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/lab.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/lab_adoption.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/lab_studies.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/lab_workspace.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/lora.py rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/modality.py (100%) rename packages/codex/mlx-agent/{skills/mlx-scout => }/src/mlx_agent/model_doctor.py (96%) rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/models.py (100%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/music.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/music_requantize_runner.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/music_runner.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/port_analysis.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/port_plan.py rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/probe_assets.py (100%) rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/project_blueprint.py (100%) rename packages/codex/mlx-agent/{skills/mlx-bench => }/src/mlx_agent/providers.py (60%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/queue_bridge.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/queue_cli.py rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/research.py (100%) rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/resources/adapter-runtime.json (100%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/backends/mflux.json create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/backends/mflux.lock create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-audio-build.lock create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-audio.json create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-audio.lock create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-embeddings.json create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-embeddings.lock create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-lm.json create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-video.json create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-video.lock create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-vlm.json create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-vlm.lock rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/resources/bench-community.json (100%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/freeze.mjs create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/legacy.mjs create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/proxy.mjs create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/rpc.mjs create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/worker.mjs create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mflux/qwen_image_21/__init__.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mflux/qwen_image_21/convert.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mflux/qwen_image_21/qwen_image_21.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/__init__.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/audio.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/audio8_asr_infinite.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/config.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-embeddings/laya/__init__.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-embeddings/laya/convert.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-embeddings/laya/laya.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-lm/deepseek_v4_dspark/__init__.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-lm/deepseek_v4_dspark/convert.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/__init__.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/convert.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/t2v.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/training.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/training_inputs.py rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/resources/references/model-families.md (100%) rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/resources/references/quantization.md (100%) rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/resources/references/troubleshooting.md (100%) rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/resources/serve-recipes.json (58%) rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/runtime_preference.py (100%) rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/scoring.py (100%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/serve.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/speak.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/speak_runner.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/strata_runtime.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/study_recipes.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/task_cli.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/task_contracts.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/task_identity.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/task_kinds.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/task_mp4_validator.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/task_png_validator.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/task_runtime.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/task_store.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/task_worker.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/taxonomy.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/training_artifacts.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/training_eval_runtime.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/training_evaluation.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/training_runner.py rename packages/codex/mlx-agent/{skills/mlx-scout => }/src/mlx_agent/transactions.py (86%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/transcribe.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/transcribe_runner.py rename packages/codex/mlx-agent/{skills/mlx-bench => }/src/mlx_agent/verification.py (96%) create mode 100644 packages/codex/mlx-agent/src/mlx_agent/video.py create mode 100644 packages/codex/mlx-agent/src/mlx_agent/video_runner.py rename packages/codex/mlx-agent/{skills/mlx-bench => }/src/mlx_agent/watch.py (97%) rename packages/codex/mlx-agent/{skills/mlx-adopt => }/src/mlx_agent/wiring.py (96%) create mode 100644 scripts/sync-codex-package.mjs create mode 100644 scripts/sync-codex-package.test.mjs diff --git a/.github/workflows/sync-codex-package.yml b/.github/workflows/sync-codex-package.yml new file mode 100644 index 0000000..21285e2 --- /dev/null +++ b/.github/workflows/sync-codex-package.yml @@ -0,0 +1,94 @@ +name: Sync Codex package + +on: + repository_dispatch: + types: [cavi-oss-release] + workflow_dispatch: + inputs: + name: + description: Plugin whose Codex package to vendor + required: true + type: choice + options: [mlx-agent, obsidian-agent] + tag: + description: Release tag to vendor, for example v0.10.0 + required: true + +permissions: + contents: read + +concurrency: + group: sync-codex-package-${{ github.event.client_payload.slug || inputs.name }} + cancel-in-progress: false + +jobs: + sync: + if: github.event_name == 'workflow_dispatch' || contains(fromJSON('["mlx-agent","obsidian-agent"]'), github.event.client_payload.slug) + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + env: + NAME: ${{ github.event.client_payload.slug || inputs.name }} + TAG: ${{ github.event.client_payload.tag || inputs.tag }} + COMMIT: ${{ github.event.client_payload.commit || '' }} + REPOSITORY: ${{ github.event.client_payload.repository || '' }} + steps: + - name: Check the release identity + run: | + set -euo pipefail + [[ "$TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || { echo "Tag must be vX.Y.Z: $TAG"; exit 1; } + if [ -n "$REPOSITORY" ] && [ "$REPOSITORY" != "cavi-ai/$NAME" ]; then echo "Repository $REPOSITORY does not match $NAME"; exit 1; fi + if [ -n "$COMMIT" ] && ! [[ "$COMMIT" =~ ^[0-9a-f]{40}$ ]]; then echo "Commit must be a full lowercase SHA: $COMMIT"; exit 1; fi + + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + repository: cavi-ai/${{ env.NAME }} + ref: refs/tags/${{ env.TAG }} + path: .upstream + persist-credentials: false + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + + - name: Vendor the tagged package + run: | + set -euo pipefail + resolved="$(git -C .upstream rev-parse HEAD)" + if [ -n "$COMMIT" ] && [ "$resolved" != "$COMMIT" ]; then echo "$TAG resolves to $resolved, not $COMMIT"; exit 1; fi + node scripts/sync-codex-package.mjs --name "$NAME" --source .upstream --commit "$resolved" + rm -rf .upstream + node --test + node scripts/validate-catalog.mjs + echo "RESOLVED=$resolved" >> "$GITHUB_ENV" + + - name: Open a pull request + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + if [ -z "$(git status --porcelain -- packages/codex)" ]; then echo "packages/codex/$NAME already matches $TAG"; exit 0; fi + branch="sync/$NAME-$TAG" + if git ls-remote --exit-code --heads origin "$branch" > /dev/null; then echo "Branch $branch already exists"; exit 1; fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git switch -c "$branch" + git add -A -- packages/codex + git commit -m "chore: vendor $NAME $TAG Codex package" + git push origin "$branch" + body="$(mktemp)" + { + echo "## Changes" + echo + echo "- \`packages/codex/$NAME\` mirrors \`cavi-ai/$NAME\` \`$TAG\` (\`$RESOLVED\`)." + echo "- \`packages/codex/expected-integrity.json\` pins that commit, its tree hash and skill inventory." + echo + echo "## Verification" + echo + echo "- \`node --test\` and \`node scripts/validate-catalog.mjs\` passed in $RUN_URL." + } > "$body" + gh pr create --base main --head "$branch" --title "chore: vendor $NAME $TAG" --body-file "$body" diff --git a/README.md b/README.md index 80f2845..ef11260 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,16 @@ codex plugin add obsidian-agent@cavi-ai The Codex projection at [`.agents/plugins/marketplace.json`](.agents/plugins/marketplace.json) uses Codex's documented marketplace-local source layout. `packages/codex/` contains minimal, self-contained projections copied from exact upstream commits. Each package includes provenance and a deterministic tree hash; catalog validation rejects local drift. -These package projections are distribution artifacts, not independent workflow sources. Refresh one only from a reviewed upstream commit, update its `provenance.json`, and run the full catalog gate. Product development remains in the linked plugin repository. +These package projections are distribution artifacts, not independent workflow sources. Product development remains in the linked plugin repository. + +A plugin release refreshes its package. The plugin's release workflow sends a `cavi-oss-release` dispatch, and [Sync Codex package](.github/workflows/sync-codex-package.yml) checks out the tag, confirms it resolves to the released commit, vendors it with `scripts/sync-codex-package.mjs`, runs the catalog gate and opens a pull request. The same workflow runs by hand from the Actions tab with a plugin name and tag. Locally: + +```sh +node scripts/sync-codex-package.mjs --name mlx-agent --source --commit +node --test && node scripts/validate-catalog.mjs +``` + +`packages/codex/expected-integrity.json` pins each package's source commit, tree hash and skill inventory; `provenance.json` inside the package must agree with it. ## Agy diff --git a/packages/codex/expected-integrity.json b/packages/codex/expected-integrity.json index c82f150..115bc19 100644 --- a/packages/codex/expected-integrity.json +++ b/packages/codex/expected-integrity.json @@ -1,14 +1,51 @@ { "mlx-agent": { "repository": "cavi-ai/mlx-agent", - "source_commit": "923627988cf98e984fa36e0b940f86e7263e2958", + "source_commit": "6e6effd57bc4fce90760d938a939aef1f18e1f49", "source_path": "providers/codex", - "tree": "c3fd7577d32e4c179d510516117a2405f80dd67e5693fd7b9835d6b84ed62d3c" + "tree": "f28e977491c286ed9b33eaae08a945daa188e899cabb1a980f84879e01351158", + "skills": [ + "mlx-adopt", + "mlx-bench", + "mlx-doctor", + "mlx-fleet", + "mlx-scout", + "mlx-watch", + "mlx-wire" + ] }, "obsidian-agent": { "repository": "cavi-ai/obsidian-agent", "source_commit": "7efecb058fce8185056cbe7868d6de1af11879a9", "source_path": ".", - "tree": "e4b9ecdc9d7d76011d955784f64d2b2f2074e5c776b5411d9fc332dd07734df2" + "tree": "e4b9ecdc9d7d76011d955784f64d2b2f2074e5c776b5411d9fc332dd07734df2", + "skills": [ + "build-retrospective", + "connection-finder", + "consistent-tagging", + "daily-rollup", + "dedup-merge", + "frontmatter-normalizer", + "manifest-content", + "manifest-core", + "manifest-feature", + "manifest-infra", + "manifest-pm", + "manifest-research", + "manifest-risk", + "manifest-vault", + "meeting-cleanup", + "moc-builder", + "note-splitter", + "outline-to-draft", + "plan-to-spec", + "source-digest", + "summarize-and-link", + "task-harvester", + "tracker-driver", + "vault-grounding", + "vault-synthesis", + "wikilink-weaver" + ] } } diff --git a/packages/codex/mlx-agent/.codex-plugin/plugin.json b/packages/codex/mlx-agent/.codex-plugin/plugin.json index a905bb6..d5f9f54 100644 --- a/packages/codex/mlx-agent/.codex-plugin/plugin.json +++ b/packages/codex/mlx-agent/.codex-plugin/plugin.json @@ -1,10 +1,10 @@ { "name": "mlx-agent", - "version": "0.4.0", + "version": "0.10.0", "description": "Structured local MLX discovery, adoption, and wiring for Apple Silicon agents.", "author": { "name": "Sasan Sotoodehfar", - "url": "https://github.com/sasan1200" + "url": "https://cavi-ai.xyz" }, "homepage": "https://github.com/cavi-ai/mlx-agent", "repository": "https://github.com/cavi-ai/mlx-agent", diff --git a/packages/codex/mlx-agent/provenance.json b/packages/codex/mlx-agent/provenance.json index f05a849..4ce83ef 100644 --- a/packages/codex/mlx-agent/provenance.json +++ b/packages/codex/mlx-agent/provenance.json @@ -1,12 +1,22 @@ { "repository": "cavi-ai/mlx-agent", - "source_commit": "923627988cf98e984fa36e0b940f86e7263e2958", + "source_commit": "6e6effd57bc4fce90760d938a939aef1f18e1f49", "source_path": "providers/codex", - "included": [".codex-plugin", "skills"], - "excluded": [".mlx-agent-generated-files.json", "skills/*/scripts/mlx-agent-mcp", "skills/*/src/mlx_agent/mcp_server.py"], + "included": [ + ".codex-plugin", + "requirements-lab-inference.txt", + "requirements-lab.txt", + "scripts", + "skills", + "src" + ], + "excluded": [ + "scripts/mlx-agent-mcp", + "src/mlx_agent/mcp_server.py" + ], "exclusion_reason": "Codex skills use the packaged CLI entrypoint; MCP transport artifacts are not required or distributed by this catalog.", "integrity": { "algorithm": "sha256", - "tree": "c3fd7577d32e4c179d510516117a2405f80dd67e5693fd7b9835d6b84ed62d3c" + "tree": "f28e977491c286ed9b33eaae08a945daa188e899cabb1a980f84879e01351158" } } diff --git a/packages/codex/mlx-agent/requirements-lab-inference.txt b/packages/codex/mlx-agent/requirements-lab-inference.txt new file mode 100644 index 0000000..8303f58 --- /dev/null +++ b/packages/codex/mlx-agent/requirements-lab-inference.txt @@ -0,0 +1,10 @@ +-r requirements-lab.txt +accelerate==1.15.0 +diffusers==0.39.0 +peft==0.21.2 +protobuf==7.36.2 +safetensors==0.8.0 +sentencepiece==0.2.2 +torch==2.14.1 +torchvision==0.29.1 +transformers==4.57.6 diff --git a/packages/codex/mlx-agent/requirements-lab.txt b/packages/codex/mlx-agent/requirements-lab.txt new file mode 100644 index 0000000..092a9b4 --- /dev/null +++ b/packages/codex/mlx-agent/requirements-lab.txt @@ -0,0 +1,9 @@ +pydantic==2.14.0 +typer==0.27.3 +click==8.5.0 +numpy==2.5.3 +pillow==13.0.0 +psutil==7.2.2 +imageio==2.38.1 +imageio-ffmpeg==0.6.0 +huggingface-hub==0.36.2 diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/scripts/mlx-agent b/packages/codex/mlx-agent/scripts/mlx-agent similarity index 54% rename from packages/codex/mlx-agent/skills/mlx-adopt/scripts/mlx-agent rename to packages/codex/mlx-agent/scripts/mlx-agent index 54a5a80..92c1a49 100755 --- a/packages/codex/mlx-agent/skills/mlx-adopt/scripts/mlx-agent +++ b/packages/codex/mlx-agent/scripts/mlx-agent @@ -1,11 +1,15 @@ #!/usr/bin/env python3 """Run the bundled MLX agent core relative to this bootstrap, never the CWD.""" +import os import sys from pathlib import Path -sys.dont_write_bytecode = True +if sys.pycache_prefix is None: + # Compiled bytecode goes to the user cache, never into the installed plugin. + sys.pycache_prefix = str(Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache").expanduser() + / "mlx-agent" / "pycache") SCRIPT_DIR = Path(__file__).resolve().parent sys.path.insert(0, str(SCRIPT_DIR.parent / "src")) diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/SKILL.md b/packages/codex/mlx-agent/skills/mlx-adopt/SKILL.md index 847174c..9472141 100644 --- a/packages/codex/mlx-agent/skills/mlx-adopt/SKILL.md +++ b/packages/codex/mlx-agent/skills/mlx-adopt/SKILL.md @@ -7,17 +7,17 @@ Use `$mlx-agent:mlx-adopt` to invoke this installed Codex skill explicitly. Code # MLX Adopt -Resolve `` as the absolute directory containing this SKILL.md. Never resolve the bundled executable from the shell working directory. +Resolve `` as the absolute directory two levels above the directory containing this SKILL.md; it holds `scripts/` and `src/`. Never resolve the bundled executable from the shell working directory. canonical capability ID: mlx-agent.adopt Use the durable adoption state owned by the structured CLI. Start with a user-visible state path and requested roles: -`python3 /scripts/mlx-agent adopt start --state --role --json` +`python3 /scripts/mlx-agent adopt start --state --role --json` If the state already exists or an earlier run was interrupted, continue it with: -`python3 /scripts/mlx-agent adopt resume --state --json` +`python3 /scripts/mlx-agent adopt resume --state --json` Report the CLI state and recommendations. Do not recreate adoption policy in this adapter. This operation must not download model weights or change configuration; any later download or mutation requires explicit user confirmation and the reviewed CLI preview. diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/__init__.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/__init__.py deleted file mode 100644 index 67dcb84..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Public API for the dependency-free MLX agent core.""" - -from .contracts import ResultEnvelope - -__all__ = ["ResultEnvelope"] -__version__ = "0.4.0" diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/bench.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/bench.py deleted file mode 100644 index ba6d8cd..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/bench.py +++ /dev/null @@ -1,431 +0,0 @@ -"""Bounded, read-only performance measurement of a locally served model. - -Bench never installs, downloads, or starts anything. It measures a model that -is already served by an already-running loopback runtime, using the same -safety envelope as verification: validated loopback URLs, absolute monotonic -deadlines, bounded responses, and redacted errors. -""" - -from __future__ import annotations - -import json -import os -import statistics -import time -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Dict, List, Optional - -from .verification import ( - EvidenceStrength, - VerificationEvidence, - VerificationStatus, - _bounded, - _is_installed, - _runtime_name, - _safe_error, - installed_model_ids, -) - - -BENCH_PROBE_ID = "bench-v1" -BENCH_PROMPT = ( - "You are a careful technical writer. Write a detailed, factual explanation " - "of how unified memory architecture on Apple Silicon affects local " - "inference of large language models. Cover memory bandwidth, quantization " - "tradeoffs, KV-cache growth with context length, and the difference " - "between dense and mixture-of-experts models. Organize the answer into " - "clear sections with concrete numbers where possible." -) -BENCH_PROMPT_TOKEN_ESTIMATE = 96 -RUNS_MIN, RUNS_MAX, RUNS_DEFAULT = 1, 10, 3 -GEN_TOKENS_MIN, GEN_TOKENS_MAX, GEN_TOKENS_DEFAULT = 16, 2048, 128 -TIMEOUT_DEFAULT = 120.0 - - -class BenchError(RuntimeError): - """Classified bench failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -@dataclass(frozen=True) -class BenchMeasurement: - """Aggregate measured performance for one model on one runtime.""" - - repo: str - runtime: str - runs: int - prompt_tokens: int - gen_tokens: int - ttft_ms: Optional[float] - decode_toks: float - prefill_toks: Optional[float] - spread_pct: float - chip: Optional[str] - measured_at: str - samples: List[Dict[str, Any]] = field(default_factory=list) - - def to_dict(self) -> Dict[str, Any]: - return { - "probe_id": BENCH_PROBE_ID, - "outcome": "measured", - "repo": self.repo, - "runtime": self.runtime, - "runs": self.runs, - "prompt_tokens": self.prompt_tokens, - "gen_tokens": self.gen_tokens, - "ttft_ms": self.ttft_ms, - "decode_toks": self.decode_toks, - "prefill_toks": self.prefill_toks, - "peak_mem_gb": None, - "spread_pct": self.spread_pct, - "chip": self.chip, - "measured_at": self.measured_at, - "samples": list(self.samples), - } - - def to_evidence(self, role="general") -> VerificationEvidence: - return VerificationEvidence( - repo=self.repo, - role=role, - strength=EvidenceStrength.RUNTIME_MEASURED, - status=VerificationStatus.VERIFIED, - available_locally=True, - loads=True, - reasoning_confirmed=None, - runtime=self.runtime, - note=_bounded( - "Measured {0} run(s) on this host; median decode " - "{1} tok/s.".format(self.runs, self.decode_toks) - ), - details=self.to_dict(), - ) - - -def validate_bench_bounds(runs, gen_tokens, timeout): - if not isinstance(runs, int) or isinstance(runs, bool): - raise TypeError("runs must be an integer") - if not RUNS_MIN <= runs <= RUNS_MAX: - raise ValueError("runs must be between {0} and {1}".format(RUNS_MIN, RUNS_MAX)) - if not isinstance(gen_tokens, int) or isinstance(gen_tokens, bool): - raise TypeError("gen_tokens must be an integer") - if not GEN_TOKENS_MIN <= gen_tokens <= GEN_TOKENS_MAX: - raise ValueError( - "gen_tokens must be between {0} and {1}".format(GEN_TOKENS_MIN, GEN_TOKENS_MAX) - ) - if not isinstance(timeout, (int, float)) or isinstance(timeout, bool): - raise TypeError("timeout must be a number") - if not 1.0 <= float(timeout) <= 600.0: - raise ValueError("timeout must be between 1 and 600 seconds") - - -def measure_runtime(repo, runtime, runs=RUNS_DEFAULT, gen_tokens=GEN_TOKENS_DEFAULT, - timeout=TIMEOUT_DEFAULT, chip=None, clock=time.monotonic, - now=None): - """Measure one already-served model; never starts or downloads anything.""" - validate_bench_bounds(runs, gen_tokens, timeout) - if not isinstance(repo, str) or not repo.strip(): - raise BenchError( - "invalid_repo", - "bench requires a non-empty model repository identifier.", - "Pass --repo as publisher/model exactly as the runtime serves it.", - ) - stream_generate = getattr(runtime, "stream_generate", None) - if not callable(stream_generate): - raise BenchError( - "unsupported_runtime", - "This runtime does not support the bounded streaming measurement.", - "Serve the model with a runtime that supports streaming generation.", - ) - runtime_name = _runtime_name(runtime) - try: - installed = installed_model_ids(runtime.list_models()) - except Exception as error: - raise BenchError( - "runtime_unreachable", - "The runtime inventory could not be read: {0}".format(_safe_error(error)), - "Confirm the local runtime is running and reachable on its loopback port.", - ) - if not _is_installed(repo, installed): - raise BenchError( - "model_not_serving", - "The model is not served by the {0} runtime.".format(runtime_name), - "Load the model in the runtime first; bench never downloads models.", - ) - - per_run_timeout = float(timeout) / runs - try: - _timed_run(stream_generate, repo, gen_tokens, per_run_timeout, clock) - samples = [ - _timed_run(stream_generate, repo, gen_tokens, per_run_timeout, clock) - for _ in range(runs) - ] - except BenchError: - raise - except Exception as error: - raise BenchError( - "measurement_failed", - "The measurement run failed: {0}".format(_safe_error(error)), - "Confirm the model responds to a normal generation request, then retry.", - ) - - decode_values = [sample["decode_toks"] for sample in samples] - median_decode = statistics.median(decode_values) - spread = 0.0 - if median_decode > 0 and len(decode_values) > 1: - spread = (max(decode_values) - min(decode_values)) / median_decode * 100.0 - ttft_values = [sample["ttft_ms"] for sample in samples if sample["ttft_ms"] is not None] - prefill_values = [ - sample["prefill_toks"] for sample in samples if sample["prefill_toks"] is not None - ] - measured_at = now() if callable(now) else _utc_now() - return BenchMeasurement( - repo=repo, - runtime=runtime_name, - runs=runs, - prompt_tokens=BENCH_PROMPT_TOKEN_ESTIMATE, - gen_tokens=gen_tokens, - ttft_ms=round(statistics.median(ttft_values), 1) if ttft_values else None, - decode_toks=round(median_decode, 1), - prefill_toks=round(statistics.median(prefill_values), 1) if prefill_values else None, - spread_pct=round(spread, 1), - chip=chip, - measured_at=measured_at, - samples=samples, - ) - - -def _timed_run(stream_generate, repo, gen_tokens, timeout, clock): - started = clock() - first_content_at = None - last_event_at = None - content_events = 0 - usage = None - for event in stream_generate(repo, BENCH_PROMPT, gen_tokens, timeout=timeout): - if not isinstance(event, dict): - continue - last_event_at = clock() - text = _stream_event_text(event) - if text: - content_events += 1 - if first_content_at is None: - first_content_at = last_event_at - event_usage = _stream_event_usage(event) - if event_usage: - usage = event_usage - if content_events == 0 and usage is None: - raise BenchError( - "empty_stream", - "The runtime streamed no tokens for the measurement prompt.", - "Confirm the model generates normally, then retry.", - ) - finished = clock() - sample = { - "content_events": content_events, - "elapsed_ms": round((finished - started) * 1000.0, 1), - } - if first_content_at is not None: - sample["ttft_ms"] = round((first_content_at - started) * 1000.0, 1) - else: - sample["ttft_ms"] = None - - if usage and usage.get("eval_count") and usage.get("eval_duration"): - sample["decode_toks"] = round( - usage["eval_count"] / (usage["eval_duration"] / 1e9), 1 - ) - if usage.get("prompt_eval_count") and usage.get("prompt_eval_duration"): - sample["prefill_toks"] = round( - usage["prompt_eval_count"] / (usage["prompt_eval_duration"] / 1e9), 1 - ) - sample["prompt_tokens"] = usage["prompt_eval_count"] - elif first_content_at is not None and content_events > 1 and last_event_at > first_content_at: - sample["decode_toks"] = round( - (content_events - 1) / (last_event_at - first_content_at), 1 - ) - ttft_seconds = first_content_at - started - if ttft_seconds > 0: - sample["prefill_toks"] = round(BENCH_PROMPT_TOKEN_ESTIMATE / ttft_seconds, 1) - elif usage and usage.get("completion_tokens") and finished > started: - sample["decode_toks"] = round(usage["completion_tokens"] / (finished - started), 1) - else: - sample["decode_toks"] = round(content_events / max(finished - started, 1e-9), 1) - sample.setdefault("prefill_toks", None) - return sample - - -def _stream_event_text(event): - message = event.get("message") - if isinstance(message, dict) and isinstance(message.get("content"), str): - return message["content"] - choices = event.get("choices") - if isinstance(choices, list) and choices and isinstance(choices[0], dict): - delta = choices[0].get("delta") - if isinstance(delta, dict) and isinstance(delta.get("content"), str): - return delta["content"] - return "" - - -def _stream_event_usage(event): - usage = event.get("usage") - normalized = {} - if isinstance(usage, dict): - if isinstance(usage.get("completion_tokens"), int): - normalized["completion_tokens"] = usage["completion_tokens"] - if isinstance(usage.get("prompt_tokens"), int): - normalized["prompt_tokens"] = usage["prompt_tokens"] - for key in ("eval_count", "eval_duration", "prompt_eval_count", "prompt_eval_duration"): - value = event.get(key) - if isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0: - normalized[key] = value - return normalized or None - - -def _utc_now(): - from datetime import datetime, timezone - - return datetime.now(timezone.utc).isoformat() - - -BENCH_EXPORT_SCHEMA_VERSION = "1.0" -BENCH_AGGREGATE_SCHEMA_VERSION = "1.0" -MAX_EXPORT_LINE_BYTES = 4096 -MAX_EXPORT_LINES_PER_FILE = 5000 -MAX_AGGREGATE_ENTRIES = 5000 -_COMMUNITY_BENCH_PATH = Path(__file__).resolve().parent / "resources" / "bench-community.json" -_community_bench_cache = None - - -def export_record(measurement, tool_version): - """Build one bounded, anonymized export line from a measurement.""" - record = { - "schema_version": BENCH_EXPORT_SCHEMA_VERSION, - "repo": measurement.repo, - "chip": measurement.chip, - "runtime": measurement.runtime, - "ttft_ms": measurement.ttft_ms, - "decode_toks": measurement.decode_toks, - "prefill_toks": measurement.prefill_toks, - "measured_at": measurement.measured_at, - "tool_version": tool_version, - } - validate_export_record(record) - return record - - -def validate_export_record(record): - if not isinstance(record, dict): - raise ValueError("bench export record must be an object") - if record.get("schema_version") != BENCH_EXPORT_SCHEMA_VERSION: - raise ValueError("bench export record has an unsupported schema version") - for key in ("repo", "chip", "runtime", "measured_at", "tool_version"): - if not isinstance(record.get(key), str) or not record[key]: - raise ValueError("bench export record {0} must be a non-empty string".format(key)) - if len(record["repo"]) > 300 or len(record["chip"]) > 120: - raise ValueError("bench export record identifiers exceed their bounds") - for key in ("ttft_ms", "decode_toks", "prefill_toks"): - value = record.get(key) - if value is None: - continue - if not isinstance(value, (int, float)) or isinstance(value, bool): - raise ValueError("bench export record {0} must be numeric or null".format(key)) - if not 0 <= value <= 1e9: - raise ValueError("bench export record {0} is outside bench bounds".format(key)) - if record.get("decode_toks") is None: - raise ValueError("bench export record requires decode_toks") - return record - - -def append_export(path, record): - """Append one validated export line; never reads or rewrites the file.""" - line = json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" - if len(line.encode("utf-8")) > MAX_EXPORT_LINE_BYTES: - raise BenchError( - "invalid_arguments", - "The export record exceeds its size bound.", - "Export only records produced by bench run.", - ) - destination = Path(path) - flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, "O_NOFOLLOW", 0) - descriptor = os.open(str(destination), flags, 0o600) - try: - os.write(descriptor, line.encode("utf-8")) - finally: - os.close(descriptor) - return destination - - -def aggregate_exports(paths): - """Deduplicate and median-aggregate validated export records.""" - newest = {} - for path in paths: - location = Path(path) - try: - handle = location.open("r", encoding="utf-8") - except OSError: - continue - with handle: - for index, line in enumerate(handle): - if index >= MAX_EXPORT_LINES_PER_FILE: - break - stripped = line.strip() - if not stripped or len(stripped.encode("utf-8")) > MAX_EXPORT_LINE_BYTES: - continue - try: - record = validate_export_record(json.loads(stripped)) - except (ValueError, json.JSONDecodeError): - continue - key = (record["repo"], record["chip"], record["runtime"]) - if key not in newest or record["measured_at"] > newest[key]["measured_at"]: - newest[key] = record - groups = {} - for record in newest.values(): - groups.setdefault((record["repo"], record["chip"]), []).append(record) - entries = [] - for (repo, chip), records in sorted(groups.items()): - if len(entries) >= MAX_AGGREGATE_ENTRIES: - break - def _median(key): - values = [item[key] for item in records if item.get(key) is not None] - return round(statistics.median(values), 1) if values else None - - entries.append({ - "repo": repo, - "chip": chip, - "decode_toks": _median("decode_toks"), - "prefill_toks": _median("prefill_toks"), - "ttft_ms": _median("ttft_ms"), - "samples": len(records), - "updated_at": max(item["measured_at"] for item in records), - }) - return {"schema_version": BENCH_AGGREGATE_SCHEMA_VERSION, "entries": entries} - - -def load_community_bench(path=None): - """Load the bundled community aggregate once; the empty stub yields {}.""" - global _community_bench_cache - if _community_bench_cache is not None and path is None: - return _community_bench_cache - location = Path(path) if path is not None else _COMMUNITY_BENCH_PATH - entries = {} - try: - value = json.loads(location.read_text(encoding="utf-8")) - if isinstance(value, dict) and isinstance(value.get("entries"), list): - for entry in value["entries"]: - if not isinstance(entry, dict): - continue - repo = entry.get("repo") - chip = entry.get("chip") - if isinstance(repo, str) and isinstance(chip, str): - entries[(repo, chip)] = { - key: entry.get(key) - for key in ("decode_toks", "prefill_toks", "ttft_ms", "samples") - } - except (OSError, ValueError, json.JSONDecodeError): - entries = {} - if path is None: - _community_bench_cache = entries - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/cli.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/cli.py deleted file mode 100644 index bbe2888..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/cli.py +++ /dev/null @@ -1,1924 +0,0 @@ -"""Command-line entry points for the dependency-free MLX agent core.""" - -import argparse -import json -import os -import sys -import urllib.parse -from datetime import datetime, timezone -from pathlib import Path - -from .adoption import ADOPTION_SCHEMA_VERSION, AdoptionRequest, AdoptionWorkflow -from .bench import ( - BENCH_PROBE_ID, - GEN_TOKENS_DEFAULT, - RUNS_DEFAULT, - TIMEOUT_DEFAULT, - BenchError, - measure_runtime, -) -from .contracts import ErrorDetail, ResultEnvelope -from .convert import ( - Q_BITS_CHOICES, - ConvertError, - plan_convert, - start_convert, - status_convert, -) -from .discovery import DiscoveryRequest, DiscoveryService -from .fleet import ( - FleetConfigAdapter, - FleetError, - assignments_from_adoption, - parse_assignments, - parse_runtime_map, -) -from .fuse import ( - FuseError, - plan_fuse, - start_fuse, - status_fuse, -) -from .host import HostInventory -from .huggingface import HuggingFaceClient -from .installer import Installer, InstallerConflictError -from .interview import build_intent, run_interview -from .lora import ( - BATCH_DEFAULT, - ITERS_DEFAULT, - LAYERS_DEFAULT, - LR_DEFAULT, - LoraError, - plan_lora, - start_lora, - status_lora, -) -from .modality import ALL_FACET_IDS, FOUNDATION_IDS, resolve_facets, resolve_modalities -from .models import DISCOVERY_ROLES, render_md, wire -from .project_blueprint import ( - build_brief, - generate_design_pack, - render_design_pack, - write_design_pack, -) -from .providers import ProviderRegistry -from .research import generate_pack, render_pack, write_pack -from .serve import ( - MAX_TOKENS_DEFAULT, - ServeError, - default_model_present, - load_recipes, - plan_start, - receipts_root, - start_serve, - status_serve, - stop_serve, - wired_port_claim, -) -from .transactions import ( - COOPERATIVE_CONCURRENCY_NOTE, - ConcurrentTransactionError, - Receipt, - Transaction, - _assert_safe_target, - _read_regular, - preview_rollback, - rollback, -) -from .verification import ( - LMStudioRuntimeClient, - MLXVLMRuntimeClient, - OllamaRuntimeClient, - OpenAICompatibleRuntimeClient, - Verifier, -) -from .watch import ( - WatchError, - build_snapshot, - collect_owned, - diff_snapshots, - read_baseline, - snapshot_candidates, - write_snapshot, -) -from .wiring import ConfigAdapter - - -FIXTURE_WARNING = { - "code": "synthetic_fixture", - "message": "Fixture-backed discovery; this is not live Hugging Face evidence.", -} - - -def _fixture_http_get(payload): - - def get(url, timeout=10.0): - del timeout - if "/api/datasets" in url: - return payload.get("datasets", []) - if "/tree/main" in url: - repo = urllib.parse.unquote(url.split("/api/models/", 1)[1].split("/tree/main", 1)[0]) - return payload["trees"].get(repo, []) - if "/api/models/" in url: - repo = urllib.parse.unquote(url.split("/api/models/", 1)[1]) - return payload["details"].get(repo, {}) - if url.startswith("http://127.0.0.1") or url.startswith("http://localhost"): - raise OSError("fixture does not emulate local runtime endpoints") - query = urllib.parse.urlsplit(url).query - params = urllib.parse.parse_qs(query) - filters = params.get("filter", []) - if "peft" in filters: - return payload.get("adapters", []) - return payload["models"] - - return get - - -def _discovery_service_from_environment(state_dir=None): - fixture = os.environ.get("MLX_AGENT_FIXTURE") - if not fixture: - return DiscoveryService(state_dir=state_dir), None, None - try: - payload = json.loads(Path(fixture).read_text()) - _validate_fixture(payload) - except (OSError, ValueError, TypeError, json.JSONDecodeError) as error: - return None, None, ResultEnvelope.fail( - "discover", "invalid_fixture", "MLX_AGENT_FIXTURE is invalid: {0}".format(error), - "Use a valid test fixture or unset MLX_AGENT_FIXTURE to run live discovery.", - ) - service = DiscoveryService( - host=HostInventory(**payload["host"]), - huggingface=HuggingFaceClient( - http_get=_fixture_http_get(payload), - card_get=lambda url, timeout=8: None, - ), - state_dir=state_dir, - cache_enabled=False, - ) - return service, FIXTURE_WARNING, None - - -def _validate_fixture(payload): - if not isinstance(payload, dict): - raise ValueError("fixture root must be an object") - if not isinstance(payload.get("models"), list): - raise ValueError("fixture.models must be a list") - for index, model in enumerate(payload["models"]): - if not isinstance(model, dict): - raise ValueError("fixture.models[{0}] must be an object".format(index)) - identifiers = [model[key] for key in ("id", "modelId") if key in model] - if not identifiers: - raise ValueError("fixture.models[{0}] must contain id or modelId".format(index)) - if any(not isinstance(identifier, str) or not identifier for identifier in identifiers): - raise ValueError("fixture.models[{0}].id and modelId must be non-empty strings".format(index)) - for counter in ("downloads", "likes"): - if counter in model and (not isinstance(model[counter], int) or isinstance(model[counter], bool)): - raise ValueError("fixture.models[{0}].{1} must be an integer".format(index, counter)) - if not isinstance(payload.get("details"), dict): - raise ValueError("fixture.details must be an object") - if not isinstance(payload.get("trees"), dict): - raise ValueError("fixture.trees must be an object") - host = payload.get("host") - if not isinstance(host, dict): - raise ValueError("fixture.host must be an object") - if set(host) != {"ram_gb", "chip", "ollama", "lmstudio"}: - raise ValueError("fixture.host must contain only ram_gb, chip, ollama, and lmstudio") - if host["ram_gb"] is not None and (not isinstance(host["ram_gb"], int) or isinstance(host["ram_gb"], bool)): - raise ValueError("fixture.host.ram_gb must be an integer or null") - if host["chip"] is not None and not isinstance(host["chip"], str): - raise ValueError("fixture.host.chip must be a string or null") - if not isinstance(host["ollama"], bool) or not isinstance(host["lmstudio"], bool): - raise ValueError("fixture.host runtime flags must be booleans") - - -def _add_discovery_arguments(parser): - parser.add_argument("--role", choices=DISCOVERY_ROLES) - parser.add_argument("--limit", type=int, default=6) - parser.add_argument("--memory-gb", type=float, help="maximum host memory budget in GB (keeps 20%% runtime headroom)") - parser.add_argument("--quantization", help="normalized quantization such as 4bit or q8") - parser.add_argument("--license", dest="licenses", action="append", help="allow only this license (repeatable)") - parser.add_argument("--publisher", dest="publishers", action="append", help="allow only this publisher (repeatable)") - parser.add_argument("--runtime", choices=["ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"], help="require a runtime compatible with the model role") - parser.add_argument("--context", dest="context_tokens", type=int, default=None, help="tighten fit checks to weights plus KV cache at this context length") - parser.add_argument("--exclude-gated", dest="include_gated", action="store_false", default=True, help="exclude gated repositories") - parser.add_argument("--include-gated", dest="include_gated", action="store_true", help="include gated repositories (the legacy default)") - cache_group = parser.add_mutually_exclusive_group() - cache_group.add_argument("--refresh", action="store_true", help="bypass a fresh cache and fetch live evidence") - cache_group.add_argument("--offline", action="store_true", help="use only a matching local cache entry") - parser.add_argument("--state-dir", help="directory for versioned discovery cache entries") - parser.add_argument("--new", action="store_true", help="sort by most-recently-updated") - parser.add_argument("--fast", action="store_true", help="skip per-model enrichment (name heuristics only)") - parser.add_argument("--json", action="store_true") - parser.add_argument("--wire", metavar="REPO", help="emit setup + config for a model, instead of discovering") - parser.add_argument("--target", choices=["ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"], default="mlx_lm") - parser.add_argument("--port", type=int, default=8080) - - -def _run_discovery(arguments, legacy): - if arguments.wire: - print(wire(arguments.wire, arguments.target, arguments.port)) - return 0 - if arguments.context_tokens is not None and not 1024 <= arguments.context_tokens <= 1048576: - result = ResultEnvelope.fail( - "discover", - "invalid_arguments", - "--context must be between 1024 and 1048576 tokens.", - "Pass a bounded context length, or omit --context for weights-only fit.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - error = result.to_dict()["error"] - print("discover failed [{0}]: {1}\nremediation: {2}".format(error["code"], error["message"], error["remediation"])) - return 2 - service, fixture_warning, fixture_error = _discovery_service_from_environment(arguments.state_dir) - result = fixture_error or service.discover(DiscoveryRequest( - role=arguments.role, - memory_gb=arguments.memory_gb, - quantization=arguments.quantization, - licenses=arguments.licenses, - include_gated=arguments.include_gated, - publishers=arguments.publishers, - runtime=arguments.runtime, - refresh=arguments.refresh, - offline=arguments.offline, - limit=arguments.limit, - new=arguments.new, - fast=arguments.fast, - context_tokens=arguments.context_tokens, - )) - if fixture_warning: - result = ResultEnvelope.ok("discover", result.data, warnings=[fixture_warning]) - print("warning: synthetic fixture-backed discovery; not live Hugging Face evidence.", file=sys.stderr) - value = result.to_dict() - if legacy: - report = value["data"] if result.status == "ok" else {"host": service.host.to_dict() if service else HostInventory().to_dict(), "error": value["error"]["message"], "roles": {}} - print(json.dumps(report, indent=2) if arguments.json else render_md(report)) - return 0 if result.status == "ok" else 2 - if arguments.json: - print(json.dumps(value, indent=2)) - elif result.status == "ok": - print(render_md(value["data"])) - else: - error = value["error"] - print("discover failed [{0}]: {1}\nremediation: {2}".format(error["code"], error["message"], error["remediation"])) - return 0 if result.status == "ok" else 2 - - -def _run_inspect_host(arguments): - host, warnings = HostInventory.inspect(HuggingFaceClient()._http_get) - result = ResultEnvelope.ok("inspect-host", host.to_dict(), warnings=warnings) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Host inventory: {0}".format(json.dumps(host.to_dict(), sort_keys=True))) - for warning in warnings: - print("warning [{0}/{1}]: {2}".format(warning["code"], warning["probe"], warning["message"])) - return 0 - - -def _adoption_state_path(arguments): - return arguments.state or os.environ.get("MLX_AGENT_ADOPTION_STATE") - - -def _emit_adoption_result(result, as_json): - value = result.to_dict() - if as_json: - print(json.dumps(value, indent=2)) - elif result.status == "ok": - state = result.data["state"] - print("Adoption {0}: {1}".format(state["status"], state["phase"])) - if state["recommendations"]: - for item in state["recommendations"]: - print("{0}: {1} [{2}]".format(item["role"], item["repo"], item["evidence_strength"])) - requested_roles = state.get("request", {}).get("roles", []) - recommended_roles = { - item.get("role") for item in state.get("recommendations", []) - } - if ( - state.get("status") == "complete" - and "tool-use" in requested_roles - and "tool-use" not in recommended_roles - ): - print("No verified tool-use model was found.") - print( - "No model was downloaded. Install a shortlisted candidate in a " - "supported local runtime and start adoption again." - ) - else: - error = value["error"] - print("{0} failed [{1}]: {2}\nremediation: {3}".format( - result.operation, error["code"], error["message"], error["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _run_adoption(arguments): - operation = "adopt-{0}".format(arguments.adopt_command) - state_path = _adoption_state_path(arguments) - if not state_path: - return _emit_adoption_result(ResultEnvelope.fail( - operation, - "state_path_required", - "No adoption state path was supplied.", - "Pass --state PATH or set MLX_AGENT_ADOPTION_STATE.", - ), arguments.json) - - if arguments.adopt_command == "status": - workflow = AdoptionWorkflow( - discovery_service=DiscoveryService(), verifier=Verifier(), state_path=state_path - ) - try: - state = workflow.status(state_path) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - return _emit_adoption_result(ResultEnvelope.fail( - operation, - "adoption_state_invalid", - "Adoption state could not be read: {0}".format(error), - "Check --state PATH and restore a schema-version {0} adoption handoff.".format( - ADOPTION_SCHEMA_VERSION - ), - ), arguments.json) - return _emit_adoption_result( - ResultEnvelope.ok(operation, {"state": state.to_dict()}), arguments.json - ) - - service, fixture_warning, fixture_error = _discovery_service_from_environment() - if fixture_error: - error = fixture_error.to_dict()["error"] - return _emit_adoption_result(ResultEnvelope.fail( - operation, - error["code"], - error["message"], - error["remediation"], - retryable=error["retryable"], - ), arguments.json) - workflow = AdoptionWorkflow( - discovery_service=service, - verifier=Verifier(metadata_client=getattr(service, "_huggingface", None)), - state_path=state_path, - ) - try: - if arguments.adopt_command == "start": - state = workflow.start(AdoptionRequest( - roles=tuple(arguments.roles or ("general",)), - state_path=state_path, - shortlist_limit=arguments.shortlist_limit, - allow_network=arguments.allow_network and not arguments.offline, - offline=arguments.offline, - refresh=arguments.refresh, - fast=arguments.fast, - )) - else: - state = workflow.resume(state_path) - while state.phase != "complete": - state = workflow.advance(state) - except (OSError, TypeError, ValueError, RuntimeError, json.JSONDecodeError) as error: - return _emit_adoption_result(ResultEnvelope.fail( - operation, - "adoption_failed", - "Adoption workflow could not continue: {0}".format(error), - "Inspect the saved state with 'adopt status', resolve the reported issue, and resume.", - ), arguments.json) - warnings = [fixture_warning] if fixture_warning else [] - return _emit_adoption_result( - ResultEnvelope.ok(operation, {"state": state.to_dict()}, warnings=warnings), - arguments.json, - ) - - -def _add_adoption_arguments(parser): - actions = parser.add_subparsers(dest="adopt_command", required=True) - start = actions.add_parser("start", help="start and durably run a model adoption workflow") - start.add_argument("--state", help="adoption handoff path") - start.add_argument("--role", dest="roles", action="append", choices=DISCOVERY_ROLES) - start.add_argument("--shortlist-limit", type=int, default=4) - start.add_argument("--offline", action="store_true", help="use cached discovery and no metadata network requests") - start.add_argument("--refresh", action="store_true", help="refresh model discovery") - start.add_argument("--fast", action="store_true", help="use heuristic-only discovery enrichment") - start.add_argument("--no-network", dest="allow_network", action="store_false", default=True, help="do not inspect missing-model metadata") - start.add_argument("--measure", action="store_true", help="measure verified shortlist candidates (bench) before ranking") - start.add_argument("--json", action="store_true") - for name in ("resume", "status"): - action = actions.add_parser(name, help="{0} an adoption handoff".format(name)) - action.add_argument("--state", help="adoption handoff path") - action.add_argument("--json", action="store_true") - - -def _add_research_arguments(parser): - parser.add_argument("--domain", help="one-line domain description (required unless --interview)") - parser.add_argument("--role", dest="roles", action="append", choices=DISCOVERY_ROLES, help="model role to research (repeatable)") - parser.add_argument("--keyword", dest="keywords", action="append", help="domain keyword to prioritize (repeatable)") - parser.add_argument( - "--modality", - dest="modalities", - action="append", - choices=list(FOUNDATION_IDS), - help="foundational modality to seed (repeatable): audio, video, document-vision", - ) - parser.add_argument( - "--facet", - dest="facets", - action="append", - choices=list(ALL_FACET_IDS), - help="modality facet to prioritize (repeatable)", - ) - parser.add_argument("--license", dest="licenses", action="append", help="allow only this license (repeatable)") - parser.add_argument("--memory-gb", type=float, help="host memory budget in GB") - parser.add_argument("--notes", default="", help="free-form constraints") - parser.add_argument("--project", default=str(Path.cwd()), help="project root; the pack is written under /mlx-research") - parser.add_argument("--limit", type=int, default=6, help="candidates fetched per role during discovery; the pack is also capped at this many total") - parser.add_argument("--interview", action="store_true", help="ask questions interactively on stdin") - parser.add_argument("--no-write", dest="write", action="store_false", default=True, help="render the pack without writing a file") - parser.add_argument("--json", action="store_true") - - -def _stdin_reader(question): - prompt = question["prompt"] - if question.get("kind") == "multi": - prompt += " [{0}]".format(", ".join(question.get("choices", ()))) - return input("{0}\n> ".format(prompt)) - - -def _emit_research(result, arguments, markdown=None): - value = result.to_dict() - if arguments.json: - print(json.dumps(value, indent=2)) - elif result.status == "ok": - if "path" in result.data: - print("Research pack written to {0}".format(result.data["path"])) - else: - print(markdown if markdown is not None else "Research pack generated.") - else: - error = value["error"] - print("research failed [{0}]: {1}\nremediation: {2}".format( - error["code"], error["message"], error["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _run_research(arguments): - operation = "research" - try: - cli_modalities = tuple(arguments.modalities or ()) - cli_facets = tuple(arguments.facets or ()) - if arguments.interview: - intent = run_interview( - _stdin_reader, - preset_modalities=cli_modalities, - preset_facets=cli_facets, - ) - elif not arguments.domain: - return _emit_research(ResultEnvelope.fail( - operation, "domain_required", - "No domain description was supplied.", - "Pass --domain \"...\" or use --interview.", - ), arguments) - else: - detect_text = " ".join( - [arguments.domain] + list(arguments.keywords or []) - ) - modalities = resolve_modalities(cli=cli_modalities, text=detect_text) - if not modalities: - return _emit_research(ResultEnvelope.fail( - operation, "modality_required", - "No foundational modality was supplied or detected from the domain text.", - "Pass --modality audio|video|document-vision (repeatable), " - "or use --interview to choose interactively.", - ), arguments) - facets = resolve_facets(modalities, cli=cli_facets, text=detect_text) - intent = build_intent({ - "domain": arguments.domain, - "roles": arguments.roles or [], - "keywords": ",".join(arguments.keywords or []), - "license": ",".join(arguments.licenses or []), - "memory_gb": arguments.memory_gb, - "notes": arguments.notes, - "modalities": list(modalities), - "facets": list(facets), - }) - except (ValueError, EOFError) as error: - return _emit_research(ResultEnvelope.fail( - operation, "invalid_intent", str(error), - "Correct the interview answers or flags and retry.", - ), arguments) - - service, fixture_warning, fixture_error = _discovery_service_from_environment() - if fixture_error: - error = fixture_error.to_dict()["error"] - return _emit_research(ResultEnvelope.fail( - operation, error["code"], error["message"], error["remediation"], - ), arguments) - hf_client = getattr(service, "_huggingface", None) or HuggingFaceClient() - moment = datetime.now(timezone.utc) - try: - pack = generate_pack(intent, service, hf_client, limit=arguments.limit, now=moment) - except (OSError, TypeError, ValueError) as error: - return _emit_research(ResultEnvelope.fail( - operation, "research_failed", - "Research could not complete: {0}".format(error), - "Check network access to huggingface.co, or unset MLX_AGENT_FIXTURE.", - ), arguments) - markdown = render_pack(pack) - data = {"pack": pack.to_dict()} - if arguments.write: - try: - data["path"] = str(write_pack( - markdown, intent, root=arguments.project, now=moment, pack=pack, - )) - except (OSError, ValueError) as error: - return _emit_research(ResultEnvelope.fail( - operation, "write_failed", - "Research pack could not be written: {0}".format(error), - "Choose a writable --project directory without a symlinked mlx-research folder.", - ), arguments) - warnings = [fixture_warning] if fixture_warning else [] - return _emit_research( - ResultEnvelope.ok(operation, data, warnings=warnings), arguments, markdown - ) - - -def _add_blueprint_arguments(parser): - parser.add_argument("--goal", help="one-line project goal (required)") - parser.add_argument( - "--modality", - dest="modalities", - action="append", - choices=list(FOUNDATION_IDS), - help="foundational modality to include (repeatable)", - ) - parser.add_argument("--memory-gb", type=float, help="host memory budget in GB") - parser.add_argument("--notes", default="", help="free-form constraints") - parser.add_argument( - "--project", - default=str(Path.cwd()), - help="project root; the pack is written under /mlx-blueprints", - ) - parser.add_argument( - "--no-write", - dest="write", - action="store_false", - default=True, - help="render the pack without writing a file", - ) - parser.add_argument("--json", action="store_true") - - -def _emit_blueprint(result, arguments, markdown=None): - value = result.to_dict() - if arguments.json: - print(json.dumps(value, indent=2)) - elif result.status == "ok": - if "path" in result.data: - print("Project design pack written to {0}".format(result.data["path"])) - else: - print(markdown if markdown is not None else "Project design pack generated.") - else: - error = value["error"] - print("blueprint failed [{0}]: {1}\nremediation: {2}".format( - error["code"], error["message"], error["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _run_blueprint(arguments): - operation = "blueprint" - try: - if not arguments.goal: - return _emit_blueprint(ResultEnvelope.fail( - operation, "goal_required", - "No project goal was supplied.", - "Pass --goal \"...\".", - ), arguments) - brief = build_brief({ - "goal": arguments.goal, - "modalities": arguments.modalities or [], - "memory_gb": arguments.memory_gb, - "notes": arguments.notes, - }) - except (ValueError, TypeError) as error: - return _emit_blueprint(ResultEnvelope.fail( - operation, "invalid_brief", str(error), - "Correct the flags and retry.", - ), arguments) - - moment = datetime.now(timezone.utc) - pack = generate_design_pack(brief, now=moment) - markdown = render_design_pack(pack) - data = {"pack": pack.to_dict()} - if arguments.write: - try: - data["path"] = str(write_design_pack( - markdown, brief, root=arguments.project, now=moment, pack=pack, - )) - except (OSError, ValueError) as error: - return _emit_blueprint(ResultEnvelope.fail( - operation, "write_failed", - "Design pack could not be written: {0}".format(error), - "Choose a writable --project directory without a symlinked mlx-blueprints folder.", - ), arguments) - return _emit_blueprint(ResultEnvelope.ok(operation, data), arguments, markdown) - - -def _emit_wire_result(result, as_json, human=None): - if as_json: - print(json.dumps(result.to_dict(), indent=2)) - elif human is not None: - print(human) - elif result.status == "ok": - print(json.dumps(result.data, indent=2)) - else: - error = result.to_dict()["error"] - print("{0} failed [{1}]: {2}\nremediation: {3}".format( - result.operation, error["code"], error["message"], error["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _receipt_data(receipt): - value = receipt.to_dict() - value["receipt_path"] = receipt.receipt_path - return value - - -def _wire_failure(operation, code, message, remediation, data=None): - return ResultEnvelope( - operation=operation, status="error", data=data or {}, - warnings=[{"code": "cooperative_concurrency", "message": COOPERATIVE_CONCURRENCY_NOTE}], - error=ErrorDetail(code, message, remediation), - ) - - -def _wire_ok(operation, data): - return ResultEnvelope.ok(operation, data, warnings=[{ - "code": "cooperative_concurrency", "message": COOPERATIVE_CONCURRENCY_NOTE, - }]) - - -def _wire_render(arguments): - path = _assert_safe_target(arguments.path) - existing = _read_regular(path).decode("utf-8") - adapter = ConfigAdapter.detect(path, runtime=arguments.target) - content = adapter.render(arguments.model, arguments.target, existing) - adapter.validate(content) - return path, adapter, content - - -def _run_wire(arguments): - operation = "wire-{0}".format(arguments.wire_command) - try: - if arguments.wire_command == "status": - location = _assert_safe_target(arguments.receipt) - receipt = Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) - return _emit_wire_result(_wire_ok(operation, {"receipt": _receipt_data(receipt)}), arguments.json) - if arguments.wire_command == "rollback": - if not arguments.confirm: - preview = preview_rollback(arguments.receipt) - result = _wire_ok( - operation, {"preview": preview, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print(json.dumps(preview, indent=2)) - print( - "Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH." - ) - return 2 - if not arguments.preview_hash: - return _emit_wire_result(_wire_failure( - operation, - "preview_hash_required", - "--confirm requires the hash from a reviewed rollback preview.", - "Run wire rollback without --confirm, inspect the current-state preview, then pass --preview-hash.", - ), arguments.json) - receipt = rollback(arguments.receipt, preview_hash=arguments.preview_hash) - result = _wire_ok(operation, {"receipt": _receipt_data(receipt)}) if receipt.status == "rolled_back" else _wire_failure( - operation, receipt.status, "Rollback did not complete; receipt status is {0}.".format(receipt.status), - "Inspect the receipt validations and restore the verified backup manually.", {"receipt": _receipt_data(receipt)}, - ) - return _emit_wire_result(result, arguments.json) - - path, adapter, content = _wire_render(arguments) - if arguments.wire_command == "render": - return _emit_wire_result(_wire_ok(operation, { - "path": str(path), "runtime": arguments.target, "config": content, - "validation": {"parse": True}, - }), arguments.json, human=content) - transaction = Transaction(receipts_dir=arguments.receipts_dir) - preview = transaction.preview([{ - "path": str(path), "content": content, "runtime": arguments.target, - "adapter": adapter, "endpoint": arguments.endpoint, - }]) - if not arguments.confirm: - result = _wire_ok(operation, {"preview": preview, "requires_confirmation": True}) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print(preview["diff"]) - print("Confirmation required: rerun with --confirm to apply this transaction.") - return 2 - if not arguments.preview_hash: - return _emit_wire_result(_wire_failure( - operation, "preview_hash_required", "--confirm requires the preview hash from a prior preview.", - "Run wire apply without --confirm, inspect the preview, then pass its --preview-hash value.", - {"preview": preview}, - ), arguments.json) - if arguments.preview_hash != preview["preview_hash"]: - return _emit_wire_result(_wire_failure( - operation, "preview_stale", "The supplied preview hash does not match the current preview.", - "Generate and inspect a new preview before confirming this mutation.", - {"preview": preview}, - ), arguments.json) - if not arguments.json: - print(preview["diff"]) - receipt = transaction.apply(arguments.preview_hash) - data = {"preview": preview, "receipt": _receipt_data(receipt)} - result = _wire_ok(operation, data) if receipt.status == "applied" else _wire_failure( - operation, receipt.status, "Wire did not apply; receipt status is {0}.".format(receipt.status), - "Inspect the receipt validation results and use the recorded recovery path before retrying.", data, - ) - return _emit_wire_result(result, arguments.json) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - code = "cooperative_lock_busy" if isinstance(error, ConcurrentTransactionError) else ("preview_stale" if str(error).startswith("preview is stale") else "wire_failed") - return _emit_wire_result(_wire_failure( - operation, code, "Wire could not complete: {0}".format(error), - "Correct the target configuration or receipt, then render a new preview.", - ), arguments.json) - - -_BENCH_RUNTIMES = ("ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm") - - -def _bench_runtime_client(name): - if name == "ollama": - return OllamaRuntimeClient() - if name == "lmstudio": - return LMStudioRuntimeClient() - if name == "mlx-vlm": - return MLXVLMRuntimeClient() - if name == "mlx_lm": - return OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080") - if name == "litellm": - return OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000") - raise ValueError("unsupported bench runtime: {0}".format(name)) - - -def _add_bench_arguments(parser): - actions = parser.add_subparsers(dest="bench_command", required=True) - run = actions.add_parser( - "run", - help="measure a model already served by a local runtime (never downloads)", - ) - run.add_argument("--repo", required=True, help="model identifier exactly as the runtime serves it") - run.add_argument("--runtime", required=True, choices=_BENCH_RUNTIMES) - run.add_argument("--role", default="general", choices=DISCOVERY_ROLES) - run.add_argument("--runs", type=int, default=RUNS_DEFAULT, help="timed runs after one warm-up (1-10)") - run.add_argument("--gen-tokens", type=int, default=GEN_TOKENS_DEFAULT, help="tokens generated per run (16-2048)") - run.add_argument("--timeout", type=float, default=TIMEOUT_DEFAULT, help="total measurement deadline in seconds") - run.add_argument("--export", default=None, help="append an anonymized result line to this JSONL file") - run.add_argument("--json", action="store_true") - aggregate = actions.add_parser( - "aggregate", - help="deduplicate and median-aggregate bench export files", - ) - aggregate.add_argument("--exports", required=True, help="directory of .jsonl export files") - aggregate.add_argument("--out", default=None, help="aggregate output path (default: print only)") - aggregate.add_argument("--json", action="store_true") - - -def _run_bench(arguments): - operation = "bench-{0}".format(arguments.bench_command) - try: - if arguments.bench_command == "aggregate": - from .bench import aggregate_exports - - exports_dir = Path(arguments.exports) - if not exports_dir.is_dir(): - raise BenchError( - "invalid_arguments", - "The exports directory does not exist: {0}".format(arguments.exports), - "Point --exports at a directory of bench export .jsonl files.", - ) - files = sorted(exports_dir.glob("*.jsonl")) - aggregate = aggregate_exports(files) - if arguments.out: - destination = Path(arguments.out) - content = (json.dumps(aggregate, indent=2, sort_keys=True) + "\n").encode("utf-8") - destination.write_bytes(content) - data = { - "entries": aggregate["entries"], - "sources": len(files), - "written": str(arguments.out) if arguments.out else None, - } - result = ResultEnvelope.ok(operation, data) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Aggregate: {0} entries from {1} export file(s){2}".format( - len(aggregate["entries"]), len(files), - " -> {0}".format(arguments.out) if arguments.out else "", - )) - return 0 - chip = None - try: - chip = HostInventory.inspect().chip - except Exception: - chip = None - measurement = measure_runtime( - arguments.repo, - _bench_runtime_client(arguments.runtime), - runs=arguments.runs, - gen_tokens=arguments.gen_tokens, - timeout=arguments.timeout, - chip=chip, - ) - evidence = measurement.to_evidence(role=arguments.role).to_dict() - data = {"measurement": measurement.to_dict(), "evidence": evidence} - if arguments.export: - from . import __version__ - from .bench import append_export, export_record - - destination = append_export( - arguments.export, export_record(measurement, __version__) - ) - data["exported"] = str(destination) - result = ResultEnvelope.ok(operation, data) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Bench: {0} on {1} ({2} runs)".format( - measurement.repo, measurement.runtime, measurement.runs - )) - ttft = "{0} ms".format(measurement.ttft_ms) if measurement.ttft_ms is not None else "n/a" - prefill = ( - "{0} tok/s".format(measurement.prefill_toks) - if measurement.prefill_toks is not None - else "n/a" - ) - print(" decode: {0} tok/s (spread {1}%)".format(measurement.decode_toks, measurement.spread_pct)) - print(" ttft: {0}".format(ttft)) - print(" prefill: {0}".format(prefill)) - return 0 - except BenchError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (TypeError, ValueError) as error: - result = ResultEnvelope.fail( - operation, - "invalid_arguments", - str(error), - "Correct the bench arguments and retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("bench failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _add_serve_arguments(parser): - actions = parser.add_subparsers(dest="serve_command", required=True) - start = actions.add_parser( - "start", - help="preview, then confirmation-gated launch of a local MLX server", - ) - start.add_argument("--repo", required=True, help="publisher/model present in the local Hugging Face cache") - start.add_argument("--runtime", required=True, choices=["mlx_lm", "mlx-vlm"]) - start.add_argument("--port", type=int, default=None, help="loopback port (defaults per runtime recipe)") - start.add_argument("--max-tokens", type=int, default=MAX_TOKENS_DEFAULT) - start.add_argument("--adapter-path", default=None, help="LoRA adapter directory (mlx_lm only)") - start.add_argument("--launchd", action="store_true", help="install a launchd agent plist instead of spawning now") - start.add_argument("--launchd-dir", default=None, help="launchd target directory (defaults to ~/Library/LaunchAgents)") - start.add_argument("--confirm", action="store_true", help="authorize this reviewed server launch") - start.add_argument("--preview-hash", help="hash returned by the separately reviewed serve start preview") - start.add_argument("--receipts-dir", default=None, help="receipt root (defaults to the current directory)") - start.add_argument("--hf-cache", default=None, help="Hugging Face cache root for the local-model gate") - start.add_argument("--json", action="store_true") - stop = actions.add_parser("stop", help="stop exactly the server a serve receipt owns") - stop.add_argument("--port", type=int, required=True) - stop.add_argument("--receipts-dir", default=None) - stop.add_argument("--json", action="store_true") - status = actions.add_parser("status", help="cross-check serve receipts against live processes") - status.add_argument("--receipts-dir", default=None) - status.add_argument("--json", action="store_true") - - -def _run_serve(arguments): - operation = "serve-{0}".format(arguments.serve_command) - try: - if arguments.serve_command == "status": - entries = status_serve(arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, {"servers": entries}), arguments.json, - human=_serve_status_human(entries), - ) - if arguments.serve_command == "stop": - outcome = stop_serve(arguments.port, arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="serve {0}: port {1} (pid {2})".format( - outcome["status"], outcome["port"], outcome["pid"] - ), - ) - recipes = load_recipes() - plan = plan_start( - arguments.repo, - arguments.runtime, - recipes, - port=arguments.port, - max_tokens=arguments.max_tokens, - adapter_path=arguments.adapter_path, - ) - if arguments.launchd: - return _run_serve_launchd(arguments, plan, operation) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Serve plan: {0} on 127.0.0.1:{1}".format(plan["repo"], plan["port"])) - print(" argv: {0}".format(" ".join(plan["argv"]))) - print(" readiness: {0}".format(plan["readiness"])) - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - wired_claims = wired_port_claim([ - arguments.receipts_dir or str(Path.cwd()) - ]) - outcome = start_serve( - plan, - receipts_dir=arguments.receipts_dir, - confirm=True, - preview_hash=arguments.preview_hash, - model_present=default_model_present(arguments.hf_cache), - wired_claims=wired_claims, - ) - receipt = outcome["receipt"] - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="serve started: {0} on 127.0.0.1:{1} (pid {2})\n log: {3}".format( - receipt["repo"], receipt["port"], receipt["pid"], receipt["log_path"] - ), - ) - except ServeError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "serve_failed", - str(error), - "Correct the serve arguments or receipt state, then retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("serve failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _run_serve_launchd(arguments, plan, operation): - """Preview-confirm installation of a launchd agent for a serve plan.""" - from .serve import LaunchdPlistAdapter, launchd_label, render_launchd_plist - - launchd_dir = Path(arguments.launchd_dir) if arguments.launchd_dir else ( - Path.home() / "Library" / "LaunchAgents" - ) - target = launchd_dir / "{0}.plist".format(launchd_label(plan["port"])) - log_path = str(receipts_root(arguments.receipts_dir) / "{0}.log".format(plan["port"])) - content = render_launchd_plist(plan, log_path=log_path) - adapter = LaunchdPlistAdapter(target) - adapter.validate(content) - if target.exists(): - result = ResultEnvelope.fail( - operation, - "output_exists", - "A launchd plist already exists at {0}.".format(target), - "Unload and remove it yourself; serve never overwrites launchd agents.", - ) - return _emit_serve_result(result, arguments.json) - if not launchd_dir.is_dir(): - result = ResultEnvelope.fail( - operation, - "invalid_arguments", - "The launchd directory does not exist: {0}".format(launchd_dir), - "Create it yourself, or pass --launchd-dir.", - ) - return _emit_serve_result(result, arguments.json) - if not default_model_present(arguments.hf_cache)(plan["repo"]): - result = ResultEnvelope.fail( - operation, - "model_not_local", - "The model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; serve never downloads.", - ) - return _emit_serve_result(result, arguments.json) - transaction = Transaction(receipts_dir=arguments.receipts_dir) - preview = transaction.preview([{ - "path": str(target), "content": content, "runtime": "launchd", - "adapter": adapter, "endpoint": None, - }]) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"preview": preview, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print(preview["diff"]) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - if not arguments.preview_hash: - result = ResultEnvelope.fail( - operation, - "preview_hash_required", - "--confirm requires the preview hash from a prior preview.", - "Run serve start --launchd without --confirm, inspect the preview, then pass --preview-hash.", - ) - return _emit_serve_result(result, arguments.json) - if arguments.preview_hash != preview["preview_hash"]: - result = ResultEnvelope.fail( - operation, - "preview_stale", - "The supplied preview hash does not match the current preview.", - "Generate and inspect a new preview before confirming this mutation.", - ) - return _emit_serve_result(result, arguments.json) - receipt = transaction.apply(arguments.preview_hash) - data = {"receipt": _receipt_data(receipt), "plist": str(target)} - result = ResultEnvelope.ok(operation, data) - return _emit_serve_result( - result, arguments.json, - human="launchd agent installed: {0}\n load it with: launchctl bootstrap gui/$(id -u) {1}".format( - target, target - ), - ) - - -def _emit_serve_result(result, as_json, human=None): - if as_json: - print(json.dumps(result.to_dict(), indent=2)) - elif human: - print(human) - elif result.status == "ok": - print(json.dumps(result.to_dict()["data"], indent=2)) - else: - payload = result.to_dict()["error"] - print("{0} failed [{1}]: {2}\nremediation: {3}".format( - result.operation, payload["code"], payload["message"], payload["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _serve_status_human(entries): - if not entries: - return "No serve receipts." - lines = [] - for entry in entries: - state = "alive" if entry["alive"] else "dead" - if entry["alive"] and not entry["argv_match"]: - state = "alive (argv mismatch)" - lines.append(" 127.0.0.1:{0} {1} ({2}) pid {3} - {4}".format( - entry["port"], entry["repo"], entry["runtime"], entry["pid"], state - )) - return "Serve receipts:\n" + "\n".join(lines) - - -def _add_fleet_arguments(parser): - actions = parser.add_subparsers(dest="fleet_command", required=True) - for name in ("render", "apply"): - action = actions.add_parser( - name, - help="{0} a one-shot per-role router configuration".format(name), - ) - action.add_argument("--path", required=True, help="target router configuration file") - action.add_argument("--assign", action="append", default=None, metavar="ROLE=REPO", - help="per-role model assignment (repeatable)") - action.add_argument("--from-adoption", default=None, metavar="STATE", - help="read role assignments from a completed adopt handoff") - action.add_argument("--runtime-map", action="append", default=None, metavar="ROLE=RUNTIME", - help="per-role runtime override: mlx_lm or mlx-vlm (repeatable)") - action.add_argument("--allow-missing", action="store_true", - help="warn instead of failing when a model is not in a local inventory") - action.add_argument("--json", action="store_true") - if name == "apply": - action.add_argument("--confirm", action="store_true", help="explicitly authorize this reviewed mutation") - action.add_argument("--preview-hash", help="hash returned by the separately reviewed fleet apply preview") - action.add_argument("--receipts-dir", help="directory for non-secret transaction receipts") - action.add_argument("--endpoint", help="optional local runtime health endpoint") - - -def _fleet_inputs(arguments): - if arguments.assign and arguments.from_adoption: - raise FleetError( - "invalid_arguments", - "--assign and --from-adoption are mutually exclusive.", - "Pick explicit assignments or an adoption handoff, not both.", - ) - if arguments.from_adoption: - assignments = assignments_from_adoption(arguments.from_adoption) - else: - assignments = parse_assignments(arguments.assign) - runtime_map = parse_runtime_map(arguments.runtime_map) - return assignments, runtime_map - - -def _fleet_missing_models(assignments): - from .model_doctor import ( - collect_runtime_inventory, - default_hf_cache, - inspect_hf_cache, - ) - - known = {item["id"].casefold() for item in inspect_hf_cache(default_hf_cache())} - runtime_clients = [ - OllamaRuntimeClient(), - LMStudioRuntimeClient(), - OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080"), - MLXVLMRuntimeClient(), - OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000"), - ] - runtime_inventory, _errors = collect_runtime_inventory(runtime_clients) - known.update(item["id"].casefold() for item in runtime_inventory) - return sorted( - repo for repo in set(assignments.values()) if repo.casefold() not in known - ) - - -def _run_fleet(arguments): - operation = "fleet-{0}".format(arguments.fleet_command) - try: - assignments, runtime_map = _fleet_inputs(arguments) - path = _assert_safe_target(arguments.path) - existing = _read_regular(path).decode("utf-8") - adapter = FleetConfigAdapter(path) - content = adapter.render(assignments, runtime_map, existing=existing) - missing = _fleet_missing_models(assignments) - warnings = [] - if missing and not arguments.allow_missing: - return _emit_wire_result(_wire_failure( - operation, - "model_not_local", - "Models are not present in a local inventory: {0}".format(", ".join(missing)), - "Download them first, or pass --allow-missing to record a reviewed warning.", - ), arguments.json) - if missing: - warnings.append({ - "code": "model_not_local", - "message": "Not in a local inventory: {0}".format(", ".join(missing)), - }) - if arguments.fleet_command == "render": - result = _wire_ok(operation, { - "path": str(path), "assignments": assignments, "config": content, - "validation": {"parse": True}, - }) - result.warnings.extend(warnings) - return _emit_wire_result(result, arguments.json, human=content) - transaction = Transaction(receipts_dir=arguments.receipts_dir) - preview = transaction.preview([{ - "path": str(path), "content": content, "runtime": "litellm", - "adapter": adapter, "endpoint": arguments.endpoint, - }]) - if not arguments.confirm: - result = _wire_ok(operation, {"preview": preview, "requires_confirmation": True}) - result.warnings.extend(warnings) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print(preview["diff"]) - print("Confirmation required: rerun with --confirm to apply this transaction.") - return 2 - if not arguments.preview_hash: - return _emit_wire_result(_wire_failure( - operation, "preview_hash_required", "--confirm requires the preview hash from a prior preview.", - "Run fleet apply without --confirm, inspect the preview, then pass its --preview-hash value.", - {"preview": preview}, - ), arguments.json) - if arguments.preview_hash != preview["preview_hash"]: - return _emit_wire_result(_wire_failure( - operation, "preview_stale", "The supplied preview hash does not match the current preview.", - "Generate and inspect a new preview before confirming this mutation.", - {"preview": preview}, - ), arguments.json) - if not arguments.json: - print(preview["diff"]) - receipt = transaction.apply(arguments.preview_hash) - data = {"preview": preview, "receipt": _receipt_data(receipt)} - result = _wire_ok(operation, data) if receipt.status == "applied" else _wire_failure( - operation, receipt.status, "Fleet did not apply; receipt status is {0}.".format(receipt.status), - "Inspect the receipt validation results and use the recorded recovery path before retrying.", data, - ) - result.warnings.extend(warnings) - return _emit_wire_result(result, arguments.json) - except FleetError as error: - return _emit_wire_result(_wire_failure( - operation, error.code, str(error), error.remediation, - ), arguments.json) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - code = "cooperative_lock_busy" if isinstance(error, ConcurrentTransactionError) else ("preview_stale" if str(error).startswith("preview is stale") else "fleet_failed") - return _emit_wire_result(_wire_failure( - operation, code, "Fleet could not complete: {0}".format(error), - "Correct the assignments or target configuration, then render a new preview.", - ), arguments.json) - - -def _watch_runtime_clients(): - return [ - OllamaRuntimeClient(), - LMStudioRuntimeClient(), - OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080"), - MLXVLMRuntimeClient(), - OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000"), - ] - - -def _add_watch_arguments(parser): - actions = parser.add_subparsers(dest="watch_command", required=True) - for name in ("snapshot", "diff"): - action = actions.add_parser( - name, - help="{0} the owned-model Hugging Face watch state".format(name), - ) - action.add_argument("--state-dir", default="./.mlx-agent-watch", - help="watch state root (default ./.mlx-agent-watch)") - action.add_argument("--limit", type=int, default=6, - help="discovery results per role (default 6)") - action.add_argument("--json", action="store_true") - - -def _run_watch(arguments): - operation = "watch-{0}".format(arguments.watch_command) - try: - service, fixture_warning, fixture_error = _discovery_service_from_environment() - if fixture_error is not None: - return _emit_wire_result(fixture_error, arguments.json) - result = service.discover(DiscoveryRequest(limit=arguments.limit)) - if result.status != "ok": - error = result.to_dict()["error"] - return _emit_wire_result(ResultEnvelope.fail( - operation, error["code"], error["message"], error["remediation"] - ), arguments.json) - owned = collect_owned(_watch_runtime_clients()) - candidates = snapshot_candidates(result.data) - if arguments.watch_command == "snapshot": - snapshot = build_snapshot(owned, candidates) - destination = write_snapshot(arguments.state_dir, snapshot) - data = { - "state": str(destination), - "owned": len(snapshot["owned"]), - "candidates": len(snapshot["candidates"]), - "rotated_previous": snapshot["previous"] is not None, - } - result = ResultEnvelope.ok(operation, data) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Watch snapshot: {0} owned, {1} tracked repos -> {2}".format( - data["owned"], data["candidates"], data["state"] - )) - return 0 - baseline = read_baseline(arguments.state_dir) - current = build_snapshot(owned, candidates) - findings = diff_snapshots(baseline, current) - data = { - "baseline_created_at": baseline["created_at"], - "findings": findings, - } - warnings = [ - {"code": finding["code"], "message": "{0}: {1}".format(finding["repo"], finding["detail"])} - for finding in findings - ] - result = ResultEnvelope.ok(operation, data, warnings=warnings) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Watch diff against {0}: {1} finding(s)".format( - baseline["created_at"], len(findings) - )) - for finding in findings: - print(" [{0}] {1} - {2}".format(finding["code"], finding["repo"], finding["detail"])) - return 0 - except WatchError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "watch_failed", - str(error), - "Correct the state directory and retry; watch only writes its own state file.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("watch failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _add_convert_arguments(parser): - actions = parser.add_subparsers(dest="convert_command", required=True) - start = actions.add_parser( - "start", - help="preview, then confirmation-gated quantization of a cached model", - ) - start.add_argument("--repo", required=True, help="publisher/model present in the local Hugging Face cache") - start.add_argument("--q-bits", type=int, default=4, choices=Q_BITS_CHOICES) - start.add_argument("--out", default=None, help="output directory (default -MLX-bit)") - start.add_argument("--confirm", action="store_true", help="authorize this reviewed conversion") - start.add_argument("--preview-hash", help="hash returned by the separately reviewed convert preview") - start.add_argument("--receipts-dir", default=None) - start.add_argument("--hf-cache", default=None) - start.add_argument("--json", action="store_true") - status = actions.add_parser("status", help="cross-check convert receipts against live processes") - status.add_argument("--receipts-dir", default=None) - status.add_argument("--json", action="store_true") - - -def _run_convert(arguments): - operation = "convert-{0}".format(arguments.convert_command) - try: - if arguments.convert_command == "status": - entries = status_convert(arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, {"jobs": entries}), arguments.json, - human=_convert_status_human(entries), - ) - plan = plan_convert(arguments.repo, q_bits=arguments.q_bits, out=arguments.out) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Convert plan: {0} -> {1} ({2}bit)".format( - plan["repo"], plan["out"], plan["q_bits"] - )) - print(" argv: {0}".format(" ".join(plan["argv"]))) - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - outcome = start_convert( - plan, - receipts_dir=arguments.receipts_dir, - confirm=True, - preview_hash=arguments.preview_hash, - model_present=default_model_present(arguments.hf_cache), - ) - receipt = outcome["receipt"] - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="convert started: {0} -> {1} (pid {2})\n log: {3}".format( - receipt["repo"], receipt["out"], receipt["pid"], receipt["log_path"] - ), - ) - except ConvertError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "convert_failed", - str(error), - "Correct the convert arguments or receipt state, then retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("convert failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _convert_status_human(entries): - if not entries: - return "No convert receipts." - lines = [] - for entry in entries: - lines.append(" {0} ({1}bit) -> {2}: {3}".format( - entry["repo"], entry["q_bits"], entry["out"], entry["state"] - )) - return "Convert jobs:\n" + "\n".join(lines) - - -def _add_lora_arguments(parser): - actions = parser.add_subparsers(dest="lora_command", required=True) - start = actions.add_parser( - "start", - help="preview, then confirmation-gated LoRA training on a cached model", - ) - start.add_argument("--repo", required=True, help="publisher/model base present in the local Hugging Face cache") - start.add_argument("--data", required=True, help="dataset directory with train.jsonl") - start.add_argument("--iters", type=int, default=ITERS_DEFAULT) - start.add_argument("--batch-size", type=int, default=BATCH_DEFAULT) - start.add_argument("--learning-rate", type=float, default=LR_DEFAULT) - start.add_argument("--num-layers", type=int, default=LAYERS_DEFAULT, help="-1 trains all layers") - start.add_argument("--out", default=None, help="adapter output directory (default -lora)") - start.add_argument("--confirm", action="store_true", help="authorize this reviewed training run") - start.add_argument("--preview-hash", help="hash returned by the separately reviewed lora preview") - start.add_argument("--receipts-dir", default=None) - start.add_argument("--hf-cache", default=None) - start.add_argument("--json", action="store_true") - status = actions.add_parser("status", help="cross-check lora receipts against live processes") - status.add_argument("--receipts-dir", default=None) - status.add_argument("--json", action="store_true") - - -def _run_lora(arguments): - operation = "lora-{0}".format(arguments.lora_command) - try: - if arguments.lora_command == "status": - entries = status_lora(arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, {"jobs": entries}), arguments.json, - human=_lora_status_human(entries), - ) - plan = plan_lora( - arguments.repo, - arguments.data, - iters=arguments.iters, - batch_size=arguments.batch_size, - learning_rate=arguments.learning_rate, - num_layers=arguments.num_layers, - out=arguments.out, - ) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("LoRA plan: {0} + {1} train lines -> {2}".format( - plan["repo"], plan["dataset"]["train_lines"], plan["out"] - )) - print(" argv: {0}".format(" ".join(plan["argv"]))) - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - outcome = start_lora( - plan, - receipts_dir=arguments.receipts_dir, - confirm=True, - preview_hash=arguments.preview_hash, - model_present=default_model_present(arguments.hf_cache), - ) - receipt = outcome["receipt"] - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="lora started: {0} -> {1} (pid {2})\n log: {3}".format( - receipt["repo"], receipt["out"], receipt["pid"], receipt["log_path"] - ), - ) - except LoraError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "lora_failed", - str(error), - "Correct the lora arguments or receipt state, then retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("lora failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _lora_status_human(entries): - if not entries: - return "No lora receipts." - lines = [] - for entry in entries: - lines.append(" {0} -> {1}: {2}".format( - entry["repo"], entry["out"], entry["state"] - )) - return "LoRA jobs:\n" + "\n".join(lines) - - -def _add_fuse_arguments(parser): - actions = parser.add_subparsers(dest="fuse_command", required=True) - start = actions.add_parser( - "start", - help="preview, then confirmation-gated LoRA fuse into the base model", - ) - start.add_argument("--repo", required=True, help="publisher/model base present in the local Hugging Face cache") - start.add_argument("--adapter", required=True, help="completed lora adapter directory") - start.add_argument("--out", default=None, help="fused output directory (default -fused)") - start.add_argument("--confirm", action="store_true", help="authorize this reviewed fuse") - start.add_argument("--preview-hash", help="hash returned by the separately reviewed fuse preview") - start.add_argument("--receipts-dir", default=None) - start.add_argument("--hf-cache", default=None) - start.add_argument("--json", action="store_true") - status = actions.add_parser("status", help="cross-check fuse receipts against live processes") - status.add_argument("--receipts-dir", default=None) - status.add_argument("--json", action="store_true") - - -def _run_fuse(arguments): - operation = "fuse-{0}".format(arguments.fuse_command) - try: - if arguments.fuse_command == "status": - entries = status_fuse(arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, {"jobs": entries}), arguments.json, - human=_fuse_status_human(entries), - ) - plan = plan_fuse(arguments.repo, arguments.adapter, out=arguments.out) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Fuse plan: {0} + {1} -> {2}".format( - plan["repo"], plan["adapter"], plan["out"] - )) - print(" argv: {0}".format(" ".join(plan["argv"]))) - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - outcome = start_fuse( - plan, - receipts_dir=arguments.receipts_dir, - confirm=True, - preview_hash=arguments.preview_hash, - model_present=default_model_present(arguments.hf_cache), - ) - receipt = outcome["receipt"] - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="fuse started: {0} -> {1} (pid {2})\n log: {3}".format( - receipt["repo"], receipt["out"], receipt["pid"], receipt["log_path"] - ), - ) - except FuseError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "fuse_failed", - str(error), - "Correct the fuse arguments or receipt state, then retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("fuse failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _fuse_status_human(entries): - if not entries: - return "No fuse receipts." - lines = [] - for entry in entries: - lines.append(" {0} -> {1}: {2}".format( - entry["repo"], entry["out"], entry["state"] - )) - return "Fuse jobs:\n" + "\n".join(lines) - - -def _add_wire_arguments(parser): - actions = parser.add_subparsers(dest="wire_command", required=True) - for name in ("render", "apply"): - action = actions.add_parser(name, help="{0} a deterministic runtime configuration ({1})".format(name, "advisory lock protects cooperative writers" if name == "apply" else "safe render")) - action.add_argument("model", help="Hugging Face model repository") - action.add_argument("--target", choices=["ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"], default="mlx_lm") - action.add_argument("--path", required=True, help="target configuration file") - action.add_argument("--json", action="store_true") - if name == "apply": - action.add_argument("--confirm", action="store_true", help="explicitly authorize this reviewed mutation") - action.add_argument("--preview-hash", help="hash returned by the separately reviewed wire apply preview") - action.add_argument("--receipts-dir", help="directory for non-secret transaction receipts") - action.add_argument("--endpoint", help="optional local runtime health endpoint") - status = actions.add_parser("status", help="inspect a Wire receipt") - status.add_argument("receipt") - status.add_argument("--json", action="store_true") - restore = actions.add_parser("rollback", help="restore a Wire receipt's exact backup") - restore.add_argument("receipt") - restore.add_argument("--confirm", action="store_true", help="explicitly authorize this rollback") - restore.add_argument("--preview-hash", help="hash returned by the separately reviewed rollback preview") - restore.add_argument("--json", action="store_true") - - -def _installer_registry(): - root = Path(__file__).resolve().parents[2] - home = os.environ.get("MLX_AGENT_HOME") - config_root = os.environ.get("MLX_AGENT_CONFIG_ROOT") or os.environ.get("XDG_STATE_HOME") - xdg_config_home = os.environ.get("XDG_CONFIG_HOME") - return ProviderRegistry( - root / "plugin.json", home=home, config_root=config_root, - xdg_config_home=xdg_config_home, - ) - - -def _add_installer_arguments(parser, include_providers=True): - if include_providers: - parser.add_argument("providers", nargs="*", help="explicit provider IDs; omit to inspect detected choices") - parser.add_argument("--scope", choices=["user", "project"], default="user") - parser.add_argument("--project", default=str(Path.cwd()), help="project root for project-scoped installation") - parser.add_argument("--dry-run", action="store_true", help="show the reviewed plan without mutating files") - parser.add_argument("--confirm", action="store_true", help="authorize a separately reviewed installer preview") - parser.add_argument("--preview-hash", help="preview hash returned by the installer dry run") - parser.add_argument("--json", action="store_true") - - -def _add_doctor_arguments(parser): - _add_installer_arguments(parser) - parser.add_argument( - "--wired-root", - action="append", - default=None, - help="root scanned for Wire-managed configs (repeatable; defaults to the project root)", - ) - parser.add_argument( - "--hf-cache", - default=None, - help="Hugging Face cache root (defaults to ~/.cache/huggingface/hub)", - ) - parser.add_argument( - "--prune", - action="store_true", - help="preview deletion of incomplete HF cache snapshots (requires --confirm --preview-hash to execute)", - ) - - -def _run_model_doctor(arguments): - from .model_doctor import ( - PruneError, - default_hf_cache, - execute_prune, - plan_prune, - run_model_doctor, - ) - - operation = "doctor-models" - wired_roots = arguments.wired_root or [arguments.project] - hf_cache = arguments.hf_cache or default_hf_cache() - if arguments.prune: - try: - plan = plan_prune(hf_cache) - outcome = execute_prune( - plan, - confirm=arguments.confirm, - preview_hash=arguments.preview_hash, - ) - except PruneError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("doctor failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - if outcome["status"] == "preview": - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - if not plan["candidates"]: - print("Nothing to prune: no incomplete cache snapshots.") - for candidate in plan["candidates"]: - print(" prune: {0} ({1:.1f} GB) - {2}".format( - candidate["repo"], candidate["bytes"] / 1e9, candidate["path"] - )) - if plan["candidates"]: - print("IRREVERSIBLE: these cache directories will be deleted permanently.") - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --prune --confirm --preview-hash PREVIEW_HASH.") - return 0 if not plan["candidates"] else 2 - result = ResultEnvelope.ok(operation, outcome) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - removed = [item for item in outcome["removed"] if item.get("removed")] - print("Pruned {0} incomplete snapshot(s).".format(len(removed))) - for item in removed: - print(" removed: {0}".format(item["repo"])) - return 0 - runtime_clients = [ - OllamaRuntimeClient(), - LMStudioRuntimeClient(), - OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080"), - MLXVLMRuntimeClient(), - OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000"), - ] - try: - report = run_model_doctor(wired_roots, hf_cache, runtime_clients) - except (OSError, TypeError, ValueError) as error: - result = ResultEnvelope.fail( - operation, - "doctor_failed", - str(error), - "Correct the scanned roots and retry; doctor never mutates anything.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("doctor failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - warnings = [ - {"code": finding["code"], "message": "{0}: {1}".format( - finding.get("path") or finding.get("model") or finding["code"], - finding["remediation"], - )} - for finding in report["findings"] - ] - result = ResultEnvelope.ok(operation, report, warnings=warnings) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - summary = report["summary"] - print("Doctor: {0} model(s), {1} wired config(s), {2} finding(s)".format( - summary["models"], summary["wired_configs"], summary["findings"] - )) - print(" hf-cache disk: {0:.1f} GB".format(summary["hf_cache_bytes"] / 1e9)) - for finding in report["findings"]: - location = finding.get("path") or finding.get("model") or "" - print(" [{0}] {1} - {2}".format(finding["code"], location, finding["remediation"])) - for endpoint in report["endpoints"]: - print(" endpoint {0}: {1}".format(endpoint["endpoint"], endpoint["status"])) - for runtime_error in report["runtime_errors"]: - print(" runtime unreachable: {0}".format(runtime_error)) - return 0 - - -def _installer_result(operation, data, as_json, status="ok", error=None): - result = ResultEnvelope.ok(operation, data) if status == "ok" else ResultEnvelope.fail( - operation, error[0], error[1], error[2] - ) - if as_json: - print(json.dumps(result.to_dict(), indent=2)) - elif status == "ok": - if data.get("selection_required"): - available = [item["id"] for item in data["providers"] if item["available"]] - print("No provider was selected. Detected providers: {0}".format(", ".join(available) if available else "none")) - elif "preview" in data: - compatibility = data.get("plan", {}).get("compatibility", []) - if compatibility: - print("Provider compatibility:") - for item in compatibility: - details = "{0}: {1}".format(item["id"], item["state"]) - if item.get("version"): - details += " (version {0})".format(item["version"]) - if item.get("error"): - details += " - {0}".format(item["error"]) - print(details) - print(data["preview"]["diff"]) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - else: - print(json.dumps(data, indent=2)) - else: - print("{0} failed [{1}]: {2}\nremediation: {3}".format(operation, error[0], error[1], error[2])) - return 0 if status == "ok" else 2 - - -def _run_installer(arguments): - operation = arguments.command - if operation == "doctor" and "models" in (getattr(arguments, "providers", None) or []): - return _run_model_doctor(arguments) - try: - installer = Installer(_installer_registry(), project_root=arguments.project) - detections = [item.to_dict() for item in installer.detected()] - selected = tuple(getattr(arguments, "providers", ()) or ()) - if operation == "providers" or not selected: - return _installer_result(operation, { - "providers": detections, - "selection_required": operation != "providers", - "mutated": False, - }, arguments.json) - plan = installer.plan(operation, selected, arguments.scope, arguments.project) - if arguments.dry_run or (operation != "doctor" and not arguments.confirm): - return _installer_result(operation, {"plan": plan.to_dict(), "preview": plan.preview, "mutated": False}, arguments.json) - if operation != "doctor" and not arguments.preview_hash: - return _installer_result(operation, {"plan": plan.to_dict(), "preview": plan.preview}, arguments.json, "error", ( - "preview_hash_required", "--confirm requires the hash from an inspected preview.", - "Run the command with --dry-run, inspect its preview, then pass --preview-hash.", - )) - if operation != "doctor" and arguments.preview_hash != plan.preview["preview_hash"]: - return _installer_result(operation, {"plan": plan.to_dict(), "preview": plan.preview}, arguments.json, "error", ( - "preview_stale", "The supplied preview hash does not match the current plan.", - "Generate and inspect a new preview before confirming this mutation.", - )) - result = installer.execute(plan, arguments.preview_hash if operation != "doctor" else False) - data = result if isinstance(result, dict) else result.to_dict() - return _installer_result(operation, {"result": data, "mutated": operation != "doctor"}, arguments.json) - except InstallerConflictError as error: - lock_code = "legacy_lock_busy" if str(error).startswith("legacy_lock_busy") else ( - "legacy_lock_recreated" if str(error).startswith("legacy_lock_recreated") else "artifact_conflict" - ) - return _installer_result(operation, {}, arguments.json, "error", ( - lock_code, str(error), - "Stop older mlx-agent processes and remove only recreated legacy locks." if lock_code != "artifact_conflict" else "Preserve the modified file or restore it to the receipt hash before retrying.", - )) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - return _installer_result(operation, {}, arguments.json, "error", ( - "installer_failed", str(error), "Inspect the provider selection and preview, then correct the reported path or manifest issue.", - )) - - -def legacy_scout_main(argv=None): - parser = argparse.ArgumentParser(description="Discover MLX models on HuggingFace for this host.") - _add_discovery_arguments(parser) - return _run_discovery(parser.parse_args(argv), legacy=True) - - -def build_parser(): - parser = argparse.ArgumentParser(description="MLX agent command-line core.") - subcommands = parser.add_subparsers(dest="command", required=True) - discover = subcommands.add_parser("discover", help="discover MLX models for this host") - _add_discovery_arguments(discover) - inspect_host = subcommands.add_parser("inspect-host", help="inspect local Apple Silicon and runtime inventory without model discovery") - inspect_host.add_argument("--json", action="store_true") - adopt = subcommands.add_parser("adopt", help="run or inspect resumable model adoption") - _add_adoption_arguments(adopt) - research = subcommands.add_parser("research", help="build a read-only domain research pack (markdown)") - _add_research_arguments(research) - blueprint = subcommands.add_parser( - "blueprint", - help="emit a read-only MLX project design pack (quant/train guidance; no scaffolding)", - ) - _add_blueprint_arguments(blueprint) - wire_command = subcommands.add_parser("wire", help="render, apply, inspect, or roll back runtime wiring") - _add_wire_arguments(wire_command) - bench_command = subcommands.add_parser("bench", help="measure a locally served model without downloading it") - _add_bench_arguments(bench_command) - serve_command = subcommands.add_parser("serve", help="preview and launch a local MLX server (confirmation-gated)") - _add_serve_arguments(serve_command) - fleet_command = subcommands.add_parser("fleet", help="render or apply a one-shot per-role router configuration") - _add_fleet_arguments(fleet_command) - watch_command = subcommands.add_parser("watch", help="snapshot and diff owned models against Hugging Face") - _add_watch_arguments(watch_command) - convert_command = subcommands.add_parser("convert", help="preview and quantize a cached model (confirmation-gated)") - _add_convert_arguments(convert_command) - lora_command = subcommands.add_parser("lora", help="preview and run LoRA training on a cached model (confirmation-gated)") - _add_lora_arguments(lora_command) - fuse_command = subcommands.add_parser("fuse", help="preview and fuse a LoRA adapter into its base (confirmation-gated)") - _add_fuse_arguments(fuse_command) - providers_command = subcommands.add_parser("providers", help="list detected supported provider CLIs") - _add_installer_arguments(providers_command, include_providers=False) - for name in ("install", "update", "uninstall", "doctor"): - installer_command = subcommands.add_parser(name, help="{0} declared provider artifacts safely".format(name)) - if name == "doctor": - _add_doctor_arguments(installer_command) - else: - _add_installer_arguments(installer_command) - return parser - - -def main(argv=None): - parser = build_parser() - arguments = parser.parse_args(argv) - if arguments.command == "adopt": - return _run_adoption(arguments) - if arguments.command == "wire": - return _run_wire(arguments) - if arguments.command == "bench": - return _run_bench(arguments) - if arguments.command == "serve": - return _run_serve(arguments) - if arguments.command == "fleet": - return _run_fleet(arguments) - if arguments.command == "watch": - return _run_watch(arguments) - if arguments.command == "convert": - return _run_convert(arguments) - if arguments.command == "lora": - return _run_lora(arguments) - if arguments.command == "fuse": - return _run_fuse(arguments) - if arguments.command in {"providers", "install", "update", "uninstall", "doctor"}: - return _run_installer(arguments) - if arguments.command == "inspect-host": - return _run_inspect_host(arguments) - if arguments.command == "research": - return _run_research(arguments) - if arguments.command == "blueprint": - return _run_blueprint(arguments) - return _run_discovery(arguments, legacy=False) diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/convert.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/convert.py deleted file mode 100644 index 908d885..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/convert.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Confirmation-gated local MLX quantization jobs.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from pathlib import Path - -from .serve import ( - _argv_matches, - _pid_alive, - _pid_command, - _terminate_pid, -) -from .transactions import _atomic_in_directory -from .wiring import _MODEL - - -CONVERT_RECEIPT_SCHEMA_VERSION = "1.0" -CONVERT_RECEIPT_KIND = "convert" -Q_BITS_CHOICES = (4, 8) -EXECUTABLE = "mlx_lm.convert" -MAX_LOG_TAIL_BYTES = 64 * 1024 - - -class ConvertError(RuntimeError): - """Classified convert failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def receipts_root(root=None, kind="convert"): - base = Path(root) if root is not None else Path.cwd() - return base / ".mlx-agent-receipts" / kind - - -def plan_convert(repo, q_bits=4, out=None): - """Render the exact conversion plan; pure and side-effect free.""" - if not isinstance(repo, str) or not _MODEL.fullmatch(repo): - raise ConvertError( - "invalid_repo", - "convert requires a safe publisher/model identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - if not isinstance(q_bits, int) or isinstance(q_bits, bool): - raise ConvertError( - "invalid_arguments", "q_bits must be an integer.", - "Pass --q-bits 4 or --q-bits 8.", - ) - if q_bits not in Q_BITS_CHOICES: - raise ConvertError( - "invalid_arguments", - "q_bits must be one of {0}.".format(list(Q_BITS_CHOICES)), - "Only bounded 4bit and 8bit recipes are supported in v1.", - ) - if out is None: - name = repo.split("/", 1)[1] - out = "{0}-MLX-{1}bit".format(name, q_bits) - plan = { - "repo": repo, - "q_bits": q_bits, - "out": str(out), - "argv": [ - EXECUTABLE, - "--hf-path", repo, - "--mlx-path", str(out), - "--q-bits", str(q_bits), - ], - } - canonical = json.dumps(plan, sort_keys=True, separators=(",", ":")) - plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() - return plan - - -def _receipt_path(root, plan): - return root / "{0}-{1}bit.json".format( - plan["repo"].split("/", 1)[1], plan["q_bits"] - ) - - -def _read_receipt(path, kind=CONVERT_RECEIPT_KIND): - try: - value = json.loads(Path(path).read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - return None - if not isinstance(value, dict) or value.get("kind") != kind: - return None - if not isinstance(value.get("pid"), int) or isinstance(value.get("pid"), bool): - return None - if not isinstance(value.get("argv"), list) or not isinstance(value.get("repo"), str): - return None - return value - - -def _write_receipt(root, receipt, filename): - content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") - _atomic_in_directory(root, filename, content, 0o600) - - -def start_convert(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, spawn=None, now=_utc_now, - pid_alive=None): - """Execute a reviewed conversion plan; the only mutating entry point.""" - from .serve import _default_spawn, _default_which - - which = which or _default_which - spawn = spawn or _default_spawn - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise ConvertError( - "preview_hash_required", - "--confirm requires the hash from a reviewed convert preview.", - "Run convert start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise ConvertError( - "preview_stale", - "The supplied preview hash does not match this convert plan.", - "Re-run convert start without --confirm and review the fresh plan.", - ) - if which(EXECUTABLE) is None: - raise ConvertError( - "runtime_not_installed", - "The {0} executable is not installed.".format(EXECUTABLE), - "Install mlx-lm yourself (pip install mlx-lm); convert never installs runtimes.", - ) - if model_present is not None and not model_present(plan["repo"]): - raise ConvertError( - "model_not_local", - "The source model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; convert never downloads.", - ) - if Path(plan["out"]).exists(): - raise ConvertError( - "output_exists", - "The output path already exists: {0}".format(plan["out"]), - "Pick a fresh --out path; convert never overwrites.", - ) - for receipt_path in sorted(root.glob("*.json")) if root.is_dir() else []: - existing = _read_receipt(receipt_path) - if existing is None or existing.get("completed_at"): - continue - if pid_alive(existing["pid"]): - raise ConvertError( - "job_in_progress", - "A convert job is still running (pid {0}).".format(existing["pid"]), - "Wait for it to finish (convert status) before starting another.", - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}-{1}bit.log".format(plan["repo"].split("/", 1)[1], plan["q_bits"]) - pid = spawn(plan["argv"], str(log_path)) - receipt = { - "schema_version": CONVERT_RECEIPT_SCHEMA_VERSION, - "kind": CONVERT_RECEIPT_KIND, - "repo": plan["repo"], - "q_bits": plan["q_bits"], - "out": plan["out"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "started_at": now(), - "preview_hash": plan["preview_hash"], - "completed_at": None, - "exit_status": None, - } - _write_receipt( - root, receipt, - "{0}-{1}bit.json".format(receipt["repo"].split("/", 1)[1], receipt["q_bits"]), - ) - return {"status": "started", "receipt": receipt} - - -def status_convert(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - now=_utc_now): - """Cross-check convert receipts against live processes; marks exits once.""" - root = receipts_root(receipts_dir) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - receipt = _read_receipt(path) - if receipt is None: - continue - entry = { - "receipt": str(path), - "repo": receipt["repo"], - "q_bits": receipt["q_bits"], - "out": receipt["out"], - "pid": receipt["pid"], - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - "completed_at": receipt.get("completed_at"), - } - if receipt.get("completed_at"): - entry["state"] = receipt.get("exit_status") or "done" - entries.append(entry) - continue - if pid_alive(receipt["pid"]): - command = pid_command(receipt["pid"]) - entry["state"] = "running" if _argv_matches( - {"argv": receipt["argv"], "port": None, "repo": receipt["repo"]}, - command, - require_port=False, - ) else "unknown" - entries.append(entry) - continue - exit_status = "done" if Path(receipt["out"]).exists() else "failed" - receipt["completed_at"] = now() - receipt["exit_status"] = exit_status - _write_receipt( - root, receipt, - "{0}-{1}bit.json".format(receipt["repo"].split("/", 1)[1], receipt["q_bits"]), - ) - entry["state"] = exit_status - entry["completed_at"] = receipt["completed_at"] - entries.append(entry) - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/fleet.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/fleet.py deleted file mode 100644 index 71de98f..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/fleet.py +++ /dev/null @@ -1,218 +0,0 @@ -"""One-shot per-role routing configuration backed by Wire transactions.""" - -from __future__ import annotations - -import json -import re -from pathlib import Path - -from .wiring import _MODEL, require_secret_free_config - - -FLEET_RUNTIMES = ("mlx_lm", "mlx-vlm") -ROLE_DEFAULT_RUNTIME = { - "general": "mlx_lm", - "coding": "mlx_lm", - "reasoning": "mlx_lm", - "embedding": "mlx_lm", - "tool-use": "mlx_lm", - "vision": "mlx-vlm", -} -RUNTIME_PORTS = {"mlx_lm": 8080, "mlx-vlm": 8083} -_ROLE = re.compile(r"^[a-z][a-z0-9-]{0,31}$") -MAX_ASSIGNMENTS = 6 - -_HEADER = "# MLX_AGENT_WIRE v1" - - -class FleetError(ValueError): - """Classified fleet failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def parse_assignments(values): - """Parse repeatable role=repo assignments into an ordered mapping.""" - assignments = {} - for value in values or []: - if not isinstance(value, str) or "=" not in value: - raise FleetError( - "invalid_assignment", - "Assignments must use role=repo form: {0}".format(value), - "Pass --assign coding=publisher/model.", - ) - role, repo = value.split("=", 1) - role = role.strip() - repo = repo.strip() - if role not in ROLE_DEFAULT_RUNTIME: - raise FleetError( - "invalid_role", - "Unknown fleet role: {0}".format(role), - "Use one of: {0}.".format(", ".join(sorted(ROLE_DEFAULT_RUNTIME))), - ) - if role in assignments: - raise FleetError( - "duplicate_role", - "Role {0} is assigned twice.".format(role), - "Assign each role at most once.", - ) - if not _MODEL.fullmatch(repo): - raise FleetError( - "invalid_repo", - "Model must be a safe publisher/model identifier: {0}".format(repo), - "Pass the Hugging Face publisher/model form.", - ) - assignments[role] = repo - if not assignments: - raise FleetError( - "missing_assignments", - "Fleet requires at least one role assignment.", - "Pass --assign role=repo or --from-adoption .", - ) - if len(assignments) > MAX_ASSIGNMENTS: - raise FleetError( - "invalid_assignment", - "Fleet supports at most {0} role assignments.".format(MAX_ASSIGNMENTS), - "Reduce the number of assigned roles.", - ) - return assignments - - -def assignments_from_adoption(state_path): - """Read role -> repo recommendations from a completed adopt handoff.""" - try: - value = json.loads(Path(state_path).read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError) as error: - raise FleetError( - "invalid_state", - "The adoption handoff could not be read: {0}".format(error), - "Pass a readable adopt state file produced by adopt start/resume.", - ) - recommendations = value.get("recommendations") - if not isinstance(recommendations, list) or not recommendations: - raise FleetError( - "empty_recommendations", - "The adoption handoff contains no recommendations.", - "Complete the adoption workflow first, then rerun fleet.", - ) - assignments = {} - for item in recommendations: - if not isinstance(item, dict): - continue - role = item.get("role") - repo = item.get("repo") - if role in ROLE_DEFAULT_RUNTIME and isinstance(repo, str) and role not in assignments: - assignments[role] = repo - return parse_assignments(["{0}={1}".format(role, repo) for role, repo in assignments.items()]) - - -def parse_runtime_map(values): - """Parse repeatable role=runtime overrides.""" - runtime_map = {} - for value in values or []: - if not isinstance(value, str) or "=" not in value: - raise FleetError( - "invalid_runtime_map", - "Runtime overrides must use role=runtime form: {0}".format(value), - "Pass --runtime-map vision=mlx-vlm.", - ) - role, runtime = value.split("=", 1) - role = role.strip() - runtime = runtime.strip() - if role not in ROLE_DEFAULT_RUNTIME: - raise FleetError( - "invalid_role", - "Unknown fleet role: {0}".format(role), - "Use one of: {0}.".format(", ".join(sorted(ROLE_DEFAULT_RUNTIME))), - ) - if runtime not in FLEET_RUNTIMES: - raise FleetError( - "unsupported_runtime", - "Fleet routes only through: {0}.".format(", ".join(FLEET_RUNTIMES)), - "Ollama and LM Studio manage their own routing; use mlx_lm or mlx-vlm.", - ) - runtime_map[role] = runtime - return runtime_map - - -class FleetConfigAdapter: - """Validate the exact bounded multi-model router subset fleet renders.""" - - version = "1.0" - runtime = "litellm-fleet" - - def __init__(self, path=None): - self.path = Path(path) if path is not None else None - - def render(self, assignments, runtime_map=None, existing=""): - runtime_map = runtime_map or {} - if not assignments: - raise FleetError( - "missing_assignments", - "Fleet requires at least one role assignment.", - "Pass --assign role=repo or --from-adoption .", - ) - if not isinstance(existing, str): - raise TypeError("existing config content must be text") - require_secret_free_config(existing) - if existing.strip() and not existing.startswith(_HEADER): - raise FleetError( - "existing_config_unmanaged", - "The target file exists and is not fleet-managed.", - "Pick a new --path or remove the unmanaged file yourself.", - ) - lines = [_HEADER, "model_list:"] - for role in sorted(assignments): - runtime = runtime_map.get(role, ROLE_DEFAULT_RUNTIME[role]) - if runtime not in FLEET_RUNTIMES: - raise FleetError( - "unsupported_runtime", - "Fleet routes only through: {0}.".format(", ".join(FLEET_RUNTIMES)), - "Use mlx_lm or mlx-vlm for every role.", - ) - port = RUNTIME_PORTS[runtime] - lines.extend([ - " - model_name: {0}".format(role), - " litellm_params:", - " model: openai/{0}".format(assignments[role]), - " api_base: http://127.0.0.1:{0}/v1".format(port), - " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY", - ]) - content = "\n".join(lines) + "\n" - self.validate(content) - return content - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("configuration content must be text") - require_secret_free_config(content) - lines = content.splitlines() - if len(lines) < 7 or (len(lines) - 2) % 5 != 0: - raise ValueError("fleet configuration must contain whole five-line entries") - if lines[0] != _HEADER or lines[1] != "model_list:": - raise ValueError("fleet configuration must start with the managed header") - roles = [] - for offset in range(2, len(lines), 5): - header, params, model, api_base, api_key = lines[offset:offset + 5] - role = re.fullmatch(r" - model_name: ([a-z][a-z0-9-]{0,31})", header) - if role is None or not _ROLE.fullmatch(role.group(1)): - raise ValueError("fleet entry has an invalid model_name") - if params != " litellm_params:": - raise ValueError("fleet entry is missing litellm_params") - model_match = re.fullmatch(r" model: openai/(" + _MODEL.pattern[1:-1] + r")", model) - if model_match is None: - raise ValueError("fleet entry has an invalid model identifier") - if api_base not in ( - " api_base: http://127.0.0.1:8080/v1", - " api_base: http://127.0.0.1:8083/v1", - ): - raise ValueError("fleet entry api_base must be a bounded loopback port") - if api_key != " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY": - raise ValueError("fleet entry must reference the managed API key env var") - roles.append(role.group(1)) - if len(roles) != len(set(roles)): - raise ValueError("fleet configuration contains duplicate roles") - return True diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/gemini_args.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/gemini_args.py deleted file mode 100644 index 886acaf..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/gemini_args.py +++ /dev/null @@ -1,340 +0,0 @@ -"""Strict argument grammar for Gemini custom-command input. - -Gemini TOML commands pass user text to the model, not directly to a process. -This module converts that opaque text into an allowlisted MLX-agent argv list -before a bundled skill may execute the structured CLI. -""" - -from __future__ import annotations - -import argparse -import json -import re -import shlex -import urllib.parse -from pathlib import Path - -from .models import ROLES - - -class GeminiArgumentError(ValueError): - """Gemini command input is outside the documented capability grammar.""" - - -_RUNTIMES = {"ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"} -_ROLES = {role for role, _keywords, _label in ROLES} -_MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,191}$") -_SAFE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+:-]{0,127}$") -_SAFE_QUANTIZATION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") -_METACHARACTERS = set(";|&`$<>") - - -def _tokens(raw): - if not isinstance(raw, str): - raise GeminiArgumentError("arguments must be text") - if len(raw) > 4096 or any(ord(character) < 32 or ord(character) == 127 for character in raw): - raise GeminiArgumentError("arguments must not contain control characters") - if any(character in _METACHARACTERS for character in raw): - raise GeminiArgumentError("shell metacharacters are not allowed") - try: - tokens = shlex.split(raw, posix=True) - except ValueError as error: - raise GeminiArgumentError("arguments must use balanced shell-style quotes: {0}".format(error)) - if len(tokens) > 32 or any(not token or len(token) > 512 for token in tokens): - raise GeminiArgumentError("arguments exceed the supported command grammar") - return tokens - - -def _value(tokens, index, flag): - if index + 1 >= len(tokens) or tokens[index + 1].startswith("--"): - raise GeminiArgumentError("{0} requires a value".format(flag)) - return tokens[index + 1] - - -def _path(value, label): - if not isinstance(value, str) or not value or value.startswith("~") or "\\" in value: - raise GeminiArgumentError("{0} must be a portable non-empty path".format(label)) - path = Path(value) - if any(part in {"", ".", ".."} for part in path.parts): - raise GeminiArgumentError("{0} must not traverse parent directories".format(label)) - return value - - -def _model(value): - if not _MODEL.fullmatch(value): - raise GeminiArgumentError("model must be a publisher/model identifier") - return value - - -def _integer(value, label, minimum, maximum): - if not re.fullmatch(r"[0-9]+", value): - raise GeminiArgumentError("{0} must be an integer".format(label)) - number = int(value) - if number < minimum or number > maximum: - raise GeminiArgumentError("{0} must be between {1} and {2}".format(label, minimum, maximum)) - return str(number) - - -def _decimal(value, label, minimum, maximum): - if not re.fullmatch(r"(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+)", value): - raise GeminiArgumentError("{0} must be a decimal number".format(label)) - number = float(value) - if number < minimum or number > maximum: - raise GeminiArgumentError("{0} must be between {1} and {2}".format(label, minimum, maximum)) - return value - - -def _endpoint(value): - try: - parsed = urllib.parse.urlsplit(value) - hostname = parsed.hostname - username = parsed.username - password = parsed.password - port = parsed.port - except (UnicodeError, ValueError) as error: - raise GeminiArgumentError("endpoint is not a valid local HTTP URL") from error - if ( - parsed.scheme not in {"http", "https"} - or hostname not in {"127.0.0.1", "localhost", "::1"} - or username is not None - or password is not None - or parsed.query - or parsed.fragment - ): - raise GeminiArgumentError("endpoint must be a credential-free local HTTP URL") - if port is not None and not 1 <= port <= 65535: - raise GeminiArgumentError("endpoint port must be between 1 and 65535") - return value - - -def _parse_scout(tokens): - argv, index = ["discover"], 0 - seen, cache = set(), None - values = { - "--role": (lambda value: value if value in _ROLES else None, "role"), - "--limit": (lambda value: _integer(value, "limit", 1, 50), "limit"), - "--memory-gb": (lambda value: _decimal(value, "memory-gb", 0.25, 1024), "memory-gb"), - "--quantization": (lambda value: value if _SAFE_QUANTIZATION.fullmatch(value) else None, "quantization"), - "--runtime": (lambda value: value if value in _RUNTIMES else None, "runtime"), - "--context": (lambda value: _integer(value, "context", 1024, 1048576), "context"), - "--state-dir": (lambda value: _path(value, "state-dir"), "state-dir"), - "--wire": (lambda value: _model(value), "wire"), - "--target": (lambda value: value if value in _RUNTIMES else None, "target"), - "--port": (lambda value: _integer(value, "port", 1, 65535), "port"), - } - repeatable = {"--license", "--publisher"} - booleans = {"--exclude-gated", "--include-gated", "--new", "--fast", "--json"} - while index < len(tokens): - flag = tokens[index] - if flag in booleans: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - if flag in {"--exclude-gated", "--include-gated"} and {"--exclude-gated", "--include-gated"} & seen: - raise GeminiArgumentError("gated-repository flags are mutually exclusive") - seen.add(flag) - argv.append(flag) - index += 1 - elif flag in {"--refresh", "--offline"}: - if cache is not None: - raise GeminiArgumentError("--refresh and --offline are mutually exclusive") - cache = flag - argv.append(flag) - index += 1 - elif flag in repeatable: - value = _value(tokens, index, flag) - if not _SAFE_NAME.fullmatch(value): - raise GeminiArgumentError("{0} must be a safe identifier".format(flag)) - argv.extend([flag, value]) - index += 2 - elif flag in values: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - value = values[flag][0](_value(tokens, index, flag)) - if value is None: - raise GeminiArgumentError("invalid {0} value".format(values[flag][1])) - seen.add(flag) - argv.extend([flag, value]) - index += 2 - else: - raise GeminiArgumentError("unsupported Scout argument: {0}".format(flag)) - return argv - - -def _parse_adopt(tokens): - if not tokens or tokens[0] not in {"start", "resume", "status"}: - raise GeminiArgumentError("Adopt requires start, resume, or status") - action, argv, index = tokens[0], ["adopt", tokens[0]], 1 - state = None - allowed_booleans = {"--json"} if action != "start" else {"--offline", "--refresh", "--fast", "--no-network", "--measure", "--json"} - while index < len(tokens): - flag = tokens[index] - if flag == "--state": - if state is not None: - raise GeminiArgumentError("duplicate flag: --state") - state = _path(_value(tokens, index, flag), "state") - argv.extend([flag, state]) - index += 2 - elif action == "start" and flag == "--role": - role = _value(tokens, index, flag) - if role not in _ROLES: - raise GeminiArgumentError("unknown adoption role") - argv.extend([flag, role]) - index += 2 - elif action == "start" and flag == "--shortlist-limit": - argv.extend([flag, _integer(_value(tokens, index, flag), "shortlist-limit", 1, 20)]) - index += 2 - elif flag in allowed_booleans and flag not in argv: - argv.append(flag) - index += 1 - else: - raise GeminiArgumentError("unsupported Adopt argument: {0}".format(flag)) - if state is None: - raise GeminiArgumentError("Adopt requires an explicit --state path") - return argv - - -def _parse_wire(tokens): - if not tokens or tokens[0] not in {"render", "apply", "status", "rollback"}: - raise GeminiArgumentError("Wire requires render, apply, status, or rollback") - action, argv, index = tokens[0], ["wire", tokens[0]], 1 - if action in {"status", "rollback"}: - if index >= len(tokens): - raise GeminiArgumentError("Wire {0} requires a receipt path".format(action)) - argv.append(_path(tokens[index], "receipt")) - index += 1 - allowed = ( - {"--json"} - if action == "status" - else {"--confirm", "--preview-hash", "--json"} - ) - else: - if index >= len(tokens): - raise GeminiArgumentError("Wire {0} requires a model".format(action)) - argv.append(_model(tokens[index])) - index += 1 - allowed = {"--target", "--path", "--json"} - if action == "apply": - allowed.update({"--confirm", "--preview-hash", "--receipts-dir", "--endpoint"}) - path_seen = False - while index < len(tokens): - flag = tokens[index] - if flag not in allowed: - raise GeminiArgumentError("unsupported Wire argument: {0}".format(flag)) - if flag in argv: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - if flag in {"--json", "--confirm"}: - argv.append(flag) - index += 1 - else: - value = _value(tokens, index, flag) - if flag == "--target": - if value not in _RUNTIMES: - raise GeminiArgumentError("unsupported Wire target") - elif flag in {"--path", "--receipts-dir"}: - value = _path(value, flag[2:]) - path_seen = path_seen or flag == "--path" - elif flag == "--preview-hash": - if not re.fullmatch(r"[0-9a-f]{64}", value): - raise GeminiArgumentError("preview-hash must be a SHA-256 digest") - elif flag == "--endpoint": - value = _endpoint(value) - argv.extend([flag, value]) - index += 2 - if action in {"render", "apply"} and not path_seen: - raise GeminiArgumentError("Wire {0} requires --path".format(action)) - return argv - - -def _parse_bench(tokens): - if not tokens or tokens[0] not in {"run", "aggregate"}: - raise GeminiArgumentError("Bench requires the run or aggregate action") - if tokens[0] == "aggregate": - return _parse_bench_aggregate(tokens) - argv, index = ["bench", "run"], 1 - seen = set() - values = { - "--repo": (lambda value: _model(value), "repo"), - "--runtime": (lambda value: value if value in _RUNTIMES else None, "runtime"), - "--role": (lambda value: value if value in _ROLES else None, "role"), - "--runs": (lambda value: _integer(value, "runs", 1, 10), "runs"), - "--gen-tokens": (lambda value: _integer(value, "gen-tokens", 16, 2048), "gen-tokens"), - "--timeout": (lambda value: _decimal(value, "timeout", 1, 600), "timeout"), - "--export": (lambda value: _path(value, "export"), "export"), - } - while index < len(tokens): - flag = tokens[index] - if flag == "--json": - if flag in seen: - raise GeminiArgumentError("duplicate flag: --json") - seen.add(flag) - argv.append(flag) - index += 1 - elif flag in values: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - value = values[flag][0](_value(tokens, index, flag)) - if value is None: - raise GeminiArgumentError("invalid {0} value".format(values[flag][1])) - seen.add(flag) - argv.extend([flag, value]) - index += 2 - else: - raise GeminiArgumentError("unsupported Bench argument: {0}".format(flag)) - for required in ("--repo", "--runtime"): - if required not in seen: - raise GeminiArgumentError("Bench run requires {0}".format(required)) - return argv - - -def _parse_bench_aggregate(tokens): - argv, index = ["bench", "aggregate"], 1 - seen = set() - while index < len(tokens): - flag = tokens[index] - if flag == "--json": - if flag in seen: - raise GeminiArgumentError("duplicate flag: --json") - seen.add(flag) - argv.append(flag) - index += 1 - elif flag in {"--exports", "--out"}: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - seen.add(flag) - argv.extend([flag, _path(_value(tokens, index, flag), flag[2:])]) - index += 2 - else: - raise GeminiArgumentError("unsupported Bench argument: {0}".format(flag)) - if "--exports" not in seen: - raise GeminiArgumentError("Bench aggregate requires --exports") - return argv - - -def parse_gemini_arguments(capability, raw): - """Return a validated argv list; this function never runs a process.""" - tokens = _tokens(raw) - if capability == "scout": - return _parse_scout(tokens) - if capability == "adopt": - return _parse_adopt(tokens) - if capability == "wire": - return _parse_wire(tokens) - if capability == "bench": - return _parse_bench(tokens) - raise GeminiArgumentError("unknown Gemini capability") - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("capability", choices=("scout", "adopt", "wire", "bench")) - parser.add_argument("raw", help="one opaque custom-command argument string") - arguments = parser.parse_args(argv) - try: - print(json.dumps({"argv": parse_gemini_arguments(arguments.capability, arguments.raw)})) - except GeminiArgumentError as error: - parser.error(str(error)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/gemini_executor.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/gemini_executor.py deleted file mode 100644 index 3a0fc12..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/gemini_executor.py +++ /dev/null @@ -1,140 +0,0 @@ -"""No-shell executor for Gemini custom-command argument files.""" - -from __future__ import annotations - -import argparse -import json -import os -import stat -import tempfile -from pathlib import Path - -from .gemini_args import GeminiArgumentError, parse_gemini_arguments - - -MAX_ARGS_BYTES = 4096 -_ARGS_DIRECTORY = "mlx-agent-gemini-command-args" - - -class GeminiCommandError(ValueError): - """A custom-command argument file is unsafe or invalid.""" - - -def command_args_root(): - """Return the private OS-temporary root reserved for Gemini argument files.""" - root = Path(tempfile.gettempdir()) / _ARGS_DIRECTORY - try: - root.mkdir(mode=0o700, parents=True, exist_ok=True) - details = os.lstat(str(root)) - except OSError as error: - raise GeminiCommandError("private argument storage is unavailable") from error - if ( - stat.S_ISLNK(details.st_mode) - or not stat.S_ISDIR(details.st_mode) - or details.st_uid != os.getuid() - or stat.S_IMODE(details.st_mode) != 0o700 - ): - raise GeminiCommandError("private argument storage is unsafe") - return root - - -def _safe_args_path(value): - root = command_args_root() - path = Path(value) - if not path.is_absolute() or path.parent != root or ".." in path.parts or not path.name: - raise GeminiCommandError("argument file path is outside private storage") - return path - - -def _discard_if_safe(path, opened=None): - """Remove only the named regular file we opened, or a rejected symlink.""" - try: - current = os.lstat(str(path)) - if stat.S_ISLNK(current.st_mode): - os.unlink(str(path)) - elif stat.S_ISREG(current.st_mode) and (opened is None or (current.st_dev, current.st_ino) == (opened.st_dev, opened.st_ino)): - os.unlink(str(path)) - except OSError: - pass - - -def _read_args_file(value): - path = _safe_args_path(value) - opened = None - descriptor = None - try: - try: - before = os.lstat(str(path)) - except OSError as error: - raise GeminiCommandError("argument file is unavailable") from error - if ( - stat.S_ISLNK(before.st_mode) - or not stat.S_ISREG(before.st_mode) - or before.st_uid != os.getuid() - or stat.S_IMODE(before.st_mode) != 0o600 - ): - raise GeminiCommandError("argument file must be a regular file") - if before.st_size > MAX_ARGS_BYTES: - raise GeminiCommandError("argument file exceeds the maximum size") - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - try: - descriptor = os.open(str(path), flags) - opened = os.fstat(descriptor) - except OSError as error: - raise GeminiCommandError("argument file cannot be opened safely") from error - if ( - not stat.S_ISREG(opened.st_mode) - or opened.st_uid != os.getuid() - or stat.S_IMODE(opened.st_mode) != 0o600 - or (opened.st_dev, opened.st_ino) != (before.st_dev, before.st_ino) - ): - raise GeminiCommandError("argument file changed before it could be read") - data = os.read(descriptor, MAX_ARGS_BYTES + 1) - if len(data) > MAX_ARGS_BYTES: - raise GeminiCommandError("argument file exceeds the maximum size") - try: - return data.decode("utf-8") - except UnicodeDecodeError as error: - raise GeminiCommandError("argument file must be UTF-8") from error - finally: - if descriptor is not None: - os.close(descriptor) - _discard_if_safe(path, opened) - - -def execute_gemini_command(capability, args_file, core=None): - """Read, validate, execute, and discard one trusted Gemini argument file.""" - try: - raw = _read_args_file(args_file) - argv = parse_gemini_arguments(capability, raw) - except (GeminiArgumentError, GeminiCommandError) as error: - raise GeminiCommandError("Gemini command arguments were rejected") from error - if core is None: - from .cli import main as core - try: - result = core(argv) - except Exception as error: - raise GeminiCommandError("Gemini command execution failed") from error - exit_code = 0 if result is None else int(result) - return { - "status": "ok" if exit_code == 0 else "error", - "capability": capability, - "exit_code": exit_code, - } - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--capability", required=True, choices=("scout", "adopt", "wire")) - parser.add_argument("--args-file", required=True) - arguments = parser.parse_args(argv) - try: - result = execute_gemini_command(arguments.capability, arguments.args_file) - print(json.dumps(result, sort_keys=True)) - except GeminiCommandError: - parser.error("Gemini command arguments were rejected") - return result["exit_code"] - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/gemini_transport.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/gemini_transport.py deleted file mode 100644 index bf7a648..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/gemini_transport.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Deterministic contract simulation for Gemini custom-command TOML transport. - -This module simulates documented ``{{args}}`` prompt substitution for tests and -local adapters. It is not a claim about Gemini model or tool behavior. -""" - -from __future__ import annotations - -import os -import subprocess -import tempfile -from pathlib import Path - -from .gemini_executor import MAX_ARGS_BYTES, GeminiCommandError, command_args_root, execute_gemini_command - - -_OPEN = "\n" -_CLOSE = "\n" - - -class GeminiTransportError(ValueError): - """The simulated TOML transport could not safely carry command data.""" - - -def substitute_toml_args(prompt, raw_args): - """Simulate one documented TOML placeholder substitution without parsing it.""" - if not isinstance(prompt, str) or not isinstance(raw_args, str) or prompt.count("{{args}}") != 1: - raise GeminiTransportError("TOML transport template is invalid") - return prompt.replace("{{args}}", raw_args) - - -def extract_untrusted_payload(rendered_prompt): - """Extract the delimited payload exactly, preserving embedded delimiter text.""" - if not isinstance(rendered_prompt, str): - raise GeminiTransportError("TOML transport payload is invalid") - start = rendered_prompt.find(_OPEN) - end = rendered_prompt.rfind(_CLOSE) - if start < 0 or end < start + len(_OPEN): - raise GeminiTransportError("TOML transport delimiters are invalid") - return rendered_prompt[start + len(_OPEN):end] - - -def write_private_args_file(payload): - """Write opaque transport data to a 0600 private file without a shell.""" - if not isinstance(payload, str): - raise GeminiTransportError("TOML transport payload is invalid") - encoded = payload.encode("utf-8") - if len(encoded) > MAX_ARGS_BYTES: - raise GeminiTransportError("TOML transport payload exceeds the maximum size") - root = command_args_root() - descriptor = None - name = None - complete = False - try: - descriptor, name = tempfile.mkstemp(prefix="transport-", suffix=".args", dir=str(root)) - os.fchmod(descriptor, 0o600) - os.write(descriptor, encoded) - complete = True - return name - except OSError as error: - raise GeminiTransportError("private transport storage is unavailable") from error - finally: - if descriptor is not None: - os.close(descriptor) - if name is not None and not complete: - try: - os.unlink(name) - except OSError: - pass - - -def build_structured_executor_call(skill_dir, capability, args_file): - """Build the only permitted non-shell Gemini executor process shape.""" - skill_path = Path(skill_dir) - argument_path = Path(args_file) - root = command_args_root() - if ( - capability not in {"scout", "adopt", "wire"} - or not skill_path.is_absolute() - or not argument_path.is_absolute() - or argument_path.parent != root - or not argument_path.name - ): - raise GeminiTransportError("structured executor fields are invalid") - return { - "program": "python3", - "argv": ["-m", "mlx_agent.gemini_executor", "--capability", capability, "--args-file", str(argument_path)], - "env": {"PYTHONPATH": str(skill_path / "src")}, - "shell": False, - } - - -def run_structured_executor_call(call, extra_env=None): - """Run a validated structured executor fixture through subprocess without a shell.""" - if not isinstance(call, dict) or set(call) != {"program", "argv", "env", "shell"}: - raise GeminiTransportError("structured executor fields are invalid") - program, argv, env, shell = call["program"], call["argv"], call["env"], call["shell"] - if ( - program != "python3" - or not isinstance(argv, list) - or len(argv) != 6 - or argv[:3] != ["-m", "mlx_agent.gemini_executor", "--capability"] - or argv[3] not in {"scout", "adopt", "wire"} - or argv[4] != "--args-file" - or not isinstance(argv[5], str) - or not isinstance(env, dict) - or set(env) != {"PYTHONPATH"} - or not isinstance(env["PYTHONPATH"], str) - ): - raise GeminiTransportError("structured executor fields are invalid") - if shell is not False or not env["PYTHONPATH"]: - raise GeminiTransportError("structured executor fields are invalid") - environment = dict(os.environ) - if extra_env is not None: - environment.update(extra_env) - environment.update(env) - return subprocess.run([program] + argv, env=environment, shell=False, text=True, capture_output=True, check=False) - - -def simulate_toml_transport(prompt, raw_args, capability, core=None): - """Simulate TOML prompt transport and dispatch only through the executor.""" - rendered = substitute_toml_args(prompt, raw_args) - payload = extract_untrusted_payload(rendered) - args_file = write_private_args_file(payload) - try: - return execute_gemini_command(capability, args_file, core=core) - except GeminiCommandError as error: - raise GeminiTransportError("TOML transport payload was rejected") from error diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/huggingface.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/huggingface.py deleted file mode 100644 index 3a84ed5..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/huggingface.py +++ /dev/null @@ -1,457 +0,0 @@ -"""Dependency-free Hugging Face Hub access for Scout.""" - -import http.client -import json -import queue -import re -import threading -import time -import urllib.parse - -from .contextfit import extract_architecture -from .models import ( - REASONER_HINTS, - TEMPLATE_REASON, - TEMPLATE_TOOL_USE, - TOOL_USE_HINTS, - TOOL_USE_TAGS, -) - - -HF_API = "https://huggingface.co/api/models" -HF_DATASETS_API = "https://huggingface.co/api/datasets" -HF_API_HOST = "huggingface.co" -UA = {"User-Agent": "mlx-scout/0.2 (+https://github.com/cavi-ai/mlx-agent)"} -HF_RESPONSE_MAX_BYTES = 8 * 1024 * 1024 -_HTTP_READ_CHUNK_BYTES = 64 * 1024 -HF_CARD_HOST = "huggingface.co" -MODEL_CARD_MAX_BYTES = 512 * 1024 -_CARD_PATH_SUFFIX = "/raw/main/README.md" - - -def _is_allowed_api_path(path): - if path == "/api/models" or path.startswith("/api/models/"): - return True - if path == "/api/datasets" or path.startswith("/api/datasets/"): - return True - return False - - -def _is_valid_card_path(path): - """Accept model or dataset README raw paths on the fixed card host.""" - if not path.endswith(_CARD_PATH_SUFFIX): - return False - slash_count = path.count("/") - if slash_count == 5 and not path.startswith("/datasets/"): - return True - if slash_count == 6 and path.startswith("/datasets/"): - return True - return False - - -def http_json( - url, - timeout=10.0, - connection_factory=None, - clock=time.monotonic, - completion_wait=None, -): - """Read bounded JSON from the fixed Hugging Face API under one deadline.""" - parsed = urllib.parse.urlsplit(url) - if parsed.scheme != "https" or parsed.hostname != HF_API_HOST: - raise ValueError("Hugging Face URL must use the fixed HTTPS API host") - if parsed.port not in (None, 443): - raise ValueError("Hugging Face URL must use the default HTTPS port") - if parsed.username is not None or parsed.password is not None: - raise ValueError("Hugging Face URL must not contain credentials") - if parsed.fragment: - raise ValueError("Hugging Face URL must not contain a fragment") - if not _is_allowed_api_path(parsed.path): - raise ValueError("Hugging Face URL must target the models or datasets API") - - deadline = clock() + timeout - remaining = _deadline_remaining(deadline, clock) - if connection_factory is None: - connection = http.client.HTTPSConnection( - HF_API_HOST, - 443, - timeout=remaining, - ) - else: - connection = connection_factory(HF_API_HOST, 443, remaining) - target = parsed.path or "/" - if parsed.query: - target = "{0}?{1}".format(target, parsed.query) - return _run_http_worker( - connection, - lambda: _http_json_operation(connection, target, deadline, clock), - deadline, - clock, - completion_wait, - ) - - -def _run_http_worker(connection, operation, deadline, clock, completion_wait): - results = queue.Queue(maxsize=1) - completion = threading.Event() - - def run(): - try: - result = (True, operation()) - except BaseException as error: # noqa: BLE001 - propagated to caller - result = (False, error) - results.put_nowait(result) - completion.set() - - worker = threading.Thread( - target=run, - name="mlx-agent-huggingface-request", - daemon=True, - ) - try: - worker.start() - except BaseException: - connection.close() - raise - - try: - remaining = _deadline_remaining(deadline, clock) - except TimeoutError: - connection.close() - raise - try: - if completion_wait is None: - completed = completion.wait(remaining) - else: - completed = completion_wait(completion, remaining) - except BaseException: - connection.close() - raise - if not completed: - connection.close() - raise TimeoutError("Hugging Face request exceeded its overall deadline") - - try: - remaining = _deadline_remaining(deadline, clock) - except TimeoutError: - connection.close() - raise - worker.join(timeout=remaining) - if worker.is_alive(): - connection.close() - raise TimeoutError("Hugging Face request exceeded its overall deadline") - - succeeded, value = results.get_nowait() - if succeeded: - return value - raise value - - -def _http_json_operation(connection, target, deadline, clock): - try: - connection.request("GET", target, headers=UA) - _set_connection_timeout( - connection, - _deadline_remaining(deadline, clock), - ) - response = connection.getresponse() - _deadline_remaining(deadline, clock) - if 300 <= response.status < 400: - raise http.client.HTTPException( - "redirect responses are not allowed for Hugging Face requests" - ) - if not 200 <= response.status < 300: - raise http.client.HTTPException( - "Hugging Face returned HTTP status {0}".format(response.status) - ) - content_length = response.headers.get("Content-Length") - if content_length is not None: - try: - declared_length = int(content_length) - except (TypeError, ValueError) as error: - raise ValueError("invalid Hugging Face Content-Length") from error - if declared_length < 0: - raise ValueError("invalid Hugging Face Content-Length") - if declared_length > HF_RESPONSE_MAX_BYTES: - raise ValueError("Hugging Face response exceeds size limit") - body = _read_bounded_body( - response, - connection, - deadline, - clock, - ) - return json.loads(body.decode("utf-8")) - finally: - connection.close() - - -def http_card_text( - url, - timeout=8.0, - connection_factory=None, - clock=time.monotonic, - completion_wait=None, -): - """Read bounded README/model-card text from the fixed card host.""" - parsed = urllib.parse.urlsplit(url) - if parsed.scheme != "https" or parsed.hostname != HF_CARD_HOST: - raise ValueError("card URL must use the fixed HTTPS card host") - if parsed.port not in (None, 443): - raise ValueError("card URL must use the default HTTPS port") - if parsed.username is not None or parsed.password is not None: - raise ValueError("card URL must not contain credentials") - if parsed.fragment: - raise ValueError("card URL must not contain a fragment") - if not _is_valid_card_path(parsed.path): - raise ValueError( - "card URL must target //raw/main/README.md " - "or datasets///raw/main/README.md" - ) - - deadline = clock() + timeout - remaining = _deadline_remaining(deadline, clock) - if connection_factory is None: - connection = http.client.HTTPSConnection(HF_CARD_HOST, 443, timeout=remaining) - else: - connection = connection_factory(HF_CARD_HOST, 443, remaining) - target = parsed.path - if parsed.query: - target = "{0}?{1}".format(target, parsed.query) - return _run_http_worker( - connection, - lambda: _http_text_operation(connection, target, deadline, clock), - deadline, - clock, - completion_wait, - ) - - -def _http_text_operation(connection, target, deadline, clock): - try: - connection.request("GET", target, headers=UA) - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - response = connection.getresponse() - _deadline_remaining(deadline, clock) - if 300 <= response.status < 400: - raise http.client.HTTPException( - "redirect responses are not allowed for card requests" - ) - if not 200 <= response.status < 300: - raise http.client.HTTPException( - "card host returned HTTP status {0}".format(response.status) - ) - content_length = response.headers.get("Content-Length") - if content_length is not None: - try: - declared_length = int(content_length) - except (TypeError, ValueError) as error: - raise ValueError("invalid card Content-Length") from error - if declared_length < 0 or declared_length > MODEL_CARD_MAX_BYTES: - raise ValueError("card response exceeds size limit") - body = _read_bounded_body( - response, - connection, - deadline, - clock, - max_bytes=MODEL_CARD_MAX_BYTES, - ) - return body.decode("utf-8", errors="replace") - finally: - connection.close() - - -def _read_bounded_body(response, connection, deadline, clock, max_bytes=HF_RESPONSE_MAX_BYTES): - chunks = [] - total = 0 - while True: - _set_connection_timeout( - connection, - _deadline_remaining(deadline, clock), - ) - chunk = response.read( - min(_HTTP_READ_CHUNK_BYTES, max_bytes - total + 1) - ) - _deadline_remaining(deadline, clock) - if not chunk: - break - total += len(chunk) - if total > max_bytes: - raise ValueError("Hugging Face response exceeds size limit") - chunks.append(chunk) - return b"".join(chunks) - - -def _deadline_remaining(deadline, clock): - remaining = deadline - clock() - if remaining <= 0: - raise TimeoutError("Hugging Face request exceeded its overall deadline") - return remaining - - -def _set_connection_timeout(connection, timeout): - sock = getattr(connection, "sock", None) - if sock is not None: - sock.settimeout(timeout) - - -class HuggingFaceClient: - def __init__(self, http_get=http_json, card_get=http_card_text): - self._http_get = http_get - self._card_get = card_get - - @property - def http_get(self): - return self._http_get - - def fetch_model_card(self, repo, timeout=8): - """Return bounded README/model-card text, or None on any failure.""" - quoted = "/".join(urllib.parse.quote(part) for part in repo.split("/")) - url = "https://{0}/{1}/raw/main/README.md".format(HF_CARD_HOST, quoted) - try: - return self._card_get(url, timeout=timeout) - except Exception: - return None - - def fetch_dataset_card(self, repo, timeout=8): - """Return bounded dataset README text, or None on any failure.""" - quoted = "/".join(urllib.parse.quote(part) for part in repo.split("/")) - url = "https://{0}/datasets/{1}/raw/main/README.md".format(HF_CARD_HOST, quoted) - try: - return self._card_get(url, timeout=timeout) - except Exception: - return None - - @staticmethod - def list_models_url(sort="trendingScore", limit_fetch=300): - query = urllib.parse.urlencode({"filter": "mlx", "sort": sort, "direction": "-1", "limit": limit_fetch}) - return "{0}?{1}".format(HF_API, query) - - @staticmethod - def list_adapters_url(search="", limit_fetch=20): - params = { - "filter": "peft", - "sort": "downloads", - "direction": "-1", - "limit": limit_fetch, - } - if search: - params["search"] = search - return "{0}?{1}".format(HF_API, urllib.parse.urlencode(params)) - - @staticmethod - def list_datasets_url(search="", limit_fetch=20): - params = { - "sort": "downloads", - "direction": "-1", - "limit": limit_fetch, - } - if search: - params["search"] = search - return "{0}?{1}".format(HF_DATASETS_API, urllib.parse.urlencode(params)) - - def list_models(self, sort="trendingScore", limit_fetch=300): - return self._http_get(self.list_models_url(sort=sort, limit_fetch=limit_fetch)) - - def list_adapters(self, search="", limit_fetch=20, timeout=10): - """List PEFT/LoRA adapter model rows from the Hub (read-only).""" - rows = self._http_get( - self.list_adapters_url(search=search, limit_fetch=limit_fetch), - timeout=timeout, - ) - return rows if isinstance(rows, list) else [] - - def list_datasets(self, search="", limit_fetch=20, timeout=10): - """List dataset rows from the Hub (read-only).""" - rows = self._http_get( - self.list_datasets_url(search=search, limit_fetch=limit_fetch), - timeout=timeout, - ) - return rows if isinstance(rows, list) else [] - - def inspect_model_metadata(self, repo, timeout=8): - """Inspect model metadata without fetching the recursive repository tree.""" - quoted = urllib.parse.quote(repo) - model_url = "{0}/{1}".format(HF_API, quoted) - tree_url = "{0}/{1}/tree/main?recursive=true".format(HF_API, quoted) - output = { - "weight_bytes": None, - "tags": [], - "gated": None, - "license": None, - "reasoning": None, - "reason_src": None, - "tool_use": None, - "tool_use_src": None, - "tool_use_confidence": "none", - "params_total": None, - "architecture": None, - "metadata_available": False, - "tree_available": False, - "metadata_url": model_url, - "tree_url": tree_url, - "repository_url": "https://huggingface.co/{0}".format(repo), - } - try: - metadata = self._http_get(model_url, timeout=timeout) - output["metadata_available"] = True - tags = metadata.get("tags", []) or [] - output["tags"] = tags - output["gated"] = bool(metadata.get("gated")) - config = metadata.get("config") or {} - card_data = metadata.get("cardData") or {} - output["license"] = card_data.get("license") or next((tag.split("license:", 1)[1] for tag in tags if tag.startswith("license:")), None) - output["params_total"] = (metadata.get("safetensors") or {}).get("total") - output["architecture"] = extract_architecture(config) - template = ((config.get("tokenizer_config") or {}).get("chat_template") or "") - lower_tags = [tag.lower() for tag in tags] - normalized_tags = { - re.sub(r"\s+", "-", str(tag).strip().lower()) - for tag in tags - } - if TEMPLATE_REASON.search(template): - output["reasoning"], output["reason_src"] = True, "chat_template" - elif any(tag in ("reasoning", "thinking", "chain-of-thought") for tag in lower_tags): - output["reasoning"], output["reason_src"] = True, "tags" - elif REASONER_HINTS.search(repo): - output["reasoning"], output["reason_src"] = True, "name" - else: - output["reasoning"], output["reason_src"] = False, "checked" - if TEMPLATE_TOOL_USE.search(template): - output.update({ - "tool_use": True, - "tool_use_src": "chat_template", - "tool_use_confidence": "explicit", - }) - elif normalized_tags & TOOL_USE_TAGS: - output.update({ - "tool_use": True, - "tool_use_src": "tags", - "tool_use_confidence": "explicit", - }) - elif TOOL_USE_HINTS.search(repo): - output.update({ - "tool_use": True, - "tool_use_src": "name", - "tool_use_confidence": "weak", - }) - else: - output.update({ - "tool_use": False, - "tool_use_src": "checked", - "tool_use_confidence": "explicit", - }) - except Exception: - pass - return output - - def inspect_model(self, repo): - output = self.inspect_model_metadata(repo, timeout=8) - tree_url = output["tree_url"] - try: - tree = self._http_get(tree_url, timeout=8) - output["tree_available"] = True - weights = sum(item.get("size", 0) for item in tree if item.get("path", "").endswith((".safetensors", ".gguf", ".bin"))) - output["weight_bytes"] = weights or None - except Exception: - pass - return output diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/lora.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/lora.py deleted file mode 100644 index 336b859..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/lora.py +++ /dev/null @@ -1,326 +0,0 @@ -"""Confirmation-gated LoRA training jobs on the convert machinery.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from pathlib import Path - -from .convert import ( - _read_receipt, - _write_receipt, - receipts_root, -) -from .serve import _argv_matches, _pid_alive, _pid_command -from .wiring import _MODEL - - -LORA_RECEIPT_SCHEMA_VERSION = "1.0" -LORA_RECEIPT_KIND = "lora" -EXECUTABLE = "mlx_lm.lora" -ITERS_MIN, ITERS_MAX, ITERS_DEFAULT = 1, 100000, 1000 -BATCH_MIN, BATCH_MAX, BATCH_DEFAULT = 1, 64, 4 -LR_MIN, LR_MAX, LR_DEFAULT = 1e-6, 1e-2, 1e-5 -LAYERS_MIN, LAYERS_MAX, LAYERS_DEFAULT = -1, 128, 16 -MAX_DATASET_LINES = 10000 -DATASET_SAMPLE_LINES = 50 - - -class LoraError(RuntimeError): - """Classified lora failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def validate_dataset(data_dir): - """Validate an mlx_lm training dataset directory; bounded and read-only.""" - root = Path(data_dir) - train = root / "train.jsonl" - if not root.is_dir() or not train.is_file(): - raise LoraError( - "dataset_invalid", - "The dataset directory must contain train.jsonl: {0}".format(data_dir), - "Provide a directory with train.jsonl (text or messages per line).", - ) - summary = {"train_lines": _validate_jsonl(train), "valid_lines": 0} - valid = root / "valid.jsonl" - if valid.is_file(): - summary["valid_lines"] = _validate_jsonl(valid) - return summary - - -def _validate_jsonl(path): - count = 0 - try: - with open(path, "r", encoding="utf-8") as handle: - for line in handle: - count += 1 - if count > MAX_DATASET_LINES: - raise LoraError( - "dataset_invalid", - "{0} exceeds {1} lines.".format(path.name, MAX_DATASET_LINES), - "Split the dataset; lora validates bounded datasets only.", - ) - if count > DATASET_SAMPLE_LINES: - continue - stripped = line.strip() - if not stripped: - continue - try: - record = json.loads(stripped) - except json.JSONDecodeError: - raise LoraError( - "dataset_invalid", - "{0} line {1} is not valid JSON.".format(path.name, count), - "Fix the dataset line; every line must be one JSON object.", - ) - _validate_record(record, path.name, count) - except OSError as error: - raise LoraError( - "dataset_invalid", - "The dataset could not be read: {0}".format(error), - "Pass a readable dataset directory.", - ) - if count == 0: - raise LoraError( - "dataset_invalid", - "{0} is empty.".format(path.name), - "Provide at least one training example.", - ) - return count - - -def _validate_record(record, name, line): - if not isinstance(record, dict): - raise LoraError( - "dataset_invalid", - "{0} line {1} must be a JSON object.".format(name, line), - "Use {\"text\": ...} or {\"messages\": [...]} per line.", - ) - text = record.get("text") - if isinstance(text, str) and text.strip(): - return - messages = record.get("messages") - if isinstance(messages, list) and messages: - for message in messages: - if ( - not isinstance(message, dict) - or not isinstance(message.get("role"), str) - or not isinstance(message.get("content"), str) - ): - raise LoraError( - "dataset_invalid", - "{0} line {1} has a malformed message.".format(name, line), - "Each message needs string role and content fields.", - ) - return - raise LoraError( - "dataset_invalid", - "{0} line {1} needs a non-empty text field or a messages array.".format(name, line), - "Use {\"text\": ...} or {\"messages\": [{\"role\": ..., \"content\": ...}]}.", - ) - - -def _validate_hyperparameters(iters, batch_size, learning_rate, num_layers): - if not isinstance(iters, int) or isinstance(iters, bool) or not ITERS_MIN <= iters <= ITERS_MAX: - raise LoraError( - "invalid_arguments", - "iters must be between {0} and {1}.".format(ITERS_MIN, ITERS_MAX), - "Pass a bounded --iters value.", - ) - if not isinstance(batch_size, int) or isinstance(batch_size, bool) or not BATCH_MIN <= batch_size <= BATCH_MAX: - raise LoraError( - "invalid_arguments", - "batch-size must be between {0} and {1}.".format(BATCH_MIN, BATCH_MAX), - "Pass a bounded --batch-size value.", - ) - if ( - not isinstance(learning_rate, (int, float)) - or isinstance(learning_rate, bool) - or not LR_MIN <= float(learning_rate) <= LR_MAX - ): - raise LoraError( - "invalid_arguments", - "learning-rate must be between {0} and {1}.".format(LR_MIN, LR_MAX), - "Pass a bounded --learning-rate value.", - ) - if not isinstance(num_layers, int) or isinstance(num_layers, bool) or not LAYERS_MIN <= num_layers <= LAYERS_MAX: - raise LoraError( - "invalid_arguments", - "num-layers must be between {0} and {1}.".format(LAYERS_MIN, LAYERS_MAX), - "Pass a bounded --num-layers value (-1 trains all layers).", - ) - - -def plan_lora(repo, data, iters=ITERS_DEFAULT, batch_size=BATCH_DEFAULT, - learning_rate=LR_DEFAULT, num_layers=LAYERS_DEFAULT, out=None): - """Validate inputs and render the exact training plan; side-effect free.""" - if not isinstance(repo, str) or not _MODEL.fullmatch(repo): - raise LoraError( - "invalid_repo", - "lora requires a safe publisher/model base identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - _validate_hyperparameters(iters, batch_size, learning_rate, num_layers) - dataset = validate_dataset(data) - if out is None: - out = "{0}-lora".format(repo.split("/", 1)[1]) - plan = { - "repo": repo, - "data": str(data), - "iters": iters, - "batch_size": batch_size, - "learning_rate": float(learning_rate), - "num_layers": num_layers, - "out": str(out), - "dataset": dataset, - "argv": [ - EXECUTABLE, - "--model", repo, - "--train", - "--data", str(data), - "--iters", str(iters), - "--batch-size", str(batch_size), - "--learning-rate", repr(float(learning_rate)), - "--num-layers", str(num_layers), - "--adapter-path", str(out), - ], - } - canonical = json.dumps( - {key: plan[key] for key in sorted(plan) if key != "preview_hash"}, - sort_keys=True, separators=(",", ":"), - ) - plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() - return plan - - -def _receipt_filename(plan): - return "{0}-lora.json".format(plan["repo"].split("/", 1)[1]) - - -def start_lora(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, spawn=None, now=_utc_now, - pid_alive=None): - """Execute a reviewed training plan; the only mutating entry point.""" - from .serve import _default_spawn, _default_which - - which = which or _default_which - spawn = spawn or _default_spawn - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir, LORA_RECEIPT_KIND) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise LoraError( - "preview_hash_required", - "--confirm requires the hash from a reviewed lora preview.", - "Run lora start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise LoraError( - "preview_stale", - "The supplied preview hash does not match this lora plan.", - "Re-run lora start without --confirm and review the fresh plan.", - ) - if which(EXECUTABLE) is None: - raise LoraError( - "runtime_not_installed", - "The {0} executable is not installed.".format(EXECUTABLE), - "Install mlx-lm yourself (pip install mlx-lm); lora never installs runtimes.", - ) - if model_present is not None and not model_present(plan["repo"]): - raise LoraError( - "model_not_local", - "The base model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; lora never downloads.", - ) - if Path(plan["out"]).exists(): - raise LoraError( - "output_exists", - "The adapter output path already exists: {0}".format(plan["out"]), - "Pick a fresh --out path; lora never overwrites.", - ) - for receipt_path in sorted(root.glob("*.json")) if root.is_dir() else []: - existing = _read_receipt(receipt_path, LORA_RECEIPT_KIND) - if existing is None or existing.get("completed_at"): - continue - if pid_alive(existing["pid"]): - raise LoraError( - "job_in_progress", - "A lora job is still running (pid {0}).".format(existing["pid"]), - "Wait for it to finish (lora status) before starting another.", - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}-lora.log".format(plan["repo"].split("/", 1)[1]) - pid = spawn(plan["argv"], str(log_path)) - receipt = { - "schema_version": LORA_RECEIPT_SCHEMA_VERSION, - "kind": LORA_RECEIPT_KIND, - "repo": plan["repo"], - "data": plan["data"], - "iters": plan["iters"], - "out": plan["out"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "started_at": now(), - "preview_hash": plan["preview_hash"], - "completed_at": None, - "exit_status": None, - } - _write_receipt(root, receipt, _receipt_filename(plan)) - return {"status": "started", "receipt": receipt} - - -def status_lora(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - now=_utc_now): - """Cross-check lora receipts against live processes; marks exits once.""" - root = receipts_root(receipts_dir, LORA_RECEIPT_KIND) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - receipt = _read_receipt(path, LORA_RECEIPT_KIND) - if receipt is None: - continue - entry = { - "receipt": str(path), - "repo": receipt["repo"], - "data": receipt.get("data"), - "out": receipt["out"], - "pid": receipt["pid"], - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - "completed_at": receipt.get("completed_at"), - } - if receipt.get("completed_at"): - entry["state"] = receipt.get("exit_status") or "done" - entries.append(entry) - continue - if pid_alive(receipt["pid"]): - command = pid_command(receipt["pid"]) - entry["state"] = "running" if _argv_matches( - {"argv": receipt["argv"], "port": None, "repo": receipt["repo"]}, - command, - require_port=False, - ) else "unknown" - entries.append(entry) - continue - exit_status = "done" if Path(receipt["out"]).exists() else "failed" - receipt["completed_at"] = now() - receipt["exit_status"] = exit_status - _write_receipt(root, receipt, path.name) - entry["state"] = exit_status - entry["completed_at"] = receipt["completed_at"] - entries.append(entry) - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/model_doctor.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/model_doctor.py deleted file mode 100644 index b10511a..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/model_doctor.py +++ /dev/null @@ -1,495 +0,0 @@ -"""Read-only model inventory, wiring drift, and endpoint health diagnostics. - -Doctor never deletes, moves, repairs, downloads, or starts anything. It reads -the Hugging Face cache, queries already-running loopback runtimes, parses -Wire-managed configuration, and reports classified findings with remediation -text. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path - -from .verification import _safe_error, installed_model_ids -from .wiring import ( - _OLLAMA, - _STATE_CHANGING_PATH_PARTS, - ConfigAdapter, - validate_health_endpoint, -) - - -MAX_INVENTORY_ENTRIES = 500 -MAX_FINDINGS = 100 -MAX_ENDPOINTS = 50 -MAX_WALK_FILES = 2000 -MAX_CONFIG_BYTES = 1024 * 1024 -MAX_RECEIPTS = 50 -_WIRE_MARKER = "MLX_AGENT_WIRE" -_SKIP_DIRECTORIES = { - ".git", ".hg", ".svn", "node_modules", "__pycache__", ".venv", "venv", - ".tox", ".mypy_cache", ".pytest_cache", "Library", -} -_CONFIG_SUFFIXES = {".json", ".yaml", ".yml", ".modelfile"} -_HEALTH_TIMEOUT = 2.0 - - -def default_hf_cache(): - return Path.home() / ".cache" / "huggingface" / "hub" - - -def inspect_hf_cache(cache_dir): - """Summarize model repos in a Hugging Face cache without reading weights.""" - root = Path(cache_dir) - inventory = [] - if not root.is_dir(): - return inventory - for entry in sorted(root.iterdir()): - if len(inventory) >= MAX_INVENTORY_ENTRIES: - break - if not entry.is_dir() or not entry.name.startswith("models--"): - continue - repo = entry.name[len("models--"):].replace("--", "/") - refs_main = _read_text_quiet(entry / "refs" / "main") - snapshots = entry / "snapshots" - revisions = [] - total_bytes = 0 - missing_blobs = 0 - seen_blobs = set() - if snapshots.is_dir(): - for revision in sorted(snapshots.iterdir()): - if not revision.is_dir(): - continue - revisions.append(revision.name) - for dirpath, dirnames, filenames in os.walk(revision): - dirnames[:] = [name for name in dirnames if not name.startswith(".")] - for filename in filenames: - location = Path(dirpath) / filename - try: - resolved = location.resolve(strict=True) - except OSError: - missing_blobs += 1 - continue - if resolved in seen_blobs: - continue - seen_blobs.add(resolved) - try: - total_bytes += resolved.stat().st_size - except OSError: - missing_blobs += 1 - complete = bool(refs_main and revisions and missing_blobs == 0) - inventory.append({ - "id": repo, - "source": "hf-cache", - "bytes": total_bytes, - "revisions": revisions[:8], - "complete": complete, - }) - return inventory - - -def _read_text_quiet(path): - try: - return path.read_text(encoding="utf-8").strip() or None - except OSError: - return None - - -def collect_runtime_inventory(runtime_clients): - """Collect served model IDs from already-running loopback runtimes.""" - inventory = [] - errors = [] - for runtime in runtime_clients: - name = str(getattr(runtime, "name", runtime.__class__.__name__)) - try: - response = runtime.list_models() - for model_id in sorted(installed_model_ids(response)): - if len(inventory) >= MAX_INVENTORY_ENTRIES: - break - inventory.append({ - "id": model_id, - "source": name, - "bytes": _runtime_model_size(response, model_id), - "complete": True, - }) - except Exception as error: - errors.append("{0}: {1}".format(name, _safe_error(error))) - return inventory, errors - - -def _runtime_model_size(response, model_id): - if not isinstance(response, dict): - return None - for record in response.get("models", []) or []: - if not isinstance(record, dict): - continue - if str(record.get("name") or record.get("id") or "") != model_id: - continue - size = record.get("size") - if isinstance(size, int) and not isinstance(size, bool) and size >= 0: - return size - return None - - -def scan_wired_configs(roots): - """Find Wire-managed configs under roots and extract their references.""" - wired = [] - for root in roots: - for location in _walk_config_files(root): - if len(wired) >= MAX_ENDPOINTS: - return wired - content = _read_bounded(location) - if content is None or _WIRE_MARKER not in content: - continue - try: - adapter = ConfigAdapter.detect(location) - reference = adapter_references(adapter, content) - except (TypeError, ValueError): - continue - if reference is None: - continue - wired.append({ - "path": str(location), - "runtime": reference.get("runtime") or adapter.runtime, - "model": reference.get("model"), - "endpoint": reference.get("endpoint"), - }) - return wired - - -def _walk_config_files(root): - root = Path(root) - if root.is_file(): - yield root - return - if not root.is_dir(): - return - yielded = 0 - for dirpath, dirnames, filenames in os.walk(root): - dirnames[:] = [ - name for name in dirnames - if name not in _SKIP_DIRECTORIES and not name.startswith(".mlx-agent-receipts") - ] - for filename in sorted(filenames): - if yielded >= MAX_WALK_FILES: - return - location = Path(dirpath) / filename - if location.suffix.lower() not in _CONFIG_SUFFIXES and filename.lower() != "modelfile": - continue - yielded += 1 - yield location - - -def _read_bounded(location): - try: - if location.stat().st_size > MAX_CONFIG_BYTES: - return None - return location.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - return None - - -def adapter_references(adapter, content): - """Extract the model and endpoint a Wire-managed config references.""" - if adapter.runtime == "ollama": - match = _OLLAMA.fullmatch(content) - if match is None: - return None - return {"model": match.group(1), "endpoint": None} - if adapter.runtime == "litellm": - model = re.search(r"(?m)^ model: openai/([^\s]+)$", content) - base = re.search(r"(?m)^ api_base: (http[^\s]+)$", content) - if model is None: - return None - return { - "model": model.group(1), - "endpoint": base.group(1) if base else None, - } - try: - document = json.loads(content) - except (TypeError, ValueError, json.JSONDecodeError): - return None - if not isinstance(document, dict): - return None - marker = document.get("mlx_agent_wire") - if not isinstance(marker, dict): - return None - provider = marker.get("provider") if isinstance(marker.get("provider"), dict) else {} - runtime = marker.get("runtime") - return { - "model": marker.get("model"), - "endpoint": provider.get("base_url"), - "runtime": runtime if isinstance(runtime, str) else None, - } - - -def scan_receipt_after_hashes(roots): - """Read after_hashes from Wire receipts without enforcing receipt validity.""" - records = [] - for root in roots: - root = Path(root) - candidates = [] - if (root / ".mlx-agent-receipts").is_dir(): - candidates.append(root / ".mlx-agent-receipts") - else: - for dirpath, dirnames, _filenames in os.walk(root): - if ".mlx-agent-receipts" in dirnames: - candidates.append(Path(dirpath) / ".mlx-agent-receipts") - dirnames.remove(".mlx-agent-receipts") - for receipts_dir in candidates: - for receipt in sorted(receipts_dir.glob("*/receipt.json")): - if len(records) >= MAX_RECEIPTS: - return records - try: - value = json.loads(receipt.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - continue - after_hashes = value.get("after_hashes") - if not isinstance(after_hashes, dict): - continue - records.append({ - "receipt": str(receipt), - "transaction_id": value.get("transaction_id"), - "status": value.get("status"), - "after_hashes": { - str(key): str(item) for key, item in after_hashes.items() - }, - }) - return records - - -def check_endpoint(endpoint, timeout=_HEALTH_TIMEOUT): - """Classify a wired loopback endpoint as ok, reachable, or down.""" - if not endpoint: - return None - try: - url = validate_health_endpoint(endpoint) - parsed_path = urllib.parse.urlparse(url).path.rstrip("/") - if parsed_path.endswith("/v1"): - url = url.rstrip("/") + "/models" - request = urllib.request.Request(url, method="GET") - opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - with opener.open(request, timeout=timeout) as response: - status = response.status - except urllib.error.HTTPError as error: - status = error.code - except (OSError, ValueError, urllib.error.URLError): - return "down" - return "ok" if 200 <= status < 400 else "reachable" - - -def run_model_doctor(wired_roots, hf_cache, runtime_clients): - """Run every read-only check and return the doctor report.""" - wired_roots = [Path(root) for root in wired_roots] - inventory = inspect_hf_cache(hf_cache) - runtime_inventory, runtime_errors = collect_runtime_inventory(runtime_clients) - inventory.extend(runtime_inventory) - known_ids = {item["id"].casefold() for item in inventory} - - wired = scan_wired_configs(wired_roots) - receipts = scan_receipt_after_hashes(wired_roots) - - findings = [] - for config in wired: - model = config.get("model") - if model and model.casefold() not in known_ids: - findings.append({ - "code": "drift_missing_model", - "path": config["path"], - "model": model, - "remediation": ( - "Re-download the model into a local inventory, or re-run " - "wire render for a model that is present." - ), - }) - - for receipt in receipts: - for target, expected in receipt["after_hashes"].items(): - location = Path(target) - if not location.is_file(): - findings.append({ - "code": "drift_missing_file", - "path": target, - "receipt_id": receipt["transaction_id"], - "remediation": ( - "The wired file is gone; restore it from the receipt " - "backup or roll the receipt back." - ), - }) - continue - try: - digest = hashlib.sha256(location.read_bytes()).hexdigest() - except OSError: - continue - if digest != expected: - findings.append({ - "code": "drift_hash_mismatch", - "path": target, - "receipt_id": receipt["transaction_id"], - "remediation": ( - "The wired file changed after wiring; inspect it, then " - "re-apply or roll back the receipt." - ), - }) - - endpoints = [] - claims = {} - for config in wired: - endpoint = config.get("endpoint") - if not endpoint: - continue - origin = _endpoint_origin_key(endpoint) - if origin is None: - continue - previous = claims.get(origin) - if previous is not None and previous["runtime"] != config["runtime"]: - findings.append({ - "code": "drift_endpoint_conflict", - "path": config["path"], - "model": config.get("model"), - "remediation": ( - "Two wired configs for different runtimes claim {0}; " - "move one runtime to its own port.".format(origin) - ), - }) - else: - claims.setdefault(origin, config) - if len(endpoints) < MAX_ENDPOINTS: - endpoints.append({ - "endpoint": origin, - "path": config["path"], - "status": check_endpoint(endpoint), - }) - - for item in inventory: - if item["source"] == "hf-cache" and not item.get("complete", True): - findings.append({ - "code": "hf_cache_incomplete", - "path": None, - "model": item["id"], - "remediation": ( - "The cached snapshot is incomplete; re-download the model " - "with the runtime's own pull command." - ), - }) - - findings = findings[:MAX_FINDINGS] - total_bytes = sum( - item["bytes"] or 0 for item in inventory if item["source"] == "hf-cache" - ) - return { - "inventory": inventory, - "summary": { - "models": len(inventory), - "hf_cache_bytes": total_bytes, - "wired_configs": len(wired), - "findings": len(findings), - }, - "wired": wired, - "endpoints": endpoints, - "findings": findings, - "runtime_errors": runtime_errors, - } - - -def _endpoint_origin_key(endpoint): - try: - url = validate_health_endpoint(endpoint) - except ValueError: - return None - parsed = urllib.parse.urlparse(url) - port = parsed.port or (443 if parsed.scheme == "https" else 80) - decoded = parsed.path or "/" - parts = {part for part in decoded.lower().split("/") if part} - if parts & _STATE_CHANGING_PATH_PARTS: - return None - return "{0}://{1}:{2}".format(parsed.scheme.lower(), parsed.hostname.lower(), port) - - -PRUNE_PREVIEW_VERSION = "1.0" -MAX_PRUNE_CANDIDATES = 100 - - -class PruneError(RuntimeError): - """Classified prune failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def plan_prune(cache_dir): - """List incomplete HF cache snapshots eligible for deletion; read-only.""" - root = Path(cache_dir) - candidates = [] - for item in inspect_hf_cache(root): - if item.get("complete", True): - continue - location = root / "models--{0}".format(item["id"].replace("/", "--")) - candidates.append({ - "repo": item["id"], - "path": str(location), - "bytes": item.get("bytes") or 0, - }) - if len(candidates) >= MAX_PRUNE_CANDIDATES: - break - canonical = json.dumps(candidates, sort_keys=True, separators=(",", ":")) - return { - "cache": str(root), - "candidates": candidates, - "total_bytes": sum(item["bytes"] for item in candidates), - "preview_hash": hashlib.sha256(canonical.encode("utf-8")).hexdigest(), - } - - -def execute_prune(plan, confirm=False, preview_hash=None, remover=None): - """Delete exactly the reviewed candidates; irreversible by design.""" - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise PruneError( - "preview_hash_required", - "--confirm requires the hash from a reviewed prune preview.", - "Run doctor models --prune without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise PruneError( - "preview_stale", - "The supplied preview hash does not match the current prune plan.", - "Re-run doctor models --prune and review the fresh candidate list.", - ) - if remover is None: - import shutil - - def remover(path): - shutil.rmtree(path) - - cache = Path(plan["cache"]).resolve() - removed = [] - for candidate in plan["candidates"]: - location = Path(candidate["path"]) - resolved = location.resolve() - if resolved == cache or cache not in resolved.parents: - raise PruneError( - "unsafe_target", - "Refusing to remove a path outside the cache: {0}".format(candidate["path"]), - "Only cache-owned model directories are ever pruned.", - ) - if not location.is_dir(): - removed.append({"repo": candidate["repo"], "removed": False, "reason": "already gone"}) - continue - remover(str(location)) - removed.append({ - "repo": candidate["repo"], - "removed": not location.exists(), - "bytes": candidate["bytes"], - }) - return {"status": "pruned", "removed": removed} diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/providers.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/providers.py deleted file mode 100644 index 8e5fad1..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/providers.py +++ /dev/null @@ -1,436 +0,0 @@ -"""Manifest-backed provider definitions and non-mutating provider detection.""" - -from __future__ import annotations - -import json -import os -import re -import selectors -import shutil -import subprocess -import time -from dataclasses import dataclass -from pathlib import Path - - -_GEMINI_EXTENSION_SUFFIX = Path(".gemini") / "extensions" / "mlx-agent" - - -@dataclass(frozen=True) -class ProviderArtifact: - source: Path - destination: Path - project_destination: Path = None - scopes: tuple = ("user", "project") - - -@dataclass(frozen=True) -class ProviderDefinition: - id: str - detect_commands: tuple - user_root: Path - project_root: Path - invocation_kind: str - invocation_prefix: str - minimum_version: str - last_tested_version: str - version_probe_args: tuple - version_probe_timeout: float - install_mode: str - artifacts: tuple - config_paths: tuple - - def destination(self, scope, project_root=None): - if scope == "user": - return self.user_root - if scope != "project": - raise ValueError("scope must be 'user' or 'project'") - if project_root is None: - raise ValueError("project scope requires a project root") - return Path(project_root).resolve() / self.project_root - - def artifact_destination(self, scope, project_root, artifact): - """Return the receipt-owned target for one declared artifact.""" - if scope == "project" and artifact.project_destination is not None: - return Path(project_root).resolve() / artifact.project_destination - return self.destination(scope, project_root) / artifact.destination - - @staticmethod - def applies_to(scope, artifact): - return scope in artifact.scopes - - -@dataclass(frozen=True) -class ProviderDetection: - id: str - available: bool - command: str = "" - command_path: str = None - state: str = "absent" - version: str = None - minimum_version: str = None - last_tested_version: str = None - error: str = None - install_mode: str = "direct" - - def to_dict(self): - return { - "id": self.id, - "available": self.available, - "command": self.command or None, - "command_path": self.command_path, - "state": self.state, - "version": self.version, - "minimum_version": self.minimum_version, - "last_tested_version": self.last_tested_version, - "error": self.error, - "install_mode": self.install_mode, - } - - -def _safe_relative(value, field, allow_current=False): - path = Path(value) - if allow_current and str(value) == ".": - return path - if path.is_absolute() or ".." in path.parts or not path.parts: - raise ValueError("{0} must be a safe relative path".format(field)) - return path - - -class ProviderRegistry: - """Load immutable installation data from the canonical plugin manifest.""" - - def __init__(self, manifest_path, home=None, config_root=None, xdg_config_home=None): - self.manifest_path = Path(manifest_path).resolve() - self.home = Path(home).expanduser().resolve() if home else Path.home().resolve() - self.config_root = ( - Path(config_root).expanduser().resolve() if config_root else self.home - ) - self.xdg_config_home = ( - Path(xdg_config_home).expanduser().resolve() - if xdg_config_home else (self.home / ".config").resolve() - ) - - def definitions(self): - try: - manifest = json.loads(self.manifest_path.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError) as error: - raise ValueError("provider manifest could not be read: {0}".format(error)) - providers = manifest.get("providers") - if not isinstance(providers, dict): - raise ValueError("provider manifest has no providers object") - definitions = {} - for provider_id, value in providers.items(): - definitions[provider_id] = self._definition(provider_id, value) - return definitions - - def _definition(self, provider_id, value): - if not isinstance(value, dict): - raise ValueError("provider {0} must be an object".format(provider_id)) - required = ( - "detect_commands", "user_root", "project_root", "artifacts", "config_paths", - "invocation", "minimum_version", "last_tested_version", "version_probe", - "install_mode", - ) - if any(key not in value for key in required): - raise ValueError("provider {0} has no installer definition".format(provider_id)) - commands = value["detect_commands"] - if not isinstance(commands, list) or not all(isinstance(item, str) and item for item in commands): - raise ValueError("provider {0}.detect_commands is invalid".format(provider_id)) - user_root = self._user_root(value["user_root"], provider_id) - project_root = self._project_root(value["project_root"], provider_id) - invocation = value["invocation"] - if not isinstance(invocation, dict) or set(invocation) != {"kind", "prefix"}: - raise ValueError("provider {0}.invocation is invalid".format(provider_id)) - invocation_kind, invocation_prefix = invocation["kind"], invocation["prefix"] - if invocation_kind not in {"command", "skill"} or not isinstance(invocation_prefix, str): - raise ValueError("provider {0}.invocation is invalid".format(provider_id)) - if invocation_kind == "command" and invocation_prefix != "/": - raise ValueError("provider {0}.invocation is invalid".format(provider_id)) - if invocation_kind == "skill" and invocation_prefix not in {"", "$"}: - raise ValueError("provider {0}.invocation is invalid".format(provider_id)) - minimum_version = value["minimum_version"] - last_tested_version = value["last_tested_version"] - for label, version in ( - ("minimum_version", minimum_version), - ("last_tested_version", last_tested_version), - ): - if version is not None and not _SEMVER.fullmatch(version): - raise ValueError("provider {0}.{1} is invalid".format(provider_id, label)) - probe = value["version_probe"] - if probe is None: - version_probe_args, version_probe_timeout = (), 0.0 - elif ( - not isinstance(probe, dict) - or set(probe) != {"args", "timeout_seconds"} - or not isinstance(probe["args"], list) - or not 1 <= len(probe["args"]) <= 4 - or not all(isinstance(item, str) and item and len(item) <= 64 for item in probe["args"]) - or not isinstance(probe["timeout_seconds"], (int, float)) - or isinstance(probe["timeout_seconds"], bool) - or not 0 < probe["timeout_seconds"] <= 5 - ): - raise ValueError("provider {0}.version_probe is invalid".format(provider_id)) - else: - version_probe_args = tuple(probe["args"]) - version_probe_timeout = float(probe["timeout_seconds"]) - install_mode = value["install_mode"] - if install_mode not in {"direct", "staged", "portable"}: - raise ValueError("provider {0}.install_mode is invalid".format(provider_id)) - if install_mode == "portable" and (commands or probe is not None or minimum_version is not None or last_tested_version is not None): - raise ValueError("portable providers must not declare CLI compatibility probes") - if install_mode != "portable" and (not commands or probe is None or minimum_version is None): - raise ValueError("native providers require a minimum version and bounded probe") - artifacts = [] - if not isinstance(value["artifacts"], list) or not value["artifacts"]: - raise ValueError("provider {0}.artifacts is invalid".format(provider_id)) - for index, artifact in enumerate(value["artifacts"]): - if not isinstance(artifact, dict) or set(artifact) not in ({"source", "destination"}, {"source", "destination", "scope"}, {"source", "destination", "project_destination"}, {"source", "destination", "project_destination", "scope"}): - raise ValueError("provider {0}.artifacts[{1}] is invalid".format(provider_id, index)) - source = _safe_relative(artifact["source"], "artifact source") - unresolved = self.manifest_path.parent / source - cursor = self.manifest_path.parent - for component in source.parts: - cursor = cursor / component - if cursor.is_symlink(): - raise ValueError( - "provider artifact source must not contain symlinks: {0}".format(source) - ) - location = unresolved.resolve() - if self.manifest_path.parent not in location.parents or not (location.is_file() or location.is_dir()): - raise ValueError("provider artifact source is outside the plugin or missing: {0}".format(source)) - destination = _safe_relative( - artifact["destination"], "artifact destination", allow_current=True - ) - project_destination = ( - _safe_relative(artifact["project_destination"], "artifact project_destination") - if "project_destination" in artifact else None - ) - scopes = (artifact.get("scope", "user"),) if "scope" in artifact else ("user", "project") - if scopes not in (("user",), ("project",)) and scopes != ("user", "project"): - raise ValueError("provider {0}.artifacts[{1}].scope is invalid".format(provider_id, index)) - if location.is_file(): - artifacts.append(ProviderArtifact(location, destination, project_destination, scopes)) - else: - for child in sorted(location.rglob("*")): - if child.is_symlink(): - raise ValueError( - "provider artifact source must not contain symlinks: {0}".format(child) - ) - resolved = child.resolve() - relative = child.relative_to(location) - if "__pycache__" in relative.parts or child.suffix == ".pyc": - continue - if child.is_file() and location in resolved.parents: - project_child = project_destination / child.relative_to(location) if project_destination else None - artifacts.append(ProviderArtifact(resolved, destination / child.relative_to(location), project_child, scopes)) - config_paths = value["config_paths"] - if not isinstance(config_paths, list) or not all(isinstance(item, str) for item in config_paths): - raise ValueError("provider {0}.config_paths is invalid".format(provider_id)) - if provider_id == "gemini": - self._validate_gemini_extension_layout(user_root, project_root, artifacts) - if provider_id == "opencode": - self._validate_opencode_layout(user_root, project_root, artifacts) - return ProviderDefinition( - id=provider_id, - detect_commands=tuple(commands), - user_root=user_root, - project_root=project_root, - invocation_kind=invocation_kind, - invocation_prefix=invocation_prefix, - minimum_version=minimum_version, - last_tested_version=last_tested_version, - version_probe_args=version_probe_args, - version_probe_timeout=version_probe_timeout, - install_mode=install_mode, - artifacts=tuple(artifacts), - config_paths=tuple(config_paths), - ) - - def _validate_gemini_extension_layout(self, user_root, project_root, artifacts): - """Keep Gemini's install roots aligned with its extension discovery layout.""" - if user_root != (self.home / _GEMINI_EXTENSION_SUFFIX).resolve(): - raise ValueError("provider gemini.user_root must target home/.gemini/extensions/mlx-agent") - if project_root != _GEMINI_EXTENSION_SUFFIX: - raise ValueError("provider gemini.project_root must target .gemini/extensions/mlx-agent") - if not any(item.destination == Path("gemini-extension.json") for item in artifacts): - raise ValueError("provider gemini must install gemini-extension.json") - for item in artifacts: - if item.project_destination is not None and item.project_destination.parts[:1] != (".gemini",): - raise ValueError("provider gemini project artifacts must stay under .gemini") - - def _validate_opencode_layout(self, user_root, project_root, artifacts): - """Keep OpenCode's documented global and project discovery locations exact.""" - if user_root != (self.xdg_config_home / "opencode").resolve(): - raise ValueError("provider opencode.user_root must target the OpenCode XDG config directory") - if project_root != Path(".opencode"): - raise ValueError("provider opencode.project_root must target .opencode") - if not any(item.destination == Path("commands/mlx-scout.md") for item in artifacts): - raise ValueError("provider opencode must install native command files") - if not any(item.destination == Path("agents/mlx-advisor.md") for item in artifacts): - raise ValueError("provider opencode must install its advisor agent") - if not any(item.destination == Path("skills/mlx-scout/SKILL.md") for item in artifacts): - raise ValueError("provider opencode must install native skills") - if any(item.destination == Path("opencode.json") for item in artifacts): - raise ValueError("provider opencode must not mutate global or project config") - if not any(item.destination == Path("plugins/mlx-agent-command.ts") for item in artifacts): - raise ValueError("provider opencode must install its native custom tool") - if not any(item.destination == Path("src/mlx_agent/command_executor.py") for item in artifacts): - raise ValueError("provider opencode must install its custom-tool runtime") - - def _user_root(self, template, provider_id): - if not isinstance(template, str) or "{project}" in template: - raise ValueError("provider {0}.user_root is invalid".format(provider_id)) - try: - path = Path(template.format( - home=str(self.home), config_root=str(self.config_root), - xdg_config_home=str(self.xdg_config_home), - )).resolve() - except (KeyError, ValueError) as error: - raise ValueError("provider {0}.user_root is invalid: {1}".format(provider_id, error)) - allowed_roots = (self.home, self.config_root, self.xdg_config_home) - if not any(root == path or root in path.parents for root in allowed_roots): - raise ValueError("provider {0}.user_root must be under an approved user configuration root".format(provider_id)) - return path - - def _project_root(self, template, provider_id): - if not isinstance(template, str) or "{project}" not in template: - raise ValueError("provider {0}.project_root is invalid".format(provider_id)) - marker = Path("/__mlx_agent_project_marker__") - try: - rendered = Path(template.format( - project=str(marker), home=str(self.home), config_root=str(self.config_root), - xdg_config_home=str(self.xdg_config_home), - )).resolve() - return rendered.relative_to(marker) - except (KeyError, ValueError): - raise ValueError("provider {0}.project_root is invalid".format(provider_id)) - - -_SEMVER = re.compile(r"(? _MAX_VERSION_OUTPUT: - raise ValueError("version probe output exceeded the bounded limit") - returncode = process.wait(timeout=max(0.01, deadline - time.monotonic())) - return subprocess.CompletedProcess(argv, returncode, bytes(output)) - finally: - selector.close() - if process.poll() is None: - process.kill() - process.wait() - if process.stdout is not None: - process.stdout.close() - - -def detect_providers(definitions, env=None, path=None, executable_lookup=None, probe_runner=None): - """Report installed provider CLIs only; this function never installs anything.""" - environment = dict(os.environ if env is None else env) - search_path = path if path is not None else environment.get("PATH", "") - lookup = executable_lookup or shutil.which - detections = [] - for definition in definitions: - if definition.install_mode == "portable": - detections.append(ProviderDetection( - definition.id, True, state="portable", install_mode="portable" - )) - continue - command_path = None - command = "" - for candidate in definition.detect_commands: - found = lookup(candidate, path=search_path) - if found: - command, command_path = candidate, str(found) - break - if not command_path: - detections.append(ProviderDetection( - definition.id, False, command, command_path, state="absent", - minimum_version=definition.minimum_version, - last_tested_version=definition.last_tested_version, - error="provider executable was not found", - install_mode=definition.install_mode, - )) - continue - try: - argv = [command_path] + list(definition.version_probe_args) - if probe_runner is None: - completed = _run_bounded_probe( - argv, - timeout=definition.version_probe_timeout, - env=environment, - ) - else: - completed = probe_runner( - argv, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=definition.version_probe_timeout, - check=False, - shell=False, - env=environment, - ) - raw = completed.stdout or b"" - if isinstance(raw, bytes): - output = raw[:_MAX_VERSION_OUTPUT].decode("utf-8", errors="replace") - else: - output = str(raw)[:_MAX_VERSION_OUTPUT] - match = _SEMVER.search(output) - if completed.returncode != 0 or match is None: - raise ValueError("version probe did not return a SemVer core") - version = match.group(1) - supported = definition.last_tested_version is not None and ( - _version_tuple(definition.minimum_version) - <= _version_tuple(version) - <= _version_tuple(definition.last_tested_version) - ) - state = "native-visible" if supported else "unsupported" - detections.append(ProviderDetection( - definition.id, supported, command, command_path, state=state, - version=version, minimum_version=definition.minimum_version, - last_tested_version=definition.last_tested_version, - error=None if supported else "provider version is outside the validated range", - install_mode=definition.install_mode, - )) - except (OSError, subprocess.SubprocessError, ValueError) as error: - detections.append(ProviderDetection( - definition.id, False, command, command_path, state="unsupported", - minimum_version=definition.minimum_version, - last_tested_version=definition.last_tested_version, - error=str(error), install_mode=definition.install_mode, - )) - return detections diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/serve.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/serve.py deleted file mode 100644 index 264cfca..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/serve.py +++ /dev/null @@ -1,528 +0,0 @@ -"""Confirmation-gated launcher for local MLX serving runtimes. - -Serve is the only component that spawns processes, so every mutation goes -through the wire-style preview -> confirm -> receipt flow. Serve never -installs runtimes, never downloads models, binds only to 127.0.0.1, and stops -only processes that it started itself, verified against their receipts. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import shutil -import signal -import subprocess -import time -import urllib.error -import urllib.request -from datetime import datetime, timezone -from pathlib import Path - -from .model_doctor import scan_wired_configs -from .transactions import _atomic_in_directory -from .wiring import require_secret_free_config, validate_health_endpoint - - -SERVE_RECEIPT_SCHEMA_VERSION = "1.0" -SERVE_RECEIPT_KIND = "serve" -MAX_TOKENS_MIN, MAX_TOKENS_MAX, MAX_TOKENS_DEFAULT = 256, 65536, 8192 -PORT_MIN, PORT_MAX = 1, 65535 -READINESS_DEADLINE_DEFAULT = 60.0 -STOP_DEADLINE_SECONDS = 10.0 -MAX_RECEIPTS = 50 - -_RECIPES_PATH = Path(__file__).resolve().parent / "resources" / "serve-recipes.json" - - -class ServeError(RuntimeError): - """Classified serve failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def load_recipes(path=None): - location = Path(path) if path is not None else _RECIPES_PATH - value = json.loads(location.read_text(encoding="utf-8")) - recipes = value.get("recipes") - if not isinstance(recipes, dict) or not recipes: - raise ValueError("serve recipe table is missing its recipes") - for name, recipe in recipes.items(): - for key in ("executable", "argv", "default_port", "readiness", "install_hint"): - if key not in recipe: - raise ValueError("serve recipe {0} is missing {1}".format(name, key)) - return recipes - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def _preview_hash(plan): - canonical = json.dumps( - {key: plan[key] for key in sorted(plan) if key != "preview_hash"}, - sort_keys=True, separators=(",", ":"), - ) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest() - - -def receipts_root(root=None): - base = Path(root) if root is not None else Path.cwd() - return base / ".mlx-agent-receipts" / "serve" - - -def plan_start(repo, runtime, recipes, port=None, max_tokens=MAX_TOKENS_DEFAULT, - adapter_path=None): - """Render the exact start plan; pure and side-effect free.""" - if not isinstance(repo, str) or not repo.strip() or "/" not in repo: - raise ServeError( - "invalid_repo", - "serve requires a publisher/model repository identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - recipe = recipes.get(runtime) - if recipe is None: - raise ServeError( - "unsupported_runtime", - "serve does not launch the {0} runtime.".format(runtime), - "Ollama and LM Studio manage their own servers; serve supports: {0}.".format( - ", ".join(sorted(recipes)) - ), - ) - if not isinstance(max_tokens, int) or isinstance(max_tokens, bool): - raise ServeError( - "invalid_arguments", "max_tokens must be an integer.", - "Pass --max-tokens between {0} and {1}.".format(MAX_TOKENS_MIN, MAX_TOKENS_MAX), - ) - if not MAX_TOKENS_MIN <= max_tokens <= MAX_TOKENS_MAX: - raise ServeError( - "invalid_arguments", - "max_tokens is outside {0}-{1}.".format(MAX_TOKENS_MIN, MAX_TOKENS_MAX), - "Pass a bounded --max-tokens value.", - ) - selected_port = recipe["default_port"] if port is None else port - if not isinstance(selected_port, int) or isinstance(selected_port, bool): - raise ServeError( - "invalid_arguments", "port must be an integer.", - "Pass --port between {0} and {1}.".format(PORT_MIN, PORT_MAX), - ) - if not PORT_MIN <= selected_port <= PORT_MAX: - raise ServeError( - "invalid_arguments", - "port is outside {0}-{1}.".format(PORT_MIN, PORT_MAX), - "Pass a valid loopback port.", - ) - if adapter_path is not None and not recipe.get("adapter_argv"): - raise ServeError( - "unsupported_runtime", - "The {0} recipe does not support adapter serving.".format(runtime), - "Serve adapters with a runtime whose recipe declares adapter support.", - ) - values = { - "executable": recipe["executable"], - "repo": repo, - "port": str(selected_port), - "max_tokens": str(max_tokens), - } - argv = [part.format(**values) for part in recipe["argv"]] - if adapter_path is not None: - argv.extend( - part.format(adapter_path=str(adapter_path)) for part in recipe["adapter_argv"] - ) - plan = { - "repo": repo, - "runtime": runtime, - "port": selected_port, - "max_tokens": max_tokens, - "adapter_path": str(adapter_path) if adapter_path is not None else None, - "argv": argv, - "readiness": recipe["readiness"].format(port=selected_port), - "bind": "127.0.0.1", - } - plan["preview_hash"] = _preview_hash(plan) - return plan - - -def _default_which(executable): - return shutil.which(executable) - - -def _default_port_free(port): - import socket - - probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - probe.settimeout(0.5) - return probe.connect_ex(("127.0.0.1", port)) != 0 - finally: - probe.close() - - -def _default_spawn(argv, log_path): - handle = open(log_path, "ab", buffering=0) - process = subprocess.Popen( - argv, - stdin=subprocess.DEVNULL, - stdout=handle, - stderr=subprocess.STDOUT, - start_new_session=True, - ) - return process.pid - - -def _default_readiness(url, deadline_seconds, clock=time.monotonic, sleep=time.sleep): - validate_health_endpoint(url) - deadline = clock() + deadline_seconds - while clock() < deadline: - try: - request = urllib.request.Request(url, method="GET") - opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - with opener.open(request, timeout=2.0) as response: - if 200 <= response.status < 400: - return True - except (OSError, ValueError, urllib.error.URLError): - pass - sleep(0.5) - return False - - -def _pid_alive(pid): - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True - except OSError: - return False - return True - - -def _pid_command(pid): - try: - result = subprocess.run( - ["ps", "-p", str(pid), "-o", "command="], - capture_output=True, text=True, timeout=5, - ) - except (OSError, subprocess.SubprocessError): - return None - if result.returncode != 0: - return None - return result.stdout.strip() or None - - -def start_serve(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, port_free=None, - wired_claims=None, spawn=None, readiness=None, - readiness_deadline=READINESS_DEADLINE_DEFAULT, now=_utc_now, - pid_alive=None): - """Execute a reviewed start plan; the only mutating serve entry point.""" - which = which or _default_which - port_free = port_free or _default_port_free - spawn = spawn or _default_spawn - readiness = readiness or _default_readiness - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise ServeError( - "preview_hash_required", - "--confirm requires the hash from a reviewed serve preview.", - "Run serve start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise ServeError( - "preview_stale", - "The supplied preview hash does not match this start plan.", - "Re-run serve start without --confirm and review the fresh plan.", - ) - - if which(plan["argv"][0]) is None: - recipe_hint = load_recipes()[plan["runtime"]]["install_hint"] - raise ServeError( - "runtime_not_installed", - "The {0} executable is not installed.".format(plan["argv"][0]), - "Install it yourself ({0}); serve never installs runtimes.".format(recipe_hint), - ) - if model_present is not None and not model_present(plan["repo"]): - raise ServeError( - "model_not_local", - "The model is not present in a local inventory.", - "Download it with the runtime's own pull command first; serve never downloads models.", - ) - if not port_free(plan["port"]): - raise ServeError( - "port_in_use", - "Port {0} is already bound on 127.0.0.1.".format(plan["port"]), - "Pick a free --port, or stop the process that owns this one.", - ) - if wired_claims is not None: - claimed = wired_claims(plan["port"], plan["runtime"]) - if claimed: - raise ServeError( - "port_in_use", - "A wired config already claims port {0}.".format(plan["port"]), - "Move this server to its own --port.", - ) - if plan["adapter_path"] is not None and not Path(plan["adapter_path"]).is_dir(): - raise ServeError( - "invalid_arguments", - "The adapter path does not exist: {0}".format(plan["adapter_path"]), - "Pass an existing --adapter-path directory produced by mlx_lm.lora.", - ) - - receipt_path = root / "{0}.json".format(plan["port"]) - if receipt_path.exists(): - existing = _read_receipt(receipt_path) - if existing is not None and pid_alive(existing.get("pid", -1)): - raise ServeError( - "port_in_use", - "A serve receipt for port {0} is still live.".format(plan["port"]), - "Run serve stop --port {0} first.".format(plan["port"]), - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}.log".format(plan["port"]) - pid = spawn(plan["argv"], str(log_path)) - if not readiness(plan["readiness"], readiness_deadline): - _terminate_pid(pid) - raise ServeError( - "readiness_timeout", - "The server did not answer its readiness endpoint in time.", - "Inspect the serve log at {0}.".format(log_path), - ) - - receipt = { - "schema_version": SERVE_RECEIPT_SCHEMA_VERSION, - "kind": SERVE_RECEIPT_KIND, - "repo": plan["repo"], - "runtime": plan["runtime"], - "port": plan["port"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "readiness": plan["readiness"], - "started_at": now(), - "preview_hash": plan["preview_hash"], - } - content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") - _atomic_in_directory(root, receipt_path.name, content, 0o600) - return {"status": "started", "receipt": receipt} - - -def _read_receipt(path): - try: - value = json.loads(Path(path).read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - return None - if not isinstance(value, dict) or value.get("kind") != SERVE_RECEIPT_KIND: - return None - if not isinstance(value.get("pid"), int) or isinstance(value.get("pid"), bool): - return None - if not isinstance(value.get("argv"), list) or not isinstance(value.get("port"), int): - return None - return value - - -def status_serve(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command): - """Cross-check serve receipts against live processes; read-only.""" - root = receipts_root(receipts_dir) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - if len(entries) >= MAX_RECEIPTS: - break - receipt = _read_receipt(path) - if receipt is None: - continue - alive = pid_alive(receipt["pid"]) - command = pid_command(receipt["pid"]) if alive else None - entries.append({ - "receipt": str(path), - "repo": receipt["repo"], - "runtime": receipt["runtime"], - "port": receipt["port"], - "pid": receipt["pid"], - "alive": alive, - "argv_match": _argv_matches(receipt, command) if alive else False, - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - }) - return entries - - -def _argv_matches(receipt, command, require_port=True): - if not command: - return False - argv = receipt.get("argv") or [] - if not argv: - return False - executable = Path(str(argv[0])).name - if executable not in command or str(receipt.get("repo")) not in command: - return False - if require_port: - return "--port {0}".format(receipt.get("port")) in command - return True - - -def _terminate_pid(pid, sig=signal.SIGTERM): - try: - os.kill(pid, sig) - except OSError: - return False - return True - - -def stop_serve(port, receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - terminate=_terminate_pid, clock=time.monotonic, sleep=time.sleep): - """Stop exactly the process a serve receipt owns; never kills by port scan.""" - root = receipts_root(receipts_dir) - receipt_path = root / "{0}.json".format(port) - receipt = _read_receipt(receipt_path) if receipt_path.is_file() else None - if receipt is None: - raise ServeError( - "receipt_not_found", - "No serve receipt exists for port {0}.".format(port), - "serve stop only stops processes that serve started; stop foreign processes yourself.", - ) - pid = receipt["pid"] - if not pid_alive(pid): - receipt_path.unlink() - return {"status": "already_stopped", "port": port, "pid": pid} - command = pid_command(pid) - if not _argv_matches(receipt, command): - raise ServeError( - "pid_argv_mismatch", - "The live pid {0} does not match the serve receipt; refusing to signal it.".format(pid), - "Inspect the process yourself; the receipt is retained at {0}.".format(receipt_path), - ) - terminate(pid, signal.SIGTERM) - deadline = clock() + STOP_DEADLINE_SECONDS - while clock() < deadline: - if not pid_alive(pid): - receipt_path.unlink() - return {"status": "stopped", "port": port, "pid": pid} - sleep(0.2) - terminate(pid, signal.SIGKILL) - if pid_alive(pid): - raise ServeError( - "stop_failed", - "The process did not exit after SIGTERM and SIGKILL.", - "Inspect pid {0} yourself; the receipt is retained at {1}.".format(pid, receipt_path), - ) - receipt_path.unlink() - return {"status": "stopped", "port": port, "pid": pid, "forced": True} - - -def wired_port_claim(roots): - """Build a wired_claims(port, runtime) gate from Wire-managed configs.""" - wired = scan_wired_configs(roots) - claims = {} - for config in wired: - endpoint = config.get("endpoint") - if not endpoint: - continue - try: - parsed = validate_health_endpoint(endpoint) - from urllib.parse import urlsplit - - parts = urlsplit(parsed) - port = parts.port or (443 if parts.scheme == "https" else 80) - except ValueError: - continue - claims.setdefault(port, config["runtime"]) - - def check(port, runtime): - owner = claims.get(port) - return owner is not None and owner != runtime - - return check - - -def default_model_present(hf_cache=None): - from .model_doctor import default_hf_cache, inspect_hf_cache - - cache = Path(hf_cache) if hf_cache is not None else default_hf_cache() - known = {item["id"].casefold() for item in inspect_hf_cache(cache)} - - def check(repo): - return repo.casefold() in known - - return check - - -LAUNCHD_LABEL_PREFIX = "com.mlx-agent.serve." -_LAUNCHD_LABEL = re.compile(r"\A" + LAUNCHD_LABEL_PREFIX.replace(".", r"\.") + r"(\d{1,5})\Z") - - -def launchd_label(port): - return "{0}{1}".format(LAUNCHD_LABEL_PREFIX, port) - - -def render_launchd_plist(plan, log_path=None): - """Render a deterministic launchd plist for a reviewed serve plan.""" - from xml.sax.saxutils import escape - - label = launchd_label(plan["port"]) - log = log_path or str(receipts_root() / "{0}.log".format(plan["port"])) - arguments = "\n".join( - " {0}".format(escape(str(part))) for part in plan["argv"] - ) - return ( - '\n' - '\n' - '\n' - '\n' - ' Label\n' - ' {label}\n' - ' ProgramArguments\n' - ' \n' - '{arguments}\n' - ' \n' - ' RunAtLoad\n' - ' \n' - ' KeepAlive\n' - ' \n' - ' StandardOutPath\n' - ' {log}\n' - ' StandardErrorPath\n' - ' {log}\n' - '\n' - '\n' - ).format(label=escape(label), arguments=arguments, log=escape(str(log))) - - -class LaunchdPlistAdapter: - """Validate the exact launchd plist subset serve renders.""" - - version = "1.0" - runtime = "launchd" - - def __init__(self, path=None): - self.path = Path(path) if path is not None else None - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("plist content must be text") - require_secret_free_config(content) - lines = content.splitlines() - if len(lines) < 8 or not lines[0].startswith("': - raise ValueError("launchd plist must use the managed XML subset") - if lines[-1] != "" or lines[-2] != "": - raise ValueError("launchd plist must close its dict and plist elements") - text = content - for required in ("Label", "ProgramArguments", "RunAtLoad"): - if required not in text: - raise ValueError("launchd plist is missing {0}".format(required)) - label_match = re.search(r"Label\s*\n\s*([^<]+)", text) - if label_match is None or _LAUNCHD_LABEL.fullmatch(label_match.group(1)) is None: - raise ValueError("launchd plist label must use the managed serve prefix") - return True diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/transactions.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/transactions.py deleted file mode 100644 index ee06116..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/transactions.py +++ /dev/null @@ -1,1343 +0,0 @@ -"""Durable, reversible configuration transactions for Wire. - -The transaction root is opened with ``O_NOFOLLOW`` where macOS/Python exposes -it. Stage, backup, receipt, and replace operations then use that directory -file descriptor. A hostile actor can still race an already-open target leaf -between validation and replacement, but ``os.replace`` replaces that leaf and -never follows it; ancestors are verified before opening their directory fd. -""" - -from __future__ import annotations - -import difflib -import errno -import fcntl -import hashlib -import json -import os -import stat -import unicodedata -import uuid -from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from contextlib import contextmanager, nullcontext - -from .wiring import ( - ConfigAdapter, - redact_endpoint, - redact_secrets, - require_secret_free_config, - validate_health_endpoint, -) - - -RECEIPT_SCHEMA_VERSION = "2.1" -MAX_PREVIEW_CHARS = 12000 -_STATUSES = {"pending", "applied", "failed", "rolled_back", "rollback_failed", "recovery_required"} -_HASH = __import__("re").compile(r"^[0-9a-f]{64}$") -_LEGACY_LOCK_NAME = __import__("re").compile(r"^\.mlx-agent-wire-([0-9a-f]{64})\.lock$") -COOPERATIVE_CONCURRENCY_NOTE = "Advisory lock protects accidental/cooperative writers; a malicious process ignoring it can still race the final rename." -LEGACY_LOCK_MIGRATION = "legacy-target-locks-v1" -_LEGACY_LOCK_MARKER = "legacy-lock-migration-v1.json" - - -class ConcurrentTransactionError(ValueError): - """A cooperative writer holds the transaction's advisory lock.""" - - -class LegacyLockError(ConcurrentTransactionError): - """A legacy target-adjacent lock prevents a safe scoped-lock upgrade.""" - - -def _sha256(value): - return hashlib.sha256(value).hexdigest() - - -def _timestamp(): - return datetime.now(timezone.utc).isoformat() - - -def _absolute(path): - value = Path(path) - if _has_parent_reference(value): - raise ValueError("path traversal is not allowed: {0}".format(path)) - return Path(os.path.abspath(str(value))) - - -def _physical_absolute(path): - result = _absolute(path) - # /var is a fixed macOS compatibility alias. Normalize before descriptor - # traversal rather than following a user-controlled symlink component. - if str(result) == "/var" or str(result).startswith("/var/"): - if not os.path.islink("/var") or os.readlink("/var") != "private/var": - raise ValueError("untrusted /var compatibility alias") - result = Path("/private/var") / result.relative_to("/var") - return result - - -def _has_parent_reference(path): - return ".." in Path(path).parts - - -def _walk_directory(path, create=False, component_hook=None): - """Open each absolute component from `/` using dir-fd + O_NOFOLLOW only.""" - if not hasattr(os, "O_NOFOLLOW"): - raise ValueError("platform lacks required no-follow directory traversal") - logical = _absolute(path) - value = _physical_absolute(logical) - flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | os.O_NOFOLLOW - descriptor = os.open("/", flags) - try: - for component in value.parts[1:]: - if component_hook is not None: - component_hook(descriptor, component) - try: - child = os.open(component, flags, dir_fd=descriptor) - except FileNotFoundError: - if not create: - raise ValueError("path component does not exist: {0}".format(component)) - os.mkdir(component, 0o700, dir_fd=descriptor) - os.fsync(descriptor) - child = os.open(component, flags, dir_fd=descriptor) - except OSError as error: - raise ValueError("refusing unsafe directory component {0}: {1}".format(component, error)) - os.close(descriptor) - descriptor = child - return logical, descriptor - except BaseException: - os.close(descriptor) - raise - - -def _assert_safe_directory(path, create=False): - value, descriptor = _walk_directory(path, create=create) - os.close(descriptor) - return value - - -def _target_lock_name(target): - """Return the target-adjacent v1 lock name used by migration checks.""" - return ".mlx-agent-wire-{0}.lock".format(_target_lock_digest(target)) - - -def _target_lock_digest(target): - """Return the path-spelling v1 identity used only for legacy migration.""" - return _sha256(str(_physical_absolute(target)).encode("utf-8")) - - -def _filesystem_target_lock_digest(target, parent_fd): - """Bind a lock to the opened parent filesystem object and canonical leaf.""" - item = os.fstat(parent_fd) - # The stdlib does not expose a stable descriptor-bound filesystem - # case-sensitivity capability. Fold conservatively rather than derive an - # identity from mutable directory contents and risk split alias locks. - leaf = unicodedata.normalize("NFC", _physical_absolute(target).name).casefold() - identity = "{0}:{1}:{2}".format(item.st_dev, item.st_ino, leaf) - return _sha256(identity.encode("utf-8")) - - -def _filesystem_target_lock_name(target, parent_fd): - return ".mlx-agent-wire-{0}.lock".format( - _filesystem_target_lock_digest(target, parent_fd) - ) - - -def _filesystem_parent_digest(directory_fd): - item = os.fstat(directory_fd) - return _sha256("{0}:{1}".format(item.st_dev, item.st_ino).encode("utf-8")) - - -def _legacy_parent_scopes(targets, create=False): - """Open and deduplicate physical target parents without following aliases.""" - scopes = {} - try: - for target in sorted({_physical_absolute(item) for item in targets}, key=str): - try: - _parent, directory_fd = _walk_directory(target.parent, create=create) - except ValueError as error: - if str(error).startswith("path component does not exist:"): - continue - raise - try: - item = os.fstat(directory_fd) - identity = (item.st_dev, item.st_ino) - if identity in scopes: - os.close(directory_fd) - scopes[identity]["targets"].append(target) - continue - scopes[identity] = { - "digest": _filesystem_parent_digest(directory_fd), - "directory_fd": directory_fd, - "identity": identity, - "parent": target.parent, - "targets": [target], - } - except BaseException: - os.close(directory_fd) - raise - return [scopes[key] for key in sorted(scopes)] - except BaseException: - for scope in scopes.values(): - os.close(scope["directory_fd"]) - raise - - -def _close_legacy_parent_scopes(scopes): - for scope in reversed(scopes): - os.close(scope["directory_fd"]) - - -def _merge_legacy_parent_scopes(scopes, additions): - """Merge newly created/opened parents without retaining duplicate fds.""" - existing = {scope["identity"]: scope for scope in scopes} - added = [] - for addition in additions: - current = existing.get(addition["identity"]) - if current is not None: - current["targets"] = sorted(set(current["targets"] + addition["targets"]), key=str) - os.close(addition["directory_fd"]) - continue - existing[addition["identity"]] = addition - scopes.append(addition) - added.append(addition) - scopes.sort(key=lambda scope: scope["identity"]) - return added - - -def _legacy_candidate_names(directory_fd): - """List only exact v1 lock names; unrelated directory entries stay opaque.""" - try: - names = os.listdir(directory_fd) - except OSError as error: - raise ValueError("could not safely enumerate legacy target transaction locks: {0}".format(error)) - candidates = [] - for name in names: - match = _LEGACY_LOCK_NAME.fullmatch(name) - if match is not None: - candidates.append((name, match.group(1))) - return sorted(candidates) - - -def _validate_legacy_candidate_item(item): - if not stat.S_ISREG(item.st_mode): - raise ValueError("legacy target transaction lock is not a regular file") - if hasattr(os, "geteuid") and item.st_uid != os.geteuid(): - raise ValueError("legacy target transaction lock is not owned by the current user") - - -def _open_legacy_candidate(scope, name, digest, writable): - descriptor = None - flags = (os.O_RDWR if writable else os.O_RDONLY) | os.O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) - try: - try: - descriptor = os.open(name, flags, dir_fd=scope["directory_fd"]) - except FileNotFoundError: - return None - except OSError as error: - raise ValueError("could not safely open legacy target transaction lock: {0}".format(error)) - item = os.fstat(descriptor) - _validate_legacy_candidate_item(item) - try: - current = os.stat(name, dir_fd=scope["directory_fd"], follow_symlinks=False) - except FileNotFoundError: - os.close(descriptor) - return None - if (current.st_dev, current.st_ino) != (item.st_dev, item.st_ino): - raise ValueError("legacy target transaction lock changed while it was opened") - return { - "descriptor": descriptor, - "digest": digest, - "identity": (item.st_dev, item.st_ino), - "name": name, - "path": str(scope["parent"] / name), - "scope": scope, - } - except BaseException: - if descriptor is not None: - os.close(descriptor) - raise - - -def _release_legacy_candidates(candidates): - for candidate in reversed(candidates): - try: - fcntl.flock(candidate["descriptor"], fcntl.LOCK_UN) - finally: - os.close(candidate["descriptor"]) - - -def _acquire_legacy_candidates(scopes): - """Acquire a stable snapshot of every strict candidate before mutation.""" - held = [] - current = {} - try: - for _attempt in range(32): - for scope in scopes: - for name, digest in _legacy_candidate_names(scope["directory_fd"]): - key = (scope["digest"], name) - existing = current.get(key) - if existing is not None: - try: - item = os.stat(name, dir_fd=scope["directory_fd"], follow_symlinks=False) - except FileNotFoundError: - current.pop(key, None) - else: - if (item.st_dev, item.st_ino) == existing["identity"]: - continue - candidate = _open_legacy_candidate(scope, name, digest, writable=True) - if candidate is None: - current.pop(key, None) - continue - try: - fcntl.flock(candidate["descriptor"], fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as error: - os.close(candidate["descriptor"]) - if error.errno in (errno.EACCES, errno.EAGAIN): - raise LegacyLockError("legacy_lock_busy: stop older mlx-agent processes before upgrading scoped locks") - raise ValueError("could not safely acquire legacy target transaction lock: {0}".format(error)) - held.append(candidate) - current[key] = candidate - - stable = True - present = set() - for scope in scopes: - for name, _digest in _legacy_candidate_names(scope["directory_fd"]): - key = (scope["digest"], name) - present.add(key) - candidate = current.get(key) - try: - item = os.stat(name, dir_fd=scope["directory_fd"], follow_symlinks=False) - except FileNotFoundError: - stable = False - continue - if candidate is None or (item.st_dev, item.st_ino) != candidate["identity"]: - stable = False - for key in list(current): - if key not in present: - current.pop(key) - stable = False - if stable: - return held, [current[key] for key in sorted(current)] - raise ValueError("legacy target transaction lock set changed repeatedly during migration") - except BaseException: - _release_legacy_candidates(held) - raise - - -def _inspect_legacy_candidates(scopes): - """Open strict candidates no-follow for a read-only doctor snapshot.""" - candidates = [] - for scope in scopes: - for name, digest in _legacy_candidate_names(scope["directory_fd"]): - candidate = _open_legacy_candidate(scope, name, digest, writable=False) - if candidate is None: - continue - os.close(candidate.pop("descriptor")) - candidates.append(candidate) - return candidates - - -def _legacy_scope_was_migrated(scope, candidates, state): - if state["unscoped"]: - return True - if scope["digest"] in state["parents"]: - return True - migrated = state["targets"] - if any(entry.get("parent") == scope["digest"] for entry in migrated.values()): - return True - relevant = {_target_lock_digest(target) for target in scope["targets"]} - relevant.update(candidate["digest"] for candidate in candidates) - if relevant.intersection(migrated): - return True - # Old markers cannot map a one-way spelling digest back to a parent. A - # strict candidate in that ambiguous upgrade state must fail closed. - return any("parent" not in entry for entry in migrated.values()) - - -def _read_legacy_migration_state(lock_root): - """Read validated parent-scoped legacy-lock migration state.""" - try: - _root, directory_fd = _walk_directory(lock_root) - except ValueError as error: - if str(error).startswith("path component does not exist:"): - return {"version": LEGACY_LOCK_MIGRATION, "targets": {}, "parents": {}, "unscoped": False} - raise - descriptor = None - try: - try: - descriptor = os.open(_LEGACY_LOCK_MARKER, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory_fd) - except FileNotFoundError: - return {"version": LEGACY_LOCK_MIGRATION, "targets": {}, "parents": {}, "unscoped": False} - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - raise ValueError("legacy lock migration marker is not a regular file") - value = json.loads(os.read(descriptor, 1048576).decode("utf-8")) - # The prior release-candidate marker had no target map. Treat it as - # an unscoped barrier: it cannot prove which physical parents migrated. - if value == {"version": LEGACY_LOCK_MIGRATION}: - return {"version": LEGACY_LOCK_MIGRATION, "targets": {}, "parents": {}, "unscoped": True} - allowed_fields = ( - {"version", "targets"}, - {"version", "targets", "parents"}, - {"version", "targets", "parents", "unscoped"}, - ) - if not isinstance(value, dict) or set(value) not in allowed_fields or value["version"] != LEGACY_LOCK_MIGRATION or not isinstance(value["targets"], dict): - raise ValueError("legacy lock migration marker is malformed") - for digest, entry in value["targets"].items(): - if not isinstance(digest, str) or not _HASH.fullmatch(digest) or not isinstance(entry, dict) or set(entry) not in ({"migrated_at"}, {"migrated_at", "parent"}) or not isinstance(entry["migrated_at"], str) or ("parent" in entry and (not isinstance(entry["parent"], str) or not _HASH.fullmatch(entry["parent"]))): - raise ValueError("legacy lock migration marker is malformed") - try: - if datetime.fromisoformat(entry["migrated_at"].replace("Z", "+00:00")).tzinfo is None: - raise ValueError("missing timezone") - except ValueError: - raise ValueError("legacy lock migration marker is malformed") - parents = value.get("parents", {}) - unscoped = value.get("unscoped", False) - if not isinstance(parents, dict) or not isinstance(unscoped, bool): - raise ValueError("legacy lock migration marker is malformed") - for digest, entry in parents.items(): - if not isinstance(digest, str) or not _HASH.fullmatch(digest) or not isinstance(entry, dict) or set(entry) != {"migrated_at"} or not isinstance(entry["migrated_at"], str): - raise ValueError("legacy lock migration marker is malformed") - try: - if datetime.fromisoformat(entry["migrated_at"].replace("Z", "+00:00")).tzinfo is None: - raise ValueError("missing timezone") - except ValueError: - raise ValueError("legacy lock migration marker is malformed") - value["parents"] = parents - value["unscoped"] = unscoped - return value - except OSError as error: - raise ValueError("could not safely read legacy lock migration marker: {0}".format(error)) - finally: - if descriptor is not None: - os.close(descriptor) - os.close(directory_fd) - - -def _write_legacy_migration_state(lock_root, state): - _atomic_in_directory( - lock_root, - _LEGACY_LOCK_MARKER, - (json.dumps(state, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8"), - 0o600, - ) - - -@contextmanager -def _legacy_migration_window(lock_root): - """Serialize parent-map upgrades so multi-target writes cannot lose entries.""" - _root, directory_fd = _walk_directory(lock_root, create=True) - descriptor = None - try: - descriptor = os.open(".legacy-lock-migration-v1.state.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600, dir_fd=directory_fd) - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - raise ValueError("legacy lock migration state lock is not a regular file") - try: - fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as error: - if error.errno in (errno.EACCES, errno.EAGAIN): - raise LegacyLockError("legacy_lock_busy: another scoped legacy-lock migration is active") - raise ValueError("could not safely acquire legacy lock migration state lock: {0}".format(error)) - yield - finally: - if descriptor is not None: - try: - fcntl.flock(descriptor, fcntl.LOCK_UN) - finally: - os.close(descriptor) - os.close(directory_fd) - - -@contextmanager -def _migrate_legacy_target_locks(targets, lock_root, create_parents=False): - """Upgrade target-adjacent legacy locks before entering scoped lock storage.""" - canonical = sorted({_physical_absolute(item) for item in targets}, key=str) - with _legacy_migration_window(lock_root): - scopes = _legacy_parent_scopes(canonical) - held = [] - try: - held, candidates = _acquire_legacy_candidates(scopes) - state = _read_legacy_migration_state(lock_root) - migrated = state["targets"] - initial_by_parent = {scope["digest"]: [] for scope in scopes} - for candidate in candidates: - initial_by_parent[candidate["scope"]["digest"]].append(candidate) - if any(initial_by_parent[scope["digest"]] and _legacy_scope_was_migrated(scope, initial_by_parent[scope["digest"]], state) for scope in scopes): - raise LegacyLockError("legacy_lock_recreated: stop older mlx-agent processes and remove recreated legacy locks before continuing") - - opened_targets = {target for scope in scopes for target in scope["targets"]} - missing_targets = [target for target in canonical if target not in opened_targets] - if create_parents and missing_targets: - additions = _legacy_parent_scopes(missing_targets, create=True) - added_scopes = _merge_legacy_parent_scopes(scopes, additions) - added_held, added_candidates = _acquire_legacy_candidates(added_scopes) - held.extend(added_held) - candidates.extend(added_candidates) - added_by_parent = {scope["digest"]: [] for scope in added_scopes} - for candidate in added_candidates: - added_by_parent[candidate["scope"]["digest"]].append(candidate) - if any(added_by_parent[scope["digest"]] and _legacy_scope_was_migrated(scope, added_by_parent[scope["digest"]], state) for scope in added_scopes): - raise LegacyLockError("legacy_lock_recreated: stop older mlx-agent processes and remove recreated legacy locks before continuing") - - by_parent = {scope["digest"]: [] for scope in scopes} - for candidate in candidates: - by_parent[candidate["scope"]["digest"]].append(candidate) - - for candidate in candidates: - try: - item = os.stat(candidate["name"], dir_fd=candidate["scope"]["directory_fd"], follow_symlinks=False) - except FileNotFoundError: - raise ValueError("legacy target transaction lock changed before migration") - if (item.st_dev, item.st_ino) != candidate["identity"]: - raise ValueError("legacy target transaction lock changed before migration") - touched = set() - for candidate in candidates: - os.unlink(candidate["name"], dir_fd=candidate["scope"]["directory_fd"]) - touched.add(candidate["scope"]["digest"]) - for scope in scopes: - if scope["digest"] in touched: - os.fsync(scope["directory_fd"]) - - stamp = _timestamp() - for scope in scopes: - state["parents"].setdefault(scope["digest"], {"migrated_at": stamp}) - for target in scope["targets"]: - digest = _target_lock_digest(target) - entry = migrated.setdefault(digest, {"migrated_at": stamp, "parent": scope["digest"]}) - entry.setdefault("parent", scope["digest"]) - for candidate in by_parent[scope["digest"]]: - entry = migrated.setdefault(candidate["digest"], {"migrated_at": stamp, "parent": scope["digest"]}) - entry.setdefault("parent", scope["digest"]) - _write_legacy_migration_state(lock_root, state) - yield - finally: - _release_legacy_candidates(held) - _close_legacy_parent_scopes(scopes) - - -def legacy_lock_problem(targets, lock_root): - """Describe legacy-lock state for installer doctor without mutating it.""" - state = _read_legacy_migration_state(lock_root) - paths = {"legacy_lock_recreated": [], "legacy_lock_migration_required": []} - scopes = _legacy_parent_scopes(targets) - try: - candidates = _inspect_legacy_candidates(scopes) - by_parent = {scope["digest"]: [] for scope in scopes} - for candidate in candidates: - by_parent[candidate["scope"]["digest"]].append(candidate) - for scope in scopes: - scope_candidates = by_parent[scope["digest"]] - if not scope_candidates: - continue - code = "legacy_lock_recreated" if _legacy_scope_was_migrated(scope, scope_candidates, state) else "legacy_lock_migration_required" - paths[code].extend(candidate["path"] for candidate in scope_candidates) - finally: - _close_legacy_parent_scopes(scopes) - return [ - { - "code": code, - "paths": sorted(set(affected)), - "remediation": "stop older mlx-agent processes; after parent-scoped legacy-lock migration, do not run an older binary and remove only recreated legacy locks.", - } - for code, affected in paths.items() if affected - ] - - -@contextmanager -def _target_locks(targets, create_parents=False, lock_root=None): - """Acquire filesystem-identity locks in canonical order to avoid deadlocks.""" - canonical = sorted({_physical_absolute(target) for target in targets}, key=str) - identities = {} - held = [] - try: - for target in canonical: - _parent, target_parent_fd = _walk_directory( - target.parent, create=create_parents - ) - digest = _filesystem_target_lock_digest(target, target_parent_fd) - if digest in identities: - os.close(target_parent_fd) - continue - identities[digest] = { - "target": target, - "target_parent_fd": target_parent_fd, - } - for digest in sorted(identities): - identity = identities[digest] - target = identity["target"] - target_parent_fd = identity["target_parent_fd"] - if lock_root is None: - directory_fd = target_parent_fd - identity["target_parent_fd"] = None - else: - os.close(target_parent_fd) - identity["target_parent_fd"] = None - _parent, directory_fd = _walk_directory(lock_root, create=True) - lock_fd = None - try: - lock_fd = os.open( - ".mlx-agent-wire-{0}.lock".format(digest), - os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, - 0o600, - dir_fd=directory_fd, - ) - if not stat.S_ISREG(os.fstat(lock_fd).st_mode): - raise ValueError("target transaction lock is not a regular file") - os.fsync(directory_fd) - fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - held.append((lock_fd, directory_fd)) - except OSError as error: - if lock_fd is not None: - os.close(lock_fd) - os.close(directory_fd) - if error.errno in (errno.EACCES, errno.EAGAIN): - raise ConcurrentTransactionError("another cooperative Wire transaction is active for a target") - raise ValueError("could not safely acquire target transaction lock: {0}".format(error)) - except BaseException: - if lock_fd is not None: - os.close(lock_fd) - os.close(directory_fd) - raise - yield canonical - finally: - for lock_fd, directory_fd in reversed(held): - try: - fcntl.flock(lock_fd, fcntl.LOCK_UN) - finally: - os.close(lock_fd) - os.close(directory_fd) - for identity in identities.values(): - if identity["target_parent_fd"] is not None: - os.close(identity["target_parent_fd"]) - - -def _assert_safe_target(path): - value = _absolute(path) - _read_target(value) - return value - - -def _open_directory(path): - _value, descriptor = _walk_directory(path) - return descriptor - - -def _fsync_directory(path_or_fd): - descriptor = path_or_fd if isinstance(path_or_fd, int) else _open_directory(path_or_fd) - close = not isinstance(path_or_fd, int) - try: - os.fsync(descriptor) - finally: - if close: - os.close(descriptor) - - -def _write_all(descriptor, content): - offset = 0 - while offset < len(content): - offset += os.write(descriptor, content[offset:]) - - -def _atomic_in_directory(directory, name, content, mode, validator=None): - """Write and fsync a stage file before atomically replacing ``name``.""" - directory = _assert_safe_directory(directory) - dir_fd = _open_directory(directory) - stage_name = ".mlx-agent-stage-{0}".format(uuid.uuid4().hex) - stage_fd = None - try: - stage_fd = os.open(stage_name, os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0), mode, dir_fd=dir_fd) - _write_all(stage_fd, content) - os.fsync(stage_fd) - os.close(stage_fd) - stage_fd = None - _fsync_directory(dir_fd) - if validator is not None: - read_fd = os.open(stage_name, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), dir_fd=dir_fd) - try: - data = b"" - while True: - chunk = os.read(read_fd, 65536) - if not chunk: - break - data += chunk - finally: - os.close(read_fd) - validator(data.decode("utf-8")) - os.chmod(stage_name, mode, dir_fd=dir_fd, follow_symlinks=False) - os.replace(stage_name, name, src_dir_fd=dir_fd, dst_dir_fd=dir_fd) - _fsync_directory(dir_fd) - except BaseException: - if stage_fd is not None: - os.close(stage_fd) - try: - os.unlink(stage_name, dir_fd=dir_fd) - _fsync_directory(dir_fd) - except OSError: - pass - raise - finally: - os.close(dir_fd) - - -def _read_target(path, component_hook=None): - """Read a regular leaf through its opened parent directory descriptor.""" - value = _absolute(path) - _parent, directory = _walk_directory(value.parent, component_hook=component_hook) - try: - try: - descriptor = os.open(value.name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory) - except FileNotFoundError: - return b"", False, None - try: - item = os.fstat(descriptor) - if not stat.S_ISREG(item.st_mode): - raise ValueError("target is not a regular file: {0}".format(value)) - parts = [] - while True: - chunk = os.read(descriptor, 65536) - if not chunk: - break - parts.append(chunk) - return b"".join(parts), True, item.st_mode & 0o777 - finally: - os.close(descriptor) - except OSError as error: - raise ValueError("refusing unsafe target leaf {0}: {1}".format(value, error)) - finally: - os.close(directory) - - -def _read_regular(path): - return _read_target(path)[0] - - -@dataclass -class Receipt: - schema_version: str - transaction_id: str - adapter_version: str - timestamp: str - transaction_root: str - targets: list - target_roots: dict - target_modes: dict - after_modes: dict - before_hashes: dict - after_hashes: dict - backup_paths: dict - validations: dict - status: str - preview: str - preview_hash: str - lock_root: str = None - lock_migration: str = None - receipt_path: str = field(default="", repr=False, compare=False) - - def to_dict(self): - value = asdict(self) - value.pop("receipt_path", None) - if value.get("lock_root") is None: - value.pop("lock_root") - if value.get("lock_migration") is None: - value.pop("lock_migration") - return value - - @classmethod - def from_dict(cls, value, receipt_path=""): - required = { - "schema_version", "transaction_id", "adapter_version", "timestamp", "transaction_root", - "targets", "target_roots", "target_modes", "after_modes", "before_hashes", "after_hashes", "backup_paths", - "validations", "status", "preview", "preview_hash", - } - if not isinstance(value, dict) or not required.issubset(value) or set(value) - (required | {"lock_root", "lock_migration"}): - raise ValueError("receipt fields are malformed or untrusted") - if value["schema_version"] != RECEIPT_SCHEMA_VERSION or value["status"] not in _STATUSES: - raise ValueError("receipt version or status is unsupported") - try: - uuid.UUID(value["transaction_id"]) - parsed_timestamp = datetime.fromisoformat(value["timestamp"].replace("Z", "+00:00")) - except (TypeError, ValueError, AttributeError): - raise ValueError("receipt ID or timestamp is malformed") - if parsed_timestamp.tzinfo is None: - raise ValueError("receipt timestamp must include a timezone") - if not isinstance(value["adapter_version"], str) or not value["adapter_version"]: - raise ValueError("receipt adapter version is malformed") - root = _assert_safe_directory(value["transaction_root"]) - if Path(value["transaction_root"]) != root: - raise ValueError("receipt transaction root must be normalized") - if receipt_path: - location = _assert_safe_target(receipt_path) - if location.parent != root or location.name != "receipt.json": - raise ValueError("receipt is outside its transaction layout") - lock_root = value.get("lock_root") - if lock_root is not None: - if not isinstance(lock_root, str) or not lock_root: - raise ValueError("receipt lock root is malformed") - safe_lock_root = _assert_safe_directory(lock_root) - if str(safe_lock_root) != lock_root: - raise ValueError("receipt lock root must be normalized") - migration = value.get("lock_migration") - if migration is not None and (lock_root is None or migration != LEGACY_LOCK_MIGRATION): - raise ValueError("receipt lock migration is malformed") - targets = value["targets"] - maps = ( - value["target_roots"], value["target_modes"], value["after_modes"], - value["before_hashes"], value["after_hashes"], value["backup_paths"], - ) - if not isinstance(targets, list) or not targets or not all(isinstance(item, str) for item in targets) or len(targets) != len(set(targets)) or not all(isinstance(item, dict) for item in maps): - raise ValueError("receipt targets are malformed") - if any(set(targets) != set(item) for item in maps): - raise ValueError("receipt target maps do not match") - for index, target_name in enumerate(targets): - target = _assert_safe_target(target_name) - if str(target) != target_name or value["target_roots"][target_name] != str(target.parent): - raise ValueError("receipt target is not rooted safely") - mode = value["target_modes"][target_name] - if mode is not None and (not isinstance(mode, int) or mode < 0 or mode > 0o777): - raise ValueError("receipt mode is malformed") - after_mode = value["after_modes"][target_name] - if after_mode is not None and ( - not isinstance(after_mode, int) or after_mode < 0 or after_mode > 0o777 - ): - raise ValueError("receipt after mode is malformed") - for hashes in (value["before_hashes"], value["after_hashes"]): - if not isinstance(hashes[target_name], str) or not _HASH.fullmatch(hashes[target_name]): - raise ValueError("receipt hash is malformed") - backup = value["backup_paths"][target_name] - expected_backup = root / "backup-{0}.bin".format(index) - if (backup is None) != (mode is None): - raise ValueError("receipt backup presence does not match before existence") - if backup is None: - if value["before_hashes"][target_name] != _sha256(b""): - raise ValueError("missing backup has a non-empty hash") - elif not isinstance(backup, str) or Path(backup) != expected_backup: - raise ValueError("receipt backup is outside its transaction layout") - validations_redacted = False - if isinstance(value["validations"], dict): - try: - validations_redacted = json.loads(redact_secrets(json.dumps(value["validations"]))) == value["validations"] - except (TypeError, ValueError, json.JSONDecodeError): - validations_redacted = False - if not isinstance(value["validations"], dict) or not validations_redacted or not isinstance(value["preview"], str) or len(value["preview"]) > MAX_PREVIEW_CHARS + 32 or redact_secrets(value["preview"]) != value["preview"] or not isinstance(value["preview_hash"], str) or not _HASH.fullmatch(value["preview_hash"]): - raise ValueError("receipt validation or preview is malformed") - return cls(receipt_path=str(receipt_path), **value) - - -class Transaction: - """Create a crash-recoverable journal before any configuration mutation.""" - - def __init__(self, receipts_dir=None, health_checker=None, fault_injector=None, receipt_writer=None, path_race_hook=None, create_target_parents=False, transaction_id=None, lock_root=None): - self.receipts_dir = Path(receipts_dir) if receipts_dir else None - self.lock_root = Path(lock_root) if lock_root is not None else None - self.health_checker = health_checker - self.fault_injector = fault_injector - self.receipt_writer = receipt_writer - self.path_race_hook = path_race_hook - self.create_target_parents = bool(create_target_parents) - if transaction_id is not None: - try: - self.transaction_id = str(uuid.UUID(str(transaction_id))) - except (TypeError, ValueError, AttributeError): - raise ValueError("transaction_id must be a UUID") - else: - self.transaction_id = None - self._changes = [] - self._preview = "" - self._preview_hash = "" - self._expected_current = None - - def preview(self, changes, expected_current=None): - if not isinstance(changes, (list, tuple)) or not changes: - raise ValueError("changes must be a non-empty list") - prepared, diffs, binding = [], [], [] - seen = set() - for change in changes: - if not isinstance(change, dict) or "path" not in change or "content" not in change: - raise ValueError("each change requires path and content") - path = _absolute(change["path"]) - if str(path) in seen: - raise ValueError("each target may appear only once") - seen.add(str(path)) - content = change["content"] - if not isinstance(content, str): - raise TypeError("change content must be text") - adapter = change.get("adapter") or ConfigAdapter.detect(path, runtime=change.get("runtime")) - try: - before, existed, mode = _read_target(path, self.path_race_hook) - except ValueError as error: - if not self.create_target_parents or not str(error).startswith("path component does not exist:"): - raise - before, existed, mode = b"", False, None - before_text = before.decode("utf-8") - if not getattr(adapter, "secret_scan_exempt", False): - require_secret_free_config(before_text) - require_secret_free_config(content) - endpoint = change.get("endpoint") - if endpoint: - validate_health_endpoint(endpoint) - adapter.validate(content) - after = content.encode("utf-8") - diffs.append("".join(difflib.unified_diff( - redact_secrets(before_text).splitlines(True), redact_secrets(content).splitlines(True), - fromfile=str(path), tofile=str(path), lineterm="", - ))) - binding.append({"path": str(path), "before": _sha256(before), "after": _sha256(after), "exists": existed, "mode": mode, "endpoint": _sha256(str(endpoint or "").encode("utf-8"))}) - prepared.append({"path": path, "content": content, "adapter": adapter, "endpoint": endpoint, "before_hash": _sha256(before), "existed": existed, "mode": mode}) - self._changes = prepared - self._expected_current = self._normalize_expected_current(expected_current) - self._preview = self._bounded("\n".join(diffs)) - self._preview_hash = _sha256(json.dumps(binding, sort_keys=True, separators=(",", ":")).encode("utf-8")) - return {"changes": len(prepared), "diff": self._preview, "preview_hash": self._preview_hash, "requires_confirmation": True} - - def apply(self, confirmation, expected_current=None): - if confirmation is not True and confirmation != self._preview_hash: - raise PermissionError("explicit confirmation for this preview is required") - if not self._changes: - raise ValueError("preview changes before applying them") - if expected_current is not None: - expected_current = self._normalize_expected_current(expected_current) - if self._expected_current is not None and expected_current != self._expected_current: - raise ValueError("apply expected_current does not match preview") - self._expected_current = expected_current - with self._advisory_lock(): - return self._apply_locked() - - @contextmanager - def _advisory_lock(self): - lock_root = self._safe_lock_root() if self.lock_root is not None else None - if lock_root is None: - with _target_locks(self._lock_targets(), create_parents=self.create_target_parents): - yield - return - with _migrate_legacy_target_locks( - self._lock_targets(), lock_root, - create_parents=self.create_target_parents, - ): - with _target_locks(self._lock_targets(), create_parents=self.create_target_parents, lock_root=lock_root): - yield - - def _safe_lock_root(self): - root, descriptor = _walk_directory(self.lock_root, create=True) - os.close(descriptor) - return root - - def _lock_targets(self): - return [str(path) for path in sorted({change["path"] for change in self._changes}, key=str)] - - def _apply_locked(self): - if self._expected_current is not None: - for change in self._changes: - current, exists, mode = _read_target(change["path"], self.path_race_hook) - expected = self._expected_current[str(change["path"])] - if _sha256(current) != expected["hash"] or exists != expected["exists"] or mode != expected["mode"]: - raise ValueError("expected current target state changed before mutation") - for change in self._changes: - current, exists, mode = _read_target(change["path"], self.path_race_hook) - if _sha256(current) != change["before_hash"] or exists != change["existed"] or mode != change["mode"]: - raise ValueError("preview is stale; target changed after preview") - receipt = self._prepare_journal() - journal_written = False - mutation_started = False - try: - self._persist(receipt) - journal_written = True - self._fault("after_pending_receipt") - for index, change in enumerate(self._changes): - path = change["path"] - self._fault("before_replace:{0}".format(index)) - current, exists, mode = _read_target(path, self.path_race_hook) - if _sha256(current) != receipt.before_hashes[str(path)] or exists != change["existed"] or mode != receipt.target_modes[str(path)]: - raise ValueError("preview is stale; target changed before replacement") - mutation_started = True - self._replace_target(path, change["content"].encode("utf-8"), change["adapter"], receipt.target_modes[str(path)]) - receipt.after_hashes[str(path)] = _sha256(change["content"].encode("utf-8")) - receipt.after_modes[str(path)] = ( - receipt.target_modes[str(path)] - if receipt.target_modes[str(path)] is not None else 0o600 - ) - self._persist(receipt) - self._fault("after_replace:{0}".format(index)) - change["adapter"].validate(_read_regular(path).decode("utf-8")) - after, after_exists, after_mode = _read_target(path) - if not after_exists or (receipt.target_modes[str(path)] is not None and after_mode != receipt.target_modes[str(path)]): - raise ValueError("replacement mode does not match the reviewed target mode") - receipt.after_hashes[str(path)] = _sha256(after) - receipt.after_modes[str(path)] = after_mode - receipt.validations[str(path)] = {"pre": True, "post": True, "passed": True} - self._persist(receipt) - receipt.validations["health_check"] = self._run_health_checks() - if receipt.validations["health_check"]["passed"]: - receipt.status = "applied" - self._persist(receipt) - else: - self._finish_restore(receipt) - except Exception as error: - receipt.validations["error"] = {"passed": False, "message": redact_secrets(str(error))} - if not journal_written and not mutation_started: - receipt.status = "failed" - return receipt - self._finish_restore(receipt) - return receipt - - def _normalize_expected_current(self, expected_current): - if expected_current is None: - return None - if not isinstance(expected_current, dict): - raise ValueError("expected_current must map every target to its current state") - targets = {str(change["path"]) for change in self._changes} - if set(expected_current) != targets: - raise ValueError("expected_current targets do not match the preview") - normalized = {} - for target in sorted(targets): - state = expected_current[target] - if not isinstance(state, dict) or set(state) != {"hash", "exists", "mode"}: - raise ValueError("expected_current state must include hash, exists, and mode") - if not isinstance(state["hash"], str) or not _HASH.match(state["hash"]): - raise ValueError("expected_current hash is invalid") - if not isinstance(state["exists"], bool) or (state["mode"] is not None and not isinstance(state["mode"], int)): - raise ValueError("expected_current existence or mode is invalid") - normalized[target] = {"hash": state["hash"], "exists": state["exists"], "mode": state["mode"]} - return normalized - - def rollback(self, receipt_path): - return rollback(receipt_path) - - def _prepare_journal(self): - self._fault("before_journal_capture") - captured = [] - for change in self._changes: - before, exists, mode = _read_target(change["path"], self.path_race_hook) - if _sha256(before) != change["before_hash"] or exists != change["existed"] or mode != change["mode"]: - raise ValueError("preview is stale; target changed before journal capture") - captured.append((before, exists, mode)) - self._fault("after_journal_capture") - for index, change in enumerate(self._changes): - before, exists, mode = _read_target(change["path"], self.path_race_hook) - if _sha256(before) != change["before_hash"] or exists != change["existed"] or mode != change["mode"]: - raise ValueError("preview is stale; target changed after journal capture") - captured[index] = (before, exists, mode) - receipts_dir, receipts_fd = _walk_directory(self.receipts_dir or self._changes[0]["path"].parent / ".mlx-agent-receipts", create=True) - transaction_id = self.transaction_id or str(uuid.uuid4()) - root = receipts_dir / transaction_id - try: - self._fault("before_transaction_root_create") - os.mkdir(transaction_id, 0o700, dir_fd=receipts_fd) - os.fsync(receipts_fd) - root_fd = os.open(transaction_id, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | os.O_NOFOLLOW, dir_fd=receipts_fd) - try: - os.fsync(root_fd) - finally: - os.close(root_fd) - finally: - os.close(receipts_fd) - targets = [str(change["path"]) for change in self._changes] - receipt = Receipt( - schema_version=RECEIPT_SCHEMA_VERSION, transaction_id=transaction_id, adapter_version=ConfigAdapter.version, - timestamp=_timestamp(), transaction_root=str(root), targets=targets, - target_roots={str(change["path"]): str(change["path"].parent) for change in self._changes}, - target_modes={str(change["path"]): captured[index][2] for index, change in enumerate(self._changes)}, - after_modes={str(change["path"]): captured[index][2] for index, change in enumerate(self._changes)}, - before_hashes={}, after_hashes={}, backup_paths={}, validations={ - "concurrency": {"scope": "advisory_cooperative_target_scoped", "note": COOPERATIVE_CONCURRENCY_NOTE, "passed": True}, - }, status="pending", - preview=self._preview, preview_hash=self._preview_hash, - lock_root=str(self._safe_lock_root()) if self.lock_root is not None else None, - lock_migration=LEGACY_LOCK_MIGRATION if self.lock_root is not None else None, - receipt_path=str(root / "receipt.json"), - ) - for index, change in enumerate(self._changes): - path = change["path"] - before, exists, mode = captured[index] - target = str(path) - receipt.before_hashes[target] = _sha256(before) - receipt.after_hashes[target] = _sha256(before) - if exists: - backup = root / "backup-{0}.bin".format(index) - _atomic_in_directory(root, backup.name, before, mode or 0o600) - receipt.backup_paths[target] = str(backup) - else: - receipt.backup_paths[target] = None - return receipt - - def _replace_target(self, path, content, adapter, mode): - _assert_safe_target(path) - _atomic_in_directory(path.parent, path.name, content, mode if mode is not None else 0o600, adapter.validate) - - def _persist(self, receipt): - root = _assert_safe_directory(receipt.transaction_root) - if self.receipt_writer is not None: - self.receipt_writer(Path(receipt.receipt_path), receipt.to_dict()) - return - _atomic_in_directory(root, "receipt.json", (json.dumps(receipt.to_dict(), indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600) - - def _finish_restore(self, receipt): - if _restore_receipt(receipt, expected_current=_receipt_after_state(receipt)): - receipt.status = "rolled_back" - else: - receipt.status = "recovery_required" - try: - self._persist(receipt) - except Exception as error: - receipt.status = "recovery_required" - receipt.validations["receipt_write"] = {"passed": False, "message": redact_secrets(str(error))} - - def _run_health_checks(self): - checks = [] - for change in self._changes: - endpoint = change.get("endpoint") - if not endpoint: - continue - passed = bool(self.health_checker(endpoint)) if self.health_checker else ConfigAdapter.health_check(endpoint) - checks.append({"endpoint": redact_endpoint(endpoint), "passed": passed}) - return {"passed": all(item["passed"] for item in checks), "endpoints": checks} - - def _fault(self, point): - if self.fault_injector is not None: - self.fault_injector(point) - - @staticmethod - def _bounded(value): - return value if len(value) <= MAX_PREVIEW_CHARS else value[:MAX_PREVIEW_CHARS] + "\n... [preview truncated]" - - -def _restore_receipt(receipt, expected_current=None): - """Preflight every backup, then restore and prove byte-for-byte hashes.""" - planned = [] - try: - root = _assert_safe_directory(receipt.transaction_root) - if expected_current is not None: - _assert_expected_states( - receipt.targets, expected_current, "rollback target changed before recovery" - ) - for index, target_name in enumerate(receipt.targets): - target = _assert_safe_target(target_name) - expected = receipt.before_hashes[target_name] - backup_name = receipt.backup_paths[target_name] - if backup_name is None: - if expected != _sha256(b""): - raise ValueError("missing backup has a non-empty hash") - planned.append((target, None, receipt.target_modes[target_name], expected)) - continue - backup = _assert_safe_target(backup_name) - backup_data, backup_exists, _backup_mode = _read_target(backup) - if backup.parent != root or backup.name != "backup-{0}.bin".format(index) or not backup_exists: - raise ValueError("backup is missing or outside transaction root") - if _sha256(backup_data) != expected: - raise ValueError("backup hash does not match receipt") - planned.append((target, backup_data, receipt.target_modes[target_name], expected)) - for target, data, mode, expected in planned: - if data is None: - _current, exists, _current_mode = _read_target(target) - if exists: - _remove_target(target) - else: - _atomic_in_directory(target.parent, target.name, data, mode if mode is not None else 0o600) - actual_content, actual_exists, actual_mode = _read_target(target) - actual = _sha256(actual_content) - if actual != expected or actual_exists != (mode is not None) or actual_mode != mode: - raise ValueError("restore hash does not match receipt") - receipt.after_hashes[str(target)] = actual - receipt.after_modes[str(target)] = actual_mode - receipt.validations["rollback"] = {"passed": True, "targets": [str(item[0]) for item in planned]} - return True - except Exception as error: - receipt.validations["rollback"] = {"passed": False, "message": redact_secrets(str(error))} - return False - - -def _remove_target(path): - _assert_safe_target(path) - _content, exists, _mode = _read_target(path) - if not exists: - return - directory = _open_directory(path.parent) - try: - item = os.stat(path.name, dir_fd=directory, follow_symlinks=False) - if not stat.S_ISREG(item.st_mode): - raise ValueError("refusing non-regular target removal") - os.unlink(path.name, dir_fd=directory) - _fsync_directory(directory) - finally: - os.close(directory) - - -def _current_state(target_name): - content, exists, mode = _read_target(target_name) - return {"hash": _sha256(content), "exists": exists, "mode": mode} - - -def _receipt_after_state(receipt): - return { - target: { - "hash": receipt.after_hashes[target], - "exists": receipt.after_modes[target] is not None, - "mode": receipt.after_modes[target], - } - for target in receipt.targets - } - - -def _rollback_restore_plan(receipt): - return [ - { - "path": target, - "root": receipt.target_roots[target], - "hash": receipt.before_hashes[target], - "exists": receipt.target_modes[target] is not None, - "mode": receipt.target_modes[target], - "backup_path": receipt.backup_paths[target], - } - for target in receipt.targets - ] - - -def _rollback_receipt_binding(receipt): - """Return every validated receipt field that can change rollback behavior.""" - return { - "transaction_id": receipt.transaction_id, - "transaction_root": receipt.transaction_root, - "targets": list(receipt.targets), - "restore": _rollback_restore_plan(receipt), - "lock_root": receipt.lock_root, - "lock_migration": receipt.lock_migration, - } - - -def _rollback_receipt_hash(receipt): - return _sha256( - json.dumps( - _rollback_receipt_binding(receipt), - sort_keys=True, - separators=(",", ":"), - ).encode("utf-8") - ) - - -def _assert_expected_states(targets, expected, message): - if not isinstance(expected, dict) or set(expected) != set(targets): - raise ValueError("reviewed rollback state does not match receipt targets") - for target in targets: - if _current_state(target) != expected[target]: - raise ValueError(message) - - -def _rollback_preview_locked(receipt, location): - if receipt.status == "rolled_back": - state = { - target: { - "hash": receipt.before_hashes[target], - "exists": receipt.target_modes[target] is not None, - "mode": receipt.target_modes[target], - } - for target in receipt.targets - } - _assert_expected_states( - receipt.targets, state, "rollback target differs from recorded before-state" - ) - else: - if receipt.status not in {"applied", "rollback_failed", "recovery_required"}: - raise ValueError("normal rollback requires an applied receipt") - state = _receipt_after_state(receipt) - _assert_expected_states( - receipt.targets, state, "rollback target differs from recorded after-state" - ) - restore = _rollback_restore_plan(receipt) - receipt_binding = _rollback_receipt_binding(receipt) - binding = { - "operation": "wire-rollback", - "receipt_path": str(location), - "receipt_status": receipt.status, - "current": state, - "operational_receipt": receipt_binding, - } - return { - "targets": [{"path": target, **state[target]} for target in receipt.targets], - "restore": restore, - "operational_receipt_hash": _rollback_receipt_hash(receipt), - "preview_hash": _sha256( - json.dumps(binding, sort_keys=True, separators=(",", ":")).encode("utf-8") - ), - "requires_confirmation": receipt.status != "rolled_back", - "already_rolled_back": receipt.status == "rolled_back", - } - - -def preview_rollback(receipt_path): - """Return a hash-bound, non-secret snapshot of the current rollback targets.""" - location = _assert_safe_target(receipt_path) - initial = Receipt.from_dict( - json.loads(_read_regular(location).decode("utf-8")), str(location) - ) - lock_context = ( - _migrate_legacy_target_locks(initial.targets, initial.lock_root) - if initial.lock_root is not None else nullcontext() - ) - with lock_context: - with _target_locks(initial.targets, lock_root=initial.lock_root): - receipt = Receipt.from_dict( - json.loads(_read_regular(location).decode("utf-8")), str(location) - ) - return _rollback_preview_locked(receipt, location) - - -def rollback( - receipt_path, - preview_hash=None, - expected_after_hashes=None, - expected_receipt_hash=None, -): - """Restore a receipt through a user preview hash or installer-internal CAS.""" - location = _assert_safe_target(receipt_path) - initial = Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) - lock_context = ( - _migrate_legacy_target_locks(initial.targets, initial.lock_root) - if initial.lock_root is not None else nullcontext() - ) - with lock_context: - with _target_locks(initial.targets, lock_root=initial.lock_root): - receipt = Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) - if ( - expected_receipt_hash is not None - and _rollback_receipt_hash(receipt) != expected_receipt_hash - ): - raise ValueError("rollback receipt changed after the reviewed plan") - if expected_after_hashes is not None: - if not isinstance(expected_after_hashes, dict) or set(expected_after_hashes) != set(receipt.targets): - raise ValueError("reviewed rollback hashes do not match receipt targets") - for target_name in receipt.targets: - current, exists, _mode = _read_target(target_name) - if not exists or _sha256(current) != expected_after_hashes[target_name]: - raise ValueError("receipt target changed after reviewed rollback preview") - else: - if preview_hash is None: - raise PermissionError( - "rollback requires the hash from a reviewed rollback preview" - ) - preview = _rollback_preview_locked(receipt, location) - if preview["already_rolled_back"]: - return receipt - if preview_hash != preview["preview_hash"]: - raise PermissionError( - "rollback preview hash does not match current target state" - ) - if receipt.status == "rolled_back": - current_matches = True - for target_name in receipt.targets: - current, exists, mode = _read_target(target_name) - expected_absent = receipt.backup_paths[target_name] is None - if _sha256(current) != receipt.before_hashes[target_name] or exists == expected_absent or mode != receipt.target_modes[target_name]: - current_matches = False - break - if current_matches: - return receipt - expected_current = _receipt_after_state(receipt) - if _restore_receipt(receipt, expected_current=expected_current): - receipt.status = "rolled_back" - else: - receipt.status = "rollback_failed" - try: - _atomic_in_directory(Path(receipt.transaction_root), "receipt.json", (json.dumps(receipt.to_dict(), indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600) - except Exception as error: - receipt.status = "recovery_required" - receipt.validations["receipt_write"] = {"passed": False, "message": redact_secrets(str(error))} - return receipt diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/verification.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/verification.py deleted file mode 100644 index 65b5e75..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/verification.py +++ /dev/null @@ -1,1163 +0,0 @@ -"""Safe, bounded verification of discovered local-model candidates.""" - -from __future__ import annotations - -import ast -import http.client -import ipaddress -import json -import math -import threading -import time -from copy import deepcopy -from dataclasses import asdict, dataclass, field -from enum import Enum -from types import MappingProxyType -from typing import Any, Dict, Optional -from urllib.parse import urlsplit - -from .probe_assets import VISION_PROBE_TEXT, vision_probe_image_base64 -from .wiring import redact_secrets - - -TOOL_USE_PROBE_ID = "tool-use-v1" -TOOL_USE_TOOL_NAME = "lookup_widget" -TOOL_USE_ARGUMENTS = MappingProxyType({"widget_id": "widget-42"}) -TOOL_USE_PROMPT = ( - "Call lookup_widget exactly once for widget-42. " - "Do not answer directly and do not call any other tool." -) -TOOL_USE_TOOLS = [ - { - "type": "function", - "function": { - "name": TOOL_USE_TOOL_NAME, - "description": "Look up one synthetic widget by its identifier.", - "parameters": { - "type": "object", - "properties": { - "widget_id": { - "type": "string", - "description": "Synthetic widget identifier.", - } - }, - "required": ["widget_id"], - "additionalProperties": False, - }, - }, - } -] -_TOOL_USE_ARGUMENTS_MAX_CHARS = 512 -INVENTORY_RESPONSE_MAX_BYTES = 1024 * 1024 -PROBE_RESPONSE_MAX_BYTES = 256 * 1024 -_HTTP_READ_CHUNK_BYTES = 16 * 1024 - -CODING_PROBE_ID = "coding-v1" -REASONING_PROBE_ID = "reasoning-v1" -VISION_PROBE_ID = "vision-v1" -EMBEDDING_PROBE_ID = "embedding-v1" -ROLE_PROBE_IDS = MappingProxyType({ - "coding": CODING_PROBE_ID, - "reasoning": REASONING_PROBE_ID, - "vision": VISION_PROBE_ID, - "embedding": EMBEDDING_PROBE_ID, -}) - -CODING_PROBE_PROMPT = ( - "Write a Python function named add that takes two arguments a and b " - "and returns their sum. Reply with only the function definition." -) -CODING_PROBE_MAX_TOKENS = 256 -_CODING_PROBE_MAX_CODE_CHARS = 2000 -_CODING_PROBE_BLOCKED_TOKENS = ( - "import", "__", "while", "for", "open", "exec", "eval", "input", - "globals", "locals", "compile", "breakpoint", "exit", "quit", - "getattr", "setattr", "delattr", "vars", "help", "print", -) -REASONING_PROBE_PROMPT = ( - "A farmer has 17 sheep. All but 9 run away. " - "How many sheep does the farmer have left? Reply with only the number." -) -REASONING_PROBE_ANSWER = 9 -REASONING_PROBE_MAX_TOKENS = 64 -VISION_PROBE_PROMPT = ( - "What text is shown in this image? Reply with only the text in the image." -) -VISION_PROBE_MAX_TOKENS = 64 -EMBEDDING_PROBE_SENTENCES = ( - "The cat sat on the warm windowsill.", - "A kitten rested on the sunny ledge.", - "Quantum chromodynamics describes the strong nuclear force.", -) -EMBEDDING_PROBE_MAX_VECTORS = 8 -_EMBEDDING_PROBE_MAX_DIMENSION = 8192 - - -def _role_probe_outcome(reason): - return {"valid": reason == "valid", "reason": reason} - - -def validate_coding_response(response): - """Validate the fixed coding probe without retaining raw model output.""" - try: - content, _hidden = _generation_text(response) - code = _extract_probe_code(content) - if code is None: - return _role_probe_outcome("missing_content") - if len(code) > _CODING_PROBE_MAX_CODE_CHARS: - return _role_probe_outcome("unsafe_content") - lowered = code.casefold() - if any(token in lowered for token in _CODING_PROBE_BLOCKED_TOKENS): - return _role_probe_outcome("unsafe_content") - try: - tree = ast.parse(code) - except (SyntaxError, ValueError, RecursionError, MemoryError): - return _role_probe_outcome("parse_failure") - function_defs = [ - node for node in tree.body - if isinstance(node, ast.FunctionDef) and node.name == "add" - ] - if not function_defs: - return _role_probe_outcome("missing_function") - namespace = {"__builtins__": {}} - try: - exec(compile(tree, "", "exec"), namespace) - add = namespace.get("add") - if not callable(add): - return _role_probe_outcome("missing_function") - if add(2, 3) != 5 or add(-1, 1) != 0: - return _role_probe_outcome("wrong_answer") - except Exception: - return _role_probe_outcome("wrong_answer") - return _role_probe_outcome("valid") - except Exception: - return _role_probe_outcome("invalid_response") - - -def _extract_probe_code(content): - if not isinstance(content, str) or not content.strip(): - return None - text = content.strip() - if "```" in text: - segments = text.split("```") - for segment in segments[1::2]: - body = segment.split("\n", 1) - candidate = body[1] if len(body) == 2 else "" - if candidate.strip(): - return candidate.strip() - return None - return text - - -def validate_reasoning_response(response): - """Validate the fixed arithmetic reasoning probe by exact answer match.""" - try: - content, _hidden = _generation_text(response) - if not isinstance(content, str) or not content.strip(): - return _role_probe_outcome("missing_content") - digits = [] - current = [] - for character in content: - if character.isdigit() or (character == "-" and not current): - current.append(character) - elif current: - digits.append("".join(current)) - current = [] - if current: - digits.append("".join(current)) - if not digits: - return _role_probe_outcome("missing_content") - try: - answer = int(digits[-1]) - except ValueError: - return _role_probe_outcome("missing_content") - if answer != REASONING_PROBE_ANSWER: - return _role_probe_outcome("wrong_answer") - return _role_probe_outcome("valid") - except Exception: - return _role_probe_outcome("invalid_response") - - -def validate_vision_response(response): - """Validate the fixed OCR probe by case-folded substring match.""" - try: - content, _hidden = _generation_text(response) - if not isinstance(content, str) or not content.strip(): - return _role_probe_outcome("missing_content") - normalized = "".join(content.split()).casefold() - if VISION_PROBE_TEXT.casefold() not in normalized: - return _role_probe_outcome("wrong_answer") - return _role_probe_outcome("valid") - except Exception: - return _role_probe_outcome("invalid_response") - - -def validate_embedding_response(response): - """Validate the fixed embedding probe by cosine ordering.""" - try: - vectors = _embedding_vectors(response) - if vectors is None or len(vectors) != len(EMBEDDING_PROBE_SENTENCES): - return _role_probe_outcome("invalid_response") - similar = _cosine_similarity(vectors[0], vectors[1]) - dissimilar = _cosine_similarity(vectors[0], vectors[2]) - if similar is None or dissimilar is None: - return _role_probe_outcome("invalid_response") - if not similar > dissimilar: - return _role_probe_outcome("order_inverted") - return _role_probe_outcome("valid") - except Exception: - return _role_probe_outcome("invalid_response") - - -def _embedding_vectors(response): - if not isinstance(response, dict): - return None - records = response.get("embeddings") - if records is None: - data = response.get("data") - if isinstance(data, list): - records = [ - item.get("embedding") if isinstance(item, dict) else None - for item in sorted( - data, - key=lambda item: item.get("index", 0) if isinstance(item, dict) else 0, - ) - ] - if not isinstance(records, list) or len(records) > EMBEDDING_PROBE_MAX_VECTORS: - return None - vectors = [] - for record in records: - if ( - not isinstance(record, list) - or not record - or len(record) > _EMBEDDING_PROBE_MAX_DIMENSION - or any(not isinstance(value, (int, float)) or isinstance(value, bool) for value in record) - ): - return None - vectors.append([float(value) for value in record]) - return vectors - - -def _cosine_similarity(left, right): - if len(left) != len(right): - return None - dot = sum(a * b for a, b in zip(left, right)) - norm_left = math.sqrt(sum(a * a for a in left)) - norm_right = math.sqrt(sum(b * b for b in right)) - if norm_left == 0 or norm_right == 0: - return None - return dot / (norm_left * norm_right) - - -def normalize_tool_call(response): - """Normalize one synthetic tool call without retaining raw model output.""" - try: - message = _tool_call_message(response) - if message is None: - return _tool_call_result("invalid_response") - - if "tool_calls" not in message or message["tool_calls"] is None: - return _tool_call_result("missing_tool_call") - tool_calls = message["tool_calls"] - if not isinstance(tool_calls, list): - return _tool_call_result("invalid_response") - if not tool_calls: - return _tool_call_result("missing_tool_call") - if len(tool_calls) != 1: - return _tool_call_result("multiple_tool_calls") - - tool_call = tool_calls[0] - if not isinstance(tool_call, dict): - return _tool_call_result("invalid_response") - function = tool_call.get("function") - if not isinstance(function, dict): - return _tool_call_result("invalid_response") - raw_name = function.get("name") - if not isinstance(raw_name, str): - return _tool_call_result("invalid_response") - if raw_name != TOOL_USE_TOOL_NAME: - return _tool_call_result("wrong_tool") - - if "arguments" not in function: - return _tool_call_result("malformed_arguments") - arguments = function["arguments"] - if isinstance(arguments, str): - if len(arguments) > _TOOL_USE_ARGUMENTS_MAX_CHARS: - return _tool_call_result("malformed_arguments") - try: - arguments = json.loads(arguments) - except (json.JSONDecodeError, RecursionError): - return _tool_call_result("malformed_arguments") - if not isinstance(arguments, dict): - return _tool_call_result("malformed_arguments") - if arguments != TOOL_USE_ARGUMENTS: - return _tool_call_result("schema_invalid") - return _tool_call_result("valid") - except Exception: - return _tool_call_result("invalid_response") - - -def _tool_call_message(response): - if not isinstance(response, dict): - return None - if "message" in response: - message = response["message"] - return message if isinstance(message, dict) else None - if "choices" not in response: - return None - choices = response["choices"] - if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict): - return None - message = choices[0].get("message") - return message if isinstance(message, dict) else None - - -def _tool_call_result(reason): - valid = reason == "valid" - return { - "valid": valid, - "tool_name": TOOL_USE_TOOL_NAME if valid else None, - "arguments": dict(TOOL_USE_ARGUMENTS) if valid else None, - "reason": reason, - } - - -class EvidenceStrength(str, Enum): - """Ordered labels describing how directly a candidate was verified.""" - - runtime_measured = "runtime_measured" - runtime_tested = "runtime_tested" - runtime_inventory = "runtime_inventory" - metadata_only = "metadata_only" - heuristic_only = "heuristic_only" - - # Conventional aliases preserve a readable internal style while the exact - # lowercase contract names remain available to API consumers. - RUNTIME_MEASURED = runtime_measured - RUNTIME_TESTED = runtime_tested - RUNTIME_INVENTORY = runtime_inventory - METADATA_ONLY = metadata_only - HEURISTIC_ONLY = heuristic_only - - -class VerificationStatus(str, Enum): - """Outcome labels independent from the source strength of the evidence.""" - - verified = "verified" - metadata_only = "metadata-only" - failed = "failed" - unsupported_runtime = "unsupported-runtime" - - VERIFIED = verified - METADATA_ONLY = metadata_only - FAILED = failed - UNSUPPORTED_RUNTIME = unsupported_runtime - - -@dataclass(frozen=True) -class VerificationEvidence: - """Portable evidence from one candidate verification attempt.""" - - repo: str - role: str - strength: EvidenceStrength - status: VerificationStatus - available_locally: bool - loads: Optional[bool] - reasoning_confirmed: Optional[bool] - runtime: Optional[str] - note: str - details: Dict[str, Any] = field(default_factory=dict) - - def to_dict(self) -> Dict[str, Any]: - value = asdict(self) - value["strength"] = self.strength.value - value["status"] = self.status.value - return value - - -def _http_json_get(url, timeout=3.0): - """Read a local runtime endpoint without installing or starting anything.""" - return _http_json_request( - url, - "GET", - timeout=timeout, - max_response_bytes=INVENTORY_RESPONSE_MAX_BYTES, - ) - - -def _http_json_post(url, payload, timeout=10.0): - """Send the single bounded generation probe used by verification.""" - return _http_json_request( - url, - "POST", - payload=payload, - timeout=timeout, - max_response_bytes=PROBE_RESPONSE_MAX_BYTES, - ) - - -def _http_event_stream(url, payload, timeout=120.0, max_response_bytes=INVENTORY_RESPONSE_MAX_BYTES, - connection_factory=None, clock=time.monotonic): - """Yield bounded JSON events from a validated loopback streaming endpoint. - - Handles both server-sent events (``data: {...}`` lines terminated by - ``data: [DONE]``) and newline-delimited JSON (Ollama-style). - """ - parsed = _validated_loopback_url(url) - deadline = clock() + timeout - port = parsed.port or (443 if parsed.scheme == "https" else 80) - if connection_factory is None: - connection_type = ( - http.client.HTTPSConnection - if parsed.scheme == "https" - else http.client.HTTPConnection - ) - connection = connection_type(parsed.hostname, port, timeout=_deadline_remaining(deadline, clock)) - else: - connection = connection_factory(parsed.scheme, parsed.hostname, port, _deadline_remaining(deadline, clock)) - try: - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - connection.request( - "POST", - parsed.path or "/", - body=json.dumps(payload).encode("utf-8"), - headers={"Content-Type": "application/json"}, - ) - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - response = connection.getresponse() - _deadline_remaining(deadline, clock) - if 300 <= response.status < 400: - raise http.client.HTTPException( - "redirect responses are not allowed for local runtime requests" - ) - if not 200 <= response.status < 300: - raise http.client.HTTPException( - "local runtime returned HTTP status {0}".format(response.status) - ) - buffer = b"" - total = 0 - done = False - while not done: - remaining = _deadline_remaining(deadline, clock) - _set_connection_timeout(connection, remaining) - chunk = response.read(_HTTP_READ_CHUNK_BYTES) - _deadline_remaining(deadline, clock) - if not chunk: - break - total += len(chunk) - if total > max_response_bytes: - raise ValueError("local runtime response exceeds size limit") - buffer += chunk - while b"\n" in buffer: - line, buffer = buffer.split(b"\n", 1) - event, done = _parse_stream_line(line) - if event is not None: - yield event - if done: - break - if not done and buffer.strip(): - event, _done = _parse_stream_line(buffer) - if event is not None: - yield event - finally: - connection.close() - - -def _parse_stream_line(line): - text = line.strip() - if not text: - return None, False - if text.startswith(b"data:"): - text = text[len(b"data:"):].strip() - if text == b"[DONE]": - return None, True - if not text: - return None, False - if not text.startswith(b"{"): - return None, False - try: - return json.loads(text.decode("utf-8")), False - except (UnicodeDecodeError, json.JSONDecodeError): - return None, False - - -def _http_json_request( - url, - method, - payload=None, - timeout=10.0, - max_response_bytes=PROBE_RESPONSE_MAX_BYTES, - connection_factory=None, - clock=time.monotonic, -): - """Exchange bounded JSON with one validated loopback endpoint.""" - parsed = _validated_loopback_url(url) - deadline = clock() + timeout - remaining = _deadline_remaining(deadline, clock) - port = parsed.port or (443 if parsed.scheme == "https" else 80) - if connection_factory is None: - connection_type = ( - http.client.HTTPSConnection - if parsed.scheme == "https" - else http.client.HTTPConnection - ) - connection = connection_type(parsed.hostname, port, timeout=remaining) - else: - connection = connection_factory( - parsed.scheme, - parsed.hostname, - port, - remaining, - ) - body = None - headers = {} - if payload is not None: - body = json.dumps(payload).encode("utf-8") - headers["Content-Type"] = "application/json" - try: - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - connection.request(method, parsed.path or "/", body=body, headers=headers) - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - response = connection.getresponse() - _deadline_remaining(deadline, clock) - if 300 <= response.status < 400: - raise http.client.HTTPException( - "redirect responses are not allowed for local runtime requests" - ) - if not 200 <= response.status < 300: - raise http.client.HTTPException( - "local runtime returned HTTP status {0}".format(response.status) - ) - response_body = _read_bounded_response( - response, - connection, - max_response_bytes, - deadline, - clock, - ) - return json.loads(response_body.decode("utf-8")) - finally: - connection.close() - - -def _read_bounded_response(response, connection, limit, deadline, clock): - if not isinstance(limit, int) or limit <= 0: - raise ValueError("response size limit must be a positive integer") - content_length = response.getheader("Content-Length") - if content_length is not None: - try: - declared_length = int(content_length) - except (TypeError, ValueError) as error: - raise ValueError("invalid Content-Length from local runtime") from error - if declared_length < 0: - raise ValueError("invalid Content-Length from local runtime") - if declared_length > limit: - raise ValueError("local runtime response exceeds size limit") - - chunks = [] - total = 0 - while True: - remaining = _deadline_remaining(deadline, clock) - _set_connection_timeout(connection, remaining) - chunk = response.read(min(_HTTP_READ_CHUNK_BYTES, limit - total + 1)) - _deadline_remaining(deadline, clock) - if not chunk: - break - total += len(chunk) - if total > limit: - raise ValueError("local runtime response exceeds size limit") - chunks.append(chunk) - return b"".join(chunks) - - -def _deadline_remaining(deadline, clock): - remaining = deadline - clock() - if remaining <= 0: - raise TimeoutError("local runtime request exceeded its overall deadline") - return remaining - - -def _set_connection_timeout(connection, timeout): - sock = getattr(connection, "sock", None) - if sock is not None: - sock.settimeout(timeout) - - -def _local_runtime_origin(base_url): - """Return a normalized credential-free HTTP(S) loopback origin.""" - parsed = _validated_loopback_url(base_url) - if parsed.path not in ("", "/"): - raise ValueError("runtime base URL must be an origin without a path") - return base_url.rstrip("/") - - -def _validated_loopback_url(url): - if not isinstance(url, str) or url != url.strip(): - raise ValueError("runtime URL must be a trimmed string") - parsed = urlsplit(url) - if parsed.scheme not in ("http", "https"): - raise ValueError("runtime URL must use HTTP(S)") - if parsed.username is not None or parsed.password is not None: - raise ValueError("runtime URL must not contain credentials") - if parsed.query or parsed.fragment: - raise ValueError("runtime URL must not contain a query or fragment") - hostname = parsed.hostname - if hostname is None: - raise ValueError("runtime URL must include a host") - try: - parsed.port - except ValueError as error: - raise ValueError("runtime URL has an invalid port") from error - if hostname.casefold() != "localhost": - try: - address = ipaddress.ip_address(hostname) - except ValueError as error: - raise ValueError("runtime URL must use a loopback host") from error - if not address.is_loopback: - raise ValueError("runtime URL must use a loopback host") - return parsed - - -def _tool_use_payload_tools(): - """Copy the fixed schema into ordinary JSON-serializable containers.""" - return deepcopy(TOOL_USE_TOOLS) - - -class OllamaRuntimeClient: - """Read-only inventory and bounded generation adapter for an existing Ollama.""" - - name = "ollama" - - def __init__(self, http_get=None, http_post=None, base_url="http://127.0.0.1:11434"): - self._http_get = http_get or _http_json_get - self._http_post = http_post or _http_json_post - self._base_url = _local_runtime_origin(base_url) - - def list_models(self): - return self._http_get("{0}/api/tags".format(self._base_url), timeout=3.0) - - def generate(self, model, prompt, max_tokens): - return self._http_post( - "{0}/api/chat".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "stream": False, - "options": {"num_predict": max_tokens}, - }, - timeout=10.0, - ) - - def probe_tool_use(self, model): - return self._http_post( - "{0}/api/chat".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": TOOL_USE_PROMPT}], - "tools": _tool_use_payload_tools(), - "stream": False, - "options": {"num_predict": 64}, - }, - timeout=10.0, - ) - - def embed(self, model, inputs): - return self._http_post( - "{0}/api/embed".format(self._base_url), - {"model": model, "input": list(inputs)}, - timeout=10.0, - ) - - def stream_generate(self, model, prompt, max_tokens, timeout=120.0): - return _http_event_stream( - "{0}/api/chat".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "stream": True, - "options": {"num_predict": max_tokens}, - }, - timeout=timeout, - ) - - -class OpenAICompatibleRuntimeClient: - """Bounded adapter for an existing local OpenAI-compatible runtime.""" - - def __init__(self, name, base_url, http_get=None, http_post=None): - self.name = name - self._http_get = http_get or _http_json_get - self._http_post = http_post or _http_json_post - self._base_url = _local_runtime_origin(base_url) - - def list_models(self): - return self._http_get("{0}/v1/models".format(self._base_url), timeout=3.0) - - def generate(self, model, prompt, max_tokens): - return self._http_post( - "{0}/v1/chat/completions".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - }, - timeout=10.0, - ) - - def probe_tool_use(self, model): - return self._http_post( - "{0}/v1/chat/completions".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": TOOL_USE_PROMPT}], - "tools": _tool_use_payload_tools(), - "tool_choice": "auto", - "max_tokens": 64, - }, - timeout=10.0, - ) - - def embed(self, model, inputs): - return self._http_post( - "{0}/v1/embeddings".format(self._base_url), - {"model": model, "input": list(inputs)}, - timeout=10.0, - ) - - def stream_generate(self, model, prompt, max_tokens, timeout=120.0): - return _http_event_stream( - "{0}/v1/chat/completions".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "stream": True, - "stream_options": {"include_usage": True}, - }, - timeout=timeout, - ) - - -class LMStudioRuntimeClient(OpenAICompatibleRuntimeClient): - """Backward-compatible adapter for an existing LM Studio runtime.""" - - name = "lmstudio" - - def __init__(self, http_get=None, http_post=None, base_url="http://127.0.0.1:1234"): - super().__init__( - "lmstudio", - base_url, - http_get=http_get, - http_post=http_post, - ) - - -class MLXVLMRuntimeClient(OpenAICompatibleRuntimeClient): - """Bounded adapter for an existing local mlx-vlm vision runtime.""" - - name = "mlx-vlm" - - def __init__(self, http_get=None, http_post=None, base_url="http://127.0.0.1:8083"): - super().__init__( - "mlx-vlm", - base_url, - http_get=http_get, - http_post=http_post, - ) - - def generate_vision(self, model, prompt, image_base64, max_tokens): - return self._http_post( - "{0}/v1/chat/completions".format(self._base_url), - { - "model": model, - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,{0}".format(image_base64) - }, - }, - ], - } - ], - "max_tokens": max_tokens, - }, - timeout=30.0, - ) - - -class Verifier: - """Verify installed candidates without ever installing or downloading them. - - Runtime clients implement ``list_models()`` and ``generate(model, prompt, - max_tokens)``; tool-capable clients also implement ``probe_tool_use(model)``. - Metadata clients implement ``inspect_model(repo)``. Keeping these protocols - implicit allows provider adapters and tests to supply clients without adding - runtime dependencies. - """ - - def __init__(self, runtime_clients=None, metadata_client=None, runtime_client=None): - if runtime_clients is None: - runtime_clients = ( - [] - if runtime_client is not None - else [ - OllamaRuntimeClient(), - LMStudioRuntimeClient(), - OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080"), - MLXVLMRuntimeClient(), - OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000"), - ] - ) - if isinstance(runtime_clients, dict): - runtime_clients = list(runtime_clients.values()) - self._runtime_clients = list(runtime_clients) - if runtime_client is not None: - self._runtime_clients.append(runtime_client) - self._metadata_client = metadata_client - self._inventory_cache = {} - self._inventory_cache_lock = threading.Lock() - - def clear_inventory_cache(self): - """Start a new verification phase with fresh runtime inventories.""" - with self._inventory_cache_lock: - self._inventory_cache.clear() - - def runtime_clients(self): - """Return the configured runtime clients without mutating them.""" - return tuple(self._runtime_clients) - - def _runtime_inventory(self, runtime): - cache_key = id(runtime) - with self._inventory_cache_lock: - if cache_key not in self._inventory_cache: - try: - result = (installed_model_ids(runtime.list_models()), None) - except Exception as error: - result = (None, _safe_error(error)) - self._inventory_cache[cache_key] = result - return self._inventory_cache[cache_key] - - def verify(self, candidate, host, allow_network=True) -> VerificationEvidence: - """Return the strongest available evidence without mutating runtimes.""" - del host # Reserved for runtime adapters whose probes depend on host facts. - repo = _candidate_repo(candidate) - role = str(candidate.get("role") or candidate.get("heuristics", {}).get("role") or "general") - inventory_errors = [] - - for runtime in self._runtime_clients: - installed, inventory_error = self._runtime_inventory(runtime) - if inventory_error is not None: - inventory_errors.append( - "{0}: {1}".format(_runtime_name(runtime), inventory_error) - ) - continue - if not _is_installed(repo, installed): - continue - runtime_name = _runtime_name(runtime) - if role == "tool-use": - return _verify_tool_use(repo, role, runtime_name, runtime) - try: - response = runtime.generate( - repo, - "Reply with the single word ready.", - max_tokens=24, - ) - content, hidden_reasoning = _generation_text(response) - reasoning_confirmed, reasoning_evidence = _reasoning_confirmation( - candidate, hidden_reasoning - ) - details = { - "visible_content": bool(content.strip()), - "hidden_reasoning": bool(hidden_reasoning), - "reasoning_evidence": reasoning_evidence, - } - if role in ROLE_PROBE_IDS: - details["probes"] = [_run_role_probe(role, repo, runtime)] - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_TESTED, - status=VerificationStatus.VERIFIED, - available_locally=True, - loads=True, - reasoning_confirmed=reasoning_confirmed, - runtime=runtime_name, - note="Runtime generation completed without downloading the model.", - details=details, - ) - except Exception as error: - safe_error = _safe_error(error) - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_INVENTORY, - status=VerificationStatus.FAILED, - available_locally=True, - loads=False, - reasoning_confirmed=None, - runtime=runtime_name, - note=_bounded( - "Installed runtime inventory found the model, but " - "generation failed: {0}".format(safe_error) - ), - details={"error": safe_error}, - ) - - if allow_network and self._metadata_client is not None: - try: - metadata = self._metadata_client.inspect_model(repo) - reasoning_confirmed, reasoning_evidence = _reasoning_confirmation( - candidate, metadata - ) - details = _bounded_metadata(metadata) - details["reasoning_evidence"] = reasoning_evidence - if role == "tool-use": - details["probe_id"] = TOOL_USE_PROBE_ID - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.METADATA_ONLY, - status=VerificationStatus.METADATA_ONLY, - available_locally=False, - loads=None, - reasoning_confirmed=reasoning_confirmed, - runtime=None, - note="Model is not installed; repository metadata was inspected without downloading it.", - details=details, - ) - except Exception as error: - inventory_errors.append("metadata: {0}".format(_safe_error(error))) - - note = "Model is not installed; only candidate heuristics were available." - if inventory_errors: - note += " Probe errors: {0}".format("; ".join(inventory_errors)) - details = { - "reasoning_heuristic": candidate.get("reasoning"), - "reasoning_source": candidate.get("reason_src"), - } - if role == "tool-use": - details["probe_id"] = TOOL_USE_PROBE_ID - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.HEURISTIC_ONLY, - status=VerificationStatus.METADATA_ONLY, - available_locally=False, - loads=None, - reasoning_confirmed=None, - runtime=None, - note=_bounded(note), - details=details, - ) - - -def _run_role_probe(role, repo, runtime): - """Run the role's deterministic probe; never raises into verification.""" - probe_id = ROLE_PROBE_IDS[role] - try: - if role == "coding": - response = runtime.generate(repo, CODING_PROBE_PROMPT, max_tokens=CODING_PROBE_MAX_TOKENS) - outcome = validate_coding_response(response) - elif role == "reasoning": - response = runtime.generate(repo, REASONING_PROBE_PROMPT, max_tokens=REASONING_PROBE_MAX_TOKENS) - outcome = validate_reasoning_response(response) - elif role == "embedding": - embed = getattr(runtime, "embed", None) - if not callable(embed): - outcome = _role_probe_outcome("unsupported_runtime") - else: - outcome = validate_embedding_response( - embed(repo, list(EMBEDDING_PROBE_SENTENCES)) - ) - elif role == "vision": - generate_vision = getattr(runtime, "generate_vision", None) - if not callable(generate_vision): - outcome = _role_probe_outcome("unsupported_runtime") - else: - response = generate_vision( - repo, - VISION_PROBE_PROMPT, - vision_probe_image_base64(), - VISION_PROBE_MAX_TOKENS, - ) - outcome = validate_vision_response(response) - else: - outcome = _role_probe_outcome("unsupported_runtime") - except Exception as error: - outcome = {"valid": False, "reason": "probe_error", "error": _bounded(_safe_error(error), 160)} - return {"probe_id": probe_id, "outcome": outcome} - - -def _verify_tool_use(repo, role, runtime_name, runtime): - if runtime_name == "mlx-vlm" or not callable(getattr(runtime, "probe_tool_use", None)): - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_INVENTORY, - status=VerificationStatus.UNSUPPORTED_RUNTIME, - available_locally=True, - loads=None, - reasoning_confirmed=None, - runtime=runtime_name, - note=_bounded( - "Installed runtime inventory found the model, but this runtime " - "does not support the bounded tool-use probe." - ), - details={ - "probe_id": TOOL_USE_PROBE_ID, - "outcome": {"reason": "unsupported_runtime"}, - }, - ) - try: - outcome = normalize_tool_call(runtime.probe_tool_use(repo)) - except Exception as error: - error_name = _redacted_error(error) - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_INVENTORY, - status=VerificationStatus.FAILED, - available_locally=True, - loads=False, - reasoning_confirmed=None, - runtime=runtime_name, - note=_bounded( - "Installed runtime inventory found the model, but the bounded " - "tool-use probe failed: {0}".format(error_name) - ), - details={ - "probe_id": TOOL_USE_PROBE_ID, - "error": _bounded(error_name, 160), - }, - ) - if outcome["valid"]: - note = "Runtime returned a schema-valid synthetic tool call." - else: - note = "Runtime responded, but the probe did not return a valid call." - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_TESTED, - status=( - VerificationStatus.VERIFIED - if outcome["valid"] - else VerificationStatus.FAILED - ), - available_locally=True, - loads=True, - reasoning_confirmed=None, - runtime=runtime_name, - note=note, - details={"probe_id": TOOL_USE_PROBE_ID, "outcome": outcome}, - ) - - -def _redacted_error(error): - name = error.__class__.__name__ - return _bounded(name if name else "runtime error", 160) - - -def _safe_error(error): - name = error.__class__.__name__ or "Error" - message = redact_secrets(str(error)) - value = "{0}: {1}".format(name, message) if message else name - return _bounded(value, 160) - - -def _candidate_repo(candidate): - if not isinstance(candidate, dict): - raise TypeError("candidate must be an object") - repo = candidate.get("repo") or candidate.get("repository") - if not isinstance(repo, str) or not repo: - raise ValueError("candidate.repo must be a non-empty string") - return repo - - -def _runtime_name(runtime): - return str(getattr(runtime, "name", runtime.__class__.__name__)) - - -def installed_model_ids(records) -> set: - """Return model IDs from a supported runtime's read-only inventory response.""" - if isinstance(records, dict): - records = records.get("models", records.get("data", [])) - models = set() - for record in records or []: - if isinstance(record, str): - models.add(record) - continue - if isinstance(record, dict): - value = record.get("repo") or record.get("id") or record.get("name") or record.get("model") - if isinstance(value, str) and value: - models.add(value) - return models - - -def _is_installed(repo, installed): - normalized = repo.casefold() - return any(item.casefold() == normalized for item in installed) - - -def _generation_text(response): - if isinstance(response, str): - return response, "" - if not isinstance(response, dict): - return "", "" - message = response.get("message") - if message is None: - choices = response.get("choices") or [] - if choices and isinstance(choices[0], dict): - message = choices[0].get("message") or choices[0] - if not isinstance(message, dict): - message = response - content = message.get("content") or message.get("response") or "" - hidden = ( - message.get("reasoning_content") - or message.get("thinking") - or message.get("reasoning") - or "" - ) - return str(content), str(hidden) - - -def _reasoning_confirmation(candidate, signal): - """Prefer any positive runtime or metadata signal over an inconclusive probe.""" - if isinstance(signal, str) and signal.strip(): - return True, "runtime_hidden" - if isinstance(signal, dict): - if signal.get("reasoning") is True: - source = signal.get("reason_src") or "field" - return True, "metadata_{0}".format(source) - tags = signal.get("tags") or [] - if any(str(tag).lower() in ("reasoning", "thinking", "chain-of-thought") for tag in tags): - return True, "metadata_tags" - if signal.get("chat_template"): - return True, "metadata_chat_template" - if candidate.get("reasoning") is True and candidate.get("reason_src") in ( - "chat_template", "tags", - ): - return True, "discovery_{0}".format(candidate["reason_src"]) - return False, "none" - - -def _bounded(value, limit=300): - text = " ".join(str(value).split()) - if len(text) <= limit: - return text - return text[: max(0, limit - 3)] + "..." - - -def _bounded_metadata(metadata): - if not isinstance(metadata, dict): - return {} - allowed = ("metadata_available", "gated", "license", "tags", "reasoning", "reason_src", "weight_bytes") - value = {key: metadata[key] for key in allowed if key in metadata} - if isinstance(value.get("tags"), list): - value["tags"] = value["tags"][:20] - return value diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/watch.py b/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/watch.py deleted file mode 100644 index 63519d1..0000000 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/watch.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Stateful Hugging Face diff digest focused on models you own.""" - -from __future__ import annotations - -import json -from datetime import datetime, timezone -from pathlib import Path - -from .model_doctor import ( - collect_runtime_inventory, - default_hf_cache, - inspect_hf_cache, - scan_wired_configs, -) -from .models import base_name -from .transactions import _atomic_in_directory - - -WATCH_SCHEMA_VERSION = "1.0" -STATE_FILENAME = "watch-state.json" -MAX_OWNED = 500 -MAX_CANDIDATES = 2000 -MAX_FINDINGS = 100 - - -class WatchError(RuntimeError): - """Classified watch failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def state_file(state_dir): - return Path(state_dir) / "watch" / STATE_FILENAME - - -def collect_owned(runtime_clients, hf_cache=None, wired_roots=None): - """Collect owned model ids from local inventories and wired configs.""" - owned = [] - seen = set() - - def add(model_id, source): - if not isinstance(model_id, str) or not model_id: - return - key = model_id.casefold() - if key in seen or len(owned) >= MAX_OWNED: - return - seen.add(key) - owned.append({"id": model_id, "source": source}) - - cache = Path(hf_cache) if hf_cache is not None else default_hf_cache() - for item in inspect_hf_cache(cache): - add(item["id"], "hf-cache") - runtime_inventory, _errors = collect_runtime_inventory(runtime_clients) - for item in runtime_inventory: - add(item["id"], item["source"]) - for config in scan_wired_configs(wired_roots or [Path.cwd()]): - add(config.get("model"), "wired") - return owned - - -def snapshot_candidates(discovery_data): - """Flatten a discovery result into a bounded repo -> signal mapping.""" - candidates = {} - roles = discovery_data.get("roles") if isinstance(discovery_data, dict) else None - if not isinstance(roles, dict): - return candidates - for records in roles.values(): - if not isinstance(records, list): - continue - for record in records: - if not isinstance(record, dict): - continue - repo = record.get("repo") - if not isinstance(repo, str) or not repo or repo in candidates: - continue - facts = record.get("facts") if isinstance(record.get("facts"), dict) else {} - candidates[repo] = { - "base": record.get("base") or base_name(repo), - "weight_bytes": facts.get("weight_bytes"), - "gated": facts.get("gated"), - "license": record.get("license"), - } - if len(candidates) >= MAX_CANDIDATES: - return candidates - return candidates - - -def build_snapshot(owned, candidates, now=None): - return { - "schema_version": WATCH_SCHEMA_VERSION, - "created_at": now or _utc_now(), - "owned": list(owned)[:MAX_OWNED], - "candidates": dict(candidates), - "previous": None, - } - - -def write_snapshot(state_dir, snapshot): - """Persist the snapshot, rotating any existing one into `previous`.""" - destination = state_file(state_dir) - destination.parent.mkdir(parents=True, exist_ok=True) - existing = None - if destination.is_file(): - try: - existing = json.loads(destination.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - existing = None - if isinstance(existing, dict) and existing.get("schema_version") == WATCH_SCHEMA_VERSION: - snapshot["previous"] = { - key: existing.get(key) - for key in ("schema_version", "created_at", "owned", "candidates") - } - snapshot["previous"]["previous"] = None - content = (json.dumps(snapshot, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") - _atomic_in_directory(destination.parent, destination.name, content, 0o600) - return destination - - -def read_baseline(state_dir): - destination = state_file(state_dir) - if not destination.is_file(): - raise WatchError( - "missing_baseline", - "No watch snapshot exists yet.", - "Run mlx-agent watch snapshot first to record a baseline.", - ) - try: - value = json.loads(destination.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError) as error: - raise WatchError( - "invalid_state", - "The watch state file could not be read: {0}".format(error), - "Remove the corrupted file and run watch snapshot again.", - ) - if not isinstance(value, dict) or value.get("schema_version") != WATCH_SCHEMA_VERSION: - raise WatchError( - "invalid_state", - "The watch state file has an unsupported schema.", - "Run watch snapshot to record a fresh baseline.", - ) - return value - - -def diff_snapshots(baseline, current): - """Classify changes relevant to owned models between two snapshots.""" - findings = [] - owned_ids = {item["id"].casefold() for item in current.get("owned", [])} - owned_bases = {base_name(item["id"]) for item in current.get("owned", [])} - baseline_candidates = baseline.get("candidates") or {} - baseline_owned = {item["id"].casefold() for item in baseline.get("owned", [])} - - for repo, info in sorted((current.get("candidates") or {}).items()): - if len(findings) >= MAX_FINDINGS: - break - base = base_name(info.get("base") or repo) - if repo.casefold() not in owned_ids and base_name(repo) not in owned_bases and base not in owned_bases: - continue - previous = baseline_candidates.get(repo) - if previous is None: - findings.append({ - "code": "new_quant_of_owned", - "repo": repo, - "base": base, - "detail": "new tracked repository matching an owned model", - }) - continue - old_bytes = previous.get("weight_bytes") - new_bytes = info.get("weight_bytes") - if old_bytes is not None and new_bytes is not None and old_bytes != new_bytes: - findings.append({ - "code": "updated_tracked_repo", - "repo": repo, - "base": base, - "detail": "weight bytes changed from {0} to {1}".format(old_bytes, new_bytes), - }) - if previous.get("gated") != info.get("gated"): - findings.append({ - "code": "gated_changed", - "repo": repo, - "base": base, - "detail": "gated changed from {0} to {1}".format( - previous.get("gated"), info.get("gated") - ), - }) - - for item in baseline.get("owned", []): - if len(findings) >= MAX_FINDINGS: - break - if item["id"].casefold() not in owned_ids: - findings.append({ - "code": "owned_missing", - "repo": item["id"], - "base": base_name(item["id"]), - "detail": "previously owned via {0}; no longer present".format( - item.get("source", "unknown") - ), - }) - return findings diff --git a/packages/codex/mlx-agent/skills/mlx-bench/SKILL.md b/packages/codex/mlx-agent/skills/mlx-bench/SKILL.md index bb98367..97620e4 100644 --- a/packages/codex/mlx-agent/skills/mlx-bench/SKILL.md +++ b/packages/codex/mlx-agent/skills/mlx-bench/SKILL.md @@ -7,12 +7,12 @@ Use `$mlx-agent:mlx-bench` to invoke this installed Codex skill explicitly. Code # MLX Bench -Resolve `` as the absolute directory containing this SKILL.md. Never resolve the bundled executable from the shell working directory. +Resolve `` as the absolute directory two levels above the directory containing this SKILL.md; it holds `scripts/` and `src/`. Never resolve the bundled executable from the shell working directory. canonical capability ID: mlx-agent.bench Measure a model already served by a running local runtime: -`python3 /scripts/mlx-agent bench run --repo --runtime --json` +`python3 /scripts/mlx-agent bench run --repo --runtime --json` Present the returned measurements as returned. Bench must not start servers, download model weights, or change configuration; the model must already be served by an existing local runtime. diff --git a/packages/codex/mlx-agent/skills/mlx-bench/scripts/mlx-agent b/packages/codex/mlx-agent/skills/mlx-bench/scripts/mlx-agent deleted file mode 100755 index 54a5a80..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/scripts/mlx-agent +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -"""Run the bundled MLX agent core relative to this bootstrap, never the CWD.""" - -import sys -from pathlib import Path - - -sys.dont_write_bytecode = True -SCRIPT_DIR = Path(__file__).resolve().parent -sys.path.insert(0, str(SCRIPT_DIR.parent / "src")) - -from mlx_agent.cli import main # noqa: E402 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/__main__.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/__main__.py deleted file mode 100644 index 94dda3d..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/__main__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Run the MLX agent core with ``python -m mlx_agent``.""" - -import sys - -from .cli import main - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/adoption.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/adoption.py deleted file mode 100644 index e7c8c48..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/adoption.py +++ /dev/null @@ -1,870 +0,0 @@ -"""Provider-neutral, resumable model adoption state machine.""" - -from __future__ import annotations - -import hashlib -import json -import uuid -from concurrent.futures import ThreadPoolExecutor -from copy import deepcopy -from dataclasses import dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Dict, List, Tuple - -from .discovery import DiscoveryRequest, DiscoveryService -from .transactions import ( - ConcurrentTransactionError, - _atomic_in_directory, - _read_target, - _target_locks, -) -from .verification import ( - EvidenceStrength, - ROLE_PROBE_IDS, - TOOL_USE_PROBE_ID, - VerificationEvidence, - VerificationStatus, - Verifier, -) - - -ADOPTION_SCHEMA_VERSION = "1.3" -LEGACY_ADOPTION_SCHEMA_VERSIONS = ("1.1", "1.2") -PHASES = ("inspect", "discover", "shortlist", "verify", "measure", "compare", "recommend", "complete") -MEASURE_LIMIT = 6 -UTILITY_ROLES = {"general", "embedding"} -ALLOWED_ROLES = { - "general", "coding", "reasoning", "vision", "embedding", "tool-use" -} -EVIDENCE_SCORES = { - EvidenceStrength.RUNTIME_MEASURED.value: 500, - EvidenceStrength.RUNTIME_TESTED.value: 400, - EvidenceStrength.RUNTIME_INVENTORY.value: 300, - EvidenceStrength.METADATA_ONLY.value: 200, - EvidenceStrength.HEURISTIC_ONLY.value: 100, -} -ROLE_PROBE_BONUS = 50 - - -class AdoptionStateConflictError(ValueError): - """A persisted handoff changed or is locked by another workflow.""" - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -@dataclass(frozen=True) -class AdoptionRequest: - """Bounded request fields that are safe to persist in a handoff.""" - - roles: Tuple[str, ...] = ("general",) - state_path: object = None - shortlist_limit: int = 4 - allow_network: bool = True - offline: bool = False - refresh: bool = False - memory_gb: object = None - quantization: object = None - licenses: object = None - include_gated: bool = True - publishers: object = None - runtime: object = None - fast: bool = False - measure: bool = False - - def __post_init__(self): - if isinstance(self.roles, str): - supplied_roles = (self.roles,) - elif isinstance(self.roles, (list, tuple)): - supplied_roles = tuple(self.roles) - else: - raise TypeError("roles must be a string or an array of strings") - if any(not isinstance(role, str) or not role for role in supplied_roles): - raise TypeError("roles must contain non-empty strings") - roles = tuple(supplied_roles) - if not roles or any(role not in ALLOWED_ROLES for role in roles) or len(roles) > 6: - raise ValueError("roles must contain one to six supported roles") - if len(set(roles)) != len(roles): - raise ValueError("roles must not contain duplicates") - if not isinstance(self.shortlist_limit, int) or isinstance(self.shortlist_limit, bool): - raise TypeError("shortlist_limit must be an integer") - if self.shortlist_limit < 1 or self.shortlist_limit > 20: - raise ValueError("shortlist_limit must be between 1 and 20") - if not isinstance(self.allow_network, bool): - raise TypeError("allow_network must be a boolean") - for name in ("offline", "refresh", "include_gated", "fast", "measure"): - if not isinstance(getattr(self, name), bool): - raise TypeError("{0} must be a boolean".format(name)) - if self.memory_gb is not None and ( - not isinstance(self.memory_gb, (int, float)) or isinstance(self.memory_gb, bool) - ): - raise TypeError("memory_gb must be a number or null") - for name in ("quantization", "runtime"): - value = getattr(self, name) - if value is not None and not isinstance(value, str): - raise TypeError("{0} must be a string or null".format(name)) - object.__setattr__(self, "roles", roles) - - def to_dict(self): - return { - "roles": list(self.roles), - "shortlist_limit": self.shortlist_limit, - "allow_network": self.allow_network, - "offline": bool(self.offline), - "refresh": bool(self.refresh), - "memory_gb": self.memory_gb, - "quantization": self.quantization, - "licenses": _string_list(self.licenses), - "include_gated": bool(self.include_gated), - "publishers": _string_list(self.publishers), - "runtime": self.runtime, - "fast": bool(self.fast), - "measure": bool(self.measure), - } - - @classmethod - def from_dict(cls, value, state_path=None): - if not isinstance(value, dict): - raise TypeError("adoption request must be an object") - allowed = { - "roles", "state_path", "state", "shortlist_limit", "limit", - "allow_network", "offline", "refresh", "memory_gb", "quantization", - "licenses", "include_gated", "publishers", "runtime", "fast", - "measure", - } - unknown = sorted(set(value) - allowed) - if unknown: - raise ValueError("adoption request has unexpected keys: {0}".format(unknown)) - return cls( - roles=value.get("roles", ("general",)), - state_path=state_path or value.get("state_path") or value.get("state"), - shortlist_limit=value.get("shortlist_limit", value.get("limit", 4)), - allow_network=value.get("allow_network", True), - offline=value.get("offline", False), - refresh=value.get("refresh", False), - memory_gb=value.get("memory_gb"), - quantization=value.get("quantization"), - licenses=value.get("licenses"), - include_gated=value.get("include_gated", True), - publishers=value.get("publishers"), - runtime=value.get("runtime"), - fast=value.get("fast", False), - measure=value.get("measure", False), - ) - - -@dataclass -class AdoptionState: - """Schema-versioned, serializable adoption handoff.""" - - workflow_id: str - phase: str - status: str - request: Dict[str, Any] - state_path: Path = field(repr=False) - revision: int = 0 - completed_phases: List[str] = field(default_factory=list) - host: Dict[str, Any] = field(default_factory=dict) - discovery: Dict[str, Any] = field(default_factory=dict) - shortlist: List[Dict[str, Any]] = field(default_factory=list) - evidence: List[Dict[str, Any]] = field(default_factory=list) - comparisons: List[Dict[str, Any]] = field(default_factory=list) - recommendations: List[Dict[str, Any]] = field(default_factory=list) - warnings: List[Dict[str, str]] = field(default_factory=list) - errors: List[Dict[str, str]] = field(default_factory=list) - created_at: str = field(default_factory=_utc_now) - updated_at: str = field(default_factory=_utc_now) - schema_version: str = ADOPTION_SCHEMA_VERSION - _source_hash: object = field(default=None, repr=False, compare=False) - - def to_dict(self): - return { - "schema_version": self.schema_version, - "workflow_id": self.workflow_id, - "revision": self.revision, - "phase": self.phase, - "status": self.status, - "request": self.request, - "completed_phases": list(self.completed_phases), - "host": self.host, - "discovery": self.discovery, - "shortlist": self.shortlist, - "evidence": self.evidence, - "comparisons": self.comparisons, - "recommendations": self.recommendations, - "warnings": self.warnings, - "errors": self.errors, - "created_at": self.created_at, - "updated_at": self.updated_at, - } - - @classmethod - def from_dict(cls, value, state_path): - value = _migrate_state(value) - _validate_state(value) - return cls( - workflow_id=value["workflow_id"], - revision=value["revision"], - phase=value["phase"], - status=value["status"], - request=dict(value["request"]), - state_path=Path(state_path), - completed_phases=list(value["completed_phases"]), - host=dict(value["host"]), - discovery=dict(value["discovery"]), - shortlist=list(value["shortlist"]), - evidence=list(value["evidence"]), - comparisons=list(value["comparisons"]), - recommendations=list(value["recommendations"]), - warnings=list(value["warnings"]), - errors=list(value["errors"]), - created_at=value["created_at"], - updated_at=value["updated_at"], - schema_version=value["schema_version"], - ) - - -class AdoptionWorkflow: - """Advance adoption one durable phase at a time.""" - - def __init__(self, discovery_service=None, verifier=None, state_path=None): - self.discovery_service = discovery_service or DiscoveryService() - metadata_client = getattr(self.discovery_service, "_huggingface", None) - self.verifier = verifier or Verifier(metadata_client=metadata_client) - self.state_path = Path(state_path) if state_path is not None else None - - def start(self, request): - if isinstance(request, dict): - request = AdoptionRequest.from_dict(request) - if not isinstance(request, AdoptionRequest): - raise TypeError("request must be an AdoptionRequest or object") - path_value = request.state_path or self.state_path - if path_value is None: - raise ValueError("a state path is required") - state = AdoptionState( - workflow_id=str(uuid.uuid4()), - phase="inspect", - status="running", - request=request.to_dict(), - state_path=Path(path_value), - ) - self._persist(state) - return state - - def advance(self, state): - if not isinstance(state, AdoptionState): - raise TypeError("state must be an AdoptionState") - state.phase = _first_incomplete(state.completed_phases) - if state.phase == "complete": - state.status = "complete" - return state - - phase = state.phase - getattr(self, "_phase_{0}".format(phase))(state) - if phase not in state.completed_phases: - state.completed_phases.append(phase) - state.phase = _first_incomplete(state.completed_phases) - state.status = "complete" if state.phase == "complete" else "running" - self._persist(state) - return state - - def resume(self, path): - state_path = Path(path) - content, exists, _mode = _read_target(state_path) - if not exists: - raise ValueError("adoption state does not exist") - value = json.loads(content.decode("utf-8")) - state = AdoptionState.from_dict(value, state_path) - state._source_hash = hashlib.sha256(content).hexdigest() - state.phase = _first_incomplete(state.completed_phases) - state.status = "complete" if state.phase == "complete" else "running" - return state - - def status(self, path): - return self.resume(path) - - def _phase_inspect(self, state): - host = self.discovery_service.host - if hasattr(host, "to_dict"): - host = host.to_dict() - if not isinstance(host, dict): - raise TypeError("host inventory must be an object") - state.host = dict(host) - - def _phase_discover(self, state): - request = state.request - roles = request["roles"] - result = self.discovery_service.discover(DiscoveryRequest( - role=roles[0] if len(roles) == 1 else None, - memory_gb=request.get("memory_gb"), - quantization=request.get("quantization"), - licenses=request.get("licenses"), - include_gated=request.get("include_gated", True), - publishers=request.get("publishers"), - runtime=request.get("runtime"), - refresh=request.get("refresh", False), - offline=request.get("offline", False), - limit=request["shortlist_limit"], - fast=request.get("fast", False), - )) - if result.status != "ok": - error = result.to_dict()["error"] - state.errors.append({key: str(error[key]) for key in ("code", "message", "remediation")}) - self._persist(state) - raise RuntimeError("discovery failed: {0}".format(error["message"])) - state.discovery = dict(result.data) - state.warnings.extend(result.warnings) - - def _phase_shortlist(self, state): - requested = set(state.request["roles"]) - limit = state.request["shortlist_limit"] - shortlisted = [] - roles = state.discovery.get("roles", {}) - if not isinstance(roles, dict): - raise TypeError("discovery roles must be an object") - for role in state.request["roles"]: - if role not in requested: - continue - for candidate in (roles.get(role) or [])[:limit]: - record = dict(candidate) - record["role"] = role - shortlisted.append(record) - state.shortlist = shortlisted - - def _phase_verify(self, state): - clear_inventory_cache = getattr(self.verifier, "clear_inventory_cache", None) - if callable(clear_inventory_cache): - clear_inventory_cache() - workers = verification_concurrency(state.host.get("ram_gb")) - - def verify_one(candidate): - try: - evidence = self.verifier.verify( - candidate, - state.host, - allow_network=state.request.get("allow_network", True), - ) - if not isinstance(evidence, VerificationEvidence): - raise TypeError("verifier returned an invalid evidence record") - return evidence.to_dict() - except Exception as error: - repo = str( - candidate.get("repo") - or candidate.get("repository") - or "unknown" - ) - role = str(candidate.get("role") or "general") - if role == "tool-use": - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.HEURISTIC_ONLY, - status=VerificationStatus.FAILED, - available_locally=False, - loads=None, - reasoning_confirmed=None, - runtime=None, - note=_bounded("Candidate tool-use verification failed."), - details={ - "probe_id": TOOL_USE_PROBE_ID, - "reason": "verification_exception", - }, - ).to_dict() - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.HEURISTIC_ONLY, - status=VerificationStatus.FAILED, - available_locally=False, - loads=None, - reasoning_confirmed=None, - runtime=None, - note=_bounded("Candidate verification failed: {0}".format(error)), - details={"error": _bounded(str(error), 160)}, - ).to_dict() - - with ThreadPoolExecutor(max_workers=workers) as executor: - state.evidence = list(executor.map(verify_one, state.shortlist)) - - def _phase_measure(self, state): - if not state.request.get("measure", False): - return - from .bench import BenchError, measure_runtime - - runtime_clients = getattr(self.verifier, "runtime_clients", None) - clients = {} - if callable(runtime_clients): - for client in runtime_clients(): - name = getattr(client, "name", None) - if isinstance(name, str): - clients[name] = client - measured = 0 - for index, candidate in enumerate(state.shortlist): - if measured >= MEASURE_LIMIT: - break - if index >= len(state.evidence): - break - evidence = state.evidence[index] - if ( - evidence.get("strength") != EvidenceStrength.RUNTIME_TESTED.value - or evidence.get("status") != VerificationStatus.VERIFIED.value - ): - continue - client = clients.get(evidence.get("runtime")) - if client is None or not callable(getattr(client, "stream_generate", None)): - continue - repo = candidate.get("repo") or candidate.get("repository") - try: - measurement = measure_runtime( - repo, client, chip=state.host.get("chip") - ) - except BenchError as error: - state.warnings.append({ - "code": "measure_skipped", - "message": _bounded( - "{0}: {1}".format(repo, error) - ), - }) - continue - except Exception as error: - state.warnings.append({ - "code": "measure_skipped", - "message": _bounded("{0}: {1}".format(repo, error)), - }) - continue - details = dict(evidence.get("details") or {}) - details["bench"] = measurement.to_dict() - evidence["details"] = details - evidence["strength"] = EvidenceStrength.RUNTIME_MEASURED.value - measured += 1 - - def _phase_compare(self, state): - evidence_by_candidate = { - (item["repo"], item["role"]): item for item in state.evidence - } - comparisons = [] - for candidate in state.shortlist: - repo = candidate.get("repo") or candidate.get("repository") - role = candidate.get("role") - evidence = evidence_by_candidate.get((repo, role), {}) - reasons = [] - if candidate.get("fits") is False: - reasons.append("memory_budget") - if evidence.get("loads") is False: - reasons.append("runtime_generation_failed") - utility_role = role in UTILITY_ROLES or state.request.get("fast", False) - if utility_role and evidence.get("reasoning_confirmed") is True: - reasons.append("confirmed_reasoner_for_utility_role") - evidence_status = evidence.get( - "status", VerificationStatus.FAILED.value - ) - if ( - role == "tool-use" - and evidence_status != VerificationStatus.VERIFIED.value - ): - reasons.append("tool_use_not_verified") - score = EVIDENCE_SCORES.get(evidence.get("strength"), 0) - score += int(candidate.get("rank_score") or 0) - score += 20 if candidate.get("trusted") else 0 - probe_outcome = _role_probe_outcome_for(role, evidence) - if probe_outcome is not None: - if probe_outcome.get("valid") is True: - score += ROLE_PROBE_BONUS - elif probe_outcome.get("reason") != "unsupported_runtime": - reasons.append("role_probe_failed") - comparisons.append({ - "repo": repo, - "role": role, - "eligible": not reasons, - "score": score, - "evidence_strength": evidence.get("strength", EvidenceStrength.HEURISTIC_ONLY.value), - "evidence_status": evidence_status, - "verification_status": evidence_status, - "rejection_reasons": reasons, - }) - state.comparisons = comparisons - - def _phase_recommend(self, state): - candidates_by_candidate = { - ( - item.get("repo") or item.get("repository"), - item.get("role"), - ): item - for item in state.shortlist - } - recommendations = [] - for role in state.request["roles"]: - eligible = [ - item for item in state.comparisons - if item["role"] == role and item["eligible"] - and ( - role != "tool-use" - or item.get("evidence_status") - == VerificationStatus.VERIFIED.value - ) - ] - eligible.sort(key=lambda item: (-item["score"], item["repo"])) - if not eligible: - continue - chosen = eligible[0] - candidate = candidates_by_candidate[(chosen["repo"], chosen["role"])] - recommendations.append({ - "role": role, - "repo": chosen["repo"], - "evidence_strength": chosen["evidence_strength"], - "evidence_status": chosen.get( - "evidence_status", - chosen.get("verification_status", VerificationStatus.FAILED.value), - ), - "estimated_ram_gb": candidate.get("est_ram_gb"), - "wiring": candidate.get("wiring"), - "alternatives": [item["repo"] for item in eligible[1:4]], - }) - state.recommendations = recommendations - - def _persist(self, state): - destination = state.state_path - try: - with _target_locks([destination], create_parents=True): - current, exists, _mode = _read_target(destination) - if state._source_hash is None: - if state.revision != 0 or exists: - raise AdoptionStateConflictError( - "adoption state already exists or lacks a CAS baseline" - ) - else: - if not exists or hashlib.sha256(current).hexdigest() != state._source_hash: - raise AdoptionStateConflictError( - "adoption state changed after it was resumed" - ) - try: - current_value = json.loads(current.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as error: - raise AdoptionStateConflictError( - "adoption state changed after it was resumed" - ) from error - if current_value.get("revision") != state.revision: - raise AdoptionStateConflictError( - "adoption state revision changed after it was resumed" - ) - - next_revision = state.revision + 1 - next_updated_at = _utc_now() - value = state.to_dict() - value["revision"] = next_revision - value["updated_at"] = next_updated_at - _validate_state(value) - content = ( - json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n" - ).encode("utf-8") - - def validate_staged(text): - _validate_state(json.loads(text)) - - _atomic_in_directory( - destination.parent, - destination.name, - content, - 0o600, - validator=validate_staged, - ) - state.revision = next_revision - state.updated_at = next_updated_at - state._source_hash = hashlib.sha256(content).hexdigest() - except ConcurrentTransactionError as error: - raise AdoptionStateConflictError( - "another adoption workflow is updating this handoff" - ) from error - - -def _role_probe_outcome_for(role, evidence): - """Return the role-fit probe outcome recorded in evidence, if any.""" - probe_id = ROLE_PROBE_IDS.get(role) - if probe_id is None or not isinstance(evidence, dict): - return None - details = evidence.get("details") - if not isinstance(details, dict): - return None - probes = details.get("probes") - if not isinstance(probes, list): - return None - for probe in probes: - if not isinstance(probe, dict) or probe.get("probe_id") != probe_id: - continue - outcome = probe.get("outcome") - return outcome if isinstance(outcome, dict) else None - return None - - -def verification_concurrency(ram_gb): - try: - memory = int(float(ram_gb)) - except (TypeError, ValueError, OverflowError): - memory = 0 - return int(min(4, max(1, memory // 16))) - - -def _first_incomplete(completed): - completed_set = set(completed) - for phase in PHASES[:-1]: - if phase not in completed_set: - return phase - return "complete" - - -def _migrate_state(value): - if not isinstance(value, dict): - raise TypeError("adoption state must be an object") - version = value.get("schema_version") - if version == ADOPTION_SCHEMA_VERSION: - return value - if version not in LEGACY_ADOPTION_SCHEMA_VERSIONS: - raise ValueError("unsupported adoption state schema version") - - migrated = deepcopy(value) - if version == "1.1": - request = migrated.get("request") - legacy_roles = ( - list(request.get("roles", [])) - if isinstance(request, dict) and isinstance(request.get("roles", []), list) - else [] - ) - for collection in ("shortlist", "evidence", "comparisons", "recommendations"): - records = migrated.get(collection) - if isinstance(records, list): - legacy_roles.extend( - item.get("role") for item in records if isinstance(item, dict) - ) - if "tool-use" in legacy_roles: - raise ValueError("legacy adoption state cannot contain tool-use") - - status_by_strength = { - EvidenceStrength.RUNTIME_TESTED.value: VerificationStatus.VERIFIED.value, - EvidenceStrength.RUNTIME_INVENTORY.value: VerificationStatus.FAILED.value, - EvidenceStrength.METADATA_ONLY.value: VerificationStatus.METADATA_ONLY.value, - EvidenceStrength.HEURISTIC_ONLY.value: VerificationStatus.METADATA_ONLY.value, - } - for item in migrated.get("evidence", []): - if not isinstance(item, dict): - continue - strength = item.get("strength") - if strength in status_by_strength: - item["status"] = status_by_strength[strength] - - request = migrated.get("request") - if isinstance(request, dict) and "measure" not in request: - request["measure"] = False - completed = migrated.get("completed_phases") - if ( - isinstance(completed, list) - and "verify" in completed - and "measure" not in completed - ): - completed.insert(completed.index("verify") + 1, "measure") - migrated["schema_version"] = ADOPTION_SCHEMA_VERSION - return migrated - - -def _validate_state(value): - required = { - "schema_version", "workflow_id", "revision", "phase", "status", "request", - "completed_phases", "host", "discovery", "shortlist", "evidence", - "comparisons", "recommendations", "warnings", "errors", "created_at", "updated_at", - } - if not isinstance(value, dict): - raise TypeError("adoption state must be an object") - missing = sorted(required - set(value)) - unexpected = sorted(set(value) - required) - if missing: - raise ValueError("adoption state is missing keys: {0}".format(missing)) - if unexpected: - raise ValueError("adoption state has unexpected keys: {0}".format(unexpected)) - if value["schema_version"] != ADOPTION_SCHEMA_VERSION: - raise ValueError("unsupported adoption state schema version") - if not isinstance(value["workflow_id"], str) or not value["workflow_id"]: - raise ValueError("workflow_id must be a non-empty string") - if ( - not isinstance(value["revision"], int) - or isinstance(value["revision"], bool) - or value["revision"] < 1 - ): - raise ValueError("revision must be a positive integer") - if value["phase"] not in PHASES: - raise ValueError("invalid adoption phase") - if value["status"] not in ("running", "complete"): - raise ValueError("invalid adoption status") - _validate_completed_phases(value) - if not isinstance(value["request"], dict): - raise TypeError("adoption request must be an object") - _validate_request_shape(value["request"]) - AdoptionRequest.from_dict(value["request"]) - for key in ("host", "discovery"): - if not isinstance(value[key], dict): - raise TypeError("{0} must be an object".format(key)) - bounds = { - "shortlist": 100, - "evidence": 100, - "comparisons": 100, - "recommendations": 6, - "warnings": 50, - "errors": 50, - } - for key, maximum in bounds.items(): - if not isinstance(value[key], list): - raise TypeError("{0} must be an array".format(key)) - if len(value[key]) > maximum: - raise ValueError("{0} exceeds its maximum size".format(key)) - _validate_timestamp(value["created_at"], "created_at") - _validate_timestamp(value["updated_at"], "updated_at") - _validate_evidence(value["evidence"]) - _validate_phase_artifacts(value) - - -def _validate_completed_phases(value): - completed = value["completed_phases"] - if not isinstance(completed, list): - raise TypeError("completed_phases must be an array") - if len(completed) > len(PHASES) - 1: - raise ValueError("completed_phases contains an invalid phase") - expected = list(PHASES[:len(completed)]) - if completed != expected: - raise ValueError("completed_phases must be a unique contiguous phase prefix") - expected_phase = PHASES[len(completed)] - if value["phase"] != expected_phase: - raise ValueError("phase does not match completed_phases") - expected_status = "complete" if expected_phase == "complete" else "running" - if value["status"] != expected_status: - raise ValueError("status does not match phase") - - -def _validate_request_shape(request): - required = { - "roles", "shortlist_limit", "allow_network", "offline", "refresh", - "memory_gb", "quantization", "licenses", "include_gated", "publishers", - "runtime", "fast", "measure", - } - if set(request) != required: - raise ValueError("adoption request does not match the persisted schema") - if not isinstance(request["roles"], list) or not request["roles"]: - raise ValueError("request.roles must be a non-empty array") - if len(request["roles"]) > 6 or len(set(request["roles"])) != len(request["roles"]): - raise ValueError("request.roles must be bounded and unique") - if any(not isinstance(role, str) or not role for role in request["roles"]): - raise TypeError("request.roles must contain non-empty strings") - if not isinstance(request["shortlist_limit"], int) or isinstance(request["shortlist_limit"], bool): - raise TypeError("request.shortlist_limit must be an integer") - if not 1 <= request["shortlist_limit"] <= 20: - raise ValueError("request.shortlist_limit is out of range") - for key in ("allow_network", "offline", "refresh", "include_gated", "fast"): - if not isinstance(request[key], bool): - raise TypeError("request.{0} must be a boolean".format(key)) - if request["memory_gb"] is not None and ( - not isinstance(request["memory_gb"], (int, float)) or isinstance(request["memory_gb"], bool) - ): - raise TypeError("request.memory_gb must be a number or null") - for key in ("quantization", "runtime"): - if request[key] is not None and not isinstance(request[key], str): - raise TypeError("request.{0} must be a string or null".format(key)) - for key in ("licenses", "publishers"): - if not isinstance(request[key], list) or len(request[key]) > 20: - raise ValueError("request.{0} must be a bounded array".format(key)) - if any(not isinstance(item, str) for item in request[key]): - raise TypeError("request.{0} must contain strings".format(key)) - - -def _validate_timestamp(timestamp, field_name): - if not isinstance(timestamp, str): - raise TypeError("{0} must be an ISO-8601 timestamp".format(field_name)) - normalized = timestamp[:-1] + "+00:00" if timestamp.endswith("Z") else timestamp - try: - parsed = datetime.fromisoformat(normalized) - except ValueError: - raise ValueError("{0} must be an ISO-8601 timestamp".format(field_name)) - if parsed.tzinfo is None: - raise ValueError("{0} must include a timezone".format(field_name)) - - -def _validate_evidence(evidence): - required = { - "repo", "role", "strength", "status", "available_locally", "loads", - "reasoning_confirmed", "runtime", "note", "details", - } - strengths = set(EVIDENCE_SCORES) - statuses = {status.value for status in VerificationStatus} - for index, item in enumerate(evidence): - prefix = "evidence[{0}]".format(index) - if not isinstance(item, dict): - raise TypeError("{0} must be an object".format(prefix)) - if set(item) != required: - raise ValueError("{0} does not match the evidence schema".format(prefix)) - for key in ("repo", "role", "note"): - if not isinstance(item[key], str) or not item[key]: - raise ValueError("{0}.{1} must be a non-empty string".format(prefix, key)) - if len(item["note"]) > 300: - raise ValueError("{0}.note exceeds its maximum length".format(prefix)) - if item["strength"] not in strengths: - raise ValueError("{0}.strength is invalid".format(prefix)) - if item["status"] not in statuses: - raise ValueError("{0}.status is invalid".format(prefix)) - if not isinstance(item["available_locally"], bool): - raise TypeError("{0}.available_locally must be a boolean".format(prefix)) - for key in ("loads", "reasoning_confirmed"): - if item[key] is not None and not isinstance(item[key], bool): - raise TypeError("{0}.{1} must be a boolean or null".format(prefix, key)) - if item["runtime"] is not None and not isinstance(item["runtime"], str): - raise TypeError("{0}.runtime must be a string or null".format(prefix)) - if not isinstance(item["details"], dict): - raise TypeError("{0}.details must be an object".format(prefix)) - - -def _validate_phase_artifacts(value): - completed = value["completed_phases"] - verify_complete = "verify" in completed - compare_complete = "compare" in completed - recommend_complete = "recommend" in completed - shortlist = value["shortlist"] - evidence = value["evidence"] - comparisons = value["comparisons"] - recommendations = value["recommendations"] - if not verify_complete and evidence: - raise ValueError("evidence is not valid before verification completes") - if verify_complete: - _validate_records_match(shortlist, evidence, "evidence") - if not compare_complete and comparisons: - raise ValueError("comparisons are not valid before comparison completes") - if compare_complete: - _validate_records_match(shortlist, comparisons, "comparisons") - if not recommend_complete and recommendations: - raise ValueError("recommendations are not valid before recommendation completes") - - -def _validate_records_match(shortlist, records, field_name): - if len(shortlist) != len(records): - raise ValueError("{0} does not cover the current shortlist".format(field_name)) - for index, (candidate, record) in enumerate(zip(shortlist, records)): - if not isinstance(candidate, dict) or not isinstance(record, dict): - raise TypeError("{0}[{1}] must be an object".format(field_name, index)) - candidate_repo = candidate.get("repo") or candidate.get("repository") - if not isinstance(candidate_repo, str) or not candidate_repo: - raise ValueError("shortlist[{0}] has no repository".format(index)) - if not isinstance(candidate.get("role"), str) or not candidate["role"]: - raise ValueError("shortlist[{0}] has no role".format(index)) - if record.get("repo") != candidate_repo or record.get("role") != candidate["role"]: - raise ValueError("{0}[{1}] does not match the current shortlist".format(field_name, index)) - - -def _string_list(value): - if value is None: - return [] - if isinstance(value, str): - return [value] - return [str(item) for item in value] - - -def _bounded(value, limit=300): - text = " ".join(str(value).split()) - if len(text) <= limit: - return text - return text[: max(0, limit - 3)] + "..." diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/bench.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/bench.py deleted file mode 100644 index ba6d8cd..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/bench.py +++ /dev/null @@ -1,431 +0,0 @@ -"""Bounded, read-only performance measurement of a locally served model. - -Bench never installs, downloads, or starts anything. It measures a model that -is already served by an already-running loopback runtime, using the same -safety envelope as verification: validated loopback URLs, absolute monotonic -deadlines, bounded responses, and redacted errors. -""" - -from __future__ import annotations - -import json -import os -import statistics -import time -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Dict, List, Optional - -from .verification import ( - EvidenceStrength, - VerificationEvidence, - VerificationStatus, - _bounded, - _is_installed, - _runtime_name, - _safe_error, - installed_model_ids, -) - - -BENCH_PROBE_ID = "bench-v1" -BENCH_PROMPT = ( - "You are a careful technical writer. Write a detailed, factual explanation " - "of how unified memory architecture on Apple Silicon affects local " - "inference of large language models. Cover memory bandwidth, quantization " - "tradeoffs, KV-cache growth with context length, and the difference " - "between dense and mixture-of-experts models. Organize the answer into " - "clear sections with concrete numbers where possible." -) -BENCH_PROMPT_TOKEN_ESTIMATE = 96 -RUNS_MIN, RUNS_MAX, RUNS_DEFAULT = 1, 10, 3 -GEN_TOKENS_MIN, GEN_TOKENS_MAX, GEN_TOKENS_DEFAULT = 16, 2048, 128 -TIMEOUT_DEFAULT = 120.0 - - -class BenchError(RuntimeError): - """Classified bench failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -@dataclass(frozen=True) -class BenchMeasurement: - """Aggregate measured performance for one model on one runtime.""" - - repo: str - runtime: str - runs: int - prompt_tokens: int - gen_tokens: int - ttft_ms: Optional[float] - decode_toks: float - prefill_toks: Optional[float] - spread_pct: float - chip: Optional[str] - measured_at: str - samples: List[Dict[str, Any]] = field(default_factory=list) - - def to_dict(self) -> Dict[str, Any]: - return { - "probe_id": BENCH_PROBE_ID, - "outcome": "measured", - "repo": self.repo, - "runtime": self.runtime, - "runs": self.runs, - "prompt_tokens": self.prompt_tokens, - "gen_tokens": self.gen_tokens, - "ttft_ms": self.ttft_ms, - "decode_toks": self.decode_toks, - "prefill_toks": self.prefill_toks, - "peak_mem_gb": None, - "spread_pct": self.spread_pct, - "chip": self.chip, - "measured_at": self.measured_at, - "samples": list(self.samples), - } - - def to_evidence(self, role="general") -> VerificationEvidence: - return VerificationEvidence( - repo=self.repo, - role=role, - strength=EvidenceStrength.RUNTIME_MEASURED, - status=VerificationStatus.VERIFIED, - available_locally=True, - loads=True, - reasoning_confirmed=None, - runtime=self.runtime, - note=_bounded( - "Measured {0} run(s) on this host; median decode " - "{1} tok/s.".format(self.runs, self.decode_toks) - ), - details=self.to_dict(), - ) - - -def validate_bench_bounds(runs, gen_tokens, timeout): - if not isinstance(runs, int) or isinstance(runs, bool): - raise TypeError("runs must be an integer") - if not RUNS_MIN <= runs <= RUNS_MAX: - raise ValueError("runs must be between {0} and {1}".format(RUNS_MIN, RUNS_MAX)) - if not isinstance(gen_tokens, int) or isinstance(gen_tokens, bool): - raise TypeError("gen_tokens must be an integer") - if not GEN_TOKENS_MIN <= gen_tokens <= GEN_TOKENS_MAX: - raise ValueError( - "gen_tokens must be between {0} and {1}".format(GEN_TOKENS_MIN, GEN_TOKENS_MAX) - ) - if not isinstance(timeout, (int, float)) or isinstance(timeout, bool): - raise TypeError("timeout must be a number") - if not 1.0 <= float(timeout) <= 600.0: - raise ValueError("timeout must be between 1 and 600 seconds") - - -def measure_runtime(repo, runtime, runs=RUNS_DEFAULT, gen_tokens=GEN_TOKENS_DEFAULT, - timeout=TIMEOUT_DEFAULT, chip=None, clock=time.monotonic, - now=None): - """Measure one already-served model; never starts or downloads anything.""" - validate_bench_bounds(runs, gen_tokens, timeout) - if not isinstance(repo, str) or not repo.strip(): - raise BenchError( - "invalid_repo", - "bench requires a non-empty model repository identifier.", - "Pass --repo as publisher/model exactly as the runtime serves it.", - ) - stream_generate = getattr(runtime, "stream_generate", None) - if not callable(stream_generate): - raise BenchError( - "unsupported_runtime", - "This runtime does not support the bounded streaming measurement.", - "Serve the model with a runtime that supports streaming generation.", - ) - runtime_name = _runtime_name(runtime) - try: - installed = installed_model_ids(runtime.list_models()) - except Exception as error: - raise BenchError( - "runtime_unreachable", - "The runtime inventory could not be read: {0}".format(_safe_error(error)), - "Confirm the local runtime is running and reachable on its loopback port.", - ) - if not _is_installed(repo, installed): - raise BenchError( - "model_not_serving", - "The model is not served by the {0} runtime.".format(runtime_name), - "Load the model in the runtime first; bench never downloads models.", - ) - - per_run_timeout = float(timeout) / runs - try: - _timed_run(stream_generate, repo, gen_tokens, per_run_timeout, clock) - samples = [ - _timed_run(stream_generate, repo, gen_tokens, per_run_timeout, clock) - for _ in range(runs) - ] - except BenchError: - raise - except Exception as error: - raise BenchError( - "measurement_failed", - "The measurement run failed: {0}".format(_safe_error(error)), - "Confirm the model responds to a normal generation request, then retry.", - ) - - decode_values = [sample["decode_toks"] for sample in samples] - median_decode = statistics.median(decode_values) - spread = 0.0 - if median_decode > 0 and len(decode_values) > 1: - spread = (max(decode_values) - min(decode_values)) / median_decode * 100.0 - ttft_values = [sample["ttft_ms"] for sample in samples if sample["ttft_ms"] is not None] - prefill_values = [ - sample["prefill_toks"] for sample in samples if sample["prefill_toks"] is not None - ] - measured_at = now() if callable(now) else _utc_now() - return BenchMeasurement( - repo=repo, - runtime=runtime_name, - runs=runs, - prompt_tokens=BENCH_PROMPT_TOKEN_ESTIMATE, - gen_tokens=gen_tokens, - ttft_ms=round(statistics.median(ttft_values), 1) if ttft_values else None, - decode_toks=round(median_decode, 1), - prefill_toks=round(statistics.median(prefill_values), 1) if prefill_values else None, - spread_pct=round(spread, 1), - chip=chip, - measured_at=measured_at, - samples=samples, - ) - - -def _timed_run(stream_generate, repo, gen_tokens, timeout, clock): - started = clock() - first_content_at = None - last_event_at = None - content_events = 0 - usage = None - for event in stream_generate(repo, BENCH_PROMPT, gen_tokens, timeout=timeout): - if not isinstance(event, dict): - continue - last_event_at = clock() - text = _stream_event_text(event) - if text: - content_events += 1 - if first_content_at is None: - first_content_at = last_event_at - event_usage = _stream_event_usage(event) - if event_usage: - usage = event_usage - if content_events == 0 and usage is None: - raise BenchError( - "empty_stream", - "The runtime streamed no tokens for the measurement prompt.", - "Confirm the model generates normally, then retry.", - ) - finished = clock() - sample = { - "content_events": content_events, - "elapsed_ms": round((finished - started) * 1000.0, 1), - } - if first_content_at is not None: - sample["ttft_ms"] = round((first_content_at - started) * 1000.0, 1) - else: - sample["ttft_ms"] = None - - if usage and usage.get("eval_count") and usage.get("eval_duration"): - sample["decode_toks"] = round( - usage["eval_count"] / (usage["eval_duration"] / 1e9), 1 - ) - if usage.get("prompt_eval_count") and usage.get("prompt_eval_duration"): - sample["prefill_toks"] = round( - usage["prompt_eval_count"] / (usage["prompt_eval_duration"] / 1e9), 1 - ) - sample["prompt_tokens"] = usage["prompt_eval_count"] - elif first_content_at is not None and content_events > 1 and last_event_at > first_content_at: - sample["decode_toks"] = round( - (content_events - 1) / (last_event_at - first_content_at), 1 - ) - ttft_seconds = first_content_at - started - if ttft_seconds > 0: - sample["prefill_toks"] = round(BENCH_PROMPT_TOKEN_ESTIMATE / ttft_seconds, 1) - elif usage and usage.get("completion_tokens") and finished > started: - sample["decode_toks"] = round(usage["completion_tokens"] / (finished - started), 1) - else: - sample["decode_toks"] = round(content_events / max(finished - started, 1e-9), 1) - sample.setdefault("prefill_toks", None) - return sample - - -def _stream_event_text(event): - message = event.get("message") - if isinstance(message, dict) and isinstance(message.get("content"), str): - return message["content"] - choices = event.get("choices") - if isinstance(choices, list) and choices and isinstance(choices[0], dict): - delta = choices[0].get("delta") - if isinstance(delta, dict) and isinstance(delta.get("content"), str): - return delta["content"] - return "" - - -def _stream_event_usage(event): - usage = event.get("usage") - normalized = {} - if isinstance(usage, dict): - if isinstance(usage.get("completion_tokens"), int): - normalized["completion_tokens"] = usage["completion_tokens"] - if isinstance(usage.get("prompt_tokens"), int): - normalized["prompt_tokens"] = usage["prompt_tokens"] - for key in ("eval_count", "eval_duration", "prompt_eval_count", "prompt_eval_duration"): - value = event.get(key) - if isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0: - normalized[key] = value - return normalized or None - - -def _utc_now(): - from datetime import datetime, timezone - - return datetime.now(timezone.utc).isoformat() - - -BENCH_EXPORT_SCHEMA_VERSION = "1.0" -BENCH_AGGREGATE_SCHEMA_VERSION = "1.0" -MAX_EXPORT_LINE_BYTES = 4096 -MAX_EXPORT_LINES_PER_FILE = 5000 -MAX_AGGREGATE_ENTRIES = 5000 -_COMMUNITY_BENCH_PATH = Path(__file__).resolve().parent / "resources" / "bench-community.json" -_community_bench_cache = None - - -def export_record(measurement, tool_version): - """Build one bounded, anonymized export line from a measurement.""" - record = { - "schema_version": BENCH_EXPORT_SCHEMA_VERSION, - "repo": measurement.repo, - "chip": measurement.chip, - "runtime": measurement.runtime, - "ttft_ms": measurement.ttft_ms, - "decode_toks": measurement.decode_toks, - "prefill_toks": measurement.prefill_toks, - "measured_at": measurement.measured_at, - "tool_version": tool_version, - } - validate_export_record(record) - return record - - -def validate_export_record(record): - if not isinstance(record, dict): - raise ValueError("bench export record must be an object") - if record.get("schema_version") != BENCH_EXPORT_SCHEMA_VERSION: - raise ValueError("bench export record has an unsupported schema version") - for key in ("repo", "chip", "runtime", "measured_at", "tool_version"): - if not isinstance(record.get(key), str) or not record[key]: - raise ValueError("bench export record {0} must be a non-empty string".format(key)) - if len(record["repo"]) > 300 or len(record["chip"]) > 120: - raise ValueError("bench export record identifiers exceed their bounds") - for key in ("ttft_ms", "decode_toks", "prefill_toks"): - value = record.get(key) - if value is None: - continue - if not isinstance(value, (int, float)) or isinstance(value, bool): - raise ValueError("bench export record {0} must be numeric or null".format(key)) - if not 0 <= value <= 1e9: - raise ValueError("bench export record {0} is outside bench bounds".format(key)) - if record.get("decode_toks") is None: - raise ValueError("bench export record requires decode_toks") - return record - - -def append_export(path, record): - """Append one validated export line; never reads or rewrites the file.""" - line = json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" - if len(line.encode("utf-8")) > MAX_EXPORT_LINE_BYTES: - raise BenchError( - "invalid_arguments", - "The export record exceeds its size bound.", - "Export only records produced by bench run.", - ) - destination = Path(path) - flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, "O_NOFOLLOW", 0) - descriptor = os.open(str(destination), flags, 0o600) - try: - os.write(descriptor, line.encode("utf-8")) - finally: - os.close(descriptor) - return destination - - -def aggregate_exports(paths): - """Deduplicate and median-aggregate validated export records.""" - newest = {} - for path in paths: - location = Path(path) - try: - handle = location.open("r", encoding="utf-8") - except OSError: - continue - with handle: - for index, line in enumerate(handle): - if index >= MAX_EXPORT_LINES_PER_FILE: - break - stripped = line.strip() - if not stripped or len(stripped.encode("utf-8")) > MAX_EXPORT_LINE_BYTES: - continue - try: - record = validate_export_record(json.loads(stripped)) - except (ValueError, json.JSONDecodeError): - continue - key = (record["repo"], record["chip"], record["runtime"]) - if key not in newest or record["measured_at"] > newest[key]["measured_at"]: - newest[key] = record - groups = {} - for record in newest.values(): - groups.setdefault((record["repo"], record["chip"]), []).append(record) - entries = [] - for (repo, chip), records in sorted(groups.items()): - if len(entries) >= MAX_AGGREGATE_ENTRIES: - break - def _median(key): - values = [item[key] for item in records if item.get(key) is not None] - return round(statistics.median(values), 1) if values else None - - entries.append({ - "repo": repo, - "chip": chip, - "decode_toks": _median("decode_toks"), - "prefill_toks": _median("prefill_toks"), - "ttft_ms": _median("ttft_ms"), - "samples": len(records), - "updated_at": max(item["measured_at"] for item in records), - }) - return {"schema_version": BENCH_AGGREGATE_SCHEMA_VERSION, "entries": entries} - - -def load_community_bench(path=None): - """Load the bundled community aggregate once; the empty stub yields {}.""" - global _community_bench_cache - if _community_bench_cache is not None and path is None: - return _community_bench_cache - location = Path(path) if path is not None else _COMMUNITY_BENCH_PATH - entries = {} - try: - value = json.loads(location.read_text(encoding="utf-8")) - if isinstance(value, dict) and isinstance(value.get("entries"), list): - for entry in value["entries"]: - if not isinstance(entry, dict): - continue - repo = entry.get("repo") - chip = entry.get("chip") - if isinstance(repo, str) and isinstance(chip, str): - entries[(repo, chip)] = { - key: entry.get(key) - for key in ("decode_toks", "prefill_toks", "ttft_ms", "samples") - } - except (OSError, ValueError, json.JSONDecodeError): - entries = {} - if path is None: - _community_bench_cache = entries - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/blueprint.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/blueprint.py deleted file mode 100644 index 483d7cf..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/blueprint.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Deterministic dataset-creation blueprint from a DomainIntent. - -Guidance only: no download, no training, no execution. Templates are fixed so -the same intent always yields the same blueprint text and structure. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Dict, Tuple - -from .interview import DomainIntent - - -@dataclass(frozen=True) -class DatasetBlueprint: - goal: str - schema_fields: Tuple[str, ...] - labeling_notes: str - split_guidance: str - license_privacy: str - mlx_next_steps: str - - def to_dict(self) -> Dict[str, object]: - return { - "goal": self.goal, - "schema_fields": list(self.schema_fields), - "labeling_notes": self.labeling_notes, - "split_guidance": self.split_guidance, - "license_privacy": self.license_privacy, - "mlx_next_steps": self.mlx_next_steps, - } - - def to_markdown(self) -> str: - lines = [ - "### Goal", - "", - self.goal, - "", - "### Suggested schema", - "", - ] - for field_name in self.schema_fields: - lines.append("- `{0}`".format(field_name)) - lines.extend([ - "", - "### Labeling notes", - "", - self.labeling_notes, - "", - "### Train / val split", - "", - self.split_guidance, - "", - "### License and privacy", - "", - self.license_privacy, - "", - "### MLX fine-tune next steps", - "", - self.mlx_next_steps, - "", - ]) - return "\n".join(lines) - - -_DEFAULT_FIELDS = ( - "id", - "input_text", - "label", - "source", - "license", - "notes", -) - - -def build_dataset_blueprint(intent: DomainIntent) -> DatasetBlueprint: - """Build a deterministic dataset blueprint for an empty catalog result.""" - roles = ", ".join(intent.roles) or "general" - keywords = ", ".join(intent.keywords) or "none" - licenses = ", ".join(intent.license_allow) or "document and respect upstream licenses" - notes = intent.notes.strip() or "none recorded" - - schema = list(_DEFAULT_FIELDS) - if "vision" in intent.roles: - schema.insert(1, "image_path") - if "embedding" in intent.roles: - schema.insert(-2, "embedding_text") - - goal = ( - "Create a small, domain-specific dataset for **{0}** covering roles " - "({1}) and keywords ({2}). Prefer quality over volume; keep examples " - "representative of real production inputs." - ).format(intent.domain, roles, keywords) - - labeling = ( - "Define clear positive/negative or multi-class labels tied to the domain. " - "Double-check borderline cases. Keywords to cover: {0}. Extra notes: {1}." - ).format(keywords, notes) - - split = ( - "Hold out 10–20% for validation with no leakage from near-duplicates. " - "Stratify by label when classes are imbalanced. Keep a tiny smoke-test " - "slice for local MLX dry-runs." - ) - - license_privacy = ( - "Allowed licenses for this intent: {0}. Do not include secrets, credentials, " - "or personal data without consent and redaction. Record provenance per row." - ).format(licenses) - - next_steps = ( - "1. Draft the schema above as JSONL or Parquet locally (no Hub upload required).\n" - "2. Re-run `mlx-agent research` after datasets appear on the Hub, or keep this " - "blueprint as the creation guide.\n" - "3. When a base MLX model is chosen, use `mlx-agent adopt start` to verify and " - "wire it — research/blueprint never downloads or trains.\n" - "4. Fine-tune offline with your preferred MLX tooling against the local dataset." - ) - - return DatasetBlueprint( - goal=goal, - schema_fields=tuple(schema), - labeling_notes=labeling, - split_guidance=split, - license_privacy=license_privacy, - mlx_next_steps=next_steps, - ) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/command_executor.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/command_executor.py deleted file mode 100644 index 737ecaf..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/command_executor.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Bounded stdin executor for native provider custom tools.""" - -from __future__ import annotations - -import argparse -import io -import json -import sys - -from .gemini_args import GeminiArgumentError, parse_gemini_arguments - - -MAX_ARGUMENT_BYTES = 4096 -_PROVIDERS = ("opencode",) -_CAPABILITIES = ("scout", "adopt", "wire", "bench") - - -class CommandExecutorError(ValueError): - """Custom-tool input or execution failed without exposing raw arguments.""" - - -def read_command_arguments(stream): - """Read one small UTF-8 argument payload from a byte stream.""" - if not hasattr(stream, "read"): - raise CommandExecutorError("custom tool input was rejected") - try: - value = stream.read(MAX_ARGUMENT_BYTES + 1) - except (OSError, ValueError, TypeError) as error: - raise CommandExecutorError("custom tool input was rejected") from error - if not isinstance(value, bytes) or len(value) > MAX_ARGUMENT_BYTES: - raise CommandExecutorError("custom tool input was rejected") - try: - return value.decode("utf-8") - except UnicodeDecodeError as error: - raise CommandExecutorError("custom tool input was rejected") from error - - -def execute_command(provider, capability, stream, core=None): - """Validate stdin and execute only the capability's allowlisted core argv.""" - if provider not in _PROVIDERS or capability not in _CAPABILITIES: - raise CommandExecutorError("custom tool request was rejected") - try: - arguments = parse_gemini_arguments(capability, read_command_arguments(stream)) - except (GeminiArgumentError, CommandExecutorError) as error: - raise CommandExecutorError("custom tool request was rejected") from error - if core is None: - from .cli import main as core - try: - result = core(arguments) - except Exception as error: - raise CommandExecutorError("custom tool execution failed") from error - try: - exit_code = 0 if result is None else int(result) - except (TypeError, ValueError) as error: - raise CommandExecutorError("custom tool execution failed") from error - return {"status": "ok", "provider": provider, "capability": capability, "exit_code": exit_code} - - -def main(argv=None, stdin=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--provider", required=True, choices=_PROVIDERS) - parser.add_argument("--capability", required=True, choices=_CAPABILITIES) - arguments = parser.parse_args(argv) - try: - result = execute_command(arguments.provider, arguments.capability, stdin or sys.stdin.buffer) - except CommandExecutorError: - print(json.dumps({"status": "error", "provider": arguments.provider, "capability": arguments.capability, "error": "command rejected"}, sort_keys=True)) - return 2 - print(json.dumps(result, sort_keys=True)) - return result["exit_code"] - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/contextfit.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/contextfit.py deleted file mode 100644 index b355316..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/contextfit.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Bounded KV-cache arithmetic for context-aware memory fit.""" - -from __future__ import annotations - - -_MAX_INT = 2 ** 63 - 1 -MAX_ARCH_VALUE = 1000000 -DEFAULT_DTYPE_BYTES = 2 -CONTEXT_MIN, CONTEXT_MAX = 1024, 1048576 - - -def extract_architecture(config): - """Return bounded architecture facts from a Hugging Face config object.""" - if not isinstance(config, dict): - return None - - def _positive_int(key): - value = config.get(key) - if ( - not isinstance(value, int) - or isinstance(value, bool) - or not 0 < value <= MAX_ARCH_VALUE - ): - return None - return value - - layers = _positive_int("num_hidden_layers") - attention_heads = _positive_int("num_attention_heads") - kv_heads = _positive_int("num_key_value_heads") or attention_heads - head_dim = _positive_int("head_dim") - if head_dim is None: - hidden_size = _positive_int("hidden_size") - if hidden_size is not None and attention_heads: - if hidden_size % attention_heads != 0: - return None - head_dim = hidden_size // attention_heads - if layers is None or kv_heads is None or head_dim is None: - return None - return { - "layers": layers, - "kv_heads": kv_heads, - "head_dim": head_dim, - "max_position_embeddings": _positive_int("max_position_embeddings"), - } - - -def kv_bytes_per_token(architecture, dtype_bytes=DEFAULT_DTYPE_BYTES): - if not isinstance(architecture, dict): - return None - layers = architecture.get("layers") - kv_heads = architecture.get("kv_heads") - head_dim = architecture.get("head_dim") - if not all(isinstance(value, int) and value > 0 for value in (layers, kv_heads, head_dim)): - return None - return min(2 * layers * kv_heads * head_dim * dtype_bytes, _MAX_INT) - - -def kv_cache_bytes(architecture, context_tokens, dtype_bytes=DEFAULT_DTYPE_BYTES): - per_token = kv_bytes_per_token(architecture, dtype_bytes) - if per_token is None: - return None - if not isinstance(context_tokens, int) or isinstance(context_tokens, bool): - raise TypeError("context_tokens must be an integer") - if not CONTEXT_MIN <= context_tokens <= CONTEXT_MAX: - raise ValueError( - "context_tokens must be between {0} and {1}".format(CONTEXT_MIN, CONTEXT_MAX) - ) - return min(per_token * context_tokens, _MAX_INT) - - -def max_context_tokens(architecture, weight_bytes, budget_bytes, dtype_bytes=DEFAULT_DTYPE_BYTES): - """Largest context whose weights + KV fit the budget; None when unknown.""" - per_token = kv_bytes_per_token(architecture, dtype_bytes) - if per_token is None: - return None - if not isinstance(weight_bytes, (int, float)) or not isinstance(budget_bytes, (int, float)): - return None - if weight_bytes < 0 or budget_bytes <= 0: - return None - headroom = budget_bytes - weight_bytes - if headroom <= 0: - return 0 - maximum = int(headroom // per_token) - cap = architecture.get("max_position_embeddings") - if isinstance(cap, int) and cap > 0: - maximum = min(maximum, cap) - return min(maximum, CONTEXT_MAX) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/contracts.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/contracts.py deleted file mode 100644 index 7f1accf..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/contracts.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Versioned result contracts shared by every provider adapter.""" - -from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional - - -SCHEMA_VERSION = "1.0" - - -def _require_non_empty_string(value: Any, name: str) -> None: - if not isinstance(value, str): - raise TypeError("{0} must be a string".format(name)) - if not value: - raise ValueError("{0} must not be empty".format(name)) - - -def _validate_warnings(warnings: Any) -> List[Dict[str, str]]: - if warnings is None: - return [] - if not isinstance(warnings, list): - raise TypeError("warnings must be a list of objects of strings") - - validated: List[Dict[str, str]] = [] - for index, warning in enumerate(warnings): - if not isinstance(warning, dict) or not all( - isinstance(item, str) for item in warning.values() - ): - raise TypeError("warnings[{0}] must be an object of strings".format(index)) - validated.append(dict(warning)) - return validated - - -@dataclass(frozen=True) -class ErrorDetail: - """A stable, actionable description of an unsuccessful operation.""" - - code: str - message: str - remediation: str - retryable: bool = False - - -@dataclass(frozen=True) -class ResultEnvelope: - """The provider-neutral result returned by MLX agent operations.""" - - operation: str - status: str - data: Dict[str, Any] = field(default_factory=dict) - warnings: List[Dict[str, str]] = field(default_factory=list) - error: Optional[ErrorDetail] = None - schema_version: str = SCHEMA_VERSION - generated_at: str = field( - default_factory=lambda: datetime.now(timezone.utc).isoformat() - ) - - @classmethod - def ok(cls, operation: str, data: Dict[str, Any], warnings=None): - """Create a successful versioned result.""" - _require_non_empty_string(operation, "operation") - if not isinstance(data, dict): - raise TypeError("data must be an object") - return cls( - operation=operation, - status="ok", - data=dict(data), - warnings=_validate_warnings(warnings), - ) - - @classmethod - def fail( - cls, - operation: str, - code: str, - message: str, - remediation: str, - retryable=False, - ): - """Create an error result with a machine-readable recovery path.""" - _require_non_empty_string(operation, "operation") - _require_non_empty_string(code, "code") - _require_non_empty_string(message, "message") - _require_non_empty_string(remediation, "remediation") - if not isinstance(retryable, bool): - raise TypeError("retryable must be a boolean") - return cls( - operation=operation, - status="error", - error=ErrorDetail(code, message, remediation, retryable), - ) - - def to_dict(self) -> Dict[str, Any]: - """Serialize the envelope without an empty error object on success.""" - value = asdict(self) - if self.error is None: - value.pop("error") - return value diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/convert.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/convert.py deleted file mode 100644 index 908d885..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/convert.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Confirmation-gated local MLX quantization jobs.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from pathlib import Path - -from .serve import ( - _argv_matches, - _pid_alive, - _pid_command, - _terminate_pid, -) -from .transactions import _atomic_in_directory -from .wiring import _MODEL - - -CONVERT_RECEIPT_SCHEMA_VERSION = "1.0" -CONVERT_RECEIPT_KIND = "convert" -Q_BITS_CHOICES = (4, 8) -EXECUTABLE = "mlx_lm.convert" -MAX_LOG_TAIL_BYTES = 64 * 1024 - - -class ConvertError(RuntimeError): - """Classified convert failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def receipts_root(root=None, kind="convert"): - base = Path(root) if root is not None else Path.cwd() - return base / ".mlx-agent-receipts" / kind - - -def plan_convert(repo, q_bits=4, out=None): - """Render the exact conversion plan; pure and side-effect free.""" - if not isinstance(repo, str) or not _MODEL.fullmatch(repo): - raise ConvertError( - "invalid_repo", - "convert requires a safe publisher/model identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - if not isinstance(q_bits, int) or isinstance(q_bits, bool): - raise ConvertError( - "invalid_arguments", "q_bits must be an integer.", - "Pass --q-bits 4 or --q-bits 8.", - ) - if q_bits not in Q_BITS_CHOICES: - raise ConvertError( - "invalid_arguments", - "q_bits must be one of {0}.".format(list(Q_BITS_CHOICES)), - "Only bounded 4bit and 8bit recipes are supported in v1.", - ) - if out is None: - name = repo.split("/", 1)[1] - out = "{0}-MLX-{1}bit".format(name, q_bits) - plan = { - "repo": repo, - "q_bits": q_bits, - "out": str(out), - "argv": [ - EXECUTABLE, - "--hf-path", repo, - "--mlx-path", str(out), - "--q-bits", str(q_bits), - ], - } - canonical = json.dumps(plan, sort_keys=True, separators=(",", ":")) - plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() - return plan - - -def _receipt_path(root, plan): - return root / "{0}-{1}bit.json".format( - plan["repo"].split("/", 1)[1], plan["q_bits"] - ) - - -def _read_receipt(path, kind=CONVERT_RECEIPT_KIND): - try: - value = json.loads(Path(path).read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - return None - if not isinstance(value, dict) or value.get("kind") != kind: - return None - if not isinstance(value.get("pid"), int) or isinstance(value.get("pid"), bool): - return None - if not isinstance(value.get("argv"), list) or not isinstance(value.get("repo"), str): - return None - return value - - -def _write_receipt(root, receipt, filename): - content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") - _atomic_in_directory(root, filename, content, 0o600) - - -def start_convert(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, spawn=None, now=_utc_now, - pid_alive=None): - """Execute a reviewed conversion plan; the only mutating entry point.""" - from .serve import _default_spawn, _default_which - - which = which or _default_which - spawn = spawn or _default_spawn - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise ConvertError( - "preview_hash_required", - "--confirm requires the hash from a reviewed convert preview.", - "Run convert start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise ConvertError( - "preview_stale", - "The supplied preview hash does not match this convert plan.", - "Re-run convert start without --confirm and review the fresh plan.", - ) - if which(EXECUTABLE) is None: - raise ConvertError( - "runtime_not_installed", - "The {0} executable is not installed.".format(EXECUTABLE), - "Install mlx-lm yourself (pip install mlx-lm); convert never installs runtimes.", - ) - if model_present is not None and not model_present(plan["repo"]): - raise ConvertError( - "model_not_local", - "The source model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; convert never downloads.", - ) - if Path(plan["out"]).exists(): - raise ConvertError( - "output_exists", - "The output path already exists: {0}".format(plan["out"]), - "Pick a fresh --out path; convert never overwrites.", - ) - for receipt_path in sorted(root.glob("*.json")) if root.is_dir() else []: - existing = _read_receipt(receipt_path) - if existing is None or existing.get("completed_at"): - continue - if pid_alive(existing["pid"]): - raise ConvertError( - "job_in_progress", - "A convert job is still running (pid {0}).".format(existing["pid"]), - "Wait for it to finish (convert status) before starting another.", - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}-{1}bit.log".format(plan["repo"].split("/", 1)[1], plan["q_bits"]) - pid = spawn(plan["argv"], str(log_path)) - receipt = { - "schema_version": CONVERT_RECEIPT_SCHEMA_VERSION, - "kind": CONVERT_RECEIPT_KIND, - "repo": plan["repo"], - "q_bits": plan["q_bits"], - "out": plan["out"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "started_at": now(), - "preview_hash": plan["preview_hash"], - "completed_at": None, - "exit_status": None, - } - _write_receipt( - root, receipt, - "{0}-{1}bit.json".format(receipt["repo"].split("/", 1)[1], receipt["q_bits"]), - ) - return {"status": "started", "receipt": receipt} - - -def status_convert(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - now=_utc_now): - """Cross-check convert receipts against live processes; marks exits once.""" - root = receipts_root(receipts_dir) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - receipt = _read_receipt(path) - if receipt is None: - continue - entry = { - "receipt": str(path), - "repo": receipt["repo"], - "q_bits": receipt["q_bits"], - "out": receipt["out"], - "pid": receipt["pid"], - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - "completed_at": receipt.get("completed_at"), - } - if receipt.get("completed_at"): - entry["state"] = receipt.get("exit_status") or "done" - entries.append(entry) - continue - if pid_alive(receipt["pid"]): - command = pid_command(receipt["pid"]) - entry["state"] = "running" if _argv_matches( - {"argv": receipt["argv"], "port": None, "repo": receipt["repo"]}, - command, - require_port=False, - ) else "unknown" - entries.append(entry) - continue - exit_status = "done" if Path(receipt["out"]).exists() else "failed" - receipt["completed_at"] = now() - receipt["exit_status"] = exit_status - _write_receipt( - root, receipt, - "{0}-{1}bit.json".format(receipt["repo"].split("/", 1)[1], receipt["q_bits"]), - ) - entry["state"] = exit_status - entry["completed_at"] = receipt["completed_at"] - entries.append(entry) - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/discovery.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/discovery.py deleted file mode 100644 index 889e34d..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/discovery.py +++ /dev/null @@ -1,528 +0,0 @@ -"""Model discovery, filtering, ranking, and cached structured Scout results.""" - -from __future__ import annotations - -import hashlib -import json -import os -import tempfile -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path - -from .contracts import ResultEnvelope -from .host import HostInventory -from .huggingface import HuggingFaceClient -from .models import ( - DISCOVERY_ROLES, - REPUTABLE, - base_name, - classify, - classify_roles, - infer_quantization, - quant_rank, - resolve_ram, - resolve_reasoning, - wiring, -) - - -CACHE_SCHEMA_VERSION = "2.0" -CACHE_TTL_SECONDS = 24 * 60 * 60 - - -@dataclass(frozen=True) -class DiscoveryRequest: - """Bounded, explicit discovery policy. Legacy ``new`` and ``fast`` remain supported.""" - - role: object = None - memory_gb: object = None - quantization: object = None - licenses: object = None - include_gated: bool = True - publishers: object = None - runtime: object = None - refresh: bool = False - offline: bool = False - limit: int = 6 - new: bool = False - fast: bool = False - context_tokens: object = None - - def cache_request(self): - """The reproducible request shape, excluding transport-only cache switches.""" - return { - "role": self.role, - "memory_gb": self.memory_gb, - "quantization": _canonical_quantization(self.quantization), - "licenses": _normalise_values(self.licenses), - "include_gated": self.include_gated, - "publishers": _normalise_values(self.publishers), - "runtime": self.runtime, - "limit": self.limit, - "new": self.new, - "fast": self.fast, - "context_tokens": self.context_tokens, - } - - -def _normalise_values(value): - if value is None: - return [] - if isinstance(value, str): - return [value.lower()] - return sorted(str(item).lower() for item in value) - - -def _canonical_quantization(value): - if value is None: - return None - normalized = infer_quantization(str(value)) - return normalized or str(value).strip().lower() - - -def _utc_now(): - return datetime.now(timezone.utc) - - -def _parse_timestamp(value): - try: - return datetime.fromisoformat(value.replace("Z", "+00:00")) - except (AttributeError, TypeError, ValueError): - return None - - -def _community_bench_for(repo, chip): - """Return the community bench entry matching repo and host chip, if any.""" - if not isinstance(repo, str) or not isinstance(chip, str) or not chip: - return None - try: - from .bench import load_community_bench - - entry = load_community_bench().get((repo, chip)) - except Exception: - return None - if not isinstance(entry, dict) or entry.get("decode_toks") is None: - return None - result = dict(entry) - result["chip"] = chip - return result - - -def _kv_estimate(architecture, ram_gb, budget_gb, context_tokens): - """Attach KV-cache math when architecture, weights, and budget are known.""" - if not isinstance(architecture, dict) or ram_gb is None or budget_gb is None: - return None - from .contextfit import ( - DEFAULT_DTYPE_BYTES, - kv_bytes_per_token, - kv_cache_bytes, - max_context_tokens, - ) - - if kv_bytes_per_token(architecture) is None: - return None - weight_bytes = float(ram_gb) * 1e9 - budget_bytes = float(budget_gb) * 0.8 * 1e9 - block = { - "max_context_tokens": max_context_tokens(architecture, weight_bytes, budget_bytes), - "dtype_bytes": DEFAULT_DTYPE_BYTES, - "src": "huggingface_model_metadata", - } - if context_tokens is not None: - kv_bytes = kv_cache_bytes(architecture, context_tokens) - block["context_tokens"] = context_tokens - block["kv_gb"] = round(kv_bytes / 1e9, 1) if kv_bytes is not None else None - return block - - -def _estimates(ram, budget, kv_block): - estimates = {"ram_gb": ram, "memory_budget_gb": budget, "headroom_fraction": 0.2} - if kv_block is not None: - estimates["kv"] = kv_block - return estimates - - -class DiscoveryService: - def __init__(self, host=None, huggingface=None, state_dir=None, cache_enabled=None, cache_ttl_seconds=CACHE_TTL_SECONDS): - self._host = host - self._huggingface = huggingface or HuggingFaceClient() - injected_state_dir = state_dir or os.environ.get("MLX_AGENT_STATE_DIR") - self._state_dir = Path(injected_state_dir) if injected_state_dir else None - self._cache_enabled = self._state_dir is not None if cache_enabled is None else cache_enabled - self._cache_ttl_seconds = cache_ttl_seconds - - @property - def host(self): - if self._host is None: - self._host = HostInventory.detect(self._huggingface.http_get) - return self._host - - def discover(self, request): - cache = self._read_cache(request) - if request.offline: - if cache is None: - return ResultEnvelope.fail( - "discover", - "offline_cache_missing", - "No cached discovery response matches this request.", - "Run discovery while online first, or remove --offline.", - ) - data, stale = cache - warnings = [_warning("stale_cache")] if stale else [] - return ResultEnvelope.ok("discover", self._response_data(request, data, "stale" if stale else "fresh"), warnings=warnings) - if cache is not None and not cache[1] and not request.refresh: - return ResultEnvelope.ok("discover", self._response_data(request, cache[0], "fresh")) - - host = self.host - list_sort = "lastModified" if request.new else "trendingScore" - list_url_builder = getattr(self._huggingface, "list_models_url", None) - list_url = ( - list_url_builder(sort=list_sort, limit_fetch=300) - if callable(list_url_builder) else None - ) - try: - raw = self._huggingface.list_models(sort=list_sort) - except Exception as error: - return ResultEnvelope.fail( - "discover", - "network_unavailable", - "HuggingFace query failed: {0}".format(error), - "Check network access to huggingface.co and retry the discovery command.", - retryable=True, - ) - - data = self._build_response(request, raw, host.to_dict(), list_url=list_url) - self._write_cache(request, data) - return ResultEnvelope.ok("discover", self._response_data(request, data, "refreshed" if request.refresh else "miss")) - - def _build_response(self, request, raw, host_data, list_url=None): - buckets = {role: [] for role in DISCOVERY_ROLES} - rejected = {} - seen_repo, seen_base = set(), {} - rejection_limit = max(1, int(request.limit or 1)) * 3 - for model in raw: - repo = model.get("id") or model.get("modelId") - if not repo or repo in seen_repo: - continue - seen_repo.add(repo) - if ( - request.role - and request.role != "tool-use" - and classify(repo) != request.role - ): - continue - enrichment = {} if request.fast else self._huggingface.inspect_model(repo) - roles, tool_use = classify_roles(repo, enrichment) - selected_roles = ( - (request.role,) if request.role in roles else () - ) if request.role else roles - if not selected_roles: - continue - accepted = False - rejected_candidate = None - for role in selected_roles: - candidate = self._candidate( - repo, - model, - role, - roles, - tool_use, - host_data, - enrichment, - request, - list_url, - ) - reasons = self._rejection_reasons(candidate, request) - candidate["rejection_reasons"] = reasons - if reasons: - rejected_candidate = rejected_candidate or candidate - continue - accepted = True - key = (role, candidate["base"]) - current = seen_base.get(key) - if current is not None: - if self._selection_key(candidate) > self._selection_key(current): - buckets[role].remove(current) - buckets[role].append(candidate) - seen_base[key] = candidate - continue - seen_base[key] = candidate - buckets[role].append(candidate) - if not accepted and rejected_candidate is not None: - if len(rejected) < rejection_limit: - rejected[repo] = rejected_candidate - - for role in buckets: - buckets[role].sort(key=self._selection_key, reverse=True) - buckets[role] = buckets[role][: max(0, int(request.limit))] - - return { - "host": host_data, - "fast": request.fast, - "roles": {role: items for role, items in buckets.items() if items}, - "rejected": rejected, - "request": request.cache_request(), - } - - def _candidate( - self, - repo, - model, - role, - roles, - tool_use, - host_data, - enrichment, - request, - list_url=None, - ): - ram, ram_source = resolve_ram(repo, enrichment) - reasoning, reason_source = resolve_reasoning(repo, enrichment) - publisher = repo.split("/", 1)[0].lower() - quantization = infer_quantization(repo) - downloads = model.get("downloads", 0) - likes = model.get("likes", 0) - trusted = publisher in REPUTABLE - budget = request.memory_gb if request.memory_gb is not None else host_data.get("ram_gb") - architecture = enrichment.get("architecture") - kv_block = _kv_estimate(architecture, ram, budget, request.context_tokens) - fits = ram is None or budget is None or ram < float(budget) * 0.8 - if request.context_tokens is not None and kv_block is not None: - fits = (ram + kv_block["kv_gb"]) < float(budget) * 0.8 - community_bench = _community_bench_for(repo, host_data.get("chip")) - rank_score = (1000000 if trusted else 0) + (quant_rank(repo) * 10000) + min(downloads, 9999) + min(likes, 999) - metadata_available = enrichment.get("metadata_available") is True - gated_status = "unknown" - if metadata_available: - gated_status = "gated" if enrichment.get("gated") else "public" - facts = { - "repository": repo, - "downloads": downloads, - "likes": likes, - "gated": gated_status, - "license": enrichment.get("license") if metadata_available else None, - } - if enrichment.get("tree_available") is True and enrichment.get("weight_bytes"): - facts["weight_bytes"] = enrichment["weight_bytes"] - candidate = { - # Legacy report keys. - "repo": repo, - "role": role, - "roles": list(roles), - "downloads": downloads, - "likes": likes, - "trusted": trusted, - "base": base_name(repo), - "qrank": quant_rank(repo), - "est_ram_gb": ram, - "ram_src": ram_source, - "fits": fits, - "reasoning": reasoning, - "reason_src": reason_source, - "gated": enrichment.get("gated") if metadata_available else None, - "license": facts["license"], - "wiring": wiring(repo, role, host_data), - # Explainable structured fields. Memory remains an estimate, even when - # the weights came from a measured file-size listing. - "facts": facts, - "estimates": _estimates(ram, budget, kv_block), - "tool_use": tool_use, - "heuristics": { - "role": role, - "primary_role": roles[0], - "roles": list(roles), - "quantization": quantization, - "trusted_publisher": trusted, - }, - "provenance": self._provenance( - repo, - role, - tool_use, - enrichment, - request.fast, - ram_source, - reason_source, - list_url, - ), - "rank_score": rank_score, - "selection_reasons": self._selection_reasons(role, quantization, trusted, fits), - "rejection_reasons": [], - } - if community_bench is not None: - candidate["community_bench"] = community_bench - return candidate - - @staticmethod - def _provenance( - repo, - role, - tool_use, - enrichment, - fast, - ram_source, - reason_source, - list_url=None, - ): - encoded = repo.replace("/", "%2F") - records = [{ - "source": "huggingface_model_list", - "url": list_url or "https://huggingface.co/api/models", - "fields": ["repository", "downloads", "likes"], - }] - if not fast and enrichment.get("metadata_available") is True: - metadata_fields = ["gated", "license"] - if reason_source in ("chat_template", "tags", "checked"): - metadata_fields.append("reasoning") - if tool_use["source"] in ("chat_template", "tags", "checked"): - metadata_fields.append("tool_use") - records.append({ - "source": "huggingface_model_metadata", - "url": enrichment.get("metadata_url") or "https://huggingface.co/api/models/{0}".format(encoded), - "fields": metadata_fields, - }) - if not fast and enrichment.get("tree_available") is True and enrichment.get("weight_bytes"): - records.append({ - "source": "huggingface_repository_tree", - "url": enrichment.get("tree_url") or "https://huggingface.co/api/models/{0}/tree/main?recursive=true".format(encoded), - "fields": ["weight_bytes"], - }) - if tool_use["source"] in ("chat_template", "tags", "checked"): - role_basis = ( - "primary role from repository name; tool-use signal from " - "Hugging Face metadata ({0})" - ).format(tool_use["source"]) - elif tool_use["source"] == "name": - role_basis = "primary role and tool-use signal from repository name" - else: - role_basis = "primary role from repository name; no tool-use signal" - records.append({ - "source": "local_role_derivation", - "fields": ["role", "roles", "primary_role"], - "basis": role_basis, - }) - name_fields = ["quantization", "trusted_publisher"] - if tool_use["source"] == "name": - name_fields.append("tool_use") - if reason_source == "name": - name_fields.append("reasoning") - records.append({ - "source": "local_name_derivation", - "fields": name_fields, - }) - records.append({ - "source": "local_memory_estimate", - "fields": ["ram_gb", "fits"], - "basis": ram_source or "unavailable", - }) - records.append({ - "source": "local_ranking_derivation", - "fields": ["rank_score", "selection_reasons", "rejection_reasons"], - }) - return records - - @staticmethod - def _selection_reasons(role, quantization, trusted, fits): - reasons = ["role membership: {0}".format(role)] - if quantization: - reasons.append("quantization inferred from repository name: {0}".format(quantization)) - if trusted: - reasons.append("publisher is in the configured reputable set") - if fits: - reasons.append("estimated weights fit within the memory budget with headroom") - return reasons - - @staticmethod - def _selection_key(candidate): - # The final repository ID makes ranking stable across equal evidence. - return (candidate["trusted"], candidate["qrank"], candidate["downloads"], candidate["likes"], candidate["repo"]) - - @staticmethod - def _rejection_reasons(candidate, request): - reasons = [] - licenses = set(_normalise_values(request.licenses)) - publishers = set(_normalise_values(request.publishers)) - wanted_quantization = _canonical_quantization(request.quantization) - if not request.include_gated and candidate["facts"]["gated"] == "gated": - reasons.append("gated") - if not request.include_gated and candidate["facts"]["gated"] == "unknown": - reasons.append("gated_status_unknown") - if licenses and (candidate["license"] or "").lower() not in licenses: - reasons.append("license") - if publishers and candidate["repo"].split("/", 1)[0].lower() not in publishers: - reasons.append("publisher") - if wanted_quantization and candidate["heuristics"]["quantization"] != wanted_quantization: - reasons.append("quantization") - if request.memory_gb is not None and not candidate["fits"]: - reasons.append("memory_budget") - if request.runtime and not HostInventory.runtime_supports(request.runtime, candidate["heuristics"]["role"]): - reasons.append("runtime") - return reasons - - def _cache_path(self, request): - encoded = json.dumps(request.cache_request(), sort_keys=True, separators=(",", ":")).encode("utf-8") - return self._state_dir / "discovery-{0}.json".format(hashlib.sha256(encoded).hexdigest()[:20]) - - def _read_cache(self, request): - if not self._cache_enabled: - return None - try: - payload = json.loads(self._cache_path(request).read_text()) - if payload.get("schema_version") != CACHE_SCHEMA_VERSION or payload.get("request") != request.cache_request(): - return None - fetched_at = _parse_timestamp(payload.get("fetched_at")) - if fetched_at is None or not isinstance(payload.get("response"), dict): - return None - stale = (_utc_now() - fetched_at).total_seconds() > self._cache_ttl_seconds - return payload["response"], stale - except (OSError, TypeError, ValueError, json.JSONDecodeError): - return None - - def _write_cache(self, request, response): - if not self._cache_enabled: - return - try: - self._state_dir.mkdir(parents=True, exist_ok=True) - payload = { - "schema_version": CACHE_SCHEMA_VERSION, - "fetched_at": _utc_now().isoformat(), - "request": request.cache_request(), - "response": response, - } - destination = self._cache_path(request) - with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=str(destination.parent), delete=False) as temporary: - json.dump(payload, temporary, sort_keys=True) - temporary.flush() - os.fsync(temporary.fileno()) - temporary_name = temporary.name - os.replace(temporary_name, destination) - except OSError: - # Discovery evidence is still useful if the local cache cannot be written. - return - - def _cached_data(self, response, status): - data = dict(response) - data["cache"] = {"status": status, "ttl_seconds": self._cache_ttl_seconds} - return data - - def _response_data(self, request, response, status): - if not self._cache_enabled and self._legacy_compatible(request): - return {key: response[key] for key in ("host", "fast", "roles")} - return self._cached_data(response, status) - - @staticmethod - def _legacy_compatible(request): - return ( - request.memory_gb is None - and request.quantization is None - and request.licenses is None - and request.include_gated - and request.publishers is None - and request.runtime is None - and not request.refresh - and not request.offline - ) - - -def _warning(code): - return {"code": code, "message": "Cached discovery response is older than the 24 hour TTL."} diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/fleet.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/fleet.py deleted file mode 100644 index 71de98f..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/fleet.py +++ /dev/null @@ -1,218 +0,0 @@ -"""One-shot per-role routing configuration backed by Wire transactions.""" - -from __future__ import annotations - -import json -import re -from pathlib import Path - -from .wiring import _MODEL, require_secret_free_config - - -FLEET_RUNTIMES = ("mlx_lm", "mlx-vlm") -ROLE_DEFAULT_RUNTIME = { - "general": "mlx_lm", - "coding": "mlx_lm", - "reasoning": "mlx_lm", - "embedding": "mlx_lm", - "tool-use": "mlx_lm", - "vision": "mlx-vlm", -} -RUNTIME_PORTS = {"mlx_lm": 8080, "mlx-vlm": 8083} -_ROLE = re.compile(r"^[a-z][a-z0-9-]{0,31}$") -MAX_ASSIGNMENTS = 6 - -_HEADER = "# MLX_AGENT_WIRE v1" - - -class FleetError(ValueError): - """Classified fleet failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def parse_assignments(values): - """Parse repeatable role=repo assignments into an ordered mapping.""" - assignments = {} - for value in values or []: - if not isinstance(value, str) or "=" not in value: - raise FleetError( - "invalid_assignment", - "Assignments must use role=repo form: {0}".format(value), - "Pass --assign coding=publisher/model.", - ) - role, repo = value.split("=", 1) - role = role.strip() - repo = repo.strip() - if role not in ROLE_DEFAULT_RUNTIME: - raise FleetError( - "invalid_role", - "Unknown fleet role: {0}".format(role), - "Use one of: {0}.".format(", ".join(sorted(ROLE_DEFAULT_RUNTIME))), - ) - if role in assignments: - raise FleetError( - "duplicate_role", - "Role {0} is assigned twice.".format(role), - "Assign each role at most once.", - ) - if not _MODEL.fullmatch(repo): - raise FleetError( - "invalid_repo", - "Model must be a safe publisher/model identifier: {0}".format(repo), - "Pass the Hugging Face publisher/model form.", - ) - assignments[role] = repo - if not assignments: - raise FleetError( - "missing_assignments", - "Fleet requires at least one role assignment.", - "Pass --assign role=repo or --from-adoption .", - ) - if len(assignments) > MAX_ASSIGNMENTS: - raise FleetError( - "invalid_assignment", - "Fleet supports at most {0} role assignments.".format(MAX_ASSIGNMENTS), - "Reduce the number of assigned roles.", - ) - return assignments - - -def assignments_from_adoption(state_path): - """Read role -> repo recommendations from a completed adopt handoff.""" - try: - value = json.loads(Path(state_path).read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError) as error: - raise FleetError( - "invalid_state", - "The adoption handoff could not be read: {0}".format(error), - "Pass a readable adopt state file produced by adopt start/resume.", - ) - recommendations = value.get("recommendations") - if not isinstance(recommendations, list) or not recommendations: - raise FleetError( - "empty_recommendations", - "The adoption handoff contains no recommendations.", - "Complete the adoption workflow first, then rerun fleet.", - ) - assignments = {} - for item in recommendations: - if not isinstance(item, dict): - continue - role = item.get("role") - repo = item.get("repo") - if role in ROLE_DEFAULT_RUNTIME and isinstance(repo, str) and role not in assignments: - assignments[role] = repo - return parse_assignments(["{0}={1}".format(role, repo) for role, repo in assignments.items()]) - - -def parse_runtime_map(values): - """Parse repeatable role=runtime overrides.""" - runtime_map = {} - for value in values or []: - if not isinstance(value, str) or "=" not in value: - raise FleetError( - "invalid_runtime_map", - "Runtime overrides must use role=runtime form: {0}".format(value), - "Pass --runtime-map vision=mlx-vlm.", - ) - role, runtime = value.split("=", 1) - role = role.strip() - runtime = runtime.strip() - if role not in ROLE_DEFAULT_RUNTIME: - raise FleetError( - "invalid_role", - "Unknown fleet role: {0}".format(role), - "Use one of: {0}.".format(", ".join(sorted(ROLE_DEFAULT_RUNTIME))), - ) - if runtime not in FLEET_RUNTIMES: - raise FleetError( - "unsupported_runtime", - "Fleet routes only through: {0}.".format(", ".join(FLEET_RUNTIMES)), - "Ollama and LM Studio manage their own routing; use mlx_lm or mlx-vlm.", - ) - runtime_map[role] = runtime - return runtime_map - - -class FleetConfigAdapter: - """Validate the exact bounded multi-model router subset fleet renders.""" - - version = "1.0" - runtime = "litellm-fleet" - - def __init__(self, path=None): - self.path = Path(path) if path is not None else None - - def render(self, assignments, runtime_map=None, existing=""): - runtime_map = runtime_map or {} - if not assignments: - raise FleetError( - "missing_assignments", - "Fleet requires at least one role assignment.", - "Pass --assign role=repo or --from-adoption .", - ) - if not isinstance(existing, str): - raise TypeError("existing config content must be text") - require_secret_free_config(existing) - if existing.strip() and not existing.startswith(_HEADER): - raise FleetError( - "existing_config_unmanaged", - "The target file exists and is not fleet-managed.", - "Pick a new --path or remove the unmanaged file yourself.", - ) - lines = [_HEADER, "model_list:"] - for role in sorted(assignments): - runtime = runtime_map.get(role, ROLE_DEFAULT_RUNTIME[role]) - if runtime not in FLEET_RUNTIMES: - raise FleetError( - "unsupported_runtime", - "Fleet routes only through: {0}.".format(", ".join(FLEET_RUNTIMES)), - "Use mlx_lm or mlx-vlm for every role.", - ) - port = RUNTIME_PORTS[runtime] - lines.extend([ - " - model_name: {0}".format(role), - " litellm_params:", - " model: openai/{0}".format(assignments[role]), - " api_base: http://127.0.0.1:{0}/v1".format(port), - " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY", - ]) - content = "\n".join(lines) + "\n" - self.validate(content) - return content - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("configuration content must be text") - require_secret_free_config(content) - lines = content.splitlines() - if len(lines) < 7 or (len(lines) - 2) % 5 != 0: - raise ValueError("fleet configuration must contain whole five-line entries") - if lines[0] != _HEADER or lines[1] != "model_list:": - raise ValueError("fleet configuration must start with the managed header") - roles = [] - for offset in range(2, len(lines), 5): - header, params, model, api_base, api_key = lines[offset:offset + 5] - role = re.fullmatch(r" - model_name: ([a-z][a-z0-9-]{0,31})", header) - if role is None or not _ROLE.fullmatch(role.group(1)): - raise ValueError("fleet entry has an invalid model_name") - if params != " litellm_params:": - raise ValueError("fleet entry is missing litellm_params") - model_match = re.fullmatch(r" model: openai/(" + _MODEL.pattern[1:-1] + r")", model) - if model_match is None: - raise ValueError("fleet entry has an invalid model identifier") - if api_base not in ( - " api_base: http://127.0.0.1:8080/v1", - " api_base: http://127.0.0.1:8083/v1", - ): - raise ValueError("fleet entry api_base must be a bounded loopback port") - if api_key != " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY": - raise ValueError("fleet entry must reference the managed API key env var") - roles.append(role.group(1)) - if len(roles) != len(set(roles)): - raise ValueError("fleet configuration contains duplicate roles") - return True diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/fuse.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/fuse.py deleted file mode 100644 index b35f13a..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/fuse.py +++ /dev/null @@ -1,213 +0,0 @@ -"""Confirmation-gated LoRA fuse jobs on the convert machinery.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from pathlib import Path - -from .convert import ( - _read_receipt, - _write_receipt, - receipts_root, -) -from .serve import _argv_matches, _pid_alive, _pid_command -from .wiring import _MODEL - - -FUSE_RECEIPT_SCHEMA_VERSION = "1.0" -FUSE_RECEIPT_KIND = "fuse" -EXECUTABLE = "mlx_lm.fuse" - - -class FuseError(RuntimeError): - """Classified fuse failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def validate_adapter(adapter_path): - """Confirm the adapter directory looks like an mlx_lm LoRA output.""" - root = Path(adapter_path) - if not root.is_dir(): - raise FuseError( - "adapter_invalid", - "The adapter path is not a directory: {0}".format(adapter_path), - "Point --adapter at a completed lora output directory.", - ) - config = root / "adapter_config.json" - if not config.is_file(): - raise FuseError( - "adapter_invalid", - "The adapter directory lacks adapter_config.json.", - "Point --adapter at a completed lora output directory.", - ) - try: - value = json.loads(config.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError) as error: - raise FuseError( - "adapter_invalid", - "adapter_config.json could not be parsed: {0}".format(error), - "Repair or regenerate the adapter before fusing.", - ) - if not isinstance(value, dict): - raise FuseError( - "adapter_invalid", - "adapter_config.json must contain a JSON object.", - "Repair or regenerate the adapter before fusing.", - ) - return {"adapter_config": True} - - -def plan_fuse(repo, adapter, out=None): - """Validate inputs and render the exact fuse plan; side-effect free.""" - if not isinstance(repo, str) or not _MODEL.fullmatch(repo): - raise FuseError( - "invalid_repo", - "fuse requires a safe publisher/model base identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - validate_adapter(adapter) - if out is None: - out = "{0}-fused".format(repo.split("/", 1)[1]) - plan = { - "repo": repo, - "adapter": str(adapter), - "out": str(out), - "argv": [ - EXECUTABLE, - "--model", repo, - "--adapter-path", str(adapter), - "--save-path", str(out), - ], - } - canonical = json.dumps(plan, sort_keys=True, separators=(",", ":")) - plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() - return plan - - -def start_fuse(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, spawn=None, now=_utc_now, - pid_alive=None): - """Execute a reviewed fuse plan; the only mutating entry point.""" - from .serve import _default_spawn, _default_which - - which = which or _default_which - spawn = spawn or _default_spawn - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir, FUSE_RECEIPT_KIND) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise FuseError( - "preview_hash_required", - "--confirm requires the hash from a reviewed fuse preview.", - "Run fuse start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise FuseError( - "preview_stale", - "The supplied preview hash does not match this fuse plan.", - "Re-run fuse start without --confirm and review the fresh plan.", - ) - if which(EXECUTABLE) is None: - raise FuseError( - "runtime_not_installed", - "The {0} executable is not installed.".format(EXECUTABLE), - "Install mlx-lm yourself (pip install mlx-lm); fuse never installs runtimes.", - ) - if model_present is not None and not model_present(plan["repo"]): - raise FuseError( - "model_not_local", - "The base model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; fuse never downloads.", - ) - if Path(plan["out"]).exists(): - raise FuseError( - "output_exists", - "The fused output path already exists: {0}".format(plan["out"]), - "Pick a fresh --out path; fuse never overwrites.", - ) - for receipt_path in sorted(root.glob("*.json")) if root.is_dir() else []: - existing = _read_receipt(receipt_path, FUSE_RECEIPT_KIND) - if existing is None or existing.get("completed_at"): - continue - if pid_alive(existing["pid"]): - raise FuseError( - "job_in_progress", - "A fuse job is still running (pid {0}).".format(existing["pid"]), - "Wait for it to finish (fuse status) before starting another.", - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}-fuse.log".format(plan["repo"].split("/", 1)[1]) - pid = spawn(plan["argv"], str(log_path)) - receipt = { - "schema_version": FUSE_RECEIPT_SCHEMA_VERSION, - "kind": FUSE_RECEIPT_KIND, - "repo": plan["repo"], - "adapter": plan["adapter"], - "out": plan["out"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "started_at": now(), - "preview_hash": plan["preview_hash"], - "completed_at": None, - "exit_status": None, - } - _write_receipt(root, receipt, "{0}-fuse.json".format(plan["repo"].split("/", 1)[1])) - return {"status": "started", "receipt": receipt} - - -def status_fuse(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - now=_utc_now): - """Cross-check fuse receipts against live processes; marks exits once.""" - root = receipts_root(receipts_dir, FUSE_RECEIPT_KIND) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - receipt = _read_receipt(path, FUSE_RECEIPT_KIND) - if receipt is None: - continue - entry = { - "receipt": str(path), - "repo": receipt["repo"], - "adapter": receipt.get("adapter"), - "out": receipt["out"], - "pid": receipt["pid"], - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - "completed_at": receipt.get("completed_at"), - } - if receipt.get("completed_at"): - entry["state"] = receipt.get("exit_status") or "done" - entries.append(entry) - continue - if pid_alive(receipt["pid"]): - command = pid_command(receipt["pid"]) - entry["state"] = "running" if _argv_matches( - {"argv": receipt["argv"], "port": None, "repo": receipt["repo"]}, - command, - require_port=False, - ) else "unknown" - entries.append(entry) - continue - exit_status = "done" if Path(receipt["out"]).exists() else "failed" - receipt["completed_at"] = now() - receipt["exit_status"] = exit_status - _write_receipt(root, receipt, path.name) - entry["state"] = exit_status - entry["completed_at"] = receipt["completed_at"] - entries.append(entry) - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/gemini_args.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/gemini_args.py deleted file mode 100644 index 886acaf..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/gemini_args.py +++ /dev/null @@ -1,340 +0,0 @@ -"""Strict argument grammar for Gemini custom-command input. - -Gemini TOML commands pass user text to the model, not directly to a process. -This module converts that opaque text into an allowlisted MLX-agent argv list -before a bundled skill may execute the structured CLI. -""" - -from __future__ import annotations - -import argparse -import json -import re -import shlex -import urllib.parse -from pathlib import Path - -from .models import ROLES - - -class GeminiArgumentError(ValueError): - """Gemini command input is outside the documented capability grammar.""" - - -_RUNTIMES = {"ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"} -_ROLES = {role for role, _keywords, _label in ROLES} -_MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,191}$") -_SAFE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+:-]{0,127}$") -_SAFE_QUANTIZATION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") -_METACHARACTERS = set(";|&`$<>") - - -def _tokens(raw): - if not isinstance(raw, str): - raise GeminiArgumentError("arguments must be text") - if len(raw) > 4096 or any(ord(character) < 32 or ord(character) == 127 for character in raw): - raise GeminiArgumentError("arguments must not contain control characters") - if any(character in _METACHARACTERS for character in raw): - raise GeminiArgumentError("shell metacharacters are not allowed") - try: - tokens = shlex.split(raw, posix=True) - except ValueError as error: - raise GeminiArgumentError("arguments must use balanced shell-style quotes: {0}".format(error)) - if len(tokens) > 32 or any(not token or len(token) > 512 for token in tokens): - raise GeminiArgumentError("arguments exceed the supported command grammar") - return tokens - - -def _value(tokens, index, flag): - if index + 1 >= len(tokens) or tokens[index + 1].startswith("--"): - raise GeminiArgumentError("{0} requires a value".format(flag)) - return tokens[index + 1] - - -def _path(value, label): - if not isinstance(value, str) or not value or value.startswith("~") or "\\" in value: - raise GeminiArgumentError("{0} must be a portable non-empty path".format(label)) - path = Path(value) - if any(part in {"", ".", ".."} for part in path.parts): - raise GeminiArgumentError("{0} must not traverse parent directories".format(label)) - return value - - -def _model(value): - if not _MODEL.fullmatch(value): - raise GeminiArgumentError("model must be a publisher/model identifier") - return value - - -def _integer(value, label, minimum, maximum): - if not re.fullmatch(r"[0-9]+", value): - raise GeminiArgumentError("{0} must be an integer".format(label)) - number = int(value) - if number < minimum or number > maximum: - raise GeminiArgumentError("{0} must be between {1} and {2}".format(label, minimum, maximum)) - return str(number) - - -def _decimal(value, label, minimum, maximum): - if not re.fullmatch(r"(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+)", value): - raise GeminiArgumentError("{0} must be a decimal number".format(label)) - number = float(value) - if number < minimum or number > maximum: - raise GeminiArgumentError("{0} must be between {1} and {2}".format(label, minimum, maximum)) - return value - - -def _endpoint(value): - try: - parsed = urllib.parse.urlsplit(value) - hostname = parsed.hostname - username = parsed.username - password = parsed.password - port = parsed.port - except (UnicodeError, ValueError) as error: - raise GeminiArgumentError("endpoint is not a valid local HTTP URL") from error - if ( - parsed.scheme not in {"http", "https"} - or hostname not in {"127.0.0.1", "localhost", "::1"} - or username is not None - or password is not None - or parsed.query - or parsed.fragment - ): - raise GeminiArgumentError("endpoint must be a credential-free local HTTP URL") - if port is not None and not 1 <= port <= 65535: - raise GeminiArgumentError("endpoint port must be between 1 and 65535") - return value - - -def _parse_scout(tokens): - argv, index = ["discover"], 0 - seen, cache = set(), None - values = { - "--role": (lambda value: value if value in _ROLES else None, "role"), - "--limit": (lambda value: _integer(value, "limit", 1, 50), "limit"), - "--memory-gb": (lambda value: _decimal(value, "memory-gb", 0.25, 1024), "memory-gb"), - "--quantization": (lambda value: value if _SAFE_QUANTIZATION.fullmatch(value) else None, "quantization"), - "--runtime": (lambda value: value if value in _RUNTIMES else None, "runtime"), - "--context": (lambda value: _integer(value, "context", 1024, 1048576), "context"), - "--state-dir": (lambda value: _path(value, "state-dir"), "state-dir"), - "--wire": (lambda value: _model(value), "wire"), - "--target": (lambda value: value if value in _RUNTIMES else None, "target"), - "--port": (lambda value: _integer(value, "port", 1, 65535), "port"), - } - repeatable = {"--license", "--publisher"} - booleans = {"--exclude-gated", "--include-gated", "--new", "--fast", "--json"} - while index < len(tokens): - flag = tokens[index] - if flag in booleans: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - if flag in {"--exclude-gated", "--include-gated"} and {"--exclude-gated", "--include-gated"} & seen: - raise GeminiArgumentError("gated-repository flags are mutually exclusive") - seen.add(flag) - argv.append(flag) - index += 1 - elif flag in {"--refresh", "--offline"}: - if cache is not None: - raise GeminiArgumentError("--refresh and --offline are mutually exclusive") - cache = flag - argv.append(flag) - index += 1 - elif flag in repeatable: - value = _value(tokens, index, flag) - if not _SAFE_NAME.fullmatch(value): - raise GeminiArgumentError("{0} must be a safe identifier".format(flag)) - argv.extend([flag, value]) - index += 2 - elif flag in values: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - value = values[flag][0](_value(tokens, index, flag)) - if value is None: - raise GeminiArgumentError("invalid {0} value".format(values[flag][1])) - seen.add(flag) - argv.extend([flag, value]) - index += 2 - else: - raise GeminiArgumentError("unsupported Scout argument: {0}".format(flag)) - return argv - - -def _parse_adopt(tokens): - if not tokens or tokens[0] not in {"start", "resume", "status"}: - raise GeminiArgumentError("Adopt requires start, resume, or status") - action, argv, index = tokens[0], ["adopt", tokens[0]], 1 - state = None - allowed_booleans = {"--json"} if action != "start" else {"--offline", "--refresh", "--fast", "--no-network", "--measure", "--json"} - while index < len(tokens): - flag = tokens[index] - if flag == "--state": - if state is not None: - raise GeminiArgumentError("duplicate flag: --state") - state = _path(_value(tokens, index, flag), "state") - argv.extend([flag, state]) - index += 2 - elif action == "start" and flag == "--role": - role = _value(tokens, index, flag) - if role not in _ROLES: - raise GeminiArgumentError("unknown adoption role") - argv.extend([flag, role]) - index += 2 - elif action == "start" and flag == "--shortlist-limit": - argv.extend([flag, _integer(_value(tokens, index, flag), "shortlist-limit", 1, 20)]) - index += 2 - elif flag in allowed_booleans and flag not in argv: - argv.append(flag) - index += 1 - else: - raise GeminiArgumentError("unsupported Adopt argument: {0}".format(flag)) - if state is None: - raise GeminiArgumentError("Adopt requires an explicit --state path") - return argv - - -def _parse_wire(tokens): - if not tokens or tokens[0] not in {"render", "apply", "status", "rollback"}: - raise GeminiArgumentError("Wire requires render, apply, status, or rollback") - action, argv, index = tokens[0], ["wire", tokens[0]], 1 - if action in {"status", "rollback"}: - if index >= len(tokens): - raise GeminiArgumentError("Wire {0} requires a receipt path".format(action)) - argv.append(_path(tokens[index], "receipt")) - index += 1 - allowed = ( - {"--json"} - if action == "status" - else {"--confirm", "--preview-hash", "--json"} - ) - else: - if index >= len(tokens): - raise GeminiArgumentError("Wire {0} requires a model".format(action)) - argv.append(_model(tokens[index])) - index += 1 - allowed = {"--target", "--path", "--json"} - if action == "apply": - allowed.update({"--confirm", "--preview-hash", "--receipts-dir", "--endpoint"}) - path_seen = False - while index < len(tokens): - flag = tokens[index] - if flag not in allowed: - raise GeminiArgumentError("unsupported Wire argument: {0}".format(flag)) - if flag in argv: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - if flag in {"--json", "--confirm"}: - argv.append(flag) - index += 1 - else: - value = _value(tokens, index, flag) - if flag == "--target": - if value not in _RUNTIMES: - raise GeminiArgumentError("unsupported Wire target") - elif flag in {"--path", "--receipts-dir"}: - value = _path(value, flag[2:]) - path_seen = path_seen or flag == "--path" - elif flag == "--preview-hash": - if not re.fullmatch(r"[0-9a-f]{64}", value): - raise GeminiArgumentError("preview-hash must be a SHA-256 digest") - elif flag == "--endpoint": - value = _endpoint(value) - argv.extend([flag, value]) - index += 2 - if action in {"render", "apply"} and not path_seen: - raise GeminiArgumentError("Wire {0} requires --path".format(action)) - return argv - - -def _parse_bench(tokens): - if not tokens or tokens[0] not in {"run", "aggregate"}: - raise GeminiArgumentError("Bench requires the run or aggregate action") - if tokens[0] == "aggregate": - return _parse_bench_aggregate(tokens) - argv, index = ["bench", "run"], 1 - seen = set() - values = { - "--repo": (lambda value: _model(value), "repo"), - "--runtime": (lambda value: value if value in _RUNTIMES else None, "runtime"), - "--role": (lambda value: value if value in _ROLES else None, "role"), - "--runs": (lambda value: _integer(value, "runs", 1, 10), "runs"), - "--gen-tokens": (lambda value: _integer(value, "gen-tokens", 16, 2048), "gen-tokens"), - "--timeout": (lambda value: _decimal(value, "timeout", 1, 600), "timeout"), - "--export": (lambda value: _path(value, "export"), "export"), - } - while index < len(tokens): - flag = tokens[index] - if flag == "--json": - if flag in seen: - raise GeminiArgumentError("duplicate flag: --json") - seen.add(flag) - argv.append(flag) - index += 1 - elif flag in values: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - value = values[flag][0](_value(tokens, index, flag)) - if value is None: - raise GeminiArgumentError("invalid {0} value".format(values[flag][1])) - seen.add(flag) - argv.extend([flag, value]) - index += 2 - else: - raise GeminiArgumentError("unsupported Bench argument: {0}".format(flag)) - for required in ("--repo", "--runtime"): - if required not in seen: - raise GeminiArgumentError("Bench run requires {0}".format(required)) - return argv - - -def _parse_bench_aggregate(tokens): - argv, index = ["bench", "aggregate"], 1 - seen = set() - while index < len(tokens): - flag = tokens[index] - if flag == "--json": - if flag in seen: - raise GeminiArgumentError("duplicate flag: --json") - seen.add(flag) - argv.append(flag) - index += 1 - elif flag in {"--exports", "--out"}: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - seen.add(flag) - argv.extend([flag, _path(_value(tokens, index, flag), flag[2:])]) - index += 2 - else: - raise GeminiArgumentError("unsupported Bench argument: {0}".format(flag)) - if "--exports" not in seen: - raise GeminiArgumentError("Bench aggregate requires --exports") - return argv - - -def parse_gemini_arguments(capability, raw): - """Return a validated argv list; this function never runs a process.""" - tokens = _tokens(raw) - if capability == "scout": - return _parse_scout(tokens) - if capability == "adopt": - return _parse_adopt(tokens) - if capability == "wire": - return _parse_wire(tokens) - if capability == "bench": - return _parse_bench(tokens) - raise GeminiArgumentError("unknown Gemini capability") - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("capability", choices=("scout", "adopt", "wire", "bench")) - parser.add_argument("raw", help="one opaque custom-command argument string") - arguments = parser.parse_args(argv) - try: - print(json.dumps({"argv": parse_gemini_arguments(arguments.capability, arguments.raw)})) - except GeminiArgumentError as error: - parser.error(str(error)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/gemini_executor.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/gemini_executor.py deleted file mode 100644 index 3a0fc12..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/gemini_executor.py +++ /dev/null @@ -1,140 +0,0 @@ -"""No-shell executor for Gemini custom-command argument files.""" - -from __future__ import annotations - -import argparse -import json -import os -import stat -import tempfile -from pathlib import Path - -from .gemini_args import GeminiArgumentError, parse_gemini_arguments - - -MAX_ARGS_BYTES = 4096 -_ARGS_DIRECTORY = "mlx-agent-gemini-command-args" - - -class GeminiCommandError(ValueError): - """A custom-command argument file is unsafe or invalid.""" - - -def command_args_root(): - """Return the private OS-temporary root reserved for Gemini argument files.""" - root = Path(tempfile.gettempdir()) / _ARGS_DIRECTORY - try: - root.mkdir(mode=0o700, parents=True, exist_ok=True) - details = os.lstat(str(root)) - except OSError as error: - raise GeminiCommandError("private argument storage is unavailable") from error - if ( - stat.S_ISLNK(details.st_mode) - or not stat.S_ISDIR(details.st_mode) - or details.st_uid != os.getuid() - or stat.S_IMODE(details.st_mode) != 0o700 - ): - raise GeminiCommandError("private argument storage is unsafe") - return root - - -def _safe_args_path(value): - root = command_args_root() - path = Path(value) - if not path.is_absolute() or path.parent != root or ".." in path.parts or not path.name: - raise GeminiCommandError("argument file path is outside private storage") - return path - - -def _discard_if_safe(path, opened=None): - """Remove only the named regular file we opened, or a rejected symlink.""" - try: - current = os.lstat(str(path)) - if stat.S_ISLNK(current.st_mode): - os.unlink(str(path)) - elif stat.S_ISREG(current.st_mode) and (opened is None or (current.st_dev, current.st_ino) == (opened.st_dev, opened.st_ino)): - os.unlink(str(path)) - except OSError: - pass - - -def _read_args_file(value): - path = _safe_args_path(value) - opened = None - descriptor = None - try: - try: - before = os.lstat(str(path)) - except OSError as error: - raise GeminiCommandError("argument file is unavailable") from error - if ( - stat.S_ISLNK(before.st_mode) - or not stat.S_ISREG(before.st_mode) - or before.st_uid != os.getuid() - or stat.S_IMODE(before.st_mode) != 0o600 - ): - raise GeminiCommandError("argument file must be a regular file") - if before.st_size > MAX_ARGS_BYTES: - raise GeminiCommandError("argument file exceeds the maximum size") - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - try: - descriptor = os.open(str(path), flags) - opened = os.fstat(descriptor) - except OSError as error: - raise GeminiCommandError("argument file cannot be opened safely") from error - if ( - not stat.S_ISREG(opened.st_mode) - or opened.st_uid != os.getuid() - or stat.S_IMODE(opened.st_mode) != 0o600 - or (opened.st_dev, opened.st_ino) != (before.st_dev, before.st_ino) - ): - raise GeminiCommandError("argument file changed before it could be read") - data = os.read(descriptor, MAX_ARGS_BYTES + 1) - if len(data) > MAX_ARGS_BYTES: - raise GeminiCommandError("argument file exceeds the maximum size") - try: - return data.decode("utf-8") - except UnicodeDecodeError as error: - raise GeminiCommandError("argument file must be UTF-8") from error - finally: - if descriptor is not None: - os.close(descriptor) - _discard_if_safe(path, opened) - - -def execute_gemini_command(capability, args_file, core=None): - """Read, validate, execute, and discard one trusted Gemini argument file.""" - try: - raw = _read_args_file(args_file) - argv = parse_gemini_arguments(capability, raw) - except (GeminiArgumentError, GeminiCommandError) as error: - raise GeminiCommandError("Gemini command arguments were rejected") from error - if core is None: - from .cli import main as core - try: - result = core(argv) - except Exception as error: - raise GeminiCommandError("Gemini command execution failed") from error - exit_code = 0 if result is None else int(result) - return { - "status": "ok" if exit_code == 0 else "error", - "capability": capability, - "exit_code": exit_code, - } - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--capability", required=True, choices=("scout", "adopt", "wire")) - parser.add_argument("--args-file", required=True) - arguments = parser.parse_args(argv) - try: - result = execute_gemini_command(arguments.capability, arguments.args_file) - print(json.dumps(result, sort_keys=True)) - except GeminiCommandError: - parser.error("Gemini command arguments were rejected") - return result["exit_code"] - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/gemini_transport.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/gemini_transport.py deleted file mode 100644 index bf7a648..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/gemini_transport.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Deterministic contract simulation for Gemini custom-command TOML transport. - -This module simulates documented ``{{args}}`` prompt substitution for tests and -local adapters. It is not a claim about Gemini model or tool behavior. -""" - -from __future__ import annotations - -import os -import subprocess -import tempfile -from pathlib import Path - -from .gemini_executor import MAX_ARGS_BYTES, GeminiCommandError, command_args_root, execute_gemini_command - - -_OPEN = "\n" -_CLOSE = "\n" - - -class GeminiTransportError(ValueError): - """The simulated TOML transport could not safely carry command data.""" - - -def substitute_toml_args(prompt, raw_args): - """Simulate one documented TOML placeholder substitution without parsing it.""" - if not isinstance(prompt, str) or not isinstance(raw_args, str) or prompt.count("{{args}}") != 1: - raise GeminiTransportError("TOML transport template is invalid") - return prompt.replace("{{args}}", raw_args) - - -def extract_untrusted_payload(rendered_prompt): - """Extract the delimited payload exactly, preserving embedded delimiter text.""" - if not isinstance(rendered_prompt, str): - raise GeminiTransportError("TOML transport payload is invalid") - start = rendered_prompt.find(_OPEN) - end = rendered_prompt.rfind(_CLOSE) - if start < 0 or end < start + len(_OPEN): - raise GeminiTransportError("TOML transport delimiters are invalid") - return rendered_prompt[start + len(_OPEN):end] - - -def write_private_args_file(payload): - """Write opaque transport data to a 0600 private file without a shell.""" - if not isinstance(payload, str): - raise GeminiTransportError("TOML transport payload is invalid") - encoded = payload.encode("utf-8") - if len(encoded) > MAX_ARGS_BYTES: - raise GeminiTransportError("TOML transport payload exceeds the maximum size") - root = command_args_root() - descriptor = None - name = None - complete = False - try: - descriptor, name = tempfile.mkstemp(prefix="transport-", suffix=".args", dir=str(root)) - os.fchmod(descriptor, 0o600) - os.write(descriptor, encoded) - complete = True - return name - except OSError as error: - raise GeminiTransportError("private transport storage is unavailable") from error - finally: - if descriptor is not None: - os.close(descriptor) - if name is not None and not complete: - try: - os.unlink(name) - except OSError: - pass - - -def build_structured_executor_call(skill_dir, capability, args_file): - """Build the only permitted non-shell Gemini executor process shape.""" - skill_path = Path(skill_dir) - argument_path = Path(args_file) - root = command_args_root() - if ( - capability not in {"scout", "adopt", "wire"} - or not skill_path.is_absolute() - or not argument_path.is_absolute() - or argument_path.parent != root - or not argument_path.name - ): - raise GeminiTransportError("structured executor fields are invalid") - return { - "program": "python3", - "argv": ["-m", "mlx_agent.gemini_executor", "--capability", capability, "--args-file", str(argument_path)], - "env": {"PYTHONPATH": str(skill_path / "src")}, - "shell": False, - } - - -def run_structured_executor_call(call, extra_env=None): - """Run a validated structured executor fixture through subprocess without a shell.""" - if not isinstance(call, dict) or set(call) != {"program", "argv", "env", "shell"}: - raise GeminiTransportError("structured executor fields are invalid") - program, argv, env, shell = call["program"], call["argv"], call["env"], call["shell"] - if ( - program != "python3" - or not isinstance(argv, list) - or len(argv) != 6 - or argv[:3] != ["-m", "mlx_agent.gemini_executor", "--capability"] - or argv[3] not in {"scout", "adopt", "wire"} - or argv[4] != "--args-file" - or not isinstance(argv[5], str) - or not isinstance(env, dict) - or set(env) != {"PYTHONPATH"} - or not isinstance(env["PYTHONPATH"], str) - ): - raise GeminiTransportError("structured executor fields are invalid") - if shell is not False or not env["PYTHONPATH"]: - raise GeminiTransportError("structured executor fields are invalid") - environment = dict(os.environ) - if extra_env is not None: - environment.update(extra_env) - environment.update(env) - return subprocess.run([program] + argv, env=environment, shell=False, text=True, capture_output=True, check=False) - - -def simulate_toml_transport(prompt, raw_args, capability, core=None): - """Simulate TOML prompt transport and dispatch only through the executor.""" - rendered = substitute_toml_args(prompt, raw_args) - payload = extract_untrusted_payload(rendered) - args_file = write_private_args_file(payload) - try: - return execute_gemini_command(capability, args_file, core=core) - except GeminiCommandError as error: - raise GeminiTransportError("TOML transport payload was rejected") from error diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/host.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/host.py deleted file mode 100644 index a7ee4c7..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/host.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Local host inventory probes used by Scout.""" - -import subprocess -from dataclasses import asdict, dataclass - - -@dataclass(frozen=True) -class HostInventory: - ram_gb: object = None - chip: object = None - ollama: bool = False - lmstudio: bool = False - - @staticmethod - def runtime_supports(runtime, role): - """Whether a runtime can serve the requested model class, independent of install state.""" - if runtime == "mlx-vlm": - return role == "vision" - if role == "vision": - return False - return runtime in ("mlx_lm", "ollama", "lmstudio", "litellm") - - @classmethod - def detect(cls, http_get, check_output=None): - return cls.inspect(http_get, check_output=check_output)[0] - - @classmethod - def inspect(cls, http_get, check_output=None): - """Return bounded local inventory plus classified unavailable-probe warnings.""" - check_output = check_output or subprocess.check_output - values = {"ram_gb": None, "chip": None, "ollama": False, "lmstudio": False} - warnings = [] - try: - values["ram_gb"] = round(int(check_output(["sysctl", "-n", "hw.memsize"]).strip()) / 1073741824) - except Exception: - warnings.append({"code": "host_probe_unavailable", "probe": "memory", "message": "Host memory probe unavailable."}) - try: - values["chip"] = check_output(["sysctl", "-n", "machdep.cpu.brand_string"], text=True).strip() - except Exception: - warnings.append({"code": "host_probe_unavailable", "probe": "chip", "message": "Host chip probe unavailable."}) - try: - http_get("http://127.0.0.1:11434/api/tags", timeout=3) - values["ollama"] = True - except Exception: - warnings.append({"code": "runtime_probe_unavailable", "probe": "ollama", "message": "Local runtime probe unavailable."}) - try: - http_get("http://localhost:1234/v1/models", timeout=3) - values["lmstudio"] = True - except Exception: - warnings.append({"code": "runtime_probe_unavailable", "probe": "lmstudio", "message": "Local runtime probe unavailable."}) - return cls(**values), warnings - - def to_dict(self): - return asdict(self) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/installer.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/installer.py deleted file mode 100644 index 06819e5..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/installer.py +++ /dev/null @@ -1,696 +0,0 @@ -"""Scoped, receipt-owned installer built on the Transaction safety boundary.""" - -from __future__ import annotations - -import hashlib -import json -import os -import uuid -from dataclasses import dataclass, field -from pathlib import Path - -from .providers import detect_providers -from .transactions import ( - LegacyLockError, Receipt, Transaction, _assert_safe_directory, _assert_safe_target, - _atomic_in_directory, _read_regular, _read_target, _walk_directory, - _rollback_receipt_hash, legacy_lock_problem, rollback, -) -from .wiring import redact_secrets - - -class InstallerConflictError(ValueError): - """A requested operation would touch an unowned or modified artifact.""" - - -class _ArtifactAdapter: - version = "installer-1.0" - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("installer artifacts must be UTF-8 text") - redacted = redact_secrets(content) - try: - contains_persisted_secret = json.loads(redacted) != json.loads(content) - except (TypeError, ValueError, json.JSONDecodeError): - contains_persisted_secret = redacted != content - if contains_persisted_secret: - raise ValueError("installer artifacts must not contain persisted secrets") - return True - - -class _SourceCodeArtifactAdapter: - """Source modules may contain secret-pattern examples but never persisted values.""" - - version = "installer-source-1.0" - secret_scan_exempt = True - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("installer artifacts must be UTF-8 text") - return True - - -@dataclass(frozen=True) -class _PlannedTransaction: - provider_id: str - changes: tuple - preview_hash: str - destinations: tuple - transaction_id: str = None - expected_receipt_path: str = None - child_index: int = None - - -@dataclass -class InstallerReceipt: - status: str - receipts: list = field(default_factory=list) - receipt_path: str = "" - targets: list = field(default_factory=list) - batch_path: str = "" - - def to_dict(self): - return { - "status": self.status, - "receipt_path": self.receipt_path or None, - "batch_path": self.batch_path or None, - "targets": list(self.targets), - "receipts": [_receipt_data(item) for item in self.receipts], - } - - -@dataclass -class InstallPlan: - action: str - provider_ids: tuple - scope: str - project_root: Path - preview: dict - transactions: list = field(default_factory=list) - rollback_receipts: list = field(default_factory=list) - uninstall_hashes: dict = field(default_factory=dict) - restore_changes: tuple = field(default_factory=tuple) - binding: dict = field(default_factory=dict) - compatibility: tuple = field(default_factory=tuple) - noop: bool = False - - def to_dict(self): - return { - "action": self.action, - "providers": list(self.provider_ids), - "scope": self.scope, - "project_root": str(self.project_root), - "preview": self.preview, - "compatibility": list(self.compatibility), - "noop": self.noop, - } - - -def _sha256(content): - return hashlib.sha256(content).hexdigest() - - -def _receipt_data(receipt): - value = receipt.to_dict() - value["receipt_path"] = receipt.receipt_path - return value - - -def _digest(value): - return _sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")) - - -def _read_optional_target(path): - """Read an artifact without following links; missing parent paths mean absent.""" - try: - content, exists, _mode = _read_target(path) - return content if exists else None - except ValueError as error: - if str(error).startswith("path component does not exist:"): - return None - raise - - -def _adapter_for_artifact(artifact): - is_runtime_source = "src" in artifact.source.parts and "mlx_agent" in artifact.source.parts - return _SourceCodeArtifactAdapter() if is_runtime_source else _ArtifactAdapter() - - -class Installer: - """Plan first; execute only the exact reviewed transaction previews.""" - - def __init__(self, registry, project_root=None, executable_lookup=None, env=None, - transaction_factory=None, rollback_func=None, probe_runner=None): - self.registry = registry - self.project_root = Path(project_root).resolve() if project_root else Path.cwd().resolve() - self.executable_lookup = executable_lookup - self.env = env - self.transaction_factory = transaction_factory or Transaction - self.rollback_func = rollback_func or rollback - self.probe_runner = probe_runner - - def detected(self): - return detect_providers( - self.registry.definitions().values(), self.env, - executable_lookup=self.executable_lookup, - probe_runner=self.probe_runner, - ) - - def _compatibility(self, provider_ids): - definitions = self.registry.definitions() - return tuple( - item.to_dict() for item in detect_providers( - [definitions[provider_id] for provider_id in provider_ids], - self.env, - executable_lookup=self.executable_lookup, - probe_runner=self.probe_runner, - ) - ) - - def plan(self, action, providers, scope="user", project_root=None): - if action not in {"install", "update", "uninstall", "doctor"}: - raise ValueError("unsupported installer action: {0}".format(action)) - if scope not in {"user", "project"}: - raise ValueError("scope must be 'user' or 'project'") - project = Path(project_root or self.project_root).resolve() - if scope == "project" and not project.is_dir(): - raise ValueError("project root must be an existing directory") - definitions = self.registry.definitions() - selected = tuple(providers or ()) - if not selected: - raise ValueError("provider selection is required; inspect detected providers first") - if len(selected) != len(set(selected)) or any(item not in definitions for item in selected): - raise ValueError("providers must be unique supported provider IDs") - if action == "doctor": - return self._doctor_plan(selected, definitions, scope, project) - if action == "uninstall": - return self._uninstall_plan(selected, definitions, scope, project) - return self._install_plan(action, selected, definitions, scope, project) - - def execute(self, plan, confirmed=False): - if plan.action == "doctor": - return self._doctor(plan) - if plan.noop: - return InstallerReceipt("noop") - if confirmed is not True and confirmed != plan.preview["preview_hash"]: - raise PermissionError("explicit confirmation for this installer preview is required") - self._verify_plan_binding(plan) - batch_path = self._create_batch(plan) - if plan.action == "uninstall": - return self._execute_uninstall(plan, batch_path) - return self._execute_install(plan, batch_path, self._batch_operations(plan, batch_path)) - - def _execute_install(self, plan, batch_path, operations): - successful, all_receipts = [], [] - try: - prepared = [(operation, *self._repreview(operation, plan.scope, plan.project_root)) for operation in operations] - except PermissionError as error: - self._batch_update(batch_path, "rolled_back", successful, error=str(error)) - raise - try: - for operation, transaction, preview in prepared: - self._batch_update(batch_path, "pending", all_receipts, active=self._active_operation(operation)) - receipt = transaction.apply(operation.preview_hash) - all_receipts.append(receipt) - self._batch_update(batch_path, "pending", all_receipts) - if receipt.status != "applied": - raise InstallerConflictError("provider transaction did not apply: {0}".format(receipt.status)) - successful.append(receipt) - self._batch_update(batch_path, "complete", successful) - return InstallerReceipt("applied", all_receipts, successful[-1].receipt_path, [target for item in successful for target in item.targets], str(batch_path)) - except LegacyLockError as error: - self._batch_update(batch_path, "rolled_back", all_receipts, error=str(error)) - raise InstallerConflictError(str(error)) - except Exception as error: - recovered, unresolved = self._compensate_install(all_receipts) - child_requires_recovery = any(item.status in {"recovery_required", "rollback_failed"} for item in all_receipts) - status = "rolled_back" if recovered and not child_requires_recovery else "recovery_required" - remediation = unresolved or ([{"paths": list(all_receipts[-1].targets), "message": "Child receipt requires manual recovery: {0}".format(all_receipts[-1].receipt_path)}] if child_requires_recovery and all_receipts else []) - self._batch_update(batch_path, status, all_receipts, error=str(error), remediation=remediation) - return InstallerReceipt(status, all_receipts, all_receipts[-1].receipt_path if all_receipts else "", [target for item in all_receipts for target in item.targets], str(batch_path)) - - def _execute_uninstall(self, plan, batch_path): - completed, attempted = [], [] - try: - for receipt in plan.rollback_receipts: - self._batch_update(batch_path, "pending", attempted, active={"provider": "uninstall", "transaction_id": receipt.transaction_id, "expected_receipt_path": receipt.receipt_path}) - result = self.rollback_func( - receipt.receipt_path, - expected_after_hashes=receipt.after_hashes, - expected_receipt_hash=_rollback_receipt_hash(receipt), - ) - recorded = self._rollback_result(result, receipt, plan.scope, plan.project_root) - attempted.append(recorded) - self._batch_update(batch_path, "pending", attempted) - if recorded.status != "rolled_back": - raise InstallerConflictError("uninstall rollback did not complete: {0}".format(recorded.status)) - completed.append(recorded) - self._cleanup_empty_artifact_dirs(plan) - self._batch_update(batch_path, "complete", attempted) - return InstallerReceipt("rolled_back", attempted, attempted[-1].receipt_path if attempted else "", [target for item in attempted for target in item.targets], str(batch_path)) - except Exception as error: - unresolved_children = [item for item in attempted if item.status != "rolled_back"] - if not completed: - status = "recovery_required" if unresolved_children else "rolled_back" - remediation = self._unresolved_child_remediation(unresolved_children) - self._batch_update(batch_path, status, attempted, error=str(error), remediation=remediation) - message = "uninstall recovery is required" if unresolved_children else "uninstall aborted before mutation" - raise InstallerConflictError("{0}: {1}".format(message, error)) - recovered, unresolved = self._compensate_uninstall(plan.restore_changes, completed, plan.scope, plan.project_root) - status = "rolled_back" if recovered and not unresolved_children else "recovery_required" - remediation = self._unresolved_child_remediation(unresolved_children) + unresolved - self._batch_update(batch_path, status, attempted, error=str(error), remediation=remediation) - if status == "recovery_required": - raise InstallerConflictError("uninstall recovery is required: {0}".format(error)) - return InstallerReceipt(status, attempted, attempted[-1].receipt_path if attempted else "", [target for item in attempted for target in item.targets], str(batch_path)) - - def _install_plan(self, action, selected, definitions, scope, project): - transactions, summary = [], [] - for provider_id in selected: - definition = definitions[provider_id] - changes = [] - for artifact in definition.artifacts: - if not definition.applies_to(scope, artifact): - continue - target = definition.artifact_destination(scope, project, artifact) - desired_bytes, source_exists, _source_mode = _read_target(artifact.source) - if not source_exists: - raise ValueError("provider artifact source disappeared: {0}".format(artifact.source)) - try: - desired = desired_bytes.decode("utf-8") - except UnicodeDecodeError as error: - raise ValueError("provider artifact source must be UTF-8") from error - current = _read_optional_target(target) - if current is not None: - history = self._artifact_history(definition, target, scope, project) - if not history or _sha256(current) != history[-1].after_hashes[str(target)]: - raise InstallerConflictError("refusing to overwrite unowned or modified artifact: {0}".format(target)) - if current == desired.encode("utf-8"): - continue - adapter = _adapter_for_artifact(artifact) - changes.append({"path": str(target), "content": desired, "adapter": adapter}) - if changes: - transaction = self._new_transaction(scope, project) - preview = transaction.preview(changes) - transactions.append(_PlannedTransaction(provider_id, tuple(changes), preview["preview_hash"], tuple(str(item["path"]) for item in changes))) - summary.append({"provider": provider_id, "changes": preview["changes"], "diff": preview["diff"]}) - return self._make_plan(action, selected, scope, project, transactions=transactions, summary=summary, definitions=definitions) - - def _uninstall_plan(self, selected, definitions, scope, project): - steps, summary, uninstall_hashes, restore_changes = [], [], {}, {} - seen = set() - for provider_id in selected: - definition = definitions[provider_id] - provider_steps = [] - for artifact in definition.artifacts: - if not definition.applies_to(scope, artifact): - continue - target = definition.artifact_destination(scope, project, artifact) - history = self._artifact_history(definition, target, scope, project) - if not history: - continue - oldest, latest = history[0], history[-1] - target_name = str(target) - if oldest.backup_paths.get(target_name) is not None: - raise InstallerConflictError("artifact was not receipt-owned at installation: {0}".format(target)) - location = _assert_safe_target(target) - content = _read_regular(location) - if _sha256(content) != latest.after_hashes[target_name]: - raise InstallerConflictError("refusing to remove user-modified artifact: {0}".format(target)) - uninstall_hashes[target_name] = latest.after_hashes[target_name] - restore_changes.setdefault(provider_id, []).append({ - "path": target_name, - "content": content.decode("utf-8"), - "adapter": _adapter_for_artifact(artifact), - }) - provider_steps.extend(history) - for receipt in sorted({item.receipt_path: item for item in provider_steps}.values(), key=lambda item: item.timestamp, reverse=True): - if receipt.receipt_path not in seen: - seen.add(receipt.receipt_path) - steps.append(receipt) - summary.append({"provider": provider_id, "changes": len(provider_steps), "diff": "remove receipt-owned artifacts"}) - return self._make_plan("uninstall", selected, scope, project, rollback_receipts=steps, - uninstall_hashes=uninstall_hashes, restore_changes=tuple((provider_id, tuple(changes)) for provider_id, changes in restore_changes.items()), - summary=summary, definitions=definitions) - - def _doctor_plan(self, selected, definitions, scope, project): - return self._make_plan("doctor", selected, scope, project, definitions=definitions) - - def _cleanup_empty_artifact_dirs(self, plan): - """Remove only emptied receipt-owned descendants; retain host-owned roots.""" - definitions = self.registry.definitions() - for provider_id in plan.provider_ids: - definition = definitions[provider_id] - provider_root = definition.destination(plan.scope, plan.project_root) - for artifact in definition.artifacts: - if not definition.applies_to(plan.scope, artifact): - continue - destination = definition.artifact_destination(plan.scope, plan.project_root, artifact) - try: - relative = destination.relative_to(provider_root) - except ValueError: - # Some providers own companion artifacts (for example, - # Gemini command files) beneath a host-wide directory - # rather than their package directory. Removing empty - # parents there could remove host-owned structure. - continue - stop = provider_root / relative.parts[0] - current = destination.parent - while current != stop: - try: - _parent, descriptor = _walk_directory(current.parent) - try: - os.rmdir(current.name, dir_fd=descriptor) - os.fsync(descriptor) - finally: - os.close(descriptor) - except (FileNotFoundError, OSError, ValueError): - break - current = current.parent - - def _doctor(self, plan): - problems, checked, provider_states = [], [], [] - definitions = self.registry.definitions() - detections = { - item.id: item for item in detect_providers( - [definitions[provider_id] for provider_id in plan.provider_ids], - self.env, - executable_lookup=self.executable_lookup, - probe_runner=self.probe_runner, - ) - } - for provider_id in plan.provider_ids: - definition = definitions[provider_id] - artifact_problems = [] - for artifact in definition.artifacts: - if not definition.applies_to(plan.scope, artifact): - continue - target = definition.artifact_destination(plan.scope, plan.project_root, artifact) - checked.append(str(target)) - try: - safe_root = _assert_safe_directory(plan.project_root if plan.scope == "project" else definition.user_root) - location = _assert_safe_target(target) - location.relative_to(safe_root) - history = self._artifact_history(definition, target, plan.scope, plan.project_root) - if not history or _sha256(_read_regular(location)) != history[-1].after_hashes[str(target)]: - raise ValueError("artifact hash is not receipt-owned") - except (OSError, ValueError): - problem = {"provider": provider_id, "code": "artifact_invalid", "path": str(target)} - problems.append(problem) - artifact_problems.append(problem) - detection = detections[provider_id] - if definition.install_mode == "portable": - integration_state = "portable" - elif definition.install_mode == "staged": - integration_state = "staged" - problems.append({ - "provider": provider_id, - "code": "provider_staged", - "message": "artifacts are staged but native host visibility is not proven", - }) - elif detection.state == "native-visible": - integration_state = "native-visible" - else: - integration_state = "staged" - code = "provider_unsupported" if detection.state == "unsupported" else "provider_absent" - problems.append({ - "provider": provider_id, - "code": code, - "message": detection.error or "provider compatibility could not be proven", - }) - provider_states.append({ - "provider": provider_id, - "artifact_state": "invalid" if artifact_problems else "installed", - "integration_state": integration_state, - "compatibility": detection.to_dict(), - }) - problems.extend(self._batch_problems(plan.scope, plan.project_root)) - lock_root = self._receipts_dir(plan.scope, plan.project_root) / "locks" - for provider_id in plan.provider_ids: - definition = definitions[provider_id] - targets = [ - str(definition.artifact_destination(plan.scope, plan.project_root, artifact)) - for artifact in definition.artifacts - if definition.applies_to(plan.scope, artifact) - ] - try: - legacy_problem = legacy_lock_problem(targets, lock_root) - except ValueError as error: - problems.append({"provider": provider_id, "code": "legacy_lock_invalid", "message": str(error)}) - continue - problems.extend({"provider": provider_id, **item} for item in legacy_problem) - return {"healthy": not problems, "problems": problems, "checked": checked, - "providers": provider_states, - "cooperative_concurrency": "Transaction advisory locks protect cooperative writers."} - - def _batch_problems(self, scope, project): - batches = self._receipts_dir(scope, project) / "batches" - try: - safe_batches = _assert_safe_directory(batches) - except ValueError: - return [] - problems = [] - for path in safe_batches.glob("*/batch.json"): - try: - batch = self._read_batch(path) - except (OSError, ValueError, json.JSONDecodeError): - problems.append({"code": "batch_recovery_required", "path": str(path)}) - continue - if batch["status"] in {"pending", "recovery_required"}: - active = batch.get("active") - if isinstance(active, dict) and active.get("expected_receipt_path"): - expected = active["expected_receipt_path"] - try: - child = _assert_safe_target(expected) - child_value = json.loads(_read_regular(child).decode("utf-8")) - if child_value.get("transaction_id") != active.get("transaction_id"): - raise ValueError("active child receipt does not match batch transaction") - except (OSError, ValueError, json.JSONDecodeError): - expected = "missing or untrusted active receipt: {0}".format(expected) - problems.append({"code": "batch_recovery_required", "path": str(path), "active_receipt": expected}) - else: - problems.append({"code": "batch_recovery_required", "path": str(path)}) - return problems - - def _make_plan(self, action, selected, scope, project, transactions=(), rollback_receipts=(), - uninstall_hashes=None, restore_changes=(), summary=(), definitions=None): - definitions = definitions or self.registry.definitions() - compatibility = self._compatibility(selected) - destinations = [{"provider": provider_id, "targets": [str(definitions[provider_id].artifact_destination(scope, project, artifact)) for artifact in definitions[provider_id].artifacts if definitions[provider_id].applies_to(scope, artifact)]} for provider_id in selected] - binding = { - "action": action, "providers": list(selected), "scope": scope, "project": str(project), - "destinations": destinations, - "inner_previews": [{"provider": item.provider_id, "hash": item.preview_hash, "targets": list(item.destinations)} for item in transactions], - "rollback_chain": [self._receipt_identity(item) for item in rollback_receipts], - "compatibility": list(compatibility), - } - preview = {"changes": sum(item["changes"] for item in summary), "providers": list(selected), - "diff": "\n".join(item["diff"] for item in summary), "preview_hash": _digest(binding), - "requires_confirmation": action != "doctor"} - return InstallPlan(action, selected, scope, project, preview, list(transactions), list(rollback_receipts), - dict(uninstall_hashes or {}), tuple(restore_changes), binding, compatibility, - noop=not transactions and not rollback_receipts) - - def _verify_plan_binding(self, plan): - definitions = self.registry.definitions() - binding = { - "action": plan.action, "providers": list(plan.provider_ids), "scope": plan.scope, - "project": str(plan.project_root), - "destinations": [{"provider": provider_id, "targets": [str(definitions[provider_id].artifact_destination(plan.scope, plan.project_root, artifact)) for artifact in definitions[provider_id].artifacts if definitions[provider_id].applies_to(plan.scope, artifact)]} for provider_id in plan.provider_ids], - "inner_previews": [{"provider": item.provider_id, "hash": item.preview_hash, "targets": list(item.destinations)} for item in plan.transactions], - "rollback_chain": [self._receipt_identity(self._load_receipt(item.receipt_path, plan.scope, plan.project_root)) for item in plan.rollback_receipts], - "compatibility": list(self._compatibility(plan.provider_ids)), - } - if binding != plan.binding or _digest(binding) != plan.preview["preview_hash"]: - raise PermissionError("installer preview is stale; regenerate and review it before confirmation") - - def _repreview(self, operation, scope, project): - transaction = self._new_transaction(scope, project, operation.transaction_id) - preview = transaction.preview(list(operation.changes)) - if preview["preview_hash"] != operation.preview_hash: - raise PermissionError("installer preview is stale; an inner preview changed") - return transaction, preview - - def _new_transaction(self, scope, project, transaction_id=None): - receipts_dir = self._receipts_dir(scope, project) - return self.transaction_factory( - receipts_dir=receipts_dir, - lock_root=receipts_dir / "locks", - create_target_parents=True, - transaction_id=transaction_id, - ) - - @staticmethod - def _operation_transaction_id(batch_id, child_index, provider_id): - return str(uuid.uuid5(uuid.UUID(batch_id), "{0}:{1}".format(child_index, provider_id))) - - def _artifact_history(self, definition, target, scope, project): - target_name = str(target) - allowed = {str(definition.artifact_destination(scope, project, artifact)) for artifact in definition.artifacts if definition.applies_to(scope, artifact)} - return [item for item in self._all_receipts(scope, project) if target_name in item.targets and set(item.targets).issubset(allowed) and item.status == "applied"] - - def _all_receipts(self, scope, project): - receipts_dir = self._receipts_dir(scope, project) - try: - safe_dir = _assert_safe_directory(receipts_dir) - except ValueError: - return [] - result = [] - for candidate in safe_dir.glob("*/receipt.json"): - try: - result.append(self._load_receipt(candidate, scope, project)) - except (OSError, ValueError, json.JSONDecodeError): - continue - return sorted(result, key=lambda item: item.timestamp) - - def _load_receipt(self, path, scope, project): - receipts_dir = _assert_safe_directory(self._receipts_dir(scope, project)) - location = _assert_safe_target(path) - if location.parent.parent != receipts_dir: - raise ValueError("installer receipt escapes its scoped receipt directory") - return Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) - - @staticmethod - def _receipt_identity(receipt): - return {"transaction_id": receipt.transaction_id, "receipt_path": receipt.receipt_path, - "targets": list(receipt.targets), "before_hashes": receipt.before_hashes, - "after_hashes": receipt.after_hashes, "backup_paths": receipt.backup_paths, - "rollback_receipt_hash": _rollback_receipt_hash(receipt)} - - def _rollback_result(self, result, planned, scope, project): - if all(hasattr(result, field) for field in ("transaction_id", "receipt_path", "targets", "before_hashes", "after_hashes", "backup_paths")): - return result - recorded = self._load_receipt(planned.receipt_path, scope, project) - recorded.status = result.status - return recorded - - @staticmethod - def _unresolved_child_remediation(receipts): - return [{"paths": list(receipt.targets), "message": "Rollback child requires manual recovery: {0}".format(receipt.receipt_path)} for receipt in receipts] - - def _compensate_install(self, receipts): - unresolved = [] - for receipt in reversed(receipts): - if receipt.status == "rolled_back": - continue - try: - expected = { - "expected_after_hashes": receipt.after_hashes, - "expected_receipt_hash": _rollback_receipt_hash(receipt), - } if receipt.status == "applied" else {} - result = self.rollback_func(receipt.receipt_path, **expected) - if result.status != "rolled_back": - unresolved.append({"paths": list(receipt.targets), "message": "Could not recover child receipt: {0}".format(receipt.receipt_path)}) - except Exception as error: - unresolved.append({"paths": list(receipt.targets), "message": redact_secrets(str(error))}) - return not unresolved, unresolved - - def _compensate_uninstall(self, changes, completed, scope, project): - if not changes: - return True, [] - expected = {} - for receipt in completed: - for target in receipt.targets: - expected[target] = {"hash": receipt.before_hashes[target], - "exists": receipt.backup_paths[target] is not None, - "mode": receipt.target_modes[target]} - unresolved = [] - try: - affected = {target for receipt in completed for target in receipt.targets} - for _provider_id, provider_changes in changes: - provider_changes = [change for change in provider_changes if change["path"] in affected] - if not provider_changes: - continue - expected_current = {change["path"]: expected[change["path"]] for change in provider_changes} - transaction = self._new_transaction(scope, project) - preview = transaction.preview(provider_changes, expected_current=expected_current) - if transaction.apply(preview["preview_hash"], expected_current=expected_current).status != "applied": - unresolved.append({"paths": [change["path"] for change in provider_changes], "message": "Compensation transaction did not apply."}) - return not unresolved, unresolved - except Exception as error: - return False, [{"paths": sorted(expected), "message": redact_secrets(str(error))}] - - def _receipts_dir(self, scope, project): - return (self.registry.config_root / "mlx-agent" / "installer-receipts") if scope == "user" else (project / ".mlx-agent" / "installer-receipts") - - def _create_batch(self, plan): - batches = self._receipts_dir(plan.scope, plan.project_root) / "batches" - directory, descriptor = _walk_directory(batches, create=True) - batch_id = str(uuid.uuid4()) - try: - os.mkdir(batch_id, 0o700, dir_fd=descriptor) - os.fsync(descriptor) - finally: - os.close(descriptor) - path = directory / batch_id / "batch.json" - operations = [] - for index, item in enumerate(plan.transactions): - operation = self._active_operation(item, batch_id, index) - operation["expected_receipt_path"] = str(self._receipts_dir(plan.scope, plan.project_root) / operation["transaction_id"] / "receipt.json") - operations.append(operation) - immutable = { - "schema_version": "1.0", "batch_id": batch_id, "action": plan.action, "scope": plan.scope, - "project_root": str(plan.project_root), "preview_hash": plan.preview["preview_hash"], - "binding": plan.binding, - "operations": operations, - } - self._write_batch(path, dict(immutable, status="pending", children=[], active=None, remediation=[])) - return path - - def _batch_update(self, path, status, receipts, active=None, error=None, remediation=None): - value = self._read_batch(path) - children = {item["receipt_path"]: item for item in value.get("children", [])} - for receipt in receipts: - children[receipt.receipt_path] = dict(self._receipt_identity(receipt), status=receipt.status) - value["children"] = list(children.values()) - value["status"] = status - value["active"] = active - if error: - value.setdefault("remediation", []).append({"paths": [], "message": redact_secrets(str(error))}) - if remediation: - value.setdefault("remediation", []).extend(remediation) - self._write_batch(path, value) - - def _read_batch(self, path): - location = _assert_safe_target(path) - value = json.loads(_read_regular(location).decode("utf-8")) - required = {"schema_version", "batch_id", "action", "scope", "project_root", "preview_hash", "binding", "operations", "status", "children", "active", "remediation"} - if not isinstance(value, dict) or set(value) != required: - raise ValueError("installer batch journal is malformed") - return value - - def _write_batch(self, path, value): - root = _assert_safe_directory(Path(path).parent) - _atomic_in_directory(root, "batch.json", (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600) - - def _batch_operations(self, plan, path): - batch = self._read_batch(path) - operations = batch["operations"] - if len(operations) != len(plan.transactions): - raise ValueError("installer batch operations do not match the confirmed plan") - result = [] - for index, (planned, operation) in enumerate(zip(plan.transactions, operations)): - expected = self._active_operation(planned, batch["batch_id"], index) - expected["expected_receipt_path"] = str(self._receipts_dir(plan.scope, plan.project_root) / expected["transaction_id"] / "receipt.json") - if operation != expected: - raise ValueError("installer batch operation binding is stale") - result.append(_PlannedTransaction(planned.provider_id, planned.changes, planned.preview_hash, - planned.destinations, operation["transaction_id"], - operation["expected_receipt_path"], operation["child_index"])) - return result - - @staticmethod - def _active_operation(operation, batch_id=None, child_index=None): - if operation.transaction_id is None: - if batch_id is None or child_index is None: - raise ValueError("batch operation identity is required") - transaction_id = Installer._operation_transaction_id(batch_id, child_index, operation.provider_id) - expected_receipt_path = None - else: - transaction_id = operation.transaction_id - expected_receipt_path = operation.expected_receipt_path - return {"provider": operation.provider_id, - "child_index": operation.child_index if operation.child_index is not None else child_index, - "transaction_id": transaction_id, - "expected_receipt_path": expected_receipt_path, - "preview_hash": operation.preview_hash, "targets": list(operation.destinations)} diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/interview.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/interview.py deleted file mode 100644 index d4711cb..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/interview.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Domain-agnostic interview engine producing a validated DomainIntent. - -The question set is a deterministic, config-driven backbone. An optional -``assist`` callable may refine answers, but every assist result is re-validated -through ``build_intent`` and can never bypass validation or inject unknown -roles. Modality foundations seed roles/keywords via ``modality`` profiles. -No network access, no LLM dependency for the deterministic path. -""" - -from __future__ import annotations - -import math - -from dataclasses import dataclass -from typing import Callable, Dict, List, Mapping, Optional, Sequence, Tuple - -from .modality import ( - FACET_CHOICES, - MODALITY_CHOICES, - apply_modality_profile, - detect_facets, - detect_modalities, - resolve_facets, - validate_facets, - validate_modalities, -) -from .models import DISCOVERY_ROLES - - -ROLE_CHOICES: Dict[str, str] = { - "Vision / OCR": "vision", - "Embeddings": "embedding", - "Coding": "coding", - "Reasoning": "reasoning", - "General chat": "general", - "Tool use / function calling": "tool-use", -} - - -QUESTIONS: Tuple[Dict[str, object], ...] = ( - { - "id": "domain", - "prompt": "What are you building? Describe the domain in a sentence.", - "kind": "text", - }, - { - "id": "roles", - "prompt": "Which model roles do you need? (choose one or more)", - "kind": "multi", - "choices": tuple(ROLE_CHOICES.keys()), - }, - { - "id": "keywords", - "prompt": "Any domain keywords to prioritize? (comma-separated, optional)", - "kind": "text", - }, - { - "id": "license", - "prompt": "Restrict to specific licenses? (comma-separated, optional)", - "kind": "text", - }, - { - "id": "memory_gb", - "prompt": "Memory budget in GB? (optional, e.g. 32)", - "kind": "text", - }, - { - "id": "notes", - "prompt": "Any other constraints or notes? (optional)", - "kind": "text", - }, -) - -_FOLLOW_ON_QUESTIONS = tuple(question for question in QUESTIONS if question["id"] != "domain") - - -@dataclass(frozen=True) -class DomainIntent: - """A validated, deterministic description of what the user wants.""" - - domain: str - roles: Tuple[str, ...] - keywords: Tuple[str, ...] = () - license_allow: Tuple[str, ...] = () - memory_gb: Optional[float] = None - notes: str = "" - modalities: Tuple[str, ...] = () - facets: Tuple[str, ...] = () - - def to_dict(self) -> Dict[str, object]: - return { - "domain": self.domain, - "roles": list(self.roles), - "keywords": list(self.keywords), - "license_allow": list(self.license_allow), - "memory_gb": self.memory_gb, - "notes": self.notes, - "modalities": list(self.modalities), - "facets": list(self.facets), - } - - -def _dedupe(values: Sequence[str]) -> Tuple[str, ...]: - seen = [] - for value in values: - if value and value not in seen: - seen.append(value) - return tuple(seen) - - -def _split_csv(raw: object) -> Tuple[str, ...]: - if raw is None: - return () - if isinstance(raw, (list, tuple)): - parts = [str(item) for item in raw] - else: - parts = str(raw).split(",") - return _dedupe([part.strip().lower() for part in parts if part.strip()]) - - -def _split_labels(raw: object) -> Tuple[str, ...]: - if raw is None or raw == "": - return () - if isinstance(raw, (list, tuple)): - return _dedupe([str(item).strip() for item in raw if str(item).strip()]) - return _dedupe([part.strip() for part in str(raw).split(",") if part.strip()]) - - -def _resolve_roles(raw: object) -> Tuple[str, ...]: - if raw is None or raw == "": - labels: Sequence[str] = () - elif isinstance(raw, (list, tuple)): - labels = [str(item) for item in raw] - else: - labels = [part.strip() for part in str(raw).split(",")] - roles = [] - for label in labels: - cleaned = label.strip() - if not cleaned: - continue - if cleaned in ROLE_CHOICES: - roles.append(ROLE_CHOICES[cleaned]) - elif cleaned in DISCOVERY_ROLES: - roles.append(cleaned) - else: - raise ValueError("unknown role: {0}".format(cleaned)) - return _dedupe(roles) - - -def _resolve_memory(raw: object) -> Optional[float]: - if raw is None: - return None - text = str(raw).strip() - if not text: - return None - try: - value = float(text) - except ValueError as error: - raise ValueError("memory_gb must be a number") from error - if not math.isfinite(value) or value <= 0: - raise ValueError("memory_gb must be a positive number") - return value - - -def _resolve_modality_ids(raw: object) -> Tuple[str, ...]: - labels = _split_labels(raw) - ids = [] - for label in labels: - if label in MODALITY_CHOICES: - ids.append(MODALITY_CHOICES[label]) - else: - ids.append(label) - return validate_modalities(ids) - - -def _resolve_facet_ids(raw: object, modalities: Sequence[str]) -> Tuple[str, ...]: - labels = _split_labels(raw) - if not labels: - return () - label_to_id = {} - for modality in modalities: - label_to_id.update(FACET_CHOICES.get(modality, {})) - ids = [] - for label in labels: - if label in label_to_id: - ids.append(label_to_id[label]) - else: - ids.append(label) - return validate_facets(ids, modalities=modalities) - - -def build_intent( - answers: Mapping[str, object], - assist: Optional[Callable[["DomainIntent"], Mapping[str, object]]] = None, -) -> DomainIntent: - """Validate raw answers into a DomainIntent. Deterministic given answers. - - If ``assist`` is provided it receives the validated intent and may return a - mapping of overrides; those overrides are merged and the intent is rebuilt - through this same validator (assist can never bypass validation). - """ - domain = str(answers.get("domain", "")).strip() - if not domain: - raise ValueError("domain is required") - - modalities_raw = answers.get("modalities") - if modalities_raw is None or modalities_raw == "": - modalities = detect_modalities(domain) - keyword_text = " ".join(_split_csv(answers.get("keywords"))) - if keyword_text: - modalities = _dedupe(list(modalities) + list(detect_modalities(keyword_text))) - else: - modalities = _resolve_modality_ids(modalities_raw) - - facets_raw = answers.get("facets") - detect_text = "{0} {1}".format(domain, " ".join(_split_csv(answers.get("keywords")))) - if facets_raw is None or facets_raw == "": - facets = resolve_facets(modalities, cli=(), text=detect_text) if modalities else () - else: - facets = _resolve_facet_ids(facets_raw, modalities) - - roles = _resolve_roles(answers.get("roles")) - keywords = _split_csv(answers.get("keywords")) - if modalities: - roles, keywords = apply_modality_profile(roles, keywords, modalities, facets) - if not roles: - roles = ("general",) - - intent = DomainIntent( - domain=domain, - roles=roles, - keywords=keywords, - license_allow=_split_csv( - answers["license"] if "license" in answers else answers.get("license_allow") - ), - memory_gb=_resolve_memory(answers.get("memory_gb")), - notes=str(answers.get("notes", "")).strip(), - modalities=modalities, - facets=facets, - ) - if assist is None: - return intent - overrides = assist(intent) - if not overrides: - return intent - merged = intent.to_dict() - merged.update(dict(overrides)) - return build_intent(merged, assist=None) - - -def run_interview( - reader: Callable[[Mapping[str, object]], object], - assist: Optional[Callable[["DomainIntent"], Mapping[str, object]]] = None, - preset_modalities: Sequence[str] = (), - preset_facets: Sequence[str] = (), -) -> DomainIntent: - """Ask questions via ``reader`` with detect-then-ask modality resolution.""" - answers: Dict[str, object] = {} - domain_question = QUESTIONS[0] - answers["domain"] = reader(domain_question) - domain = str(answers["domain"]).strip() - - modalities = validate_modalities(preset_modalities) if preset_modalities else detect_modalities(domain) - if not modalities: - modality_question = { - "id": "modalities", - "prompt": ( - "Which foundational modalities apply? " - "(audio, video, document extraction / vision)" - ), - "kind": "multi", - "choices": tuple(MODALITY_CHOICES.keys()), - } - modalities = _resolve_modality_ids(reader(modality_question)) - if not modalities: - raise ValueError("at least one foundational modality is required") - answers["modalities"] = list(modalities) - - facets: List[str] = list(validate_facets(preset_facets, modalities=modalities)) if preset_facets else [] - if not facets: - for modality in modalities: - detected = detect_facets(modality, domain) - if detected: - facets.extend(detected) - else: - facet_question = { - "id": "facets:{0}".format(modality), - "prompt": "Which {0} facets matter? (optional multi-select)".format(modality), - "kind": "multi", - "choices": tuple(FACET_CHOICES[modality].keys()), - } - response = reader(facet_question) - if response is None or response == "": - continue - facets.extend(_resolve_facet_ids(response, (modality,))) - if not facets and modalities: - facets = list(resolve_facets(modalities, cli=(), text=domain)) - answers["facets"] = list(_dedupe(facets)) - - for question in _FOLLOW_ON_QUESTIONS: - answers[str(question["id"])] = reader(question) - return build_intent(answers, assist=assist) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/lora.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/lora.py deleted file mode 100644 index 336b859..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/lora.py +++ /dev/null @@ -1,326 +0,0 @@ -"""Confirmation-gated LoRA training jobs on the convert machinery.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from pathlib import Path - -from .convert import ( - _read_receipt, - _write_receipt, - receipts_root, -) -from .serve import _argv_matches, _pid_alive, _pid_command -from .wiring import _MODEL - - -LORA_RECEIPT_SCHEMA_VERSION = "1.0" -LORA_RECEIPT_KIND = "lora" -EXECUTABLE = "mlx_lm.lora" -ITERS_MIN, ITERS_MAX, ITERS_DEFAULT = 1, 100000, 1000 -BATCH_MIN, BATCH_MAX, BATCH_DEFAULT = 1, 64, 4 -LR_MIN, LR_MAX, LR_DEFAULT = 1e-6, 1e-2, 1e-5 -LAYERS_MIN, LAYERS_MAX, LAYERS_DEFAULT = -1, 128, 16 -MAX_DATASET_LINES = 10000 -DATASET_SAMPLE_LINES = 50 - - -class LoraError(RuntimeError): - """Classified lora failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def validate_dataset(data_dir): - """Validate an mlx_lm training dataset directory; bounded and read-only.""" - root = Path(data_dir) - train = root / "train.jsonl" - if not root.is_dir() or not train.is_file(): - raise LoraError( - "dataset_invalid", - "The dataset directory must contain train.jsonl: {0}".format(data_dir), - "Provide a directory with train.jsonl (text or messages per line).", - ) - summary = {"train_lines": _validate_jsonl(train), "valid_lines": 0} - valid = root / "valid.jsonl" - if valid.is_file(): - summary["valid_lines"] = _validate_jsonl(valid) - return summary - - -def _validate_jsonl(path): - count = 0 - try: - with open(path, "r", encoding="utf-8") as handle: - for line in handle: - count += 1 - if count > MAX_DATASET_LINES: - raise LoraError( - "dataset_invalid", - "{0} exceeds {1} lines.".format(path.name, MAX_DATASET_LINES), - "Split the dataset; lora validates bounded datasets only.", - ) - if count > DATASET_SAMPLE_LINES: - continue - stripped = line.strip() - if not stripped: - continue - try: - record = json.loads(stripped) - except json.JSONDecodeError: - raise LoraError( - "dataset_invalid", - "{0} line {1} is not valid JSON.".format(path.name, count), - "Fix the dataset line; every line must be one JSON object.", - ) - _validate_record(record, path.name, count) - except OSError as error: - raise LoraError( - "dataset_invalid", - "The dataset could not be read: {0}".format(error), - "Pass a readable dataset directory.", - ) - if count == 0: - raise LoraError( - "dataset_invalid", - "{0} is empty.".format(path.name), - "Provide at least one training example.", - ) - return count - - -def _validate_record(record, name, line): - if not isinstance(record, dict): - raise LoraError( - "dataset_invalid", - "{0} line {1} must be a JSON object.".format(name, line), - "Use {\"text\": ...} or {\"messages\": [...]} per line.", - ) - text = record.get("text") - if isinstance(text, str) and text.strip(): - return - messages = record.get("messages") - if isinstance(messages, list) and messages: - for message in messages: - if ( - not isinstance(message, dict) - or not isinstance(message.get("role"), str) - or not isinstance(message.get("content"), str) - ): - raise LoraError( - "dataset_invalid", - "{0} line {1} has a malformed message.".format(name, line), - "Each message needs string role and content fields.", - ) - return - raise LoraError( - "dataset_invalid", - "{0} line {1} needs a non-empty text field or a messages array.".format(name, line), - "Use {\"text\": ...} or {\"messages\": [{\"role\": ..., \"content\": ...}]}.", - ) - - -def _validate_hyperparameters(iters, batch_size, learning_rate, num_layers): - if not isinstance(iters, int) or isinstance(iters, bool) or not ITERS_MIN <= iters <= ITERS_MAX: - raise LoraError( - "invalid_arguments", - "iters must be between {0} and {1}.".format(ITERS_MIN, ITERS_MAX), - "Pass a bounded --iters value.", - ) - if not isinstance(batch_size, int) or isinstance(batch_size, bool) or not BATCH_MIN <= batch_size <= BATCH_MAX: - raise LoraError( - "invalid_arguments", - "batch-size must be between {0} and {1}.".format(BATCH_MIN, BATCH_MAX), - "Pass a bounded --batch-size value.", - ) - if ( - not isinstance(learning_rate, (int, float)) - or isinstance(learning_rate, bool) - or not LR_MIN <= float(learning_rate) <= LR_MAX - ): - raise LoraError( - "invalid_arguments", - "learning-rate must be between {0} and {1}.".format(LR_MIN, LR_MAX), - "Pass a bounded --learning-rate value.", - ) - if not isinstance(num_layers, int) or isinstance(num_layers, bool) or not LAYERS_MIN <= num_layers <= LAYERS_MAX: - raise LoraError( - "invalid_arguments", - "num-layers must be between {0} and {1}.".format(LAYERS_MIN, LAYERS_MAX), - "Pass a bounded --num-layers value (-1 trains all layers).", - ) - - -def plan_lora(repo, data, iters=ITERS_DEFAULT, batch_size=BATCH_DEFAULT, - learning_rate=LR_DEFAULT, num_layers=LAYERS_DEFAULT, out=None): - """Validate inputs and render the exact training plan; side-effect free.""" - if not isinstance(repo, str) or not _MODEL.fullmatch(repo): - raise LoraError( - "invalid_repo", - "lora requires a safe publisher/model base identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - _validate_hyperparameters(iters, batch_size, learning_rate, num_layers) - dataset = validate_dataset(data) - if out is None: - out = "{0}-lora".format(repo.split("/", 1)[1]) - plan = { - "repo": repo, - "data": str(data), - "iters": iters, - "batch_size": batch_size, - "learning_rate": float(learning_rate), - "num_layers": num_layers, - "out": str(out), - "dataset": dataset, - "argv": [ - EXECUTABLE, - "--model", repo, - "--train", - "--data", str(data), - "--iters", str(iters), - "--batch-size", str(batch_size), - "--learning-rate", repr(float(learning_rate)), - "--num-layers", str(num_layers), - "--adapter-path", str(out), - ], - } - canonical = json.dumps( - {key: plan[key] for key in sorted(plan) if key != "preview_hash"}, - sort_keys=True, separators=(",", ":"), - ) - plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() - return plan - - -def _receipt_filename(plan): - return "{0}-lora.json".format(plan["repo"].split("/", 1)[1]) - - -def start_lora(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, spawn=None, now=_utc_now, - pid_alive=None): - """Execute a reviewed training plan; the only mutating entry point.""" - from .serve import _default_spawn, _default_which - - which = which or _default_which - spawn = spawn or _default_spawn - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir, LORA_RECEIPT_KIND) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise LoraError( - "preview_hash_required", - "--confirm requires the hash from a reviewed lora preview.", - "Run lora start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise LoraError( - "preview_stale", - "The supplied preview hash does not match this lora plan.", - "Re-run lora start without --confirm and review the fresh plan.", - ) - if which(EXECUTABLE) is None: - raise LoraError( - "runtime_not_installed", - "The {0} executable is not installed.".format(EXECUTABLE), - "Install mlx-lm yourself (pip install mlx-lm); lora never installs runtimes.", - ) - if model_present is not None and not model_present(plan["repo"]): - raise LoraError( - "model_not_local", - "The base model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; lora never downloads.", - ) - if Path(plan["out"]).exists(): - raise LoraError( - "output_exists", - "The adapter output path already exists: {0}".format(plan["out"]), - "Pick a fresh --out path; lora never overwrites.", - ) - for receipt_path in sorted(root.glob("*.json")) if root.is_dir() else []: - existing = _read_receipt(receipt_path, LORA_RECEIPT_KIND) - if existing is None or existing.get("completed_at"): - continue - if pid_alive(existing["pid"]): - raise LoraError( - "job_in_progress", - "A lora job is still running (pid {0}).".format(existing["pid"]), - "Wait for it to finish (lora status) before starting another.", - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}-lora.log".format(plan["repo"].split("/", 1)[1]) - pid = spawn(plan["argv"], str(log_path)) - receipt = { - "schema_version": LORA_RECEIPT_SCHEMA_VERSION, - "kind": LORA_RECEIPT_KIND, - "repo": plan["repo"], - "data": plan["data"], - "iters": plan["iters"], - "out": plan["out"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "started_at": now(), - "preview_hash": plan["preview_hash"], - "completed_at": None, - "exit_status": None, - } - _write_receipt(root, receipt, _receipt_filename(plan)) - return {"status": "started", "receipt": receipt} - - -def status_lora(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - now=_utc_now): - """Cross-check lora receipts against live processes; marks exits once.""" - root = receipts_root(receipts_dir, LORA_RECEIPT_KIND) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - receipt = _read_receipt(path, LORA_RECEIPT_KIND) - if receipt is None: - continue - entry = { - "receipt": str(path), - "repo": receipt["repo"], - "data": receipt.get("data"), - "out": receipt["out"], - "pid": receipt["pid"], - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - "completed_at": receipt.get("completed_at"), - } - if receipt.get("completed_at"): - entry["state"] = receipt.get("exit_status") or "done" - entries.append(entry) - continue - if pid_alive(receipt["pid"]): - command = pid_command(receipt["pid"]) - entry["state"] = "running" if _argv_matches( - {"argv": receipt["argv"], "port": None, "repo": receipt["repo"]}, - command, - require_port=False, - ) else "unknown" - entries.append(entry) - continue - exit_status = "done" if Path(receipt["out"]).exists() else "failed" - receipt["completed_at"] = now() - receipt["exit_status"] = exit_status - _write_receipt(root, receipt, path.name) - entry["state"] = exit_status - entry["completed_at"] = receipt["completed_at"] - entries.append(entry) - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/modality.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/modality.py deleted file mode 100644 index 03205fb..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/modality.py +++ /dev/null @@ -1,283 +0,0 @@ -"""Foundational modality layers for research intent seeding. - -Three foundations (audio, video, document-vision) seed existing discovery roles -and keywords. No new discovery roles or runtimes. Detection is deterministic -lexicon matching only — no LLM classification. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Dict, Iterable, Mapping, Optional, Sequence, Tuple - - -@dataclass(frozen=True) -class FacetProfile: - id: str - label: str - lexicon: Tuple[str, ...] - seed_keywords: Tuple[str, ...] - - -@dataclass(frozen=True) -class ModalityProfile: - id: str - label: str - lexicon: Tuple[str, ...] - default_roles: Tuple[str, ...] - seed_keywords: Tuple[str, ...] - facets: Tuple[FacetProfile, ...] - - -MODALITY_PROFILES: Tuple[ModalityProfile, ...] = ( - ModalityProfile( - id="audio", - label="Audio (ASR / TTS / music)", - lexicon=( - "audio", "speech", "whisper", "asr", "tts", "voice", "music", - "sound", "transcri", - ), - default_roles=("general",), - seed_keywords=("audio", "speech", "whisper"), - facets=( - FacetProfile( - id="asr", - label="ASR / speech-to-text", - lexicon=("asr", "speech-to-text", "speech to text", "transcri", "whisper", "stt"), - seed_keywords=("asr", "whisper", "transcription", "speech-to-text"), - ), - FacetProfile( - id="tts", - label="TTS / text-to-speech", - lexicon=("tts", "text-to-speech", "text to speech", "narrat", "voice synthes"), - seed_keywords=("tts", "text-to-speech", "speech synthesis"), - ), - FacetProfile( - id="music", - label="Music", - lexicon=("music", "melody", "song", "soundtrack", "audio generation"), - seed_keywords=("music", "melody", "song"), - ), - ), - ), - ModalityProfile( - id="video", - label="Video", - lexicon=("video", "footage", "clip", "mp4", "frames"), - default_roles=("vision", "general"), - seed_keywords=("video",), - facets=( - FacetProfile( - id="understanding", - label="Understanding / captioning", - lexicon=("understanding", "caption", "describe video", "video-llm", "vlm video"), - seed_keywords=("video", "caption", "understanding"), - ), - FacetProfile( - id="generation", - label="Generation", - lexicon=("generation", "generate video", "diffusion", "text-to-video", "text to video"), - seed_keywords=("video", "generation", "diffusion", "text-to-video"), - ), - FacetProfile( - id="action", - label="Action / tracking", - lexicon=("action recognition", "action", "tracking", "pose", "motion"), - seed_keywords=("video", "action", "tracking"), - ), - ), - ), - ModalityProfile( - id="document-vision", - label="Document extraction / vision", - lexicon=( - "ocr", "document", "pdf", "scanned", "invoice", "layout", - "handwriting", "vision", "vlm", "image understanding", - ), - default_roles=("vision",), - seed_keywords=("ocr", "document", "vision"), - facets=( - FacetProfile( - id="ocr", - label="OCR / text extraction", - lexicon=("ocr", "optical character", "scanned", "handwriting", "text extraction"), - seed_keywords=("ocr", "handwriting", "scanned"), - ), - FacetProfile( - id="layout", - label="Layout / structure", - lexicon=("layout", "form parsing", "table extraction", "document structure"), - seed_keywords=("layout", "document", "pdf"), - ), - FacetProfile( - id="general-vision", - label="General vision / VLM", - lexicon=("general vision", "vlm", "image caption", "visual qa", "vision-language"), - seed_keywords=("vision", "vlm", "image"), - ), - ), - ), -) - -FOUNDATION_IDS: Tuple[str, ...] = tuple(profile.id for profile in MODALITY_PROFILES) -ALL_FACET_IDS: Tuple[str, ...] = tuple( - facet.id for profile in MODALITY_PROFILES for facet in profile.facets -) -MODALITY_CHOICES: Dict[str, str] = {profile.label: profile.id for profile in MODALITY_PROFILES} -FACET_CHOICES: Dict[str, Dict[str, str]] = { - profile.id: {facet.label: facet.id for facet in profile.facets} - for profile in MODALITY_PROFILES -} - -_PROFILES_BY_ID: Mapping[str, ModalityProfile] = {profile.id: profile for profile in MODALITY_PROFILES} -_FACETS_BY_ID: Mapping[str, FacetProfile] = { - facet.id: facet for profile in MODALITY_PROFILES for facet in profile.facets -} -_FACET_TO_MODALITY: Mapping[str, str] = { - facet.id: profile.id for profile in MODALITY_PROFILES for facet in profile.facets -} - - -def _dedupe(values: Iterable[str]) -> Tuple[str, ...]: - seen = [] - for value in values: - if value and value not in seen: - seen.append(value) - return tuple(seen) - - -def _haystack(text: str) -> str: - return " ".join(str(text or "").lower().split()) - - -def _lexicon_hit(haystack: str, lexicon: Sequence[str]) -> bool: - return any(token in haystack for token in lexicon) - - -def detect_modalities(text: str) -> Tuple[str, ...]: - """Return foundation ids matched by deterministic lexicon hits.""" - haystack = _haystack(text) - if not haystack: - return () - hits = [] - for profile in MODALITY_PROFILES: - if _lexicon_hit(haystack, profile.lexicon): - hits.append(profile.id) - return tuple(hits) - - -def detect_facets(modality: str, text: str) -> Tuple[str, ...]: - """Return facet ids for one foundation matched in text.""" - profile = _PROFILES_BY_ID.get(modality) - if profile is None: - raise ValueError("unknown modality: {0}".format(modality)) - haystack = _haystack(text) - if not haystack: - return () - hits = [] - for facet in profile.facets: - if _lexicon_hit(haystack, facet.lexicon): - hits.append(facet.id) - return tuple(hits) - - -def validate_modalities(modalities: Sequence[str]) -> Tuple[str, ...]: - resolved = _dedupe([str(item).strip() for item in modalities if str(item).strip()]) - for modality in resolved: - if modality not in _PROFILES_BY_ID: - raise ValueError("unknown modality: {0}".format(modality)) - return resolved - - -def validate_facets(facets: Sequence[str], modalities: Optional[Sequence[str]] = None) -> Tuple[str, ...]: - resolved = _dedupe([str(item).strip() for item in facets if str(item).strip()]) - allowed = None - if modalities is not None: - allowed = set() - for modality in validate_modalities(modalities): - allowed.update(facet.id for facet in _PROFILES_BY_ID[modality].facets) - for facet in resolved: - if facet not in _FACETS_BY_ID: - raise ValueError("unknown facet: {0}".format(facet)) - if allowed is not None and facet not in allowed: - raise ValueError( - "facet {0} is not valid for modalities {1}".format( - facet, ",".join(modalities) or "none" - ) - ) - return resolved - - -def resolve_modalities(cli: Sequence[str] = (), text: str = "") -> Tuple[str, ...]: - """CLI modalities win; otherwise detect from text.""" - if cli: - return validate_modalities(cli) - return detect_modalities(text) - - -def resolve_facets( - modalities: Sequence[str], - cli: Sequence[str] = (), - text: str = "", -) -> Tuple[str, ...]: - """Resolve facets: CLI wins; else detect; else seed all facets for each modality.""" - modalities = validate_modalities(modalities) - if not modalities: - return () - if cli: - return validate_facets(cli, modalities=modalities) - detected = [] - for modality in modalities: - detected.extend(detect_facets(modality, text)) - detected = _dedupe(detected) - if detected: - return validate_facets(detected, modalities=modalities) - # Lock: seed all facet ids for selected foundations when none detected. - all_facets = [] - for modality in modalities: - all_facets.extend(facet.id for facet in _PROFILES_BY_ID[modality].facets) - return tuple(all_facets) - - -def apply_modality_profile( - roles: Sequence[str], - keywords: Sequence[str], - modalities: Sequence[str], - facets: Sequence[str] = (), -) -> Tuple[Tuple[str, ...], Tuple[str, ...]]: - """Union user roles/keywords with profile seeds. User values are preserved.""" - modalities = validate_modalities(modalities) - facets = validate_facets(facets, modalities=modalities) if facets else () - seeded_roles = list(roles) - seeded_keywords = list(keywords) - for modality in modalities: - profile = _PROFILES_BY_ID[modality] - for role in profile.default_roles: - if role not in seeded_roles: - seeded_roles.append(role) - for keyword in profile.seed_keywords: - if keyword not in seeded_keywords: - seeded_keywords.append(keyword) - facet_ids = facets - if modalities and not facet_ids: - facet_ids = resolve_facets(modalities, cli=(), text="") - for facet_id in facet_ids: - facet = _FACETS_BY_ID[facet_id] - for keyword in facet.seed_keywords: - if keyword not in seeded_keywords: - seeded_keywords.append(keyword) - return tuple(seeded_roles), tuple(seeded_keywords) - - -def modality_label(modality_id: str) -> str: - profile = _PROFILES_BY_ID.get(modality_id) - if profile is None: - raise ValueError("unknown modality: {0}".format(modality_id)) - return profile.label - - -def facet_label(facet_id: str) -> str: - facet = _FACETS_BY_ID.get(facet_id) - if facet is None: - raise ValueError("unknown facet: {0}".format(facet_id)) - return facet.label diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/model_doctor.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/model_doctor.py deleted file mode 100644 index b10511a..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/model_doctor.py +++ /dev/null @@ -1,495 +0,0 @@ -"""Read-only model inventory, wiring drift, and endpoint health diagnostics. - -Doctor never deletes, moves, repairs, downloads, or starts anything. It reads -the Hugging Face cache, queries already-running loopback runtimes, parses -Wire-managed configuration, and reports classified findings with remediation -text. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path - -from .verification import _safe_error, installed_model_ids -from .wiring import ( - _OLLAMA, - _STATE_CHANGING_PATH_PARTS, - ConfigAdapter, - validate_health_endpoint, -) - - -MAX_INVENTORY_ENTRIES = 500 -MAX_FINDINGS = 100 -MAX_ENDPOINTS = 50 -MAX_WALK_FILES = 2000 -MAX_CONFIG_BYTES = 1024 * 1024 -MAX_RECEIPTS = 50 -_WIRE_MARKER = "MLX_AGENT_WIRE" -_SKIP_DIRECTORIES = { - ".git", ".hg", ".svn", "node_modules", "__pycache__", ".venv", "venv", - ".tox", ".mypy_cache", ".pytest_cache", "Library", -} -_CONFIG_SUFFIXES = {".json", ".yaml", ".yml", ".modelfile"} -_HEALTH_TIMEOUT = 2.0 - - -def default_hf_cache(): - return Path.home() / ".cache" / "huggingface" / "hub" - - -def inspect_hf_cache(cache_dir): - """Summarize model repos in a Hugging Face cache without reading weights.""" - root = Path(cache_dir) - inventory = [] - if not root.is_dir(): - return inventory - for entry in sorted(root.iterdir()): - if len(inventory) >= MAX_INVENTORY_ENTRIES: - break - if not entry.is_dir() or not entry.name.startswith("models--"): - continue - repo = entry.name[len("models--"):].replace("--", "/") - refs_main = _read_text_quiet(entry / "refs" / "main") - snapshots = entry / "snapshots" - revisions = [] - total_bytes = 0 - missing_blobs = 0 - seen_blobs = set() - if snapshots.is_dir(): - for revision in sorted(snapshots.iterdir()): - if not revision.is_dir(): - continue - revisions.append(revision.name) - for dirpath, dirnames, filenames in os.walk(revision): - dirnames[:] = [name for name in dirnames if not name.startswith(".")] - for filename in filenames: - location = Path(dirpath) / filename - try: - resolved = location.resolve(strict=True) - except OSError: - missing_blobs += 1 - continue - if resolved in seen_blobs: - continue - seen_blobs.add(resolved) - try: - total_bytes += resolved.stat().st_size - except OSError: - missing_blobs += 1 - complete = bool(refs_main and revisions and missing_blobs == 0) - inventory.append({ - "id": repo, - "source": "hf-cache", - "bytes": total_bytes, - "revisions": revisions[:8], - "complete": complete, - }) - return inventory - - -def _read_text_quiet(path): - try: - return path.read_text(encoding="utf-8").strip() or None - except OSError: - return None - - -def collect_runtime_inventory(runtime_clients): - """Collect served model IDs from already-running loopback runtimes.""" - inventory = [] - errors = [] - for runtime in runtime_clients: - name = str(getattr(runtime, "name", runtime.__class__.__name__)) - try: - response = runtime.list_models() - for model_id in sorted(installed_model_ids(response)): - if len(inventory) >= MAX_INVENTORY_ENTRIES: - break - inventory.append({ - "id": model_id, - "source": name, - "bytes": _runtime_model_size(response, model_id), - "complete": True, - }) - except Exception as error: - errors.append("{0}: {1}".format(name, _safe_error(error))) - return inventory, errors - - -def _runtime_model_size(response, model_id): - if not isinstance(response, dict): - return None - for record in response.get("models", []) or []: - if not isinstance(record, dict): - continue - if str(record.get("name") or record.get("id") or "") != model_id: - continue - size = record.get("size") - if isinstance(size, int) and not isinstance(size, bool) and size >= 0: - return size - return None - - -def scan_wired_configs(roots): - """Find Wire-managed configs under roots and extract their references.""" - wired = [] - for root in roots: - for location in _walk_config_files(root): - if len(wired) >= MAX_ENDPOINTS: - return wired - content = _read_bounded(location) - if content is None or _WIRE_MARKER not in content: - continue - try: - adapter = ConfigAdapter.detect(location) - reference = adapter_references(adapter, content) - except (TypeError, ValueError): - continue - if reference is None: - continue - wired.append({ - "path": str(location), - "runtime": reference.get("runtime") or adapter.runtime, - "model": reference.get("model"), - "endpoint": reference.get("endpoint"), - }) - return wired - - -def _walk_config_files(root): - root = Path(root) - if root.is_file(): - yield root - return - if not root.is_dir(): - return - yielded = 0 - for dirpath, dirnames, filenames in os.walk(root): - dirnames[:] = [ - name for name in dirnames - if name not in _SKIP_DIRECTORIES and not name.startswith(".mlx-agent-receipts") - ] - for filename in sorted(filenames): - if yielded >= MAX_WALK_FILES: - return - location = Path(dirpath) / filename - if location.suffix.lower() not in _CONFIG_SUFFIXES and filename.lower() != "modelfile": - continue - yielded += 1 - yield location - - -def _read_bounded(location): - try: - if location.stat().st_size > MAX_CONFIG_BYTES: - return None - return location.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - return None - - -def adapter_references(adapter, content): - """Extract the model and endpoint a Wire-managed config references.""" - if adapter.runtime == "ollama": - match = _OLLAMA.fullmatch(content) - if match is None: - return None - return {"model": match.group(1), "endpoint": None} - if adapter.runtime == "litellm": - model = re.search(r"(?m)^ model: openai/([^\s]+)$", content) - base = re.search(r"(?m)^ api_base: (http[^\s]+)$", content) - if model is None: - return None - return { - "model": model.group(1), - "endpoint": base.group(1) if base else None, - } - try: - document = json.loads(content) - except (TypeError, ValueError, json.JSONDecodeError): - return None - if not isinstance(document, dict): - return None - marker = document.get("mlx_agent_wire") - if not isinstance(marker, dict): - return None - provider = marker.get("provider") if isinstance(marker.get("provider"), dict) else {} - runtime = marker.get("runtime") - return { - "model": marker.get("model"), - "endpoint": provider.get("base_url"), - "runtime": runtime if isinstance(runtime, str) else None, - } - - -def scan_receipt_after_hashes(roots): - """Read after_hashes from Wire receipts without enforcing receipt validity.""" - records = [] - for root in roots: - root = Path(root) - candidates = [] - if (root / ".mlx-agent-receipts").is_dir(): - candidates.append(root / ".mlx-agent-receipts") - else: - for dirpath, dirnames, _filenames in os.walk(root): - if ".mlx-agent-receipts" in dirnames: - candidates.append(Path(dirpath) / ".mlx-agent-receipts") - dirnames.remove(".mlx-agent-receipts") - for receipts_dir in candidates: - for receipt in sorted(receipts_dir.glob("*/receipt.json")): - if len(records) >= MAX_RECEIPTS: - return records - try: - value = json.loads(receipt.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - continue - after_hashes = value.get("after_hashes") - if not isinstance(after_hashes, dict): - continue - records.append({ - "receipt": str(receipt), - "transaction_id": value.get("transaction_id"), - "status": value.get("status"), - "after_hashes": { - str(key): str(item) for key, item in after_hashes.items() - }, - }) - return records - - -def check_endpoint(endpoint, timeout=_HEALTH_TIMEOUT): - """Classify a wired loopback endpoint as ok, reachable, or down.""" - if not endpoint: - return None - try: - url = validate_health_endpoint(endpoint) - parsed_path = urllib.parse.urlparse(url).path.rstrip("/") - if parsed_path.endswith("/v1"): - url = url.rstrip("/") + "/models" - request = urllib.request.Request(url, method="GET") - opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - with opener.open(request, timeout=timeout) as response: - status = response.status - except urllib.error.HTTPError as error: - status = error.code - except (OSError, ValueError, urllib.error.URLError): - return "down" - return "ok" if 200 <= status < 400 else "reachable" - - -def run_model_doctor(wired_roots, hf_cache, runtime_clients): - """Run every read-only check and return the doctor report.""" - wired_roots = [Path(root) for root in wired_roots] - inventory = inspect_hf_cache(hf_cache) - runtime_inventory, runtime_errors = collect_runtime_inventory(runtime_clients) - inventory.extend(runtime_inventory) - known_ids = {item["id"].casefold() for item in inventory} - - wired = scan_wired_configs(wired_roots) - receipts = scan_receipt_after_hashes(wired_roots) - - findings = [] - for config in wired: - model = config.get("model") - if model and model.casefold() not in known_ids: - findings.append({ - "code": "drift_missing_model", - "path": config["path"], - "model": model, - "remediation": ( - "Re-download the model into a local inventory, or re-run " - "wire render for a model that is present." - ), - }) - - for receipt in receipts: - for target, expected in receipt["after_hashes"].items(): - location = Path(target) - if not location.is_file(): - findings.append({ - "code": "drift_missing_file", - "path": target, - "receipt_id": receipt["transaction_id"], - "remediation": ( - "The wired file is gone; restore it from the receipt " - "backup or roll the receipt back." - ), - }) - continue - try: - digest = hashlib.sha256(location.read_bytes()).hexdigest() - except OSError: - continue - if digest != expected: - findings.append({ - "code": "drift_hash_mismatch", - "path": target, - "receipt_id": receipt["transaction_id"], - "remediation": ( - "The wired file changed after wiring; inspect it, then " - "re-apply or roll back the receipt." - ), - }) - - endpoints = [] - claims = {} - for config in wired: - endpoint = config.get("endpoint") - if not endpoint: - continue - origin = _endpoint_origin_key(endpoint) - if origin is None: - continue - previous = claims.get(origin) - if previous is not None and previous["runtime"] != config["runtime"]: - findings.append({ - "code": "drift_endpoint_conflict", - "path": config["path"], - "model": config.get("model"), - "remediation": ( - "Two wired configs for different runtimes claim {0}; " - "move one runtime to its own port.".format(origin) - ), - }) - else: - claims.setdefault(origin, config) - if len(endpoints) < MAX_ENDPOINTS: - endpoints.append({ - "endpoint": origin, - "path": config["path"], - "status": check_endpoint(endpoint), - }) - - for item in inventory: - if item["source"] == "hf-cache" and not item.get("complete", True): - findings.append({ - "code": "hf_cache_incomplete", - "path": None, - "model": item["id"], - "remediation": ( - "The cached snapshot is incomplete; re-download the model " - "with the runtime's own pull command." - ), - }) - - findings = findings[:MAX_FINDINGS] - total_bytes = sum( - item["bytes"] or 0 for item in inventory if item["source"] == "hf-cache" - ) - return { - "inventory": inventory, - "summary": { - "models": len(inventory), - "hf_cache_bytes": total_bytes, - "wired_configs": len(wired), - "findings": len(findings), - }, - "wired": wired, - "endpoints": endpoints, - "findings": findings, - "runtime_errors": runtime_errors, - } - - -def _endpoint_origin_key(endpoint): - try: - url = validate_health_endpoint(endpoint) - except ValueError: - return None - parsed = urllib.parse.urlparse(url) - port = parsed.port or (443 if parsed.scheme == "https" else 80) - decoded = parsed.path or "/" - parts = {part for part in decoded.lower().split("/") if part} - if parts & _STATE_CHANGING_PATH_PARTS: - return None - return "{0}://{1}:{2}".format(parsed.scheme.lower(), parsed.hostname.lower(), port) - - -PRUNE_PREVIEW_VERSION = "1.0" -MAX_PRUNE_CANDIDATES = 100 - - -class PruneError(RuntimeError): - """Classified prune failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def plan_prune(cache_dir): - """List incomplete HF cache snapshots eligible for deletion; read-only.""" - root = Path(cache_dir) - candidates = [] - for item in inspect_hf_cache(root): - if item.get("complete", True): - continue - location = root / "models--{0}".format(item["id"].replace("/", "--")) - candidates.append({ - "repo": item["id"], - "path": str(location), - "bytes": item.get("bytes") or 0, - }) - if len(candidates) >= MAX_PRUNE_CANDIDATES: - break - canonical = json.dumps(candidates, sort_keys=True, separators=(",", ":")) - return { - "cache": str(root), - "candidates": candidates, - "total_bytes": sum(item["bytes"] for item in candidates), - "preview_hash": hashlib.sha256(canonical.encode("utf-8")).hexdigest(), - } - - -def execute_prune(plan, confirm=False, preview_hash=None, remover=None): - """Delete exactly the reviewed candidates; irreversible by design.""" - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise PruneError( - "preview_hash_required", - "--confirm requires the hash from a reviewed prune preview.", - "Run doctor models --prune without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise PruneError( - "preview_stale", - "The supplied preview hash does not match the current prune plan.", - "Re-run doctor models --prune and review the fresh candidate list.", - ) - if remover is None: - import shutil - - def remover(path): - shutil.rmtree(path) - - cache = Path(plan["cache"]).resolve() - removed = [] - for candidate in plan["candidates"]: - location = Path(candidate["path"]) - resolved = location.resolve() - if resolved == cache or cache not in resolved.parents: - raise PruneError( - "unsafe_target", - "Refusing to remove a path outside the cache: {0}".format(candidate["path"]), - "Only cache-owned model directories are ever pruned.", - ) - if not location.is_dir(): - removed.append({"repo": candidate["repo"], "removed": False, "reason": "already gone"}) - continue - remover(str(location)) - removed.append({ - "repo": candidate["repo"], - "removed": not location.exists(), - "bytes": candidate["bytes"], - }) - return {"status": "pruned", "removed": removed} diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/models.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/models.py deleted file mode 100644 index ef170de..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/models.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Pure model classification, sizing, wiring, and rendering helpers.""" - -from __future__ import annotations - -import json -import re - -from .runtime_preference import prefer_runtime_for_role, wiring_for_preference - - -ROLES = [ - ("vision", ["-vl", "vision", "vlm", "ocr", "-omni"], "Vision / OCR (needs mlx-vlm)"), - ("embedding", ["embed", "embedding", "bge", "nomic", "gte"], "Embeddings"), - ("coding", ["coder", "-code", "devstral", "codestral", "starcoder"], "Coding"), - ("reasoning", ["gpt-oss", "-a3b", "thinking", "reason", "qwq", "-r1", "deepseek-r"], "Reasoning"), - ("general", ["instruct", "-it", "chat", "mistral", "gemma", "qwen", "llama", "phi"], "General"), -] -REASONER_HINTS = re.compile(r"(thinking|reason|-a3b\b|qwq|-r1\b|deepseek-r|gpt-oss)", re.I) -TEMPLATE_REASON = re.compile(r"(reasoning_effort||<\|think|channel\|>analysis)", re.I) -TOOL_USE_HINTS = re.compile( - r"\b(?:tool[-_]?use|(?:function|tool)(?:[-_]call|[ -]calling))\b", - re.I, -) -TEMPLATE_TOOL_USE = re.compile( - r"(?:|\btool_calls\b|\bfunction_call\b|\bavailable_tools\b|" - r"(?:\{\{|\{%)[^}]*\btools\b[^}]*(?:\}\}|%\}))", - re.I, -) -TOOL_USE_TAGS = { - "tool-use", - "tool_use", - "function-calling", - "function_calling", - "tools", -} -DISCOVERY_ROLES = tuple(role for role, _keywords, _label in ROLES) + ("tool-use",) -REPUTABLE = {"mlx-community", "lmstudio-community", "unsloth", "mlx-omni", "mlxvlm", "qwen", "google", "mistralai", "meta-llama", "nvidia"} -QUANT_TOK = re.compile(r"[-_.]?(mlx|mxfp4|nvfp4|dwq|optiq|turboquant|rotorquant|oq4e?|mtp|q\d(_k(_[ms])?)?|int[48]|fp16|fp8|bf16|\d+\.?\d*bpw|\d+bit|e[24]b|4bit|8bit)$", re.I) -PARAM_RE = re.compile(r"(\d+(?:\.\d+)?)\s*b(?![a-z])", re.I) -QUANT_GB_PER_B = [ - (re.compile(r"(fp16|bf16|-16bit|f16)", re.I), 2.0), - (re.compile(r"(8bit|-q8|int8|fp8|mxfp8)", re.I), 1.0), - (re.compile(r"(6bit|-q6)", re.I), 0.75), - (re.compile(r"(4bit|-q4|mxfp4|nvfp4|int4)", re.I), 0.55), - (re.compile(r"(3bit|-q3)", re.I), 0.45), - (re.compile(r"(2bit|-q2)", re.I), 0.35), -] -QUANT_RANK = [("bf16", 6), ("fp16", 6), ("8bit", 5), ("q8", 5), ("6bit", 4), ("q6", 4), ("4bit", 3), ("q4", 3), ("mxfp4", 3), ("3bit", 2), ("2bit", 1)] - - -def infer_quantization(value): - """Return a normalized name-derived quantization, never a measured property.""" - low = value.lower() - for name, patterns in [ - ("fp16", ("fp16", "bf16", "f16")), - ("8bit", ("8bit", "q8", "int8", "fp8")), - ("6bit", ("6bit", "q6")), - ("4bit", ("4bit", "q4", "int4", "mxfp4", "nvfp4")), - ("3bit", ("3bit", "q3")), - ("2bit", ("2bit", "q2")), - ]: - if any(pattern in low for pattern in patterns): - return name - return None - - -def name_ram_gb(repo): - match = PARAM_RE.search(repo) - if not match: - return None - for pattern, value in QUANT_GB_PER_B: - if pattern.search(repo): - return round(float(match.group(1)) * value, 1) - return round(float(match.group(1)) * 0.55, 1) - - -def classify(repo): - low = repo.lower() - for role, keywords, _label in ROLES: - if any(keyword in low for keyword in keywords): - return role - return "general" - - -def classify_roles(repo, enrichment): - """Return ordered role memberships and the canonical tool-use signal.""" - primary = classify(repo) - explicit = enrichment.get("tool_use") - if explicit is not None: - source = enrichment.get("tool_use_src") or "checked" - signal = { - "supported": bool(explicit), - "source": source, - "confidence": "weak" if source == "name" else "explicit", - } - elif TOOL_USE_HINTS.search(repo): - signal = { - "supported": True, - "source": "name", - "confidence": "weak", - } - else: - signal = { - "supported": False, - "source": "none", - "confidence": "none", - } - roles = (primary, "tool-use") if signal["supported"] else (primary,) - return roles, signal - - -def base_name(repo): - name = repo.split("/")[-1] - previous = None - while name != previous: - previous = name - name = QUANT_TOK.sub("", name).rstrip("-_.") - return name.lower() - - -def quant_rank(repo): - low = repo.lower() - for token, rank in QUANT_RANK: - if token in low: - return rank - return 0 - - -def resolve_ram(repo, enrichment): - if enrichment.get("weight_bytes"): - return round(enrichment["weight_bytes"] / 1e9, 1), "estimated_from_weight_bytes" - if enrichment.get("params_total"): - for pattern, value in QUANT_GB_PER_B: - if pattern.search(repo): - return round(enrichment["params_total"] * value / 1e9, 1), "estimated_from_parameter_count" - return round(enrichment["params_total"] * 0.55 / 1e9, 1), "estimated_from_parameter_count" - estimate = name_ram_gb(repo) - return (estimate, "estimated_from_name") if estimate is not None else (None, None) - - -def resolve_reasoning(repo, enrichment): - if enrichment.get("reasoning") is not None: - return enrichment["reasoning"], enrichment.get("reason_src") - return bool(REASONER_HINTS.search(repo)), "name" - - -def wiring(repo, role, host): - preference = prefer_runtime_for_role(role, host or {}) - return wiring_for_preference(repo, role, preference) - - -def render_md(report): - host = report["host"] - out = ["# mlx-scout report", ""] - out.append("**Host:** {0} · {1}GB · Ollama {2} · LM Studio {3}".format( - host.get("chip") or "?", host.get("ram_gb") or "?", "✓" if host["ollama"] else "✗", "✓" if host["lmstudio"] else "✗") + (" _(fast mode: name heuristics)_" if report.get("fast") else "")) - if report.get("error"): - return "\n".join(out + ["\n> ⚠ {0}".format(report["error"])]) - labels = {role: label for role, _keywords, label in ROLES} - for role, items in report["roles"].items(): - out.extend(["\n## {0}".format(labels.get(role, role)), "| model | RAM | reasoning | fits | license | wiring |", "|---|---|---|---|---|---|"]) - for item in items: - ram = "{0}GB".format(item["est_ram_gb"]) if item.get("est_ram_gb") is not None else "?" - reason = "⚠ {0}".format(item.get("reason_src") or "") if item.get("reasoning") else "no" - license_name = (item.get("license") or "?") + (" 🔒" if item.get("gated") else "") - out.append("| `{0}`{1} | {2} | {3} | {4} | {5} | {6} |".format(item["repo"], " ⭐" if item.get("trusted") else "", ram, reason, "✓" if item["fits"] else "✗", license_name, item["wiring"])) - out.append("\n_RAM is always an estimate; measured model-file bytes, when available, are retained separately as evidence. Add KV-cache headroom (~1–4GB) beyond weights._") - out.append("_`reasoning ⚠` sources: chat_template > tags > name (weakest). Verify with a test-generate._") - return "\n".join(out) - - -def wire(repo, target, port): - short = repo.split("/")[-1] - if target == "ollama": - return "\n".join(["# Ollama runs curated tags / HF GGUF repos (not arbitrary mlx-community weights).", "ollama pull hf.co/{0} # GGUF repos; else use the matching library tag".format(repo), "# agent ref: ollama/"]) - if target == "lmstudio": - return "\n".join(["lms get {0} # download into LM Studio".format(repo), "lms server start # OpenAI-compatible API on http://localhost:1234/v1", "# agent ref: lmstudio/{0}".format(short.lower())]) - if target == "mlx_lm": - return "\n".join(["pip install mlx-lm", "mlx_lm.server --model {0} --port {1} --max-tokens 8192".format(repo, port), "# openai-compatible provider:", json.dumps({"id": "mlxlm", "type": "openai", "baseURL": "http://127.0.0.1:{0}/v1".format(port), "apiKey": "local"}, indent=2), "# agent ref: mlxlm/{0}".format(short)]) - if target == "mlx-vlm": - return "\n".join(["pip install mlx-vlm", "mlx_vlm.server --model {0} --port {1}".format(repo, port), json.dumps({"id": "mlxvlm", "type": "openai", "baseURL": "http://127.0.0.1:{0}/v1".format(port), "apiKey": "local"}, indent=2), "# agent ref: mlxvlm/{0}".format(short)]) - if target == "litellm": - return "\n".join(["# litellm config.yaml (run a native server first, e.g. mlx_lm.server):", "model_list:", " - model_name: {0}".format(short.lower()), " litellm_params:", " model: openai/{0}".format(repo), " api_base: http://127.0.0.1:{0}/v1".format(port), " api_key: local"]) - return "unknown target: {0}".format(target) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/probe_assets.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/probe_assets.py deleted file mode 100644 index 52e0c98..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/probe_assets.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Deterministic synthetic assets for role-fit verification probes.""" - -from __future__ import annotations - -import base64 -import struct -import zlib - - -VISION_PROBE_TEXT = "MLX42" - -_FONT_5X7 = { - "M": ( - "X...X", - "XX.XX", - "X.X.X", - "X.X.X", - "X...X", - "X...X", - "X...X", - ), - "L": ( - "X....", - "X....", - "X....", - "X....", - "X....", - "X....", - "XXXXX", - ), - "X": ( - "X...X", - "X...X", - ".X.X.", - "..X..", - ".X.X.", - "X...X", - "X...X", - ), - "4": ( - "...X.", - "..XX.", - ".X.X.", - "X..X.", - "XXXXX", - "...X.", - "...X.", - ), - "2": ( - ".XXX.", - "X...X", - "....X", - "...X.", - "..X..", - ".X...", - "XXXXX", - ), -} - -_cached_image_b64 = None - - -def _png_chunk(tag, data): - return ( - struct.pack(">I", len(data)) - + tag - + data - + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) - ) - - -def _png_gray(width, height, rows): - ihdr = struct.pack(">IIBBBBB", width, height, 8, 0, 0, 0, 0) - raw = b"".join(b"\x00" + bytes(row) for row in rows) - return ( - b"\x89PNG\r\n\x1a\n" - + _png_chunk(b"IHDR", ihdr) - + _png_chunk(b"IDAT", zlib.compress(raw)) - + _png_chunk(b"IEND", b"") - ) - - -def _render_text_bitmap(text, scale=6, padding=8): - glyphs = [_FONT_5X7[character] for character in text] - glyph_width = 5 - spacing = 1 - bitmap_width = len(glyphs) * (glyph_width + spacing) - spacing - bitmap_height = 7 - width = bitmap_width * scale + 2 * padding - height = bitmap_height * scale + 2 * padding - rows = [bytearray(width) for _ in range(height)] - for index, glyph in enumerate(glyphs): - origin_x = padding + index * (glyph_width + spacing) * scale - for gy, line in enumerate(glyph): - for gx, mark in enumerate(line): - if mark != "X": - continue - for dy in range(scale): - row = rows[padding + gy * scale + dy] - start = origin_x + gx * scale - for dx in range(scale): - row[start + dx] = 255 - return width, height, rows - - -def vision_probe_image_png(): - """Render the fixed probe word as a small deterministic grayscale PNG.""" - width, height, rows = _render_text_bitmap(VISION_PROBE_TEXT) - return _png_gray(width, height, rows) - - -def vision_probe_image_base64(): - global _cached_image_b64 - if _cached_image_b64 is None: - _cached_image_b64 = base64.b64encode(vision_probe_image_png()).decode("ascii") - return _cached_image_b64 diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/project_blueprint.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/project_blueprint.py deleted file mode 100644 index c0872fb..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/project_blueprint.py +++ /dev/null @@ -1,257 +0,0 @@ -"""MLX project design packs: guidance-only blueprints for downstream agents. - -No scaffolding, no downloads, no training. Deterministic templates from a short -project brief. Distinct from dataset blueprints in ``blueprint.py``. -""" - -from __future__ import annotations - -import json -import math -import os -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from typing import Dict, List, Mapping, Optional, Tuple - -from .modality import validate_modalities -from .research import slugify - - -@dataclass(frozen=True) -class ProjectBrief: - goal: str - modalities: Tuple[str, ...] = () - notes: str = "" - memory_gb: Optional[float] = None - - def to_dict(self) -> Dict[str, object]: - return { - "goal": self.goal, - "modalities": list(self.modalities), - "notes": self.notes, - "memory_gb": self.memory_gb, - } - - -@dataclass(frozen=True) -class ProjectDesignPack: - brief: ProjectBrief - generated_at: str - quantization_ideas: Tuple[str, ...] - training_loop: Tuple[str, ...] - lora_notes: Tuple[str, ...] - mtx_notes: Tuple[str, ...] - study_materials: Tuple[str, ...] - stack_path: Tuple[str, ...] - next_steps: Tuple[str, ...] - - def to_dict(self) -> Dict[str, object]: - return { - "brief": self.brief.to_dict(), - "generated_at": self.generated_at, - "quantization_ideas": list(self.quantization_ideas), - "training_loop": list(self.training_loop), - "lora_notes": list(self.lora_notes), - "mtx_notes": list(self.mtx_notes), - "study_materials": list(self.study_materials), - "stack_path": list(self.stack_path), - "next_steps": list(self.next_steps), - } - - -def _resolve_memory(raw: object) -> Optional[float]: - if raw is None: - return None - text = str(raw).strip() - if not text: - return None - try: - value = float(text) - except ValueError as error: - raise ValueError("memory_gb must be a number") from error - if not math.isfinite(value) or value <= 0: - raise ValueError("memory_gb must be a positive number") - return value - - -def build_brief(answers: Mapping[str, object]) -> ProjectBrief: - goal = str(answers.get("goal") or answers.get("domain") or "").strip() - if not goal: - raise ValueError("goal is required") - modalities_raw = answers.get("modalities") or () - if isinstance(modalities_raw, str): - modalities_raw = [part.strip() for part in modalities_raw.split(",") if part.strip()] - modalities = validate_modalities(list(modalities_raw)) if modalities_raw else () - return ProjectBrief( - goal=goal, - modalities=modalities, - notes=str(answers.get("notes", "")).strip(), - memory_gb=_resolve_memory(answers.get("memory_gb")), - ) - - -def generate_design_pack( - brief: ProjectBrief, - now: Optional[datetime] = None, -) -> ProjectDesignPack: - """Build a deterministic project design pack. Guidance only.""" - moment = now or datetime.now(timezone.utc) - visionish = any(item in brief.modalities for item in ("document-vision", "video")) - audio = "audio" in brief.modalities - memory = brief.memory_gb - - quant: List[str] = [ - "Start from published MLX community 4-bit / 8-bit weights when available.", - "Prefer 4-bit for chat/tooling under tight unified memory; keep 8-bit for quality-sensitive OCR or reasoning.", - "Record the exact Hub repo + quant tag in the research pack before fine-tuning.", - ] - if memory is not None: - quant.append( - "Respect the {0:g} GB budget with ~20% runtime headroom beyond weight size.".format(memory) - ) - if visionish: - quant.append( - "Vision stacks need mlx-vlm-compatible VLM quants; do not plan Ollama-only vision paths." - ) - if audio: - quant.append( - "Audio ASR/TTS often ships as specialty checkpoints — verify native MLX serve path before training plans." - ) - - training = ( - "Define a tiny offline JSONL/Parquet dataset before any train loop.", - "Dry-run one batch on Apple Silicon with mlx-lm / project tooling; abort on OOM.", - "Use LoRA/QLoRA-style adapters before full fine-tunes; keep base weights frozen.", - "Hold out 10–20% validation with no near-duplicate leakage.", - "This blueprint does not train or download — execution stays outside mlx-agent.", - ) - - lora = ( - "Search adapters in `mlx-agent research` (PEFT/LoRA section) before training new ones.", - "Serve adapters with mlx_lm `--adapter-path` after verify; keep base + adapter versions pinned.", - "If no Hub adapter fits, use the research dataset blueprint to draft labels locally.", - ) - - mtx = ( - "Treat experimental MTX / mixed-precision experiments as optional research notes, not production defaults.", - "Document any experimental kernel or quant recipe separately from the shipping wire config.", - "Prefer proven mlx-community quants for the first vertical slice; revisit MTX only after a working baseline.", - ) - - study = ( - "Apple MLX documentation: model convert / quantize / generate flows.", - "mlx-lm and mlx-vlm server READMEs for OpenAI-compatible local serving.", - "mlx-scout runtimes reference: LM Studio vs mlx_lm vs Ollama trade-offs.", - "PEFT/LoRA primers focused on low-rank adapters for domain dial-in.", - ) - - stack = ( - "1. Run `mlx-agent research` with the same goal/modalities to rank models, adapters, and datasets.", - "2. Follow the pack's Runtime preference (mlx-vlm / LM Studio / mlx_lm; Ollama only as curated alternate).", - "3. `mlx-agent adopt start` to verify a candidate on a local runtime.", - "4. `mlx-agent wire` to apply a confirmation-gated config — never skip preview hashes.", - ) - - next_steps = ( - "Freeze the goal and modalities above; do not expand scope mid-slice.", - "Generate a research pack and pick one base model + optional adapter.", - "Create or obtain a small labeled dataset offline.", - "Implement training outside mlx-agent; keep this design pack as the agent-ingestible plan.", - ) - if brief.notes: - next_steps = next_steps + ("Honor project notes: {0}".format(brief.notes),) - - return ProjectDesignPack( - brief=brief, - generated_at=moment.isoformat(), - quantization_ideas=tuple(quant), - training_loop=training, - lora_notes=lora, - mtx_notes=mtx, - study_materials=study, - stack_path=stack, - next_steps=next_steps, - ) - - -def render_design_pack(pack: ProjectDesignPack) -> str: - brief = pack.brief - lines = [ - "# MLX Project Design Pack: {0}".format(brief.goal), - "", - "- Generated: {0}".format(pack.generated_at), - "- Modalities: {0}".format(", ".join(brief.modalities) or "none"), - "- Memory budget (GB): {0}".format( - brief.memory_gb if brief.memory_gb is not None else "none" - ), - ] - if brief.notes: - lines.append("- Notes: {0}".format(brief.notes)) - lines.extend([ - "", - "## Goal", - "", - brief.goal, - "", - "## Recommended stack path", - "", - ]) - for item in pack.stack_path: - lines.append("- {0}".format(item)) - lines.extend(["", "## Quantization ideas", ""]) - for item in pack.quantization_ideas: - lines.append("- {0}".format(item)) - lines.extend(["", "## Training loop sketch", ""]) - for item in pack.training_loop: - lines.append("- {0}".format(item)) - lines.extend(["", "## LoRA / adapter notes", ""]) - for item in pack.lora_notes: - lines.append("- {0}".format(item)) - lines.extend(["", "## Experimental MTX notes", ""]) - for item in pack.mtx_notes: - lines.append("- {0}".format(item)) - lines.extend(["", "## Study materials", ""]) - for item in pack.study_materials: - lines.append("- {0}".format(item)) - lines.extend(["", "## Next steps", ""]) - for index, item in enumerate(pack.next_steps, start=1): - lines.append("{0}. {1}".format(index, item)) - lines.extend([ - "", - "## Notes", - "", - "This pack is guidance only for downstream agents. " - "mlx-agent blueprint does not scaffold repositories, download weights, or run training.", - "", - ]) - return "\n".join(lines) - - -def write_design_pack( - markdown: str, - brief: ProjectBrief, - root: Optional[object] = None, - now: Optional[datetime] = None, - pack: Optional[ProjectDesignPack] = None, -) -> Path: - """Write under ``/mlx-blueprints`` and never outside it.""" - moment = now or datetime.now(timezone.utc) - base = Path(root) if root is not None else Path.cwd() - out_dir = (base / "mlx-blueprints").resolve() - unresolved_dir = base / "mlx-blueprints" - if unresolved_dir.is_symlink(): - raise ValueError("refusing to write design pack through a symlinked folder") - timestamp = moment.strftime("%Y%m%dT%H%M%SZ") - filename = "{0}-{1}.md".format(slugify(brief.goal), timestamp) - path = (out_dir / filename).resolve() - if os.path.commonpath([str(out_dir), str(path)]) != str(out_dir): - raise ValueError("refusing to write design pack outside the project folder") - out_dir.mkdir(parents=True, exist_ok=True) - path.write_text(markdown, encoding="utf-8") - if pack is not None: - sidecar = path.with_suffix(".json") - if os.path.commonpath([str(out_dir), str(sidecar.resolve())]) != str(out_dir): - raise ValueError("refusing to write design pack outside the project folder") - sidecar.write_text(json.dumps(pack.to_dict(), indent=2) + "\n", encoding="utf-8") - return path diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/research.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/research.py deleted file mode 100644 index c817b79..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/research.py +++ /dev/null @@ -1,502 +0,0 @@ -"""Read-only domain research packs: discovery -> scoring -> project-local markdown. - -No verification, no wiring, no downloads. Discovery and the Hugging Face client -are injected so this module is fully testable offline. Output is written only -inside the project-local ``mlx-research`` folder. -""" - -from __future__ import annotations - -import json -import os -import re -from dataclasses import dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -from .blueprint import DatasetBlueprint, build_dataset_blueprint -from .discovery import DiscoveryRequest -from .interview import DomainIntent -from .modality import FACET_CHOICES, facet_label, modality_label -from .runtime_preference import RuntimePreference, prefer_runtime, wiring_for_preference -from .scoring import ScoreResult, rank_scored, score_candidate - - -_SLUG_RE = re.compile(r"[^a-z0-9]+") -_CARD_EXCERPT_CHARS = 240 -_DEFAULT_CATALOG_LIMIT = 5 - - -@dataclass(frozen=True) -class ResearchCandidate: - repo: str - role: str - score: float - wiring: str - signals: Tuple[dict, ...] - provenance: Tuple[dict, ...] - card_present: bool - card_excerpt: str - - def to_dict(self): - return { - "repo": self.repo, - "role": self.role, - "score": self.score, - "wiring": self.wiring, - "signals": [dict(signal) for signal in self.signals], - "provenance": [dict(record) for record in self.provenance], - "card_present": self.card_present, - "card_excerpt": self.card_excerpt, - } - - -@dataclass(frozen=True) -class CatalogItem: - repo: str - kind: str # adapter | dataset - score: float - signals: Tuple[dict, ...] - provenance: Tuple[dict, ...] - card_excerpt: str - record: Dict[str, object] - - def to_dict(self): - return { - "repo": self.repo, - "kind": self.kind, - "score": self.score, - "signals": [dict(signal) for signal in self.signals], - "provenance": [dict(record) for record in self.provenance], - "card_excerpt": self.card_excerpt, - "record": dict(self.record), - } - - -@dataclass(frozen=True) -class ResearchPack: - intent: DomainIntent - candidates: Tuple[ResearchCandidate, ...] - generated_at: str - warnings: Tuple[dict, ...] = field(default_factory=tuple) - adapters: Tuple[CatalogItem, ...] = field(default_factory=tuple) - datasets: Tuple[CatalogItem, ...] = field(default_factory=tuple) - dataset_blueprint: Optional[DatasetBlueprint] = None - runtime_preference: Optional[RuntimePreference] = None - - def to_dict(self): - return { - "intent": self.intent.to_dict(), - "candidates": [candidate.to_dict() for candidate in self.candidates], - "generated_at": self.generated_at, - "warnings": [dict(warning) for warning in self.warnings], - "adapters": [item.to_dict() for item in self.adapters], - "datasets": [item.to_dict() for item in self.datasets], - "dataset_blueprint": ( - self.dataset_blueprint.to_dict() if self.dataset_blueprint else None - ), - "runtime_preference": ( - self.runtime_preference.to_dict() if self.runtime_preference else None - ), - } - - -def slugify(domain: str) -> str: - slug = _SLUG_RE.sub("-", domain.strip().lower()).strip("-") - return slug or "domain" - - -def candidate_metadata(candidate: Dict[str, object]) -> Dict[str, object]: - """Map a discovery candidate to the scoring core's metadata shape.""" - roles = candidate.get("roles") or ([candidate["role"]] if candidate.get("role") else []) - return { - "roles": list(roles), - "license": candidate.get("license"), - "downloads": candidate.get("downloads", 0), - "likes": candidate.get("likes", 0), - "est_ram_gb": candidate.get("est_ram_gb"), - "tags": candidate.get("tags", []), - } - - -def hub_row_metadata(row: Dict[str, object], intent: DomainIntent) -> Dict[str, object]: - """Map a Hub adapter/dataset row into the scoring metadata shape.""" - tags = list(row.get("tags") or []) - license_name = row.get("license") - if not license_name: - for tag in tags: - text = str(tag) - if text.startswith("license:"): - license_name = text.split(":", 1)[1] - break - return { - "roles": list(intent.roles), - "license": license_name, - "downloads": row.get("downloads", 0), - "likes": row.get("likes", 0), - "est_ram_gb": None, - "tags": tags, - } - - -def catalog_search_query(intent: DomainIntent) -> str: - parts = list(intent.keywords) - if intent.domain and intent.domain.lower() not in {p.lower() for p in parts}: - parts.append(intent.domain) - return " ".join(parts).strip() - - -def _excerpt(card_text: Optional[str]) -> str: - if not card_text: - return "" - collapsed = " ".join(card_text.split()) - return collapsed[:_CARD_EXCERPT_CHARS] - - -def _repo_id(row: Dict[str, object]) -> Optional[str]: - repo = row.get("id") or row.get("modelId") or row.get("repo") - if not repo: - return None - return str(repo) - - -def _score_catalog_rows( - intent: DomainIntent, - rows: List[Dict[str, object]], - kind: str, - fetch_card, - catalog_limit: int, -) -> Tuple[CatalogItem, ...]: - scored: List[Tuple[str, ScoreResult]] = [] - detail: Dict[str, Dict[str, object]] = {} - for row in rows: - repo = _repo_id(row) - if not repo or repo in detail: - continue - try: - card_text = fetch_card(repo) - except Exception: - card_text = None - result = score_candidate(intent, hub_row_metadata(row, intent), card_text) - scored.append((repo, result)) - detail[repo] = {"row": row, "card_text": card_text} - - ranked = rank_scored(scored)[:max(0, catalog_limit)] - items: List[CatalogItem] = [] - for repo, result in ranked: - row = detail[repo]["row"] - card_text = detail[repo]["card_text"] - items.append(CatalogItem( - repo=repo, - kind=kind, - score=result.score, - signals=tuple(signal.to_dict() for signal in result.signals), - provenance=result.provenance, - card_excerpt=_excerpt(card_text), - record=dict(row), - )) - return tuple(items) - - -def generate_pack( - intent: DomainIntent, - discovery_service, - hf_client, - limit: int = 6, - catalog_limit: int = _DEFAULT_CATALOG_LIMIT, - now: Optional[datetime] = None, -) -> ResearchPack: - """Build a research pack. Read-only: never verifies, wires, or downloads.""" - moment = now or datetime.now(timezone.utc) - seen: Dict[str, Dict[str, object]] = {} - warnings: List[dict] = [] - host: Dict[str, object] = {} - for role in intent.roles: - request = DiscoveryRequest( - role=role, - memory_gb=intent.memory_gb, - licenses=list(intent.license_allow) or None, - limit=limit, - ) - envelope = discovery_service.discover(request) - if envelope.status != "ok": - detail = envelope.error - warnings.append({ - "code": detail.code if detail else "discovery_failed", - "message": "{0}: {1}".format(role, detail.message if detail else "discovery failed"), - }) - continue - if not host and isinstance(envelope.data.get("host"), dict): - host = dict(envelope.data["host"]) - for bucket in envelope.data.get("roles", {}).values(): - for candidate in bucket: - repo = candidate.get("repo") - if repo and repo not in seen: - seen[repo] = candidate - - preference = prefer_runtime( - host, - roles=intent.roles, - modalities=intent.modalities, - ) - - scored: List[Tuple[str, ScoreResult]] = [] - detail_by_repo: Dict[str, Dict[str, object]] = {} - for repo, candidate in seen.items(): - try: - card_text = hf_client.fetch_model_card(repo) - except Exception: - card_text = None - result = score_candidate(intent, candidate_metadata(candidate), card_text) - scored.append((repo, result)) - detail_by_repo[repo] = {"candidate": candidate, "card_text": card_text} - - ranked = rank_scored(scored)[:max(0, limit)] - candidates = [] - for repo, result in ranked: - candidate = detail_by_repo[repo]["candidate"] - card_text = detail_by_repo[repo]["card_text"] - role = candidate.get("role", "") - candidates.append(ResearchCandidate( - repo=repo, - role=role, - score=result.score, - wiring=wiring_for_preference(repo, role, preference), - signals=tuple(signal.to_dict() for signal in result.signals), - provenance=result.provenance, - card_present=bool(card_text), - card_excerpt=_excerpt(card_text), - )) - - search = catalog_search_query(intent) - try: - adapter_rows = list(hf_client.list_adapters(search=search, limit_fetch=max(catalog_limit * 4, 20))) - except Exception as error: - warnings.append({ - "code": "adapter_search_failed", - "message": "adapter search failed: {0}".format(error), - }) - adapter_rows = [] - try: - dataset_rows = list(hf_client.list_datasets(search=search, limit_fetch=max(catalog_limit * 4, 20))) - except Exception as error: - warnings.append({ - "code": "dataset_search_failed", - "message": "dataset search failed: {0}".format(error), - }) - dataset_rows = [] - - adapters = _score_catalog_rows( - intent, - adapter_rows, - "adapter", - hf_client.fetch_model_card, - catalog_limit, - ) - datasets = _score_catalog_rows( - intent, - dataset_rows, - "dataset", - hf_client.fetch_dataset_card, - catalog_limit, - ) - blueprint = build_dataset_blueprint(intent) if not datasets else None - - return ResearchPack( - intent=intent, - candidates=tuple(candidates), - generated_at=moment.isoformat(), - warnings=tuple(warnings), - adapters=adapters, - datasets=datasets, - dataset_blueprint=blueprint, - runtime_preference=preference, - ) - - -def _render_catalog_section(title: str, items: Tuple[CatalogItem, ...], empty_message: str) -> List[str]: - lines = ["## {0}".format(title), ""] - if not items: - lines.append(empty_message) - lines.append("") - return lines - for index, item in enumerate(items, start=1): - lines.append("### {0}. `{1}` — score {2}/100".format(index, item.repo, item.score)) - lines.append("") - lines.append("- Kind: {0}".format(item.kind)) - matched = [s for s in item.signals if s["applicable"] and s["matched"]] - if matched: - lines.append("- Why it ranked:") - for signal in matched: - lines.append(" - {0}: {1}".format(signal["id"], signal["detail"])) - if item.card_excerpt: - lines.append("- Card excerpt: {0}".format(item.card_excerpt)) - lines.append("") - return lines - - -def render_pack(pack: ResearchPack) -> str: - intent = pack.intent - lines = [ - "# MLX Research Pack: {0}".format(intent.domain), - "", - "- Generated: {0}".format(pack.generated_at), - "- Roles: {0}".format(", ".join(intent.roles) or "none"), - "- Keywords: {0}".format(", ".join(intent.keywords) or "none"), - "- License filter: {0}".format(", ".join(intent.license_allow) or "none"), - "- Memory budget (GB): {0}".format(intent.memory_gb if intent.memory_gb is not None else "none"), - "- Modalities: {0}".format(", ".join(intent.modalities) or "none"), - "- Facets: {0}".format(", ".join(intent.facets) or "none"), - ] - if intent.notes: - lines.append("- Notes: {0}".format(intent.notes)) - lines.extend(["", "## Summary", ""]) - if pack.candidates: - lines.append( - "{0} candidate(s) ranked by transparent scoring. Scores are read-only " - "estimates from metadata and model-card text; no model was downloaded, " - "verified, or wired.".format(len(pack.candidates)) - ) - else: - lines.append("No candidates were found for this intent. See warnings below.") - summary_bits = [] - if pack.adapters: - summary_bits.append("{0} adapter(s)".format(len(pack.adapters))) - if pack.datasets: - summary_bits.append("{0} dataset(s)".format(len(pack.datasets))) - if pack.dataset_blueprint is not None: - summary_bits.append("dataset blueprint (no Hub datasets matched)") - if summary_bits: - lines.append("Catalog enrichment: {0}.".format("; ".join(summary_bits))) - if pack.warnings: - lines.extend(["", "## Warnings", ""]) - for warning in pack.warnings: - lines.append("- [{0}] {1}".format(warning.get("code", "warning"), warning.get("message", ""))) - lines.extend(["", "## Modality foundations", ""]) - if intent.modalities: - for modality in intent.modalities: - allowed = set(FACET_CHOICES.get(modality, {}).values()) - facet_ids = [facet for facet in intent.facets if facet in allowed] - lines.append("- `{0}` — {1}".format(modality, modality_label(modality))) - if facet_ids: - lines.append(" - Facets: {0}".format( - ", ".join( - "`{0}` ({1})".format(facet, facet_label(facet)) - for facet in facet_ids - ) - )) - lines.append("") - else: - lines.append("No foundational modalities were selected for this pack.") - lines.append("") - lines.extend(["", "## Runtime preference", ""]) - preference = pack.runtime_preference - if preference is not None: - lines.append("- Preferred: `{0}`".format(preference.preferred)) - lines.append( - "- Alternates: {0}".format( - ", ".join("`{0}`".format(item) for item in preference.alternates) - if preference.alternates - else "none" - ) - ) - lines.append("- Rationale: {0}".format(preference.rationale)) - snapshot = preference.host_snapshot - lines.append( - "- Host inventory: Ollama {0}; LM Studio {1}".format( - "up" if snapshot.get("ollama") else "down", - "up" if snapshot.get("lmstudio") else "down", - ) - ) - lines.append("") - else: - lines.append("No runtime preference was computed for this pack.") - lines.append("") - lines.extend(["", "## Candidates", ""]) - for index, candidate in enumerate(pack.candidates, start=1): - lines.append("### {0}. `{1}` — score {2}/100".format(index, candidate.repo, candidate.score)) - lines.append("") - lines.append("- Role: {0}".format(candidate.role)) - lines.append("- Suggested wiring: {0}".format(candidate.wiring)) - lines.append("- Model card: {0}".format("present" if candidate.card_present else "not found")) - matched = [s for s in candidate.signals if s["applicable"] and s["matched"]] - if matched: - lines.append("- Why it ranked:") - for signal in matched: - lines.append(" - {0}: {1}".format(signal["id"], signal["detail"])) - if candidate.card_excerpt: - lines.append("- Card excerpt: {0}".format(candidate.card_excerpt)) - lines.append("") - lines.extend(_render_catalog_section( - "Adapters / LoRAs", - pack.adapters, - "No PEFT/LoRA adapters matched this intent on the Hub (read-only search).", - )) - lines.extend(_render_catalog_section( - "Datasets", - pack.datasets, - "No datasets matched this intent on the Hub. See the dataset blueprint below.", - )) - if pack.dataset_blueprint is not None: - lines.extend([ - "## Dataset blueprint", - "", - "No Hub datasets ranked for this intent. Use this guidance-only blueprint " - "to create a local dataset — nothing is downloaded or trained by research.", - "", - pack.dataset_blueprint.to_markdown().rstrip(), - "", - ]) - lines.extend([ - "## Next steps", - "", - "1. Review candidates, adapters, and datasets above and pick a local stack.", - "2. Prefer the runtime in **Runtime preference** above" - + ( - " (`{0}`)".format(pack.runtime_preference.preferred) - if pack.runtime_preference is not None - else "" - ) - + "; Ollama remains a valid alternate for curated tags when listed.", - "3. Install a base model in that runtime (LM Studio, mlx_lm/mlx-vlm, or Ollama).", - "4. Run `mlx-agent adopt start` to verify and wire it — research does not verify or download.", - "5. If no datasets matched, follow the dataset blueprint to create labeled data locally.", - "", - "## Notes", - "", - "This pack is a read-only research artifact intended for ingestion by other agents. " - "All scores are estimates with per-signal provenance; verify capability before relying on any model. " - "Runtime preference is guidance from host inventory and modality/role rules — it does not change scores.", - "", - ]) - return "\n".join(lines) - - -def write_pack( - markdown: str, - intent: DomainIntent, - root: Optional[object] = None, - now: Optional[datetime] = None, - pack: Optional[ResearchPack] = None, -) -> Path: - """Write the pack under ``/mlx-research`` and never outside it.""" - moment = now or datetime.now(timezone.utc) - base = Path(root) if root is not None else Path.cwd() - out_dir = (base / "mlx-research").resolve() - unresolved_dir = base / "mlx-research" - if unresolved_dir.is_symlink(): - raise ValueError("refusing to write research pack through a symlinked folder") - timestamp = moment.strftime("%Y%m%dT%H%M%SZ") - filename = "{0}-{1}.md".format(slugify(intent.domain), timestamp) - path = (out_dir / filename).resolve() - if os.path.commonpath([str(out_dir), str(path)]) != str(out_dir): - raise ValueError("refusing to write research pack outside the project folder") - out_dir.mkdir(parents=True, exist_ok=True) - path.write_text(markdown, encoding="utf-8") - if pack is not None: - sidecar = path.with_suffix(".json") - if os.path.commonpath([str(out_dir), str(sidecar.resolve())]) != str(out_dir): - raise ValueError("refusing to write research pack outside the project folder") - sidecar.write_text(json.dumps(pack.to_dict(), indent=2) + "\n", encoding="utf-8") - return path diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/adapter-runtime.json b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/adapter-runtime.json deleted file mode 100644 index b0e6f55..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/adapter-runtime.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "schema_version": "1.0", - "purpose": "Bundled adapter runtime resource contract" -} diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/bench-community.json b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/bench-community.json deleted file mode 100644 index 4c3573d..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/bench-community.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "schema_version": "1.0", - "entries": [] -} diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/references/model-families.md b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/references/model-families.md deleted file mode 100644 index e331847..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/references/model-families.md +++ /dev/null @@ -1,35 +0,0 @@ -# Model family quirks (chat templates, tool calling, reasoning) - -Family-specific behaviors that change how you wire, prompt, and verify. Read this before debugging a "broken" model — it is usually a template mismatch. - -## Qwen (Qwen3 and later) - -- Ships **hybrid reasoning**: `enable_thinking` in the chat template toggles hidden `` blocks. `mlx-agent` detects this from the template and tags reasoning models so they stay out of fast/utility roles. -- Tool calling uses the Hermes-style `` JSON blocks; works well through `mlx_lm.server` and LM Studio. Verified tool-use is still per-model — template support is not behavior proof. -- MoE variants (`A3B`, `A35B`) are the speed pick on Apple Silicon; dense variants are the consistency pick. - -## Gemma - -- **No system role.** A system prompt must be folded into the first user turn; runtimes that auto-inject a system role produce degraded or refused outputs. -- Google ships **QAT (quantization-aware trained)** builds — their 4bit is unusually strong; prefer publisher QAT over generic 4bit re-quants. -- Tool calling is template-level and inconsistent across sizes; verify before assigning a tool-use role. - -## gpt-oss (OpenAI open weights) - -- **MXFP4 native** — use the MXFP4 builds; do not re-quantize to generic 4bit. -- Uses the **harmony** response format with `reasoning_effort` levels (low/medium/high). These are reasoning models by design: expect hidden analysis channels; never put them in a fast/cheap slot. -- Strong tool calling when served through runtimes that parse harmony (recent `mlx-lm`, LM Studio). - -## Llama (Meta, MLX ports) - -- Solid generic instruction following; tool calling depends heavily on the exact instruct template version — verify, don't assume. -- Community MLX ports vary in template fidelity; check the repo's `chat_template` presence before wiring (missing template → runtime default, often wrong). - -## Vision (Qwen-VL and friends) - -- Vision models need `mlx-vlm`; Ollama's engine does not run them. Wire them on their own port (`:8083` by convention) so they never collide with the text server. -- OCR quality varies more with input resolution handling than with quant; keep 4bit unless the documents are dense. - -## Embedding models - -- Served via `/v1/embeddings` (OpenAI-compatible) or `/api/embed` (Ollama). They cannot chat; a "ready" generation probe is not meaningful for them — rely on the embedding probe, not text generation. diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/references/quantization.md b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/references/quantization.md deleted file mode 100644 index 7600943..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/references/quantization.md +++ /dev/null @@ -1,25 +0,0 @@ -# MLX quantization guide - -How to pick a quant on Apple Silicon. Read this before recommending a specific quantized repo. - -## The tradeoff ladder - -| Quant | Size vs bf16 | Quality | When to pick | -| --- | --- | --- | --- | -| `bf16` / `fp16` | 1.0× | reference | Fine-tuning source, quality ceiling checks. Rarely for serving. | -| `8bit` | ~0.5× | near-reference | RAM is comfortable and the task is quality-sensitive (reasoning, code review). | -| `4bit` | ~0.25× | good | Default serving choice. Best size/quality point for most models. | -| `MXFP4` | ~0.25× | good | Native format for gpt-oss; use the publisher's own MXFP4 builds. | -| `3bit` and below | ≤0.2× | noticeably worse | Only when nothing else fits; expect degradation on reasoning and tool calling. | - -Rules of thumb: - -- **Spend RAM on a bigger model at 4bit, not a smaller model at 8bit.** A 32B at 4bit (~18 GB) usually beats a 14B at 8bit (~15 GB) on hard tasks. -- **Mixed MoE quants matter.** MoE models (e.g. Qwen3-A3B) activate few parameters per token, so at the same RAM they decode much faster than dense. On memory-bandwidth-bound Apple Silicon, active parameters set decode speed. -- **Weights are not the whole budget.** KV cache grows linearly with context: `2 × layers × kv_heads × head_dim × context × 2 bytes` (fp16). A model that "fits" can still die at long context. `mlx-agent discover --context N` computes this for you; the candidate's `estimates.kv.max_context_tokens` is the honest ceiling. -- **Publisher quants beat community re-quants** for quality-sensitive roles: mlx-community and the original publisher calibrate with different data and care. When both exist, prefer the publisher or `mlx-community` (trusted), then unsloth. -- **LoRA preserves quant.** `mlx_lm.lora` trains on the quantized base and `mlx_lm.fuse` keeps the quant, so you do not need bf16 weights to fine-tune. - -## Reasoning models and quant - -Reasoning models emit long hidden chains, so they are more sensitive to aggressive quant than chat models (errors compound over thousands of generated tokens). Prefer 8bit for a reasoning role when RAM allows; 4bit is acceptable for MXFP4-native families. diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/references/troubleshooting.md b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/references/troubleshooting.md deleted file mode 100644 index 4ae6b5c..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/references/troubleshooting.md +++ /dev/null @@ -1,33 +0,0 @@ -# Troubleshooting local MLX serving - -Symptom-first playbook. Each entry: cause, then the fix. Run `mlx-agent doctor models` first — half of these are visible there. - -## "It was working yesterday" - -1. `mlx-agent doctor models` — look for `drift_missing_model` (deleted weights), `drift_hash_mismatch` (hand-edited config), `endpoint_down`. -2. `mlx-agent serve status` — a serve receipt whose pid died, or an argv mismatch after a manual restart. - -## Metal / memory allocation failures at load or mid-generation - -- Cause: weights + KV cache exceed unified memory under pressure. macOS will swap-thrash before it errors cleanly. -- Fix: drop context length (KV is linear in context — check `estimates.kv.max_context_tokens` from discovery), close other GPU consumers (browsers, other servers), or drop a quant tier. Two loaded 30B+ models do not coexist on ≤64 GB. - -## Slow tokens/sec - -- Expected ceiling: decode tok/s ≈ memory bandwidth ÷ active bytes per token. Dense 32B 4bit on a high-end M-series is tens of tok/s, not hundreds. -- Checklist: MoE instead of dense (2–5× at the same RAM); native `mlx_lm` on native MLX weights instead of Ollama for MoE; shorter context (KV reads grow); don't bench while another server is loaded — measurements contend. -- Prompt processing (prefill) is compute-bound, not bandwidth-bound; long prompts are slow on every runtime. Measure with `mlx-agent bench run` before blaming the model. - -## Server answers but agent calls fail - -- Port collision: two configs on one port (`doctor models` → `drift_endpoint_conflict`). Convention: mlx_lm `:8080`, mlx-vlm `:8083`, LiteLLM `:4000`, Ollama `:11434`, LM Studio `:1234`. -- Auth header: wired configs reference `MLX_AGENT_LOCAL_API_KEY` by design; export any value, but the variable must exist for OpenAI-compatible clients that send a key. -- Chat template missing/wrong: outputs look like raw completion text, tool calls never parse. Check the repo ships a `chat_template`; see `references/model-families.md` for family quirks (Gemma has no system role). - -## Tool calling fails verification - -- Metadata ≠ behavior. Only a schema-valid synthetic call counts. If the probe fails with `malformed_arguments` or `missing_tool_call`, the model (or its template at that quant) does not do reliable tool use — pick a different model for the tool-use role; do not lower the bar. - -## Gated repositories - -- `gated: true` means HF auth is required before any runtime can pull. Accept the terms on the Hub page with the account that owns the local `HF_TOKEN`; nothing in this pack downloads, so nothing here can accept terms for you. diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/serve-recipes.json b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/serve-recipes.json deleted file mode 100644 index d297d5e..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/resources/serve-recipes.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "schema_version": "1.0", - "recipes": { - "mlx_lm": { - "executable": "mlx_lm.server", - "argv": ["{executable}", "--model", "{repo}", "--port", "{port}", "--max-tokens", "{max_tokens}"], - "adapter_argv": ["--adapter-path", "{adapter_path}"], - "default_port": 8080, - "readiness": "http://127.0.0.1:{port}/v1/models", - "install_hint": "pip install mlx-lm" - }, - "mlx-vlm": { - "executable": "mlx_vlm.server", - "argv": ["{executable}", "--model", "{repo}", "--port", "{port}"], - "adapter_argv": null, - "default_port": 8083, - "readiness": "http://127.0.0.1:{port}/v1/models", - "install_hint": "pip install mlx-vlm" - } - } -} diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/runtime_preference.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/runtime_preference.py deleted file mode 100644 index d458cc1..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/runtime_preference.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Justified MLX-native runtime preference for research packs and discovery wiring. - -Reporting only: does not change scoring. Uses existing HostInventory flags -(ollama, lmstudio) — no new runtime probes. Ollama remains a valid alternate -for text workloads and is never removed from wire/CLI surfaces. - -This module stays free of interview/models imports to avoid cycles. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Mapping, Sequence, Tuple - - -_VISION_MODALITIES = frozenset({"document-vision", "video"}) - - -@dataclass(frozen=True) -class RuntimePreference: - preferred: str - alternates: Tuple[str, ...] - rationale: str - host_snapshot: Mapping[str, object] - - def to_dict(self): - return { - "preferred": self.preferred, - "alternates": list(self.alternates), - "rationale": self.rationale, - "host_snapshot": { - "ollama": bool(self.host_snapshot.get("ollama")), - "lmstudio": bool(self.host_snapshot.get("lmstudio")), - "chip": self.host_snapshot.get("chip"), - "ram_gb": self.host_snapshot.get("ram_gb"), - }, - } - - -def _needs_vision_path(roles: Sequence[str], modalities: Sequence[str]) -> bool: - if "vision" in roles: - return True - return any(modality in _VISION_MODALITIES for modality in modalities) - - -def prefer_runtime( - host: Mapping[str, object], - roles: Sequence[str] = (), - modalities: Sequence[str] = (), -) -> RuntimePreference: - """Return a deterministic preferred runtime with alternates and rationale.""" - ollama = bool(host.get("ollama")) - lmstudio = bool(host.get("lmstudio")) - snapshot = { - "ollama": ollama, - "lmstudio": lmstudio, - "chip": host.get("chip"), - "ram_gb": host.get("ram_gb"), - } - - if _needs_vision_path(roles, modalities): - alternates = [] - if lmstudio: - alternates.append("lmstudio") - return RuntimePreference( - preferred="mlx-vlm", - alternates=tuple(alternates), - rationale=( - "Vision / document / video workloads need native mlx-vlm; " - "Ollama does not run VLMs. LM Studio can help download weights " - "when its server is up." - ), - host_snapshot=snapshot, - ) - - audio = "audio" in modalities - if lmstudio: - alternates = ["mlx_lm"] - if ollama: - alternates.append("ollama") - rationale = ( - "LM Studio is the lowest-friction native MLX path for arbitrary Hub " - "repos when its local server is up." - ) - if audio: - rationale += ( - " Audio packs still prefer native MLX weights; Ollama is limited " - "to curated tags." - ) - return RuntimePreference( - preferred="lmstudio", - alternates=tuple(alternates), - rationale=rationale, - host_snapshot=snapshot, - ) - - alternates = [] - if ollama: - alternates.append("ollama") - rationale = ( - "Prefer headless mlx_lm for native MLX quants and Hub repos when LM Studio " - "is not detected." - ) - if audio: - rationale += ( - " Audio packs benefit from native MLX serving; Ollama remains a valid " - "alternate for curated tags only." - ) - elif ollama: - rationale += ( - " Ollama remains a valid first choice for curated library tags." - ) - else: - rationale += " Start LM Studio or install mlx-lm to serve local models." - - return RuntimePreference( - preferred="mlx_lm", - alternates=tuple(alternates), - rationale=rationale, - host_snapshot=snapshot, - ) - - -def wiring_for_preference(repo: str, role: str, preference: RuntimePreference) -> str: - """Render a suggested wiring string for the preferred runtime.""" - del role - short = repo.split("/")[-1].lower() - name = repo.split("/")[-1] - preferred = preference.preferred - if preferred == "mlx-vlm": - return "mlx-vlm server → `mlxvlm/{0}`".format(name) - if preferred == "lmstudio": - return "LM Studio → `lmstudio/{0}`".format(short) - if preferred == "ollama": - return "Ollama → `ollama/` (curated / GGUF; not arbitrary mlx-community)" - return "`mlx_lm.server --model {0}` → custom provider".format(repo) - - -def prefer_runtime_for_role(role: str, host: Mapping[str, object]) -> RuntimePreference: - """Scout/discovery helper: preference from a single role + host inventory.""" - modalities = ("document-vision",) if role == "vision" else () - return prefer_runtime(host, roles=(role or "general",), modalities=modalities) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/scoring.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/scoring.py deleted file mode 100644 index 2bad358..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/scoring.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Deterministic, transparent scoring of a model against a DomainIntent. - -Pure and side-effect free. Signals that do not apply to the intent are excluded -from both the numerator and denominator so a candidate is never penalized for a -constraint the user did not request. No network access. -""" - -from __future__ import annotations - -import math -from dataclasses import dataclass -from typing import List, Mapping, Optional, Sequence, Tuple - -from .interview import DomainIntent - - -SIGNAL_WEIGHTS = { - "role_match": 30.0, - "keyword_match": 25.0, - "popularity": 15.0, - "license_ok": 10.0, - "memory_fit": 10.0, - "card_quality": 10.0, -} - -_SIGNAL_SOURCES = { - "role_match": "local_role_derivation", - "keyword_match": "card_text_and_tags", - "popularity": "huggingface_model_list", - "license_ok": "huggingface_model_metadata", - "memory_fit": "local_memory_estimate", - "card_quality": "card_text", -} - -_POPULARITY_LOG_CEILING = 6.0 # ~1e6 downloads saturates the signal. -_MEMORY_HEADROOM_FRACTION = 0.8 - - -@dataclass(frozen=True) -class Signal: - id: str - weight: float - applicable: bool - matched: Optional[bool] - contribution: float - source: str - detail: str - - def to_dict(self): - return { - "id": self.id, - "weight": self.weight, - "applicable": self.applicable, - "matched": self.matched, - "contribution": round(self.contribution, 3), - "source": self.source, - "detail": self.detail, - } - - -@dataclass(frozen=True) -class ScoreResult: - score: float - signals: Tuple[Signal, ...] - provenance: Tuple[dict, ...] - - def to_dict(self): - return { - "score": self.score, - "signals": [signal.to_dict() for signal in self.signals], - "provenance": [dict(record) for record in self.provenance], - } - - -def _signal(signal_id, applicable, matched, fraction, detail): - weight = SIGNAL_WEIGHTS[signal_id] - contribution = weight * fraction if applicable else 0.0 - return Signal( - id=signal_id, - weight=weight, - applicable=applicable, - matched=matched if applicable else None, - contribution=contribution, - source=_SIGNAL_SOURCES[signal_id], - detail=detail, - ) - - -def _role_signal(intent: DomainIntent, metadata: Mapping[str, object]) -> Signal: - model_roles = set(metadata.get("roles") or []) - matched = [role for role in intent.roles if role in model_roles] - fraction = len(matched) / len(intent.roles) if intent.roles else 0.0 - return _signal( - "role_match", True, bool(matched), fraction, - "matched roles: {0}".format(", ".join(matched) or "none"), - ) - - -def _keyword_signal( - intent: DomainIntent, metadata: Mapping[str, object], card_text: Optional[str] -) -> Signal: - if not intent.keywords: - return _signal("keyword_match", False, None, 0.0, "no keywords requested") - tags = " ".join(str(tag).lower() for tag in (metadata.get("tags") or [])) - haystack = "{0} {1}".format((card_text or "").lower(), tags) - matched = [keyword for keyword in intent.keywords if keyword in haystack] - fraction = len(matched) / len(intent.keywords) - return _signal( - "keyword_match", True, bool(matched), fraction, - "matched keywords: {0}".format(", ".join(matched) or "none"), - ) - - -def _popularity_signal(metadata: Mapping[str, object]) -> Signal: - downloads = metadata.get("downloads") or 0 - try: - downloads = max(0, int(downloads)) - except (TypeError, ValueError): - downloads = 0 - fraction = min(1.0, math.log10(downloads + 1) / _POPULARITY_LOG_CEILING) - return _signal( - "popularity", True, downloads > 0, fraction, - "downloads: {0}".format(downloads), - ) - - -def _license_signal(intent: DomainIntent, metadata: Mapping[str, object]) -> Signal: - if not intent.license_allow: - return _signal("license_ok", False, None, 0.0, "no license restriction") - license_name = (metadata.get("license") or "").lower() - matched = license_name in set(intent.license_allow) - return _signal( - "license_ok", True, matched, 1.0 if matched else 0.0, - "license: {0}".format(license_name or "unknown"), - ) - - -def _memory_signal(intent: DomainIntent, metadata: Mapping[str, object]) -> Signal: - est_ram = metadata.get("est_ram_gb") - if intent.memory_gb is None or est_ram is None: - return _signal("memory_fit", False, None, 0.0, "no budget or estimate") - fits = float(est_ram) < intent.memory_gb * _MEMORY_HEADROOM_FRACTION - return _signal( - "memory_fit", True, fits, 1.0 if fits else 0.0, - "est_ram_gb {0} vs budget {1}".format(est_ram, intent.memory_gb), - ) - - -def _card_quality_signal(card_text: Optional[str]) -> Signal: - if not card_text: - return _signal("card_quality", True, False, 0.0, "no model card text") - quality = 0.5 - lowered = card_text.lower() - if "```" in card_text or "usage" in lowered or "example" in lowered or "how to" in lowered: - quality += 0.5 - return _signal( - "card_quality", True, True, quality, - "card present; usage/example: {0}".format(quality > 0.5), - ) - - -def score_candidate( - intent: DomainIntent, - metadata: Mapping[str, object], - card_text: Optional[str], -) -> ScoreResult: - signals = ( - _role_signal(intent, metadata), - _keyword_signal(intent, metadata, card_text), - _popularity_signal(metadata), - _license_signal(intent, metadata), - _memory_signal(intent, metadata), - _card_quality_signal(card_text), - ) - applicable = [signal for signal in signals if signal.applicable] - denominator = sum(signal.weight for signal in applicable) - numerator = sum(signal.contribution for signal in applicable) - score = round(100.0 * numerator / denominator, 1) if denominator else 0.0 - provenance = tuple( - { - "source": signal.source, - "fields": [signal.id], - "basis": signal.detail, - } - for signal in applicable - ) - return ScoreResult(score=score, signals=signals, provenance=provenance) - - -def rank_scored( - entries: Sequence[Tuple[str, ScoreResult]] -) -> List[Tuple[str, ScoreResult]]: - """Sort (repo, ScoreResult) entries by score desc, then repo asc.""" - return sorted(entries, key=lambda entry: (-entry[1].score, entry[0])) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/serve.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/serve.py deleted file mode 100644 index 264cfca..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/serve.py +++ /dev/null @@ -1,528 +0,0 @@ -"""Confirmation-gated launcher for local MLX serving runtimes. - -Serve is the only component that spawns processes, so every mutation goes -through the wire-style preview -> confirm -> receipt flow. Serve never -installs runtimes, never downloads models, binds only to 127.0.0.1, and stops -only processes that it started itself, verified against their receipts. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import shutil -import signal -import subprocess -import time -import urllib.error -import urllib.request -from datetime import datetime, timezone -from pathlib import Path - -from .model_doctor import scan_wired_configs -from .transactions import _atomic_in_directory -from .wiring import require_secret_free_config, validate_health_endpoint - - -SERVE_RECEIPT_SCHEMA_VERSION = "1.0" -SERVE_RECEIPT_KIND = "serve" -MAX_TOKENS_MIN, MAX_TOKENS_MAX, MAX_TOKENS_DEFAULT = 256, 65536, 8192 -PORT_MIN, PORT_MAX = 1, 65535 -READINESS_DEADLINE_DEFAULT = 60.0 -STOP_DEADLINE_SECONDS = 10.0 -MAX_RECEIPTS = 50 - -_RECIPES_PATH = Path(__file__).resolve().parent / "resources" / "serve-recipes.json" - - -class ServeError(RuntimeError): - """Classified serve failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def load_recipes(path=None): - location = Path(path) if path is not None else _RECIPES_PATH - value = json.loads(location.read_text(encoding="utf-8")) - recipes = value.get("recipes") - if not isinstance(recipes, dict) or not recipes: - raise ValueError("serve recipe table is missing its recipes") - for name, recipe in recipes.items(): - for key in ("executable", "argv", "default_port", "readiness", "install_hint"): - if key not in recipe: - raise ValueError("serve recipe {0} is missing {1}".format(name, key)) - return recipes - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def _preview_hash(plan): - canonical = json.dumps( - {key: plan[key] for key in sorted(plan) if key != "preview_hash"}, - sort_keys=True, separators=(",", ":"), - ) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest() - - -def receipts_root(root=None): - base = Path(root) if root is not None else Path.cwd() - return base / ".mlx-agent-receipts" / "serve" - - -def plan_start(repo, runtime, recipes, port=None, max_tokens=MAX_TOKENS_DEFAULT, - adapter_path=None): - """Render the exact start plan; pure and side-effect free.""" - if not isinstance(repo, str) or not repo.strip() or "/" not in repo: - raise ServeError( - "invalid_repo", - "serve requires a publisher/model repository identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - recipe = recipes.get(runtime) - if recipe is None: - raise ServeError( - "unsupported_runtime", - "serve does not launch the {0} runtime.".format(runtime), - "Ollama and LM Studio manage their own servers; serve supports: {0}.".format( - ", ".join(sorted(recipes)) - ), - ) - if not isinstance(max_tokens, int) or isinstance(max_tokens, bool): - raise ServeError( - "invalid_arguments", "max_tokens must be an integer.", - "Pass --max-tokens between {0} and {1}.".format(MAX_TOKENS_MIN, MAX_TOKENS_MAX), - ) - if not MAX_TOKENS_MIN <= max_tokens <= MAX_TOKENS_MAX: - raise ServeError( - "invalid_arguments", - "max_tokens is outside {0}-{1}.".format(MAX_TOKENS_MIN, MAX_TOKENS_MAX), - "Pass a bounded --max-tokens value.", - ) - selected_port = recipe["default_port"] if port is None else port - if not isinstance(selected_port, int) or isinstance(selected_port, bool): - raise ServeError( - "invalid_arguments", "port must be an integer.", - "Pass --port between {0} and {1}.".format(PORT_MIN, PORT_MAX), - ) - if not PORT_MIN <= selected_port <= PORT_MAX: - raise ServeError( - "invalid_arguments", - "port is outside {0}-{1}.".format(PORT_MIN, PORT_MAX), - "Pass a valid loopback port.", - ) - if adapter_path is not None and not recipe.get("adapter_argv"): - raise ServeError( - "unsupported_runtime", - "The {0} recipe does not support adapter serving.".format(runtime), - "Serve adapters with a runtime whose recipe declares adapter support.", - ) - values = { - "executable": recipe["executable"], - "repo": repo, - "port": str(selected_port), - "max_tokens": str(max_tokens), - } - argv = [part.format(**values) for part in recipe["argv"]] - if adapter_path is not None: - argv.extend( - part.format(adapter_path=str(adapter_path)) for part in recipe["adapter_argv"] - ) - plan = { - "repo": repo, - "runtime": runtime, - "port": selected_port, - "max_tokens": max_tokens, - "adapter_path": str(adapter_path) if adapter_path is not None else None, - "argv": argv, - "readiness": recipe["readiness"].format(port=selected_port), - "bind": "127.0.0.1", - } - plan["preview_hash"] = _preview_hash(plan) - return plan - - -def _default_which(executable): - return shutil.which(executable) - - -def _default_port_free(port): - import socket - - probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - probe.settimeout(0.5) - return probe.connect_ex(("127.0.0.1", port)) != 0 - finally: - probe.close() - - -def _default_spawn(argv, log_path): - handle = open(log_path, "ab", buffering=0) - process = subprocess.Popen( - argv, - stdin=subprocess.DEVNULL, - stdout=handle, - stderr=subprocess.STDOUT, - start_new_session=True, - ) - return process.pid - - -def _default_readiness(url, deadline_seconds, clock=time.monotonic, sleep=time.sleep): - validate_health_endpoint(url) - deadline = clock() + deadline_seconds - while clock() < deadline: - try: - request = urllib.request.Request(url, method="GET") - opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - with opener.open(request, timeout=2.0) as response: - if 200 <= response.status < 400: - return True - except (OSError, ValueError, urllib.error.URLError): - pass - sleep(0.5) - return False - - -def _pid_alive(pid): - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True - except OSError: - return False - return True - - -def _pid_command(pid): - try: - result = subprocess.run( - ["ps", "-p", str(pid), "-o", "command="], - capture_output=True, text=True, timeout=5, - ) - except (OSError, subprocess.SubprocessError): - return None - if result.returncode != 0: - return None - return result.stdout.strip() or None - - -def start_serve(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, port_free=None, - wired_claims=None, spawn=None, readiness=None, - readiness_deadline=READINESS_DEADLINE_DEFAULT, now=_utc_now, - pid_alive=None): - """Execute a reviewed start plan; the only mutating serve entry point.""" - which = which or _default_which - port_free = port_free or _default_port_free - spawn = spawn or _default_spawn - readiness = readiness or _default_readiness - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise ServeError( - "preview_hash_required", - "--confirm requires the hash from a reviewed serve preview.", - "Run serve start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise ServeError( - "preview_stale", - "The supplied preview hash does not match this start plan.", - "Re-run serve start without --confirm and review the fresh plan.", - ) - - if which(plan["argv"][0]) is None: - recipe_hint = load_recipes()[plan["runtime"]]["install_hint"] - raise ServeError( - "runtime_not_installed", - "The {0} executable is not installed.".format(plan["argv"][0]), - "Install it yourself ({0}); serve never installs runtimes.".format(recipe_hint), - ) - if model_present is not None and not model_present(plan["repo"]): - raise ServeError( - "model_not_local", - "The model is not present in a local inventory.", - "Download it with the runtime's own pull command first; serve never downloads models.", - ) - if not port_free(plan["port"]): - raise ServeError( - "port_in_use", - "Port {0} is already bound on 127.0.0.1.".format(plan["port"]), - "Pick a free --port, or stop the process that owns this one.", - ) - if wired_claims is not None: - claimed = wired_claims(plan["port"], plan["runtime"]) - if claimed: - raise ServeError( - "port_in_use", - "A wired config already claims port {0}.".format(plan["port"]), - "Move this server to its own --port.", - ) - if plan["adapter_path"] is not None and not Path(plan["adapter_path"]).is_dir(): - raise ServeError( - "invalid_arguments", - "The adapter path does not exist: {0}".format(plan["adapter_path"]), - "Pass an existing --adapter-path directory produced by mlx_lm.lora.", - ) - - receipt_path = root / "{0}.json".format(plan["port"]) - if receipt_path.exists(): - existing = _read_receipt(receipt_path) - if existing is not None and pid_alive(existing.get("pid", -1)): - raise ServeError( - "port_in_use", - "A serve receipt for port {0} is still live.".format(plan["port"]), - "Run serve stop --port {0} first.".format(plan["port"]), - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}.log".format(plan["port"]) - pid = spawn(plan["argv"], str(log_path)) - if not readiness(plan["readiness"], readiness_deadline): - _terminate_pid(pid) - raise ServeError( - "readiness_timeout", - "The server did not answer its readiness endpoint in time.", - "Inspect the serve log at {0}.".format(log_path), - ) - - receipt = { - "schema_version": SERVE_RECEIPT_SCHEMA_VERSION, - "kind": SERVE_RECEIPT_KIND, - "repo": plan["repo"], - "runtime": plan["runtime"], - "port": plan["port"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "readiness": plan["readiness"], - "started_at": now(), - "preview_hash": plan["preview_hash"], - } - content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") - _atomic_in_directory(root, receipt_path.name, content, 0o600) - return {"status": "started", "receipt": receipt} - - -def _read_receipt(path): - try: - value = json.loads(Path(path).read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - return None - if not isinstance(value, dict) or value.get("kind") != SERVE_RECEIPT_KIND: - return None - if not isinstance(value.get("pid"), int) or isinstance(value.get("pid"), bool): - return None - if not isinstance(value.get("argv"), list) or not isinstance(value.get("port"), int): - return None - return value - - -def status_serve(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command): - """Cross-check serve receipts against live processes; read-only.""" - root = receipts_root(receipts_dir) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - if len(entries) >= MAX_RECEIPTS: - break - receipt = _read_receipt(path) - if receipt is None: - continue - alive = pid_alive(receipt["pid"]) - command = pid_command(receipt["pid"]) if alive else None - entries.append({ - "receipt": str(path), - "repo": receipt["repo"], - "runtime": receipt["runtime"], - "port": receipt["port"], - "pid": receipt["pid"], - "alive": alive, - "argv_match": _argv_matches(receipt, command) if alive else False, - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - }) - return entries - - -def _argv_matches(receipt, command, require_port=True): - if not command: - return False - argv = receipt.get("argv") or [] - if not argv: - return False - executable = Path(str(argv[0])).name - if executable not in command or str(receipt.get("repo")) not in command: - return False - if require_port: - return "--port {0}".format(receipt.get("port")) in command - return True - - -def _terminate_pid(pid, sig=signal.SIGTERM): - try: - os.kill(pid, sig) - except OSError: - return False - return True - - -def stop_serve(port, receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - terminate=_terminate_pid, clock=time.monotonic, sleep=time.sleep): - """Stop exactly the process a serve receipt owns; never kills by port scan.""" - root = receipts_root(receipts_dir) - receipt_path = root / "{0}.json".format(port) - receipt = _read_receipt(receipt_path) if receipt_path.is_file() else None - if receipt is None: - raise ServeError( - "receipt_not_found", - "No serve receipt exists for port {0}.".format(port), - "serve stop only stops processes that serve started; stop foreign processes yourself.", - ) - pid = receipt["pid"] - if not pid_alive(pid): - receipt_path.unlink() - return {"status": "already_stopped", "port": port, "pid": pid} - command = pid_command(pid) - if not _argv_matches(receipt, command): - raise ServeError( - "pid_argv_mismatch", - "The live pid {0} does not match the serve receipt; refusing to signal it.".format(pid), - "Inspect the process yourself; the receipt is retained at {0}.".format(receipt_path), - ) - terminate(pid, signal.SIGTERM) - deadline = clock() + STOP_DEADLINE_SECONDS - while clock() < deadline: - if not pid_alive(pid): - receipt_path.unlink() - return {"status": "stopped", "port": port, "pid": pid} - sleep(0.2) - terminate(pid, signal.SIGKILL) - if pid_alive(pid): - raise ServeError( - "stop_failed", - "The process did not exit after SIGTERM and SIGKILL.", - "Inspect pid {0} yourself; the receipt is retained at {1}.".format(pid, receipt_path), - ) - receipt_path.unlink() - return {"status": "stopped", "port": port, "pid": pid, "forced": True} - - -def wired_port_claim(roots): - """Build a wired_claims(port, runtime) gate from Wire-managed configs.""" - wired = scan_wired_configs(roots) - claims = {} - for config in wired: - endpoint = config.get("endpoint") - if not endpoint: - continue - try: - parsed = validate_health_endpoint(endpoint) - from urllib.parse import urlsplit - - parts = urlsplit(parsed) - port = parts.port or (443 if parts.scheme == "https" else 80) - except ValueError: - continue - claims.setdefault(port, config["runtime"]) - - def check(port, runtime): - owner = claims.get(port) - return owner is not None and owner != runtime - - return check - - -def default_model_present(hf_cache=None): - from .model_doctor import default_hf_cache, inspect_hf_cache - - cache = Path(hf_cache) if hf_cache is not None else default_hf_cache() - known = {item["id"].casefold() for item in inspect_hf_cache(cache)} - - def check(repo): - return repo.casefold() in known - - return check - - -LAUNCHD_LABEL_PREFIX = "com.mlx-agent.serve." -_LAUNCHD_LABEL = re.compile(r"\A" + LAUNCHD_LABEL_PREFIX.replace(".", r"\.") + r"(\d{1,5})\Z") - - -def launchd_label(port): - return "{0}{1}".format(LAUNCHD_LABEL_PREFIX, port) - - -def render_launchd_plist(plan, log_path=None): - """Render a deterministic launchd plist for a reviewed serve plan.""" - from xml.sax.saxutils import escape - - label = launchd_label(plan["port"]) - log = log_path or str(receipts_root() / "{0}.log".format(plan["port"])) - arguments = "\n".join( - " {0}".format(escape(str(part))) for part in plan["argv"] - ) - return ( - '\n' - '\n' - '\n' - '\n' - ' Label\n' - ' {label}\n' - ' ProgramArguments\n' - ' \n' - '{arguments}\n' - ' \n' - ' RunAtLoad\n' - ' \n' - ' KeepAlive\n' - ' \n' - ' StandardOutPath\n' - ' {log}\n' - ' StandardErrorPath\n' - ' {log}\n' - '\n' - '\n' - ).format(label=escape(label), arguments=arguments, log=escape(str(log))) - - -class LaunchdPlistAdapter: - """Validate the exact launchd plist subset serve renders.""" - - version = "1.0" - runtime = "launchd" - - def __init__(self, path=None): - self.path = Path(path) if path is not None else None - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("plist content must be text") - require_secret_free_config(content) - lines = content.splitlines() - if len(lines) < 8 or not lines[0].startswith("': - raise ValueError("launchd plist must use the managed XML subset") - if lines[-1] != "" or lines[-2] != "": - raise ValueError("launchd plist must close its dict and plist elements") - text = content - for required in ("Label", "ProgramArguments", "RunAtLoad"): - if required not in text: - raise ValueError("launchd plist is missing {0}".format(required)) - label_match = re.search(r"Label\s*\n\s*([^<]+)", text) - if label_match is None or _LAUNCHD_LABEL.fullmatch(label_match.group(1)) is None: - raise ValueError("launchd plist label must use the managed serve prefix") - return True diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/transactions.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/transactions.py deleted file mode 100644 index ee06116..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/transactions.py +++ /dev/null @@ -1,1343 +0,0 @@ -"""Durable, reversible configuration transactions for Wire. - -The transaction root is opened with ``O_NOFOLLOW`` where macOS/Python exposes -it. Stage, backup, receipt, and replace operations then use that directory -file descriptor. A hostile actor can still race an already-open target leaf -between validation and replacement, but ``os.replace`` replaces that leaf and -never follows it; ancestors are verified before opening their directory fd. -""" - -from __future__ import annotations - -import difflib -import errno -import fcntl -import hashlib -import json -import os -import stat -import unicodedata -import uuid -from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from contextlib import contextmanager, nullcontext - -from .wiring import ( - ConfigAdapter, - redact_endpoint, - redact_secrets, - require_secret_free_config, - validate_health_endpoint, -) - - -RECEIPT_SCHEMA_VERSION = "2.1" -MAX_PREVIEW_CHARS = 12000 -_STATUSES = {"pending", "applied", "failed", "rolled_back", "rollback_failed", "recovery_required"} -_HASH = __import__("re").compile(r"^[0-9a-f]{64}$") -_LEGACY_LOCK_NAME = __import__("re").compile(r"^\.mlx-agent-wire-([0-9a-f]{64})\.lock$") -COOPERATIVE_CONCURRENCY_NOTE = "Advisory lock protects accidental/cooperative writers; a malicious process ignoring it can still race the final rename." -LEGACY_LOCK_MIGRATION = "legacy-target-locks-v1" -_LEGACY_LOCK_MARKER = "legacy-lock-migration-v1.json" - - -class ConcurrentTransactionError(ValueError): - """A cooperative writer holds the transaction's advisory lock.""" - - -class LegacyLockError(ConcurrentTransactionError): - """A legacy target-adjacent lock prevents a safe scoped-lock upgrade.""" - - -def _sha256(value): - return hashlib.sha256(value).hexdigest() - - -def _timestamp(): - return datetime.now(timezone.utc).isoformat() - - -def _absolute(path): - value = Path(path) - if _has_parent_reference(value): - raise ValueError("path traversal is not allowed: {0}".format(path)) - return Path(os.path.abspath(str(value))) - - -def _physical_absolute(path): - result = _absolute(path) - # /var is a fixed macOS compatibility alias. Normalize before descriptor - # traversal rather than following a user-controlled symlink component. - if str(result) == "/var" or str(result).startswith("/var/"): - if not os.path.islink("/var") or os.readlink("/var") != "private/var": - raise ValueError("untrusted /var compatibility alias") - result = Path("/private/var") / result.relative_to("/var") - return result - - -def _has_parent_reference(path): - return ".." in Path(path).parts - - -def _walk_directory(path, create=False, component_hook=None): - """Open each absolute component from `/` using dir-fd + O_NOFOLLOW only.""" - if not hasattr(os, "O_NOFOLLOW"): - raise ValueError("platform lacks required no-follow directory traversal") - logical = _absolute(path) - value = _physical_absolute(logical) - flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | os.O_NOFOLLOW - descriptor = os.open("/", flags) - try: - for component in value.parts[1:]: - if component_hook is not None: - component_hook(descriptor, component) - try: - child = os.open(component, flags, dir_fd=descriptor) - except FileNotFoundError: - if not create: - raise ValueError("path component does not exist: {0}".format(component)) - os.mkdir(component, 0o700, dir_fd=descriptor) - os.fsync(descriptor) - child = os.open(component, flags, dir_fd=descriptor) - except OSError as error: - raise ValueError("refusing unsafe directory component {0}: {1}".format(component, error)) - os.close(descriptor) - descriptor = child - return logical, descriptor - except BaseException: - os.close(descriptor) - raise - - -def _assert_safe_directory(path, create=False): - value, descriptor = _walk_directory(path, create=create) - os.close(descriptor) - return value - - -def _target_lock_name(target): - """Return the target-adjacent v1 lock name used by migration checks.""" - return ".mlx-agent-wire-{0}.lock".format(_target_lock_digest(target)) - - -def _target_lock_digest(target): - """Return the path-spelling v1 identity used only for legacy migration.""" - return _sha256(str(_physical_absolute(target)).encode("utf-8")) - - -def _filesystem_target_lock_digest(target, parent_fd): - """Bind a lock to the opened parent filesystem object and canonical leaf.""" - item = os.fstat(parent_fd) - # The stdlib does not expose a stable descriptor-bound filesystem - # case-sensitivity capability. Fold conservatively rather than derive an - # identity from mutable directory contents and risk split alias locks. - leaf = unicodedata.normalize("NFC", _physical_absolute(target).name).casefold() - identity = "{0}:{1}:{2}".format(item.st_dev, item.st_ino, leaf) - return _sha256(identity.encode("utf-8")) - - -def _filesystem_target_lock_name(target, parent_fd): - return ".mlx-agent-wire-{0}.lock".format( - _filesystem_target_lock_digest(target, parent_fd) - ) - - -def _filesystem_parent_digest(directory_fd): - item = os.fstat(directory_fd) - return _sha256("{0}:{1}".format(item.st_dev, item.st_ino).encode("utf-8")) - - -def _legacy_parent_scopes(targets, create=False): - """Open and deduplicate physical target parents without following aliases.""" - scopes = {} - try: - for target in sorted({_physical_absolute(item) for item in targets}, key=str): - try: - _parent, directory_fd = _walk_directory(target.parent, create=create) - except ValueError as error: - if str(error).startswith("path component does not exist:"): - continue - raise - try: - item = os.fstat(directory_fd) - identity = (item.st_dev, item.st_ino) - if identity in scopes: - os.close(directory_fd) - scopes[identity]["targets"].append(target) - continue - scopes[identity] = { - "digest": _filesystem_parent_digest(directory_fd), - "directory_fd": directory_fd, - "identity": identity, - "parent": target.parent, - "targets": [target], - } - except BaseException: - os.close(directory_fd) - raise - return [scopes[key] for key in sorted(scopes)] - except BaseException: - for scope in scopes.values(): - os.close(scope["directory_fd"]) - raise - - -def _close_legacy_parent_scopes(scopes): - for scope in reversed(scopes): - os.close(scope["directory_fd"]) - - -def _merge_legacy_parent_scopes(scopes, additions): - """Merge newly created/opened parents without retaining duplicate fds.""" - existing = {scope["identity"]: scope for scope in scopes} - added = [] - for addition in additions: - current = existing.get(addition["identity"]) - if current is not None: - current["targets"] = sorted(set(current["targets"] + addition["targets"]), key=str) - os.close(addition["directory_fd"]) - continue - existing[addition["identity"]] = addition - scopes.append(addition) - added.append(addition) - scopes.sort(key=lambda scope: scope["identity"]) - return added - - -def _legacy_candidate_names(directory_fd): - """List only exact v1 lock names; unrelated directory entries stay opaque.""" - try: - names = os.listdir(directory_fd) - except OSError as error: - raise ValueError("could not safely enumerate legacy target transaction locks: {0}".format(error)) - candidates = [] - for name in names: - match = _LEGACY_LOCK_NAME.fullmatch(name) - if match is not None: - candidates.append((name, match.group(1))) - return sorted(candidates) - - -def _validate_legacy_candidate_item(item): - if not stat.S_ISREG(item.st_mode): - raise ValueError("legacy target transaction lock is not a regular file") - if hasattr(os, "geteuid") and item.st_uid != os.geteuid(): - raise ValueError("legacy target transaction lock is not owned by the current user") - - -def _open_legacy_candidate(scope, name, digest, writable): - descriptor = None - flags = (os.O_RDWR if writable else os.O_RDONLY) | os.O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) - try: - try: - descriptor = os.open(name, flags, dir_fd=scope["directory_fd"]) - except FileNotFoundError: - return None - except OSError as error: - raise ValueError("could not safely open legacy target transaction lock: {0}".format(error)) - item = os.fstat(descriptor) - _validate_legacy_candidate_item(item) - try: - current = os.stat(name, dir_fd=scope["directory_fd"], follow_symlinks=False) - except FileNotFoundError: - os.close(descriptor) - return None - if (current.st_dev, current.st_ino) != (item.st_dev, item.st_ino): - raise ValueError("legacy target transaction lock changed while it was opened") - return { - "descriptor": descriptor, - "digest": digest, - "identity": (item.st_dev, item.st_ino), - "name": name, - "path": str(scope["parent"] / name), - "scope": scope, - } - except BaseException: - if descriptor is not None: - os.close(descriptor) - raise - - -def _release_legacy_candidates(candidates): - for candidate in reversed(candidates): - try: - fcntl.flock(candidate["descriptor"], fcntl.LOCK_UN) - finally: - os.close(candidate["descriptor"]) - - -def _acquire_legacy_candidates(scopes): - """Acquire a stable snapshot of every strict candidate before mutation.""" - held = [] - current = {} - try: - for _attempt in range(32): - for scope in scopes: - for name, digest in _legacy_candidate_names(scope["directory_fd"]): - key = (scope["digest"], name) - existing = current.get(key) - if existing is not None: - try: - item = os.stat(name, dir_fd=scope["directory_fd"], follow_symlinks=False) - except FileNotFoundError: - current.pop(key, None) - else: - if (item.st_dev, item.st_ino) == existing["identity"]: - continue - candidate = _open_legacy_candidate(scope, name, digest, writable=True) - if candidate is None: - current.pop(key, None) - continue - try: - fcntl.flock(candidate["descriptor"], fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as error: - os.close(candidate["descriptor"]) - if error.errno in (errno.EACCES, errno.EAGAIN): - raise LegacyLockError("legacy_lock_busy: stop older mlx-agent processes before upgrading scoped locks") - raise ValueError("could not safely acquire legacy target transaction lock: {0}".format(error)) - held.append(candidate) - current[key] = candidate - - stable = True - present = set() - for scope in scopes: - for name, _digest in _legacy_candidate_names(scope["directory_fd"]): - key = (scope["digest"], name) - present.add(key) - candidate = current.get(key) - try: - item = os.stat(name, dir_fd=scope["directory_fd"], follow_symlinks=False) - except FileNotFoundError: - stable = False - continue - if candidate is None or (item.st_dev, item.st_ino) != candidate["identity"]: - stable = False - for key in list(current): - if key not in present: - current.pop(key) - stable = False - if stable: - return held, [current[key] for key in sorted(current)] - raise ValueError("legacy target transaction lock set changed repeatedly during migration") - except BaseException: - _release_legacy_candidates(held) - raise - - -def _inspect_legacy_candidates(scopes): - """Open strict candidates no-follow for a read-only doctor snapshot.""" - candidates = [] - for scope in scopes: - for name, digest in _legacy_candidate_names(scope["directory_fd"]): - candidate = _open_legacy_candidate(scope, name, digest, writable=False) - if candidate is None: - continue - os.close(candidate.pop("descriptor")) - candidates.append(candidate) - return candidates - - -def _legacy_scope_was_migrated(scope, candidates, state): - if state["unscoped"]: - return True - if scope["digest"] in state["parents"]: - return True - migrated = state["targets"] - if any(entry.get("parent") == scope["digest"] for entry in migrated.values()): - return True - relevant = {_target_lock_digest(target) for target in scope["targets"]} - relevant.update(candidate["digest"] for candidate in candidates) - if relevant.intersection(migrated): - return True - # Old markers cannot map a one-way spelling digest back to a parent. A - # strict candidate in that ambiguous upgrade state must fail closed. - return any("parent" not in entry for entry in migrated.values()) - - -def _read_legacy_migration_state(lock_root): - """Read validated parent-scoped legacy-lock migration state.""" - try: - _root, directory_fd = _walk_directory(lock_root) - except ValueError as error: - if str(error).startswith("path component does not exist:"): - return {"version": LEGACY_LOCK_MIGRATION, "targets": {}, "parents": {}, "unscoped": False} - raise - descriptor = None - try: - try: - descriptor = os.open(_LEGACY_LOCK_MARKER, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory_fd) - except FileNotFoundError: - return {"version": LEGACY_LOCK_MIGRATION, "targets": {}, "parents": {}, "unscoped": False} - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - raise ValueError("legacy lock migration marker is not a regular file") - value = json.loads(os.read(descriptor, 1048576).decode("utf-8")) - # The prior release-candidate marker had no target map. Treat it as - # an unscoped barrier: it cannot prove which physical parents migrated. - if value == {"version": LEGACY_LOCK_MIGRATION}: - return {"version": LEGACY_LOCK_MIGRATION, "targets": {}, "parents": {}, "unscoped": True} - allowed_fields = ( - {"version", "targets"}, - {"version", "targets", "parents"}, - {"version", "targets", "parents", "unscoped"}, - ) - if not isinstance(value, dict) or set(value) not in allowed_fields or value["version"] != LEGACY_LOCK_MIGRATION or not isinstance(value["targets"], dict): - raise ValueError("legacy lock migration marker is malformed") - for digest, entry in value["targets"].items(): - if not isinstance(digest, str) or not _HASH.fullmatch(digest) or not isinstance(entry, dict) or set(entry) not in ({"migrated_at"}, {"migrated_at", "parent"}) or not isinstance(entry["migrated_at"], str) or ("parent" in entry and (not isinstance(entry["parent"], str) or not _HASH.fullmatch(entry["parent"]))): - raise ValueError("legacy lock migration marker is malformed") - try: - if datetime.fromisoformat(entry["migrated_at"].replace("Z", "+00:00")).tzinfo is None: - raise ValueError("missing timezone") - except ValueError: - raise ValueError("legacy lock migration marker is malformed") - parents = value.get("parents", {}) - unscoped = value.get("unscoped", False) - if not isinstance(parents, dict) or not isinstance(unscoped, bool): - raise ValueError("legacy lock migration marker is malformed") - for digest, entry in parents.items(): - if not isinstance(digest, str) or not _HASH.fullmatch(digest) or not isinstance(entry, dict) or set(entry) != {"migrated_at"} or not isinstance(entry["migrated_at"], str): - raise ValueError("legacy lock migration marker is malformed") - try: - if datetime.fromisoformat(entry["migrated_at"].replace("Z", "+00:00")).tzinfo is None: - raise ValueError("missing timezone") - except ValueError: - raise ValueError("legacy lock migration marker is malformed") - value["parents"] = parents - value["unscoped"] = unscoped - return value - except OSError as error: - raise ValueError("could not safely read legacy lock migration marker: {0}".format(error)) - finally: - if descriptor is not None: - os.close(descriptor) - os.close(directory_fd) - - -def _write_legacy_migration_state(lock_root, state): - _atomic_in_directory( - lock_root, - _LEGACY_LOCK_MARKER, - (json.dumps(state, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8"), - 0o600, - ) - - -@contextmanager -def _legacy_migration_window(lock_root): - """Serialize parent-map upgrades so multi-target writes cannot lose entries.""" - _root, directory_fd = _walk_directory(lock_root, create=True) - descriptor = None - try: - descriptor = os.open(".legacy-lock-migration-v1.state.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600, dir_fd=directory_fd) - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - raise ValueError("legacy lock migration state lock is not a regular file") - try: - fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as error: - if error.errno in (errno.EACCES, errno.EAGAIN): - raise LegacyLockError("legacy_lock_busy: another scoped legacy-lock migration is active") - raise ValueError("could not safely acquire legacy lock migration state lock: {0}".format(error)) - yield - finally: - if descriptor is not None: - try: - fcntl.flock(descriptor, fcntl.LOCK_UN) - finally: - os.close(descriptor) - os.close(directory_fd) - - -@contextmanager -def _migrate_legacy_target_locks(targets, lock_root, create_parents=False): - """Upgrade target-adjacent legacy locks before entering scoped lock storage.""" - canonical = sorted({_physical_absolute(item) for item in targets}, key=str) - with _legacy_migration_window(lock_root): - scopes = _legacy_parent_scopes(canonical) - held = [] - try: - held, candidates = _acquire_legacy_candidates(scopes) - state = _read_legacy_migration_state(lock_root) - migrated = state["targets"] - initial_by_parent = {scope["digest"]: [] for scope in scopes} - for candidate in candidates: - initial_by_parent[candidate["scope"]["digest"]].append(candidate) - if any(initial_by_parent[scope["digest"]] and _legacy_scope_was_migrated(scope, initial_by_parent[scope["digest"]], state) for scope in scopes): - raise LegacyLockError("legacy_lock_recreated: stop older mlx-agent processes and remove recreated legacy locks before continuing") - - opened_targets = {target for scope in scopes for target in scope["targets"]} - missing_targets = [target for target in canonical if target not in opened_targets] - if create_parents and missing_targets: - additions = _legacy_parent_scopes(missing_targets, create=True) - added_scopes = _merge_legacy_parent_scopes(scopes, additions) - added_held, added_candidates = _acquire_legacy_candidates(added_scopes) - held.extend(added_held) - candidates.extend(added_candidates) - added_by_parent = {scope["digest"]: [] for scope in added_scopes} - for candidate in added_candidates: - added_by_parent[candidate["scope"]["digest"]].append(candidate) - if any(added_by_parent[scope["digest"]] and _legacy_scope_was_migrated(scope, added_by_parent[scope["digest"]], state) for scope in added_scopes): - raise LegacyLockError("legacy_lock_recreated: stop older mlx-agent processes and remove recreated legacy locks before continuing") - - by_parent = {scope["digest"]: [] for scope in scopes} - for candidate in candidates: - by_parent[candidate["scope"]["digest"]].append(candidate) - - for candidate in candidates: - try: - item = os.stat(candidate["name"], dir_fd=candidate["scope"]["directory_fd"], follow_symlinks=False) - except FileNotFoundError: - raise ValueError("legacy target transaction lock changed before migration") - if (item.st_dev, item.st_ino) != candidate["identity"]: - raise ValueError("legacy target transaction lock changed before migration") - touched = set() - for candidate in candidates: - os.unlink(candidate["name"], dir_fd=candidate["scope"]["directory_fd"]) - touched.add(candidate["scope"]["digest"]) - for scope in scopes: - if scope["digest"] in touched: - os.fsync(scope["directory_fd"]) - - stamp = _timestamp() - for scope in scopes: - state["parents"].setdefault(scope["digest"], {"migrated_at": stamp}) - for target in scope["targets"]: - digest = _target_lock_digest(target) - entry = migrated.setdefault(digest, {"migrated_at": stamp, "parent": scope["digest"]}) - entry.setdefault("parent", scope["digest"]) - for candidate in by_parent[scope["digest"]]: - entry = migrated.setdefault(candidate["digest"], {"migrated_at": stamp, "parent": scope["digest"]}) - entry.setdefault("parent", scope["digest"]) - _write_legacy_migration_state(lock_root, state) - yield - finally: - _release_legacy_candidates(held) - _close_legacy_parent_scopes(scopes) - - -def legacy_lock_problem(targets, lock_root): - """Describe legacy-lock state for installer doctor without mutating it.""" - state = _read_legacy_migration_state(lock_root) - paths = {"legacy_lock_recreated": [], "legacy_lock_migration_required": []} - scopes = _legacy_parent_scopes(targets) - try: - candidates = _inspect_legacy_candidates(scopes) - by_parent = {scope["digest"]: [] for scope in scopes} - for candidate in candidates: - by_parent[candidate["scope"]["digest"]].append(candidate) - for scope in scopes: - scope_candidates = by_parent[scope["digest"]] - if not scope_candidates: - continue - code = "legacy_lock_recreated" if _legacy_scope_was_migrated(scope, scope_candidates, state) else "legacy_lock_migration_required" - paths[code].extend(candidate["path"] for candidate in scope_candidates) - finally: - _close_legacy_parent_scopes(scopes) - return [ - { - "code": code, - "paths": sorted(set(affected)), - "remediation": "stop older mlx-agent processes; after parent-scoped legacy-lock migration, do not run an older binary and remove only recreated legacy locks.", - } - for code, affected in paths.items() if affected - ] - - -@contextmanager -def _target_locks(targets, create_parents=False, lock_root=None): - """Acquire filesystem-identity locks in canonical order to avoid deadlocks.""" - canonical = sorted({_physical_absolute(target) for target in targets}, key=str) - identities = {} - held = [] - try: - for target in canonical: - _parent, target_parent_fd = _walk_directory( - target.parent, create=create_parents - ) - digest = _filesystem_target_lock_digest(target, target_parent_fd) - if digest in identities: - os.close(target_parent_fd) - continue - identities[digest] = { - "target": target, - "target_parent_fd": target_parent_fd, - } - for digest in sorted(identities): - identity = identities[digest] - target = identity["target"] - target_parent_fd = identity["target_parent_fd"] - if lock_root is None: - directory_fd = target_parent_fd - identity["target_parent_fd"] = None - else: - os.close(target_parent_fd) - identity["target_parent_fd"] = None - _parent, directory_fd = _walk_directory(lock_root, create=True) - lock_fd = None - try: - lock_fd = os.open( - ".mlx-agent-wire-{0}.lock".format(digest), - os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, - 0o600, - dir_fd=directory_fd, - ) - if not stat.S_ISREG(os.fstat(lock_fd).st_mode): - raise ValueError("target transaction lock is not a regular file") - os.fsync(directory_fd) - fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - held.append((lock_fd, directory_fd)) - except OSError as error: - if lock_fd is not None: - os.close(lock_fd) - os.close(directory_fd) - if error.errno in (errno.EACCES, errno.EAGAIN): - raise ConcurrentTransactionError("another cooperative Wire transaction is active for a target") - raise ValueError("could not safely acquire target transaction lock: {0}".format(error)) - except BaseException: - if lock_fd is not None: - os.close(lock_fd) - os.close(directory_fd) - raise - yield canonical - finally: - for lock_fd, directory_fd in reversed(held): - try: - fcntl.flock(lock_fd, fcntl.LOCK_UN) - finally: - os.close(lock_fd) - os.close(directory_fd) - for identity in identities.values(): - if identity["target_parent_fd"] is not None: - os.close(identity["target_parent_fd"]) - - -def _assert_safe_target(path): - value = _absolute(path) - _read_target(value) - return value - - -def _open_directory(path): - _value, descriptor = _walk_directory(path) - return descriptor - - -def _fsync_directory(path_or_fd): - descriptor = path_or_fd if isinstance(path_or_fd, int) else _open_directory(path_or_fd) - close = not isinstance(path_or_fd, int) - try: - os.fsync(descriptor) - finally: - if close: - os.close(descriptor) - - -def _write_all(descriptor, content): - offset = 0 - while offset < len(content): - offset += os.write(descriptor, content[offset:]) - - -def _atomic_in_directory(directory, name, content, mode, validator=None): - """Write and fsync a stage file before atomically replacing ``name``.""" - directory = _assert_safe_directory(directory) - dir_fd = _open_directory(directory) - stage_name = ".mlx-agent-stage-{0}".format(uuid.uuid4().hex) - stage_fd = None - try: - stage_fd = os.open(stage_name, os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0), mode, dir_fd=dir_fd) - _write_all(stage_fd, content) - os.fsync(stage_fd) - os.close(stage_fd) - stage_fd = None - _fsync_directory(dir_fd) - if validator is not None: - read_fd = os.open(stage_name, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), dir_fd=dir_fd) - try: - data = b"" - while True: - chunk = os.read(read_fd, 65536) - if not chunk: - break - data += chunk - finally: - os.close(read_fd) - validator(data.decode("utf-8")) - os.chmod(stage_name, mode, dir_fd=dir_fd, follow_symlinks=False) - os.replace(stage_name, name, src_dir_fd=dir_fd, dst_dir_fd=dir_fd) - _fsync_directory(dir_fd) - except BaseException: - if stage_fd is not None: - os.close(stage_fd) - try: - os.unlink(stage_name, dir_fd=dir_fd) - _fsync_directory(dir_fd) - except OSError: - pass - raise - finally: - os.close(dir_fd) - - -def _read_target(path, component_hook=None): - """Read a regular leaf through its opened parent directory descriptor.""" - value = _absolute(path) - _parent, directory = _walk_directory(value.parent, component_hook=component_hook) - try: - try: - descriptor = os.open(value.name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory) - except FileNotFoundError: - return b"", False, None - try: - item = os.fstat(descriptor) - if not stat.S_ISREG(item.st_mode): - raise ValueError("target is not a regular file: {0}".format(value)) - parts = [] - while True: - chunk = os.read(descriptor, 65536) - if not chunk: - break - parts.append(chunk) - return b"".join(parts), True, item.st_mode & 0o777 - finally: - os.close(descriptor) - except OSError as error: - raise ValueError("refusing unsafe target leaf {0}: {1}".format(value, error)) - finally: - os.close(directory) - - -def _read_regular(path): - return _read_target(path)[0] - - -@dataclass -class Receipt: - schema_version: str - transaction_id: str - adapter_version: str - timestamp: str - transaction_root: str - targets: list - target_roots: dict - target_modes: dict - after_modes: dict - before_hashes: dict - after_hashes: dict - backup_paths: dict - validations: dict - status: str - preview: str - preview_hash: str - lock_root: str = None - lock_migration: str = None - receipt_path: str = field(default="", repr=False, compare=False) - - def to_dict(self): - value = asdict(self) - value.pop("receipt_path", None) - if value.get("lock_root") is None: - value.pop("lock_root") - if value.get("lock_migration") is None: - value.pop("lock_migration") - return value - - @classmethod - def from_dict(cls, value, receipt_path=""): - required = { - "schema_version", "transaction_id", "adapter_version", "timestamp", "transaction_root", - "targets", "target_roots", "target_modes", "after_modes", "before_hashes", "after_hashes", "backup_paths", - "validations", "status", "preview", "preview_hash", - } - if not isinstance(value, dict) or not required.issubset(value) or set(value) - (required | {"lock_root", "lock_migration"}): - raise ValueError("receipt fields are malformed or untrusted") - if value["schema_version"] != RECEIPT_SCHEMA_VERSION or value["status"] not in _STATUSES: - raise ValueError("receipt version or status is unsupported") - try: - uuid.UUID(value["transaction_id"]) - parsed_timestamp = datetime.fromisoformat(value["timestamp"].replace("Z", "+00:00")) - except (TypeError, ValueError, AttributeError): - raise ValueError("receipt ID or timestamp is malformed") - if parsed_timestamp.tzinfo is None: - raise ValueError("receipt timestamp must include a timezone") - if not isinstance(value["adapter_version"], str) or not value["adapter_version"]: - raise ValueError("receipt adapter version is malformed") - root = _assert_safe_directory(value["transaction_root"]) - if Path(value["transaction_root"]) != root: - raise ValueError("receipt transaction root must be normalized") - if receipt_path: - location = _assert_safe_target(receipt_path) - if location.parent != root or location.name != "receipt.json": - raise ValueError("receipt is outside its transaction layout") - lock_root = value.get("lock_root") - if lock_root is not None: - if not isinstance(lock_root, str) or not lock_root: - raise ValueError("receipt lock root is malformed") - safe_lock_root = _assert_safe_directory(lock_root) - if str(safe_lock_root) != lock_root: - raise ValueError("receipt lock root must be normalized") - migration = value.get("lock_migration") - if migration is not None and (lock_root is None or migration != LEGACY_LOCK_MIGRATION): - raise ValueError("receipt lock migration is malformed") - targets = value["targets"] - maps = ( - value["target_roots"], value["target_modes"], value["after_modes"], - value["before_hashes"], value["after_hashes"], value["backup_paths"], - ) - if not isinstance(targets, list) or not targets or not all(isinstance(item, str) for item in targets) or len(targets) != len(set(targets)) or not all(isinstance(item, dict) for item in maps): - raise ValueError("receipt targets are malformed") - if any(set(targets) != set(item) for item in maps): - raise ValueError("receipt target maps do not match") - for index, target_name in enumerate(targets): - target = _assert_safe_target(target_name) - if str(target) != target_name or value["target_roots"][target_name] != str(target.parent): - raise ValueError("receipt target is not rooted safely") - mode = value["target_modes"][target_name] - if mode is not None and (not isinstance(mode, int) or mode < 0 or mode > 0o777): - raise ValueError("receipt mode is malformed") - after_mode = value["after_modes"][target_name] - if after_mode is not None and ( - not isinstance(after_mode, int) or after_mode < 0 or after_mode > 0o777 - ): - raise ValueError("receipt after mode is malformed") - for hashes in (value["before_hashes"], value["after_hashes"]): - if not isinstance(hashes[target_name], str) or not _HASH.fullmatch(hashes[target_name]): - raise ValueError("receipt hash is malformed") - backup = value["backup_paths"][target_name] - expected_backup = root / "backup-{0}.bin".format(index) - if (backup is None) != (mode is None): - raise ValueError("receipt backup presence does not match before existence") - if backup is None: - if value["before_hashes"][target_name] != _sha256(b""): - raise ValueError("missing backup has a non-empty hash") - elif not isinstance(backup, str) or Path(backup) != expected_backup: - raise ValueError("receipt backup is outside its transaction layout") - validations_redacted = False - if isinstance(value["validations"], dict): - try: - validations_redacted = json.loads(redact_secrets(json.dumps(value["validations"]))) == value["validations"] - except (TypeError, ValueError, json.JSONDecodeError): - validations_redacted = False - if not isinstance(value["validations"], dict) or not validations_redacted or not isinstance(value["preview"], str) or len(value["preview"]) > MAX_PREVIEW_CHARS + 32 or redact_secrets(value["preview"]) != value["preview"] or not isinstance(value["preview_hash"], str) or not _HASH.fullmatch(value["preview_hash"]): - raise ValueError("receipt validation or preview is malformed") - return cls(receipt_path=str(receipt_path), **value) - - -class Transaction: - """Create a crash-recoverable journal before any configuration mutation.""" - - def __init__(self, receipts_dir=None, health_checker=None, fault_injector=None, receipt_writer=None, path_race_hook=None, create_target_parents=False, transaction_id=None, lock_root=None): - self.receipts_dir = Path(receipts_dir) if receipts_dir else None - self.lock_root = Path(lock_root) if lock_root is not None else None - self.health_checker = health_checker - self.fault_injector = fault_injector - self.receipt_writer = receipt_writer - self.path_race_hook = path_race_hook - self.create_target_parents = bool(create_target_parents) - if transaction_id is not None: - try: - self.transaction_id = str(uuid.UUID(str(transaction_id))) - except (TypeError, ValueError, AttributeError): - raise ValueError("transaction_id must be a UUID") - else: - self.transaction_id = None - self._changes = [] - self._preview = "" - self._preview_hash = "" - self._expected_current = None - - def preview(self, changes, expected_current=None): - if not isinstance(changes, (list, tuple)) or not changes: - raise ValueError("changes must be a non-empty list") - prepared, diffs, binding = [], [], [] - seen = set() - for change in changes: - if not isinstance(change, dict) or "path" not in change or "content" not in change: - raise ValueError("each change requires path and content") - path = _absolute(change["path"]) - if str(path) in seen: - raise ValueError("each target may appear only once") - seen.add(str(path)) - content = change["content"] - if not isinstance(content, str): - raise TypeError("change content must be text") - adapter = change.get("adapter") or ConfigAdapter.detect(path, runtime=change.get("runtime")) - try: - before, existed, mode = _read_target(path, self.path_race_hook) - except ValueError as error: - if not self.create_target_parents or not str(error).startswith("path component does not exist:"): - raise - before, existed, mode = b"", False, None - before_text = before.decode("utf-8") - if not getattr(adapter, "secret_scan_exempt", False): - require_secret_free_config(before_text) - require_secret_free_config(content) - endpoint = change.get("endpoint") - if endpoint: - validate_health_endpoint(endpoint) - adapter.validate(content) - after = content.encode("utf-8") - diffs.append("".join(difflib.unified_diff( - redact_secrets(before_text).splitlines(True), redact_secrets(content).splitlines(True), - fromfile=str(path), tofile=str(path), lineterm="", - ))) - binding.append({"path": str(path), "before": _sha256(before), "after": _sha256(after), "exists": existed, "mode": mode, "endpoint": _sha256(str(endpoint or "").encode("utf-8"))}) - prepared.append({"path": path, "content": content, "adapter": adapter, "endpoint": endpoint, "before_hash": _sha256(before), "existed": existed, "mode": mode}) - self._changes = prepared - self._expected_current = self._normalize_expected_current(expected_current) - self._preview = self._bounded("\n".join(diffs)) - self._preview_hash = _sha256(json.dumps(binding, sort_keys=True, separators=(",", ":")).encode("utf-8")) - return {"changes": len(prepared), "diff": self._preview, "preview_hash": self._preview_hash, "requires_confirmation": True} - - def apply(self, confirmation, expected_current=None): - if confirmation is not True and confirmation != self._preview_hash: - raise PermissionError("explicit confirmation for this preview is required") - if not self._changes: - raise ValueError("preview changes before applying them") - if expected_current is not None: - expected_current = self._normalize_expected_current(expected_current) - if self._expected_current is not None and expected_current != self._expected_current: - raise ValueError("apply expected_current does not match preview") - self._expected_current = expected_current - with self._advisory_lock(): - return self._apply_locked() - - @contextmanager - def _advisory_lock(self): - lock_root = self._safe_lock_root() if self.lock_root is not None else None - if lock_root is None: - with _target_locks(self._lock_targets(), create_parents=self.create_target_parents): - yield - return - with _migrate_legacy_target_locks( - self._lock_targets(), lock_root, - create_parents=self.create_target_parents, - ): - with _target_locks(self._lock_targets(), create_parents=self.create_target_parents, lock_root=lock_root): - yield - - def _safe_lock_root(self): - root, descriptor = _walk_directory(self.lock_root, create=True) - os.close(descriptor) - return root - - def _lock_targets(self): - return [str(path) for path in sorted({change["path"] for change in self._changes}, key=str)] - - def _apply_locked(self): - if self._expected_current is not None: - for change in self._changes: - current, exists, mode = _read_target(change["path"], self.path_race_hook) - expected = self._expected_current[str(change["path"])] - if _sha256(current) != expected["hash"] or exists != expected["exists"] or mode != expected["mode"]: - raise ValueError("expected current target state changed before mutation") - for change in self._changes: - current, exists, mode = _read_target(change["path"], self.path_race_hook) - if _sha256(current) != change["before_hash"] or exists != change["existed"] or mode != change["mode"]: - raise ValueError("preview is stale; target changed after preview") - receipt = self._prepare_journal() - journal_written = False - mutation_started = False - try: - self._persist(receipt) - journal_written = True - self._fault("after_pending_receipt") - for index, change in enumerate(self._changes): - path = change["path"] - self._fault("before_replace:{0}".format(index)) - current, exists, mode = _read_target(path, self.path_race_hook) - if _sha256(current) != receipt.before_hashes[str(path)] or exists != change["existed"] or mode != receipt.target_modes[str(path)]: - raise ValueError("preview is stale; target changed before replacement") - mutation_started = True - self._replace_target(path, change["content"].encode("utf-8"), change["adapter"], receipt.target_modes[str(path)]) - receipt.after_hashes[str(path)] = _sha256(change["content"].encode("utf-8")) - receipt.after_modes[str(path)] = ( - receipt.target_modes[str(path)] - if receipt.target_modes[str(path)] is not None else 0o600 - ) - self._persist(receipt) - self._fault("after_replace:{0}".format(index)) - change["adapter"].validate(_read_regular(path).decode("utf-8")) - after, after_exists, after_mode = _read_target(path) - if not after_exists or (receipt.target_modes[str(path)] is not None and after_mode != receipt.target_modes[str(path)]): - raise ValueError("replacement mode does not match the reviewed target mode") - receipt.after_hashes[str(path)] = _sha256(after) - receipt.after_modes[str(path)] = after_mode - receipt.validations[str(path)] = {"pre": True, "post": True, "passed": True} - self._persist(receipt) - receipt.validations["health_check"] = self._run_health_checks() - if receipt.validations["health_check"]["passed"]: - receipt.status = "applied" - self._persist(receipt) - else: - self._finish_restore(receipt) - except Exception as error: - receipt.validations["error"] = {"passed": False, "message": redact_secrets(str(error))} - if not journal_written and not mutation_started: - receipt.status = "failed" - return receipt - self._finish_restore(receipt) - return receipt - - def _normalize_expected_current(self, expected_current): - if expected_current is None: - return None - if not isinstance(expected_current, dict): - raise ValueError("expected_current must map every target to its current state") - targets = {str(change["path"]) for change in self._changes} - if set(expected_current) != targets: - raise ValueError("expected_current targets do not match the preview") - normalized = {} - for target in sorted(targets): - state = expected_current[target] - if not isinstance(state, dict) or set(state) != {"hash", "exists", "mode"}: - raise ValueError("expected_current state must include hash, exists, and mode") - if not isinstance(state["hash"], str) or not _HASH.match(state["hash"]): - raise ValueError("expected_current hash is invalid") - if not isinstance(state["exists"], bool) or (state["mode"] is not None and not isinstance(state["mode"], int)): - raise ValueError("expected_current existence or mode is invalid") - normalized[target] = {"hash": state["hash"], "exists": state["exists"], "mode": state["mode"]} - return normalized - - def rollback(self, receipt_path): - return rollback(receipt_path) - - def _prepare_journal(self): - self._fault("before_journal_capture") - captured = [] - for change in self._changes: - before, exists, mode = _read_target(change["path"], self.path_race_hook) - if _sha256(before) != change["before_hash"] or exists != change["existed"] or mode != change["mode"]: - raise ValueError("preview is stale; target changed before journal capture") - captured.append((before, exists, mode)) - self._fault("after_journal_capture") - for index, change in enumerate(self._changes): - before, exists, mode = _read_target(change["path"], self.path_race_hook) - if _sha256(before) != change["before_hash"] or exists != change["existed"] or mode != change["mode"]: - raise ValueError("preview is stale; target changed after journal capture") - captured[index] = (before, exists, mode) - receipts_dir, receipts_fd = _walk_directory(self.receipts_dir or self._changes[0]["path"].parent / ".mlx-agent-receipts", create=True) - transaction_id = self.transaction_id or str(uuid.uuid4()) - root = receipts_dir / transaction_id - try: - self._fault("before_transaction_root_create") - os.mkdir(transaction_id, 0o700, dir_fd=receipts_fd) - os.fsync(receipts_fd) - root_fd = os.open(transaction_id, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | os.O_NOFOLLOW, dir_fd=receipts_fd) - try: - os.fsync(root_fd) - finally: - os.close(root_fd) - finally: - os.close(receipts_fd) - targets = [str(change["path"]) for change in self._changes] - receipt = Receipt( - schema_version=RECEIPT_SCHEMA_VERSION, transaction_id=transaction_id, adapter_version=ConfigAdapter.version, - timestamp=_timestamp(), transaction_root=str(root), targets=targets, - target_roots={str(change["path"]): str(change["path"].parent) for change in self._changes}, - target_modes={str(change["path"]): captured[index][2] for index, change in enumerate(self._changes)}, - after_modes={str(change["path"]): captured[index][2] for index, change in enumerate(self._changes)}, - before_hashes={}, after_hashes={}, backup_paths={}, validations={ - "concurrency": {"scope": "advisory_cooperative_target_scoped", "note": COOPERATIVE_CONCURRENCY_NOTE, "passed": True}, - }, status="pending", - preview=self._preview, preview_hash=self._preview_hash, - lock_root=str(self._safe_lock_root()) if self.lock_root is not None else None, - lock_migration=LEGACY_LOCK_MIGRATION if self.lock_root is not None else None, - receipt_path=str(root / "receipt.json"), - ) - for index, change in enumerate(self._changes): - path = change["path"] - before, exists, mode = captured[index] - target = str(path) - receipt.before_hashes[target] = _sha256(before) - receipt.after_hashes[target] = _sha256(before) - if exists: - backup = root / "backup-{0}.bin".format(index) - _atomic_in_directory(root, backup.name, before, mode or 0o600) - receipt.backup_paths[target] = str(backup) - else: - receipt.backup_paths[target] = None - return receipt - - def _replace_target(self, path, content, adapter, mode): - _assert_safe_target(path) - _atomic_in_directory(path.parent, path.name, content, mode if mode is not None else 0o600, adapter.validate) - - def _persist(self, receipt): - root = _assert_safe_directory(receipt.transaction_root) - if self.receipt_writer is not None: - self.receipt_writer(Path(receipt.receipt_path), receipt.to_dict()) - return - _atomic_in_directory(root, "receipt.json", (json.dumps(receipt.to_dict(), indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600) - - def _finish_restore(self, receipt): - if _restore_receipt(receipt, expected_current=_receipt_after_state(receipt)): - receipt.status = "rolled_back" - else: - receipt.status = "recovery_required" - try: - self._persist(receipt) - except Exception as error: - receipt.status = "recovery_required" - receipt.validations["receipt_write"] = {"passed": False, "message": redact_secrets(str(error))} - - def _run_health_checks(self): - checks = [] - for change in self._changes: - endpoint = change.get("endpoint") - if not endpoint: - continue - passed = bool(self.health_checker(endpoint)) if self.health_checker else ConfigAdapter.health_check(endpoint) - checks.append({"endpoint": redact_endpoint(endpoint), "passed": passed}) - return {"passed": all(item["passed"] for item in checks), "endpoints": checks} - - def _fault(self, point): - if self.fault_injector is not None: - self.fault_injector(point) - - @staticmethod - def _bounded(value): - return value if len(value) <= MAX_PREVIEW_CHARS else value[:MAX_PREVIEW_CHARS] + "\n... [preview truncated]" - - -def _restore_receipt(receipt, expected_current=None): - """Preflight every backup, then restore and prove byte-for-byte hashes.""" - planned = [] - try: - root = _assert_safe_directory(receipt.transaction_root) - if expected_current is not None: - _assert_expected_states( - receipt.targets, expected_current, "rollback target changed before recovery" - ) - for index, target_name in enumerate(receipt.targets): - target = _assert_safe_target(target_name) - expected = receipt.before_hashes[target_name] - backup_name = receipt.backup_paths[target_name] - if backup_name is None: - if expected != _sha256(b""): - raise ValueError("missing backup has a non-empty hash") - planned.append((target, None, receipt.target_modes[target_name], expected)) - continue - backup = _assert_safe_target(backup_name) - backup_data, backup_exists, _backup_mode = _read_target(backup) - if backup.parent != root or backup.name != "backup-{0}.bin".format(index) or not backup_exists: - raise ValueError("backup is missing or outside transaction root") - if _sha256(backup_data) != expected: - raise ValueError("backup hash does not match receipt") - planned.append((target, backup_data, receipt.target_modes[target_name], expected)) - for target, data, mode, expected in planned: - if data is None: - _current, exists, _current_mode = _read_target(target) - if exists: - _remove_target(target) - else: - _atomic_in_directory(target.parent, target.name, data, mode if mode is not None else 0o600) - actual_content, actual_exists, actual_mode = _read_target(target) - actual = _sha256(actual_content) - if actual != expected or actual_exists != (mode is not None) or actual_mode != mode: - raise ValueError("restore hash does not match receipt") - receipt.after_hashes[str(target)] = actual - receipt.after_modes[str(target)] = actual_mode - receipt.validations["rollback"] = {"passed": True, "targets": [str(item[0]) for item in planned]} - return True - except Exception as error: - receipt.validations["rollback"] = {"passed": False, "message": redact_secrets(str(error))} - return False - - -def _remove_target(path): - _assert_safe_target(path) - _content, exists, _mode = _read_target(path) - if not exists: - return - directory = _open_directory(path.parent) - try: - item = os.stat(path.name, dir_fd=directory, follow_symlinks=False) - if not stat.S_ISREG(item.st_mode): - raise ValueError("refusing non-regular target removal") - os.unlink(path.name, dir_fd=directory) - _fsync_directory(directory) - finally: - os.close(directory) - - -def _current_state(target_name): - content, exists, mode = _read_target(target_name) - return {"hash": _sha256(content), "exists": exists, "mode": mode} - - -def _receipt_after_state(receipt): - return { - target: { - "hash": receipt.after_hashes[target], - "exists": receipt.after_modes[target] is not None, - "mode": receipt.after_modes[target], - } - for target in receipt.targets - } - - -def _rollback_restore_plan(receipt): - return [ - { - "path": target, - "root": receipt.target_roots[target], - "hash": receipt.before_hashes[target], - "exists": receipt.target_modes[target] is not None, - "mode": receipt.target_modes[target], - "backup_path": receipt.backup_paths[target], - } - for target in receipt.targets - ] - - -def _rollback_receipt_binding(receipt): - """Return every validated receipt field that can change rollback behavior.""" - return { - "transaction_id": receipt.transaction_id, - "transaction_root": receipt.transaction_root, - "targets": list(receipt.targets), - "restore": _rollback_restore_plan(receipt), - "lock_root": receipt.lock_root, - "lock_migration": receipt.lock_migration, - } - - -def _rollback_receipt_hash(receipt): - return _sha256( - json.dumps( - _rollback_receipt_binding(receipt), - sort_keys=True, - separators=(",", ":"), - ).encode("utf-8") - ) - - -def _assert_expected_states(targets, expected, message): - if not isinstance(expected, dict) or set(expected) != set(targets): - raise ValueError("reviewed rollback state does not match receipt targets") - for target in targets: - if _current_state(target) != expected[target]: - raise ValueError(message) - - -def _rollback_preview_locked(receipt, location): - if receipt.status == "rolled_back": - state = { - target: { - "hash": receipt.before_hashes[target], - "exists": receipt.target_modes[target] is not None, - "mode": receipt.target_modes[target], - } - for target in receipt.targets - } - _assert_expected_states( - receipt.targets, state, "rollback target differs from recorded before-state" - ) - else: - if receipt.status not in {"applied", "rollback_failed", "recovery_required"}: - raise ValueError("normal rollback requires an applied receipt") - state = _receipt_after_state(receipt) - _assert_expected_states( - receipt.targets, state, "rollback target differs from recorded after-state" - ) - restore = _rollback_restore_plan(receipt) - receipt_binding = _rollback_receipt_binding(receipt) - binding = { - "operation": "wire-rollback", - "receipt_path": str(location), - "receipt_status": receipt.status, - "current": state, - "operational_receipt": receipt_binding, - } - return { - "targets": [{"path": target, **state[target]} for target in receipt.targets], - "restore": restore, - "operational_receipt_hash": _rollback_receipt_hash(receipt), - "preview_hash": _sha256( - json.dumps(binding, sort_keys=True, separators=(",", ":")).encode("utf-8") - ), - "requires_confirmation": receipt.status != "rolled_back", - "already_rolled_back": receipt.status == "rolled_back", - } - - -def preview_rollback(receipt_path): - """Return a hash-bound, non-secret snapshot of the current rollback targets.""" - location = _assert_safe_target(receipt_path) - initial = Receipt.from_dict( - json.loads(_read_regular(location).decode("utf-8")), str(location) - ) - lock_context = ( - _migrate_legacy_target_locks(initial.targets, initial.lock_root) - if initial.lock_root is not None else nullcontext() - ) - with lock_context: - with _target_locks(initial.targets, lock_root=initial.lock_root): - receipt = Receipt.from_dict( - json.loads(_read_regular(location).decode("utf-8")), str(location) - ) - return _rollback_preview_locked(receipt, location) - - -def rollback( - receipt_path, - preview_hash=None, - expected_after_hashes=None, - expected_receipt_hash=None, -): - """Restore a receipt through a user preview hash or installer-internal CAS.""" - location = _assert_safe_target(receipt_path) - initial = Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) - lock_context = ( - _migrate_legacy_target_locks(initial.targets, initial.lock_root) - if initial.lock_root is not None else nullcontext() - ) - with lock_context: - with _target_locks(initial.targets, lock_root=initial.lock_root): - receipt = Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) - if ( - expected_receipt_hash is not None - and _rollback_receipt_hash(receipt) != expected_receipt_hash - ): - raise ValueError("rollback receipt changed after the reviewed plan") - if expected_after_hashes is not None: - if not isinstance(expected_after_hashes, dict) or set(expected_after_hashes) != set(receipt.targets): - raise ValueError("reviewed rollback hashes do not match receipt targets") - for target_name in receipt.targets: - current, exists, _mode = _read_target(target_name) - if not exists or _sha256(current) != expected_after_hashes[target_name]: - raise ValueError("receipt target changed after reviewed rollback preview") - else: - if preview_hash is None: - raise PermissionError( - "rollback requires the hash from a reviewed rollback preview" - ) - preview = _rollback_preview_locked(receipt, location) - if preview["already_rolled_back"]: - return receipt - if preview_hash != preview["preview_hash"]: - raise PermissionError( - "rollback preview hash does not match current target state" - ) - if receipt.status == "rolled_back": - current_matches = True - for target_name in receipt.targets: - current, exists, mode = _read_target(target_name) - expected_absent = receipt.backup_paths[target_name] is None - if _sha256(current) != receipt.before_hashes[target_name] or exists == expected_absent or mode != receipt.target_modes[target_name]: - current_matches = False - break - if current_matches: - return receipt - expected_current = _receipt_after_state(receipt) - if _restore_receipt(receipt, expected_current=expected_current): - receipt.status = "rolled_back" - else: - receipt.status = "rollback_failed" - try: - _atomic_in_directory(Path(receipt.transaction_root), "receipt.json", (json.dumps(receipt.to_dict(), indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600) - except Exception as error: - receipt.status = "recovery_required" - receipt.validations["receipt_write"] = {"passed": False, "message": redact_secrets(str(error))} - return receipt diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/wiring.py b/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/wiring.py deleted file mode 100644 index a0e753d..0000000 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/wiring.py +++ /dev/null @@ -1,353 +0,0 @@ -"""Deterministic, injection-safe configuration renderers for Wire.""" - -from __future__ import annotations - -import ipaddress -import json -import re -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path - - -_RUNTIMES = {"ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"} -_MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,191}$") -_SECRET_NAMES = { - "accesskey", "accesstoken", "apikey", "auth", "authorization", "clientsecret", "credential", - "credentials", "idtoken", "passwd", "password", "privatekey", - "refreshtoken", "secret", "secretkey", "token", -} -_ASSIGNMENT_SECRET = re.compile( - r"(?im)(^|[\s,{&?;])([\"']?(?:" - r"[a-z0-9_-]*(?:token|secret|passw(?:or)?d|authorization|credentials?|auth)|" - r"[a-z0-9_-]*(?:api|access|private)[_-]?key" - r")[\"']?\s*[:=]\s*)" - r"([^\s,}\]\r\n]+|\"(?:\\.|[^\"])*\"|'(?:\\.|[^'])*')" -) -_URL_USERINFO = re.compile(r"(https?://)([^/@\s]+@)", re.I) -_OLLAMA = re.compile(r"\A# MLX_AGENT_WIRE v1\nFROM (" + _MODEL.pattern[1:-1] + r")\n\Z") -_ENV_REFERENCE = re.compile( - r"^(?:os\.environ(?:\[[\"'][A-Z][A-Z0-9_]{0,127}[\"']\]|/[A-Z][A-Z0-9_]{0,127})|" - r"\$\{[A-Z][A-Z0-9_]{0,127}\}|\$[A-Z][A-Z0-9_]{0,127}|env:[A-Z][A-Z0-9_]{0,127})$" -) -_STATE_CHANGING_PATH_PARTS = { - "admin", "create", "delete", "install", "kill", "mutate", "pull", "push", - "reload", "remove", "reset", "restart", "shutdown", "update", "write", -} - - -def _secret_name(name): - raw = str(name).lower() - collapsed = re.sub(r"[^a-z0-9]", "", raw) - if collapsed in _SECRET_NAMES: - return True - segments = [item for item in re.split(r"[^a-z0-9]+", raw) if item] - if any( - item in {"auth", "authorization", "credential", "credentials", "passwd", "password", "secret", "token"} - for item in segments - ): - return True - return collapsed.endswith( - ("accesskey", "apikey", "authorization", "credential", "credentials", "passwd", "password", "privatekey", "secret", "token") - ) - - -def _environment_reference(value): - return isinstance(value, str) and _ENV_REFERENCE.fullmatch(value.strip()) is not None - - -def _json_contains_resolved_secret(value): - if isinstance(value, dict): - for key, item in value.items(): - if _secret_name(key) and not _environment_reference(item): - return True - if _json_contains_resolved_secret(item): - return True - return False - if isinstance(value, list): - return any(_json_contains_resolved_secret(item) for item in value) - if isinstance(value, str): - try: - parsed = urllib.parse.urlsplit(value) - except ValueError: - return False - if parsed.scheme not in {"http", "https"}: - return False - if parsed.username is not None or parsed.password is not None: - return True - return any( - _secret_name(key) and not _environment_reference(item) - for key, item in urllib.parse.parse_qsl(parsed.query, keep_blank_values=True) - ) - return False - - -def contains_resolved_secrets(content): - """Return whether configuration text contains a resolved credential value.""" - if not isinstance(content, str): - raise TypeError("configuration content must be text") - try: - parsed = json.loads(content) - except (TypeError, ValueError, json.JSONDecodeError): - parsed = None - if parsed is not None and _json_contains_resolved_secret(parsed): - return True - if _URL_USERINFO.search(content): - return True - for match in _ASSIGNMENT_SECRET.finditer(content): - raw = match.group(3).strip() - if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {"\"", "'"}: - raw = raw[1:-1] - if not _environment_reference(raw): - return True - return False - - -def require_secret_free_config(content): - """Fail closed without echoing a resolved secret-bearing value.""" - if contains_resolved_secrets(content): - raise ValueError( - "existing configuration contains resolved secret-bearing fields; " - "move secrets to environment references or a secret-free managed fragment" - ) - return content - - -def _redact_json(value): - if isinstance(value, dict): - return {key: "" if _secret_name(key) else _redact_json(item) for key, item in value.items()} - if isinstance(value, list): - return [_redact_json(item) for item in value] - if isinstance(value, str): - return re.sub(r"https?://[^\s\"']+", lambda match: redact_endpoint(match.group(0)), value) - return value - - -def redact_endpoint(endpoint): - """Return only a non-credential origin/path representation of an endpoint.""" - try: - parsed = urllib.parse.urlsplit(str(endpoint)) - if parsed.scheme not in {"http", "https"} or not parsed.hostname: - return "" - host = parsed.hostname - if ":" in host and not host.startswith("["): - host = "[{0}]".format(host) - origin = "{0}://{1}".format(parsed.scheme, host) - if parsed.port is not None: - origin += ":{0}".format(parsed.port) - return origin + (parsed.path or "/") - except ValueError: - return "" - - -def redact_secrets(content): - """Redact common credential forms before they can enter previews or receipts.""" - text = str(content) - try: - parsed = json.loads(text) - except (TypeError, ValueError, json.JSONDecodeError): - parsed = None - if parsed is not None: - return json.dumps(_redact_json(parsed), sort_keys=True, separators=(",", ": ")) - text = _URL_USERINFO.sub(r"\1@", text) - def redact_assignment(match): - value = match.group(3) - replacement = value if "" in value else "" - return match.group(1) + match.group(2) + replacement - text = _ASSIGNMENT_SECRET.sub(redact_assignment, text) - def redact_url(match): - return redact_endpoint(match.group(0)) - text = re.sub(r"https?://[^\s\"']+", redact_url, text) - return text - - -def validate_health_endpoint(endpoint): - """Validate one credential-free, read-only loopback HTTP(S) health URL.""" - if not isinstance(endpoint, str) or not endpoint: - raise ValueError("health endpoint must be a non-empty URL") - try: - parsed = urllib.parse.urlsplit(endpoint) - port = parsed.port - except ValueError as error: - raise ValueError("health endpoint has an invalid port") from error - if parsed.scheme not in {"http", "https"} or not parsed.hostname: - raise ValueError("health endpoint must use HTTP(S)") - if parsed.username is not None or parsed.password is not None or parsed.query or parsed.fragment: - raise ValueError("health endpoint must not contain credentials, query, or fragment") - if port is not None and not 1 <= port <= 65535: - raise ValueError("health endpoint port is outside 1-65535") - hostname = parsed.hostname.lower().rstrip(".") - if hostname != "localhost": - try: - address = ipaddress.ip_address(hostname) - except ValueError as error: - raise ValueError("health endpoint must use localhost or a loopback IP literal") from error - if not address.is_loopback: - raise ValueError("health endpoint must resolve only to loopback") - decoded_path = urllib.parse.unquote(parsed.path or "/").lower() - parts = {part for part in decoded_path.split("/") if part} - if parts & _STATE_CHANGING_PATH_PARTS: - raise ValueError("health endpoint path appears state-changing") - return endpoint - - -def _endpoint_origin(endpoint): - parsed = urllib.parse.urlsplit(validate_health_endpoint(endpoint)) - port = parsed.port or (443 if parsed.scheme == "https" else 80) - return parsed.scheme.lower(), parsed.hostname.lower().rstrip("."), port - - -class _SameLoopbackOriginRedirect(urllib.request.HTTPRedirectHandler): - def __init__(self, origin): - super().__init__() - self.origin = origin - - def redirect_request(self, request, file_pointer, code, message, headers, new_url): - destination = urllib.parse.urljoin(request.full_url, new_url) - if _endpoint_origin(destination) != self.origin: - raise urllib.error.URLError("health redirect left the reviewed loopback origin") - return super().redirect_request( - request, file_pointer, code, message, headers, destination - ) - - -class ConfigAdapter: - """A small format adapter using only documented, exact stdlib-safe subsets.""" - - version = "2.0" - - def __init__(self, runtime, path=None): - if runtime not in _RUNTIMES: - raise ValueError("unsupported runtime: {0}".format(runtime)) - self.runtime = runtime - self.path = Path(path) if path is not None else None - - @classmethod - def detect(cls, path, runtime=None): - location = Path(path) - selected = runtime - if selected is None: - name = location.name.lower() - if name == "modelfile" or location.suffix.lower() == ".modelfile": - selected = "ollama" - elif location.suffix.lower() in {".yaml", ".yml"}: - selected = "litellm" - elif "lmstudio" in name: - selected = "lmstudio" - else: - selected = "mlx_lm" - return cls(selected, location) - - def render(self, model, runtime=None, existing=""): - selected = runtime or self.runtime - if selected != self.runtime: - return ConfigAdapter(selected, self.path).render(model, existing=existing) - self._validate_model(model) - if not isinstance(existing, str): - raise TypeError("existing config content must be text") - require_secret_free_config(existing) - if selected == "ollama": - if existing and existing != self._render_ollama(model): - raise ValueError("Ollama Wire accepts only its exact managed configuration") - return self._render_ollama(model) - if selected == "litellm": - if existing: - self._validate_litellm(existing) - return self._render_litellm(model) - return self._render_json(model, existing) - - @staticmethod - def _validate_model(model): - if not isinstance(model, str) or not _MODEL.fullmatch(model): - raise ValueError("model must be a safe publisher/model identifier") - - def _render_json(self, model, existing): - if existing.strip(): - try: - document = json.loads(existing) - except json.JSONDecodeError as error: - raise ValueError("existing JSON configuration is invalid: {0}".format(error)) - if not isinstance(document, dict): - raise ValueError("existing JSON configuration must be an object") - else: - document = {} - port = 8081 if self.runtime == "mlx-vlm" else 8080 - provider_id = "mlxvlm" if self.runtime == "mlx-vlm" else self.runtime.replace("_", "") - document["mlx_agent_wire"] = { - "marker": "MLX_AGENT_WIRE", - "version": self.version, - "runtime": self.runtime, - "model": model, - "provider": { - "id": provider_id, - "type": "openai", - "base_url": "http://127.0.0.1:{0}/v1".format(port), - "api_key_env": "MLX_AGENT_LOCAL_API_KEY", - }, - } - return json.dumps(document, indent=2, sort_keys=True) + "\n" - - @staticmethod - def _render_ollama(model): - return "# MLX_AGENT_WIRE v1\nFROM {0}\n".format(model) - - @staticmethod - def _render_litellm(model): - short = model.split("/", 1)[1].lower() - return ( - "# MLX_AGENT_WIRE v1\n" - "model_list:\n" - " - model_name: {0}\n" - " litellm_params:\n" - " model: openai/{1}\n" - " api_base: http://127.0.0.1:8080/v1\n" - " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY\n" - ).format(short, model) - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("configuration content must be text") - require_secret_free_config(content) - if self.runtime in {"lmstudio", "mlx_lm", "mlx-vlm"}: - value = json.loads(content) - if not isinstance(value, dict): - raise ValueError("JSON configuration must be an object") - return True - if self.runtime == "ollama": - match = _OLLAMA.fullmatch(content) - if match is None: - raise ValueError("Ollama configuration must match the exact Wire grammar") - self._validate_model(match.group(1)) - return True - self._validate_litellm(content) - return True - - @staticmethod - def _validate_litellm(content): - lines = content.splitlines() - if len(lines) != 7 or lines[0] != "# MLX_AGENT_WIRE v1" or lines[1] != "model_list:": - raise ValueError("LiteLLM configuration must use the supported Wire YAML subset") - model_name = re.fullmatch(r" - model_name: ([A-Za-z0-9][A-Za-z0-9._-]{0,191})", lines[2]) - model = re.fullmatch(r" model: openai/(" + _MODEL.pattern[1:-1] + r")", lines[4]) - if lines[3] != " litellm_params:" or lines[5] != " api_base: http://127.0.0.1:8080/v1" or lines[6] != " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY" or not model_name or not model: - raise ValueError("LiteLLM configuration must use the supported Wire YAML subset") - ConfigAdapter._validate_model(model.group(1)) - if model_name.group(1) != model.group(1).split("/", 1)[1].lower(): - raise ValueError("LiteLLM model_name does not match the model identifier") - - @staticmethod - def health_check(endpoint, timeout=2.0): - if not endpoint: - return True - try: - endpoint = validate_health_endpoint(endpoint) - request = urllib.request.Request(endpoint, method="GET") - opener = urllib.request.build_opener( - urllib.request.ProxyHandler({}), - _SameLoopbackOriginRedirect(_endpoint_origin(endpoint)) - ) - with opener.open(request, timeout=timeout) as response: - return 200 <= response.status < 400 - except (OSError, ValueError, urllib.error.URLError): - return False diff --git a/packages/codex/mlx-agent/skills/mlx-doctor/SKILL.md b/packages/codex/mlx-agent/skills/mlx-doctor/SKILL.md new file mode 100644 index 0000000..df63bc0 --- /dev/null +++ b/packages/codex/mlx-agent/skills/mlx-doctor/SKILL.md @@ -0,0 +1,18 @@ +--- +name: "mlx-doctor" +description: "Diagnose local model inventories, wiring drift, and endpoint health." +--- + +Use `$mlx-agent:mlx-doctor` to invoke this installed Codex skill explicitly. Codex does not support custom `/mlx-*` slash commands. + +# MLX Doctor + +Resolve `` as the absolute directory two levels above the directory containing this SKILL.md; it holds `scripts/` and `src/`. Never resolve the bundled executable from the shell working directory. + +canonical capability ID: mlx-agent.doctor + +Run the read-only model diagnostics: + +`python3 /scripts/mlx-agent doctor models --json` + +Present the inventory, drift findings, and endpoint health as returned. Doctor must not delete, move, or repair anything; the confirmation-gated prune of incomplete cache snapshots requires an explicit reviewed preview from the user first. diff --git a/packages/codex/mlx-agent/skills/mlx-fleet/SKILL.md b/packages/codex/mlx-agent/skills/mlx-fleet/SKILL.md new file mode 100644 index 0000000..0eb69f6 --- /dev/null +++ b/packages/codex/mlx-agent/skills/mlx-fleet/SKILL.md @@ -0,0 +1,26 @@ +--- +name: "mlx-fleet" +description: "Preview and confirm a one-shot per-role router configuration." +--- + +Use `$mlx-agent:mlx-fleet` to invoke this installed Codex skill explicitly. Codex does not support custom `/mlx-*` slash commands. + +# MLX Fleet + +Resolve `` as the absolute directory two levels above the directory containing this SKILL.md; it holds `scripts/` and `src/`. Never resolve the bundled executable from the shell working directory. + +canonical capability ID: mlx-agent.fleet + +Render the one-shot per-role router configuration without mutation: + +`python3 /scripts/mlx-agent fleet render --path --assign --json` + +Then request the exact transaction diff and preview hash without confirmation: + +`python3 /scripts/mlx-agent fleet apply --path --assign --json` + +Show that returned diff and preview hash. Do not write configuration files directly. Only after the user explicitly confirms that exact preview, run: + +`python3 /scripts/mlx-agent fleet apply --path --assign --confirm --preview-hash --json` + +Never download model weights without an explicit confirmation. Report the transaction receipt returned by the CLI. diff --git a/packages/codex/mlx-agent/skills/mlx-scout/SKILL.md b/packages/codex/mlx-agent/skills/mlx-scout/SKILL.md index c51a39b..f662382 100644 --- a/packages/codex/mlx-agent/skills/mlx-scout/SKILL.md +++ b/packages/codex/mlx-agent/skills/mlx-scout/SKILL.md @@ -7,14 +7,14 @@ Use `$mlx-agent:mlx-scout` to invoke this installed Codex skill explicitly. Code # MLX Scout -Resolve `` as the absolute directory containing this SKILL.md. Never resolve the bundled executable from the shell working directory. +Resolve `` as the absolute directory two levels above the directory containing this SKILL.md; it holds `scripts/` and `src/`. Never resolve the bundled executable from the shell working directory. canonical capability ID: mlx-agent.scout Run the provider-neutral discovery command: -`python3 /scripts/mlx-agent discover ` +`python3 /scripts/mlx-agent discover ` -Present its evidence and recommendations as returned. Discovery must not download model weights or change configuration. If a later download or configuration mutation would help, describe the exact CLI preview first and obtain explicit user confirmation before it. Bundled references (read when relevant): `src/mlx_agent/resources/references/quantization.md` for quant tradeoffs, `src/mlx_agent/resources/references/model-families.md` for chat-template and tool-calling quirks, `src/mlx_agent/resources/references/troubleshooting.md` for serving symptoms. +Present its evidence and recommendations as returned. Discovery must not download model weights or change configuration. If a later download or configuration mutation would help, describe the exact CLI preview first and obtain explicit user confirmation before it. Bundled references (read when relevant): `/src/mlx_agent/resources/references/quantization.md` for quant tradeoffs, `/src/mlx_agent/resources/references/model-families.md` for chat-template and tool-calling quirks, `/src/mlx_agent/resources/references/troubleshooting.md` for serving symptoms. Tool-use is canonical; agentic is descriptive only. Models verified to invoke supplied tools with schema-valid arguments. Tool-use membership is additional, so a model may retain its primary role. Its recommendation minimum is verified: metadata is not verification, and recommendation requires verified evidence from a schema-valid synthetic runtime tool call. Manifest safety says automatic model downloads are disabled; verification must not pull, install, or download models. Report unsupported runtimes explicitly. If none is verified, recommend none; never use a fallback. diff --git a/packages/codex/mlx-agent/skills/mlx-scout/scripts/mlx-agent b/packages/codex/mlx-agent/skills/mlx-scout/scripts/mlx-agent deleted file mode 100755 index 54a5a80..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/scripts/mlx-agent +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -"""Run the bundled MLX agent core relative to this bootstrap, never the CWD.""" - -import sys -from pathlib import Path - - -sys.dont_write_bytecode = True -SCRIPT_DIR = Path(__file__).resolve().parent -sys.path.insert(0, str(SCRIPT_DIR.parent / "src")) - -from mlx_agent.cli import main # noqa: E402 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/__init__.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/__init__.py deleted file mode 100644 index 67dcb84..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Public API for the dependency-free MLX agent core.""" - -from .contracts import ResultEnvelope - -__all__ = ["ResultEnvelope"] -__version__ = "0.4.0" diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/__main__.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/__main__.py deleted file mode 100644 index 94dda3d..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/__main__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Run the MLX agent core with ``python -m mlx_agent``.""" - -import sys - -from .cli import main - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/adoption.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/adoption.py deleted file mode 100644 index e7c8c48..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/adoption.py +++ /dev/null @@ -1,870 +0,0 @@ -"""Provider-neutral, resumable model adoption state machine.""" - -from __future__ import annotations - -import hashlib -import json -import uuid -from concurrent.futures import ThreadPoolExecutor -from copy import deepcopy -from dataclasses import dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Dict, List, Tuple - -from .discovery import DiscoveryRequest, DiscoveryService -from .transactions import ( - ConcurrentTransactionError, - _atomic_in_directory, - _read_target, - _target_locks, -) -from .verification import ( - EvidenceStrength, - ROLE_PROBE_IDS, - TOOL_USE_PROBE_ID, - VerificationEvidence, - VerificationStatus, - Verifier, -) - - -ADOPTION_SCHEMA_VERSION = "1.3" -LEGACY_ADOPTION_SCHEMA_VERSIONS = ("1.1", "1.2") -PHASES = ("inspect", "discover", "shortlist", "verify", "measure", "compare", "recommend", "complete") -MEASURE_LIMIT = 6 -UTILITY_ROLES = {"general", "embedding"} -ALLOWED_ROLES = { - "general", "coding", "reasoning", "vision", "embedding", "tool-use" -} -EVIDENCE_SCORES = { - EvidenceStrength.RUNTIME_MEASURED.value: 500, - EvidenceStrength.RUNTIME_TESTED.value: 400, - EvidenceStrength.RUNTIME_INVENTORY.value: 300, - EvidenceStrength.METADATA_ONLY.value: 200, - EvidenceStrength.HEURISTIC_ONLY.value: 100, -} -ROLE_PROBE_BONUS = 50 - - -class AdoptionStateConflictError(ValueError): - """A persisted handoff changed or is locked by another workflow.""" - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -@dataclass(frozen=True) -class AdoptionRequest: - """Bounded request fields that are safe to persist in a handoff.""" - - roles: Tuple[str, ...] = ("general",) - state_path: object = None - shortlist_limit: int = 4 - allow_network: bool = True - offline: bool = False - refresh: bool = False - memory_gb: object = None - quantization: object = None - licenses: object = None - include_gated: bool = True - publishers: object = None - runtime: object = None - fast: bool = False - measure: bool = False - - def __post_init__(self): - if isinstance(self.roles, str): - supplied_roles = (self.roles,) - elif isinstance(self.roles, (list, tuple)): - supplied_roles = tuple(self.roles) - else: - raise TypeError("roles must be a string or an array of strings") - if any(not isinstance(role, str) or not role for role in supplied_roles): - raise TypeError("roles must contain non-empty strings") - roles = tuple(supplied_roles) - if not roles or any(role not in ALLOWED_ROLES for role in roles) or len(roles) > 6: - raise ValueError("roles must contain one to six supported roles") - if len(set(roles)) != len(roles): - raise ValueError("roles must not contain duplicates") - if not isinstance(self.shortlist_limit, int) or isinstance(self.shortlist_limit, bool): - raise TypeError("shortlist_limit must be an integer") - if self.shortlist_limit < 1 or self.shortlist_limit > 20: - raise ValueError("shortlist_limit must be between 1 and 20") - if not isinstance(self.allow_network, bool): - raise TypeError("allow_network must be a boolean") - for name in ("offline", "refresh", "include_gated", "fast", "measure"): - if not isinstance(getattr(self, name), bool): - raise TypeError("{0} must be a boolean".format(name)) - if self.memory_gb is not None and ( - not isinstance(self.memory_gb, (int, float)) or isinstance(self.memory_gb, bool) - ): - raise TypeError("memory_gb must be a number or null") - for name in ("quantization", "runtime"): - value = getattr(self, name) - if value is not None and not isinstance(value, str): - raise TypeError("{0} must be a string or null".format(name)) - object.__setattr__(self, "roles", roles) - - def to_dict(self): - return { - "roles": list(self.roles), - "shortlist_limit": self.shortlist_limit, - "allow_network": self.allow_network, - "offline": bool(self.offline), - "refresh": bool(self.refresh), - "memory_gb": self.memory_gb, - "quantization": self.quantization, - "licenses": _string_list(self.licenses), - "include_gated": bool(self.include_gated), - "publishers": _string_list(self.publishers), - "runtime": self.runtime, - "fast": bool(self.fast), - "measure": bool(self.measure), - } - - @classmethod - def from_dict(cls, value, state_path=None): - if not isinstance(value, dict): - raise TypeError("adoption request must be an object") - allowed = { - "roles", "state_path", "state", "shortlist_limit", "limit", - "allow_network", "offline", "refresh", "memory_gb", "quantization", - "licenses", "include_gated", "publishers", "runtime", "fast", - "measure", - } - unknown = sorted(set(value) - allowed) - if unknown: - raise ValueError("adoption request has unexpected keys: {0}".format(unknown)) - return cls( - roles=value.get("roles", ("general",)), - state_path=state_path or value.get("state_path") or value.get("state"), - shortlist_limit=value.get("shortlist_limit", value.get("limit", 4)), - allow_network=value.get("allow_network", True), - offline=value.get("offline", False), - refresh=value.get("refresh", False), - memory_gb=value.get("memory_gb"), - quantization=value.get("quantization"), - licenses=value.get("licenses"), - include_gated=value.get("include_gated", True), - publishers=value.get("publishers"), - runtime=value.get("runtime"), - fast=value.get("fast", False), - measure=value.get("measure", False), - ) - - -@dataclass -class AdoptionState: - """Schema-versioned, serializable adoption handoff.""" - - workflow_id: str - phase: str - status: str - request: Dict[str, Any] - state_path: Path = field(repr=False) - revision: int = 0 - completed_phases: List[str] = field(default_factory=list) - host: Dict[str, Any] = field(default_factory=dict) - discovery: Dict[str, Any] = field(default_factory=dict) - shortlist: List[Dict[str, Any]] = field(default_factory=list) - evidence: List[Dict[str, Any]] = field(default_factory=list) - comparisons: List[Dict[str, Any]] = field(default_factory=list) - recommendations: List[Dict[str, Any]] = field(default_factory=list) - warnings: List[Dict[str, str]] = field(default_factory=list) - errors: List[Dict[str, str]] = field(default_factory=list) - created_at: str = field(default_factory=_utc_now) - updated_at: str = field(default_factory=_utc_now) - schema_version: str = ADOPTION_SCHEMA_VERSION - _source_hash: object = field(default=None, repr=False, compare=False) - - def to_dict(self): - return { - "schema_version": self.schema_version, - "workflow_id": self.workflow_id, - "revision": self.revision, - "phase": self.phase, - "status": self.status, - "request": self.request, - "completed_phases": list(self.completed_phases), - "host": self.host, - "discovery": self.discovery, - "shortlist": self.shortlist, - "evidence": self.evidence, - "comparisons": self.comparisons, - "recommendations": self.recommendations, - "warnings": self.warnings, - "errors": self.errors, - "created_at": self.created_at, - "updated_at": self.updated_at, - } - - @classmethod - def from_dict(cls, value, state_path): - value = _migrate_state(value) - _validate_state(value) - return cls( - workflow_id=value["workflow_id"], - revision=value["revision"], - phase=value["phase"], - status=value["status"], - request=dict(value["request"]), - state_path=Path(state_path), - completed_phases=list(value["completed_phases"]), - host=dict(value["host"]), - discovery=dict(value["discovery"]), - shortlist=list(value["shortlist"]), - evidence=list(value["evidence"]), - comparisons=list(value["comparisons"]), - recommendations=list(value["recommendations"]), - warnings=list(value["warnings"]), - errors=list(value["errors"]), - created_at=value["created_at"], - updated_at=value["updated_at"], - schema_version=value["schema_version"], - ) - - -class AdoptionWorkflow: - """Advance adoption one durable phase at a time.""" - - def __init__(self, discovery_service=None, verifier=None, state_path=None): - self.discovery_service = discovery_service or DiscoveryService() - metadata_client = getattr(self.discovery_service, "_huggingface", None) - self.verifier = verifier or Verifier(metadata_client=metadata_client) - self.state_path = Path(state_path) if state_path is not None else None - - def start(self, request): - if isinstance(request, dict): - request = AdoptionRequest.from_dict(request) - if not isinstance(request, AdoptionRequest): - raise TypeError("request must be an AdoptionRequest or object") - path_value = request.state_path or self.state_path - if path_value is None: - raise ValueError("a state path is required") - state = AdoptionState( - workflow_id=str(uuid.uuid4()), - phase="inspect", - status="running", - request=request.to_dict(), - state_path=Path(path_value), - ) - self._persist(state) - return state - - def advance(self, state): - if not isinstance(state, AdoptionState): - raise TypeError("state must be an AdoptionState") - state.phase = _first_incomplete(state.completed_phases) - if state.phase == "complete": - state.status = "complete" - return state - - phase = state.phase - getattr(self, "_phase_{0}".format(phase))(state) - if phase not in state.completed_phases: - state.completed_phases.append(phase) - state.phase = _first_incomplete(state.completed_phases) - state.status = "complete" if state.phase == "complete" else "running" - self._persist(state) - return state - - def resume(self, path): - state_path = Path(path) - content, exists, _mode = _read_target(state_path) - if not exists: - raise ValueError("adoption state does not exist") - value = json.loads(content.decode("utf-8")) - state = AdoptionState.from_dict(value, state_path) - state._source_hash = hashlib.sha256(content).hexdigest() - state.phase = _first_incomplete(state.completed_phases) - state.status = "complete" if state.phase == "complete" else "running" - return state - - def status(self, path): - return self.resume(path) - - def _phase_inspect(self, state): - host = self.discovery_service.host - if hasattr(host, "to_dict"): - host = host.to_dict() - if not isinstance(host, dict): - raise TypeError("host inventory must be an object") - state.host = dict(host) - - def _phase_discover(self, state): - request = state.request - roles = request["roles"] - result = self.discovery_service.discover(DiscoveryRequest( - role=roles[0] if len(roles) == 1 else None, - memory_gb=request.get("memory_gb"), - quantization=request.get("quantization"), - licenses=request.get("licenses"), - include_gated=request.get("include_gated", True), - publishers=request.get("publishers"), - runtime=request.get("runtime"), - refresh=request.get("refresh", False), - offline=request.get("offline", False), - limit=request["shortlist_limit"], - fast=request.get("fast", False), - )) - if result.status != "ok": - error = result.to_dict()["error"] - state.errors.append({key: str(error[key]) for key in ("code", "message", "remediation")}) - self._persist(state) - raise RuntimeError("discovery failed: {0}".format(error["message"])) - state.discovery = dict(result.data) - state.warnings.extend(result.warnings) - - def _phase_shortlist(self, state): - requested = set(state.request["roles"]) - limit = state.request["shortlist_limit"] - shortlisted = [] - roles = state.discovery.get("roles", {}) - if not isinstance(roles, dict): - raise TypeError("discovery roles must be an object") - for role in state.request["roles"]: - if role not in requested: - continue - for candidate in (roles.get(role) or [])[:limit]: - record = dict(candidate) - record["role"] = role - shortlisted.append(record) - state.shortlist = shortlisted - - def _phase_verify(self, state): - clear_inventory_cache = getattr(self.verifier, "clear_inventory_cache", None) - if callable(clear_inventory_cache): - clear_inventory_cache() - workers = verification_concurrency(state.host.get("ram_gb")) - - def verify_one(candidate): - try: - evidence = self.verifier.verify( - candidate, - state.host, - allow_network=state.request.get("allow_network", True), - ) - if not isinstance(evidence, VerificationEvidence): - raise TypeError("verifier returned an invalid evidence record") - return evidence.to_dict() - except Exception as error: - repo = str( - candidate.get("repo") - or candidate.get("repository") - or "unknown" - ) - role = str(candidate.get("role") or "general") - if role == "tool-use": - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.HEURISTIC_ONLY, - status=VerificationStatus.FAILED, - available_locally=False, - loads=None, - reasoning_confirmed=None, - runtime=None, - note=_bounded("Candidate tool-use verification failed."), - details={ - "probe_id": TOOL_USE_PROBE_ID, - "reason": "verification_exception", - }, - ).to_dict() - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.HEURISTIC_ONLY, - status=VerificationStatus.FAILED, - available_locally=False, - loads=None, - reasoning_confirmed=None, - runtime=None, - note=_bounded("Candidate verification failed: {0}".format(error)), - details={"error": _bounded(str(error), 160)}, - ).to_dict() - - with ThreadPoolExecutor(max_workers=workers) as executor: - state.evidence = list(executor.map(verify_one, state.shortlist)) - - def _phase_measure(self, state): - if not state.request.get("measure", False): - return - from .bench import BenchError, measure_runtime - - runtime_clients = getattr(self.verifier, "runtime_clients", None) - clients = {} - if callable(runtime_clients): - for client in runtime_clients(): - name = getattr(client, "name", None) - if isinstance(name, str): - clients[name] = client - measured = 0 - for index, candidate in enumerate(state.shortlist): - if measured >= MEASURE_LIMIT: - break - if index >= len(state.evidence): - break - evidence = state.evidence[index] - if ( - evidence.get("strength") != EvidenceStrength.RUNTIME_TESTED.value - or evidence.get("status") != VerificationStatus.VERIFIED.value - ): - continue - client = clients.get(evidence.get("runtime")) - if client is None or not callable(getattr(client, "stream_generate", None)): - continue - repo = candidate.get("repo") or candidate.get("repository") - try: - measurement = measure_runtime( - repo, client, chip=state.host.get("chip") - ) - except BenchError as error: - state.warnings.append({ - "code": "measure_skipped", - "message": _bounded( - "{0}: {1}".format(repo, error) - ), - }) - continue - except Exception as error: - state.warnings.append({ - "code": "measure_skipped", - "message": _bounded("{0}: {1}".format(repo, error)), - }) - continue - details = dict(evidence.get("details") or {}) - details["bench"] = measurement.to_dict() - evidence["details"] = details - evidence["strength"] = EvidenceStrength.RUNTIME_MEASURED.value - measured += 1 - - def _phase_compare(self, state): - evidence_by_candidate = { - (item["repo"], item["role"]): item for item in state.evidence - } - comparisons = [] - for candidate in state.shortlist: - repo = candidate.get("repo") or candidate.get("repository") - role = candidate.get("role") - evidence = evidence_by_candidate.get((repo, role), {}) - reasons = [] - if candidate.get("fits") is False: - reasons.append("memory_budget") - if evidence.get("loads") is False: - reasons.append("runtime_generation_failed") - utility_role = role in UTILITY_ROLES or state.request.get("fast", False) - if utility_role and evidence.get("reasoning_confirmed") is True: - reasons.append("confirmed_reasoner_for_utility_role") - evidence_status = evidence.get( - "status", VerificationStatus.FAILED.value - ) - if ( - role == "tool-use" - and evidence_status != VerificationStatus.VERIFIED.value - ): - reasons.append("tool_use_not_verified") - score = EVIDENCE_SCORES.get(evidence.get("strength"), 0) - score += int(candidate.get("rank_score") or 0) - score += 20 if candidate.get("trusted") else 0 - probe_outcome = _role_probe_outcome_for(role, evidence) - if probe_outcome is not None: - if probe_outcome.get("valid") is True: - score += ROLE_PROBE_BONUS - elif probe_outcome.get("reason") != "unsupported_runtime": - reasons.append("role_probe_failed") - comparisons.append({ - "repo": repo, - "role": role, - "eligible": not reasons, - "score": score, - "evidence_strength": evidence.get("strength", EvidenceStrength.HEURISTIC_ONLY.value), - "evidence_status": evidence_status, - "verification_status": evidence_status, - "rejection_reasons": reasons, - }) - state.comparisons = comparisons - - def _phase_recommend(self, state): - candidates_by_candidate = { - ( - item.get("repo") or item.get("repository"), - item.get("role"), - ): item - for item in state.shortlist - } - recommendations = [] - for role in state.request["roles"]: - eligible = [ - item for item in state.comparisons - if item["role"] == role and item["eligible"] - and ( - role != "tool-use" - or item.get("evidence_status") - == VerificationStatus.VERIFIED.value - ) - ] - eligible.sort(key=lambda item: (-item["score"], item["repo"])) - if not eligible: - continue - chosen = eligible[0] - candidate = candidates_by_candidate[(chosen["repo"], chosen["role"])] - recommendations.append({ - "role": role, - "repo": chosen["repo"], - "evidence_strength": chosen["evidence_strength"], - "evidence_status": chosen.get( - "evidence_status", - chosen.get("verification_status", VerificationStatus.FAILED.value), - ), - "estimated_ram_gb": candidate.get("est_ram_gb"), - "wiring": candidate.get("wiring"), - "alternatives": [item["repo"] for item in eligible[1:4]], - }) - state.recommendations = recommendations - - def _persist(self, state): - destination = state.state_path - try: - with _target_locks([destination], create_parents=True): - current, exists, _mode = _read_target(destination) - if state._source_hash is None: - if state.revision != 0 or exists: - raise AdoptionStateConflictError( - "adoption state already exists or lacks a CAS baseline" - ) - else: - if not exists or hashlib.sha256(current).hexdigest() != state._source_hash: - raise AdoptionStateConflictError( - "adoption state changed after it was resumed" - ) - try: - current_value = json.loads(current.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as error: - raise AdoptionStateConflictError( - "adoption state changed after it was resumed" - ) from error - if current_value.get("revision") != state.revision: - raise AdoptionStateConflictError( - "adoption state revision changed after it was resumed" - ) - - next_revision = state.revision + 1 - next_updated_at = _utc_now() - value = state.to_dict() - value["revision"] = next_revision - value["updated_at"] = next_updated_at - _validate_state(value) - content = ( - json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n" - ).encode("utf-8") - - def validate_staged(text): - _validate_state(json.loads(text)) - - _atomic_in_directory( - destination.parent, - destination.name, - content, - 0o600, - validator=validate_staged, - ) - state.revision = next_revision - state.updated_at = next_updated_at - state._source_hash = hashlib.sha256(content).hexdigest() - except ConcurrentTransactionError as error: - raise AdoptionStateConflictError( - "another adoption workflow is updating this handoff" - ) from error - - -def _role_probe_outcome_for(role, evidence): - """Return the role-fit probe outcome recorded in evidence, if any.""" - probe_id = ROLE_PROBE_IDS.get(role) - if probe_id is None or not isinstance(evidence, dict): - return None - details = evidence.get("details") - if not isinstance(details, dict): - return None - probes = details.get("probes") - if not isinstance(probes, list): - return None - for probe in probes: - if not isinstance(probe, dict) or probe.get("probe_id") != probe_id: - continue - outcome = probe.get("outcome") - return outcome if isinstance(outcome, dict) else None - return None - - -def verification_concurrency(ram_gb): - try: - memory = int(float(ram_gb)) - except (TypeError, ValueError, OverflowError): - memory = 0 - return int(min(4, max(1, memory // 16))) - - -def _first_incomplete(completed): - completed_set = set(completed) - for phase in PHASES[:-1]: - if phase not in completed_set: - return phase - return "complete" - - -def _migrate_state(value): - if not isinstance(value, dict): - raise TypeError("adoption state must be an object") - version = value.get("schema_version") - if version == ADOPTION_SCHEMA_VERSION: - return value - if version not in LEGACY_ADOPTION_SCHEMA_VERSIONS: - raise ValueError("unsupported adoption state schema version") - - migrated = deepcopy(value) - if version == "1.1": - request = migrated.get("request") - legacy_roles = ( - list(request.get("roles", [])) - if isinstance(request, dict) and isinstance(request.get("roles", []), list) - else [] - ) - for collection in ("shortlist", "evidence", "comparisons", "recommendations"): - records = migrated.get(collection) - if isinstance(records, list): - legacy_roles.extend( - item.get("role") for item in records if isinstance(item, dict) - ) - if "tool-use" in legacy_roles: - raise ValueError("legacy adoption state cannot contain tool-use") - - status_by_strength = { - EvidenceStrength.RUNTIME_TESTED.value: VerificationStatus.VERIFIED.value, - EvidenceStrength.RUNTIME_INVENTORY.value: VerificationStatus.FAILED.value, - EvidenceStrength.METADATA_ONLY.value: VerificationStatus.METADATA_ONLY.value, - EvidenceStrength.HEURISTIC_ONLY.value: VerificationStatus.METADATA_ONLY.value, - } - for item in migrated.get("evidence", []): - if not isinstance(item, dict): - continue - strength = item.get("strength") - if strength in status_by_strength: - item["status"] = status_by_strength[strength] - - request = migrated.get("request") - if isinstance(request, dict) and "measure" not in request: - request["measure"] = False - completed = migrated.get("completed_phases") - if ( - isinstance(completed, list) - and "verify" in completed - and "measure" not in completed - ): - completed.insert(completed.index("verify") + 1, "measure") - migrated["schema_version"] = ADOPTION_SCHEMA_VERSION - return migrated - - -def _validate_state(value): - required = { - "schema_version", "workflow_id", "revision", "phase", "status", "request", - "completed_phases", "host", "discovery", "shortlist", "evidence", - "comparisons", "recommendations", "warnings", "errors", "created_at", "updated_at", - } - if not isinstance(value, dict): - raise TypeError("adoption state must be an object") - missing = sorted(required - set(value)) - unexpected = sorted(set(value) - required) - if missing: - raise ValueError("adoption state is missing keys: {0}".format(missing)) - if unexpected: - raise ValueError("adoption state has unexpected keys: {0}".format(unexpected)) - if value["schema_version"] != ADOPTION_SCHEMA_VERSION: - raise ValueError("unsupported adoption state schema version") - if not isinstance(value["workflow_id"], str) or not value["workflow_id"]: - raise ValueError("workflow_id must be a non-empty string") - if ( - not isinstance(value["revision"], int) - or isinstance(value["revision"], bool) - or value["revision"] < 1 - ): - raise ValueError("revision must be a positive integer") - if value["phase"] not in PHASES: - raise ValueError("invalid adoption phase") - if value["status"] not in ("running", "complete"): - raise ValueError("invalid adoption status") - _validate_completed_phases(value) - if not isinstance(value["request"], dict): - raise TypeError("adoption request must be an object") - _validate_request_shape(value["request"]) - AdoptionRequest.from_dict(value["request"]) - for key in ("host", "discovery"): - if not isinstance(value[key], dict): - raise TypeError("{0} must be an object".format(key)) - bounds = { - "shortlist": 100, - "evidence": 100, - "comparisons": 100, - "recommendations": 6, - "warnings": 50, - "errors": 50, - } - for key, maximum in bounds.items(): - if not isinstance(value[key], list): - raise TypeError("{0} must be an array".format(key)) - if len(value[key]) > maximum: - raise ValueError("{0} exceeds its maximum size".format(key)) - _validate_timestamp(value["created_at"], "created_at") - _validate_timestamp(value["updated_at"], "updated_at") - _validate_evidence(value["evidence"]) - _validate_phase_artifacts(value) - - -def _validate_completed_phases(value): - completed = value["completed_phases"] - if not isinstance(completed, list): - raise TypeError("completed_phases must be an array") - if len(completed) > len(PHASES) - 1: - raise ValueError("completed_phases contains an invalid phase") - expected = list(PHASES[:len(completed)]) - if completed != expected: - raise ValueError("completed_phases must be a unique contiguous phase prefix") - expected_phase = PHASES[len(completed)] - if value["phase"] != expected_phase: - raise ValueError("phase does not match completed_phases") - expected_status = "complete" if expected_phase == "complete" else "running" - if value["status"] != expected_status: - raise ValueError("status does not match phase") - - -def _validate_request_shape(request): - required = { - "roles", "shortlist_limit", "allow_network", "offline", "refresh", - "memory_gb", "quantization", "licenses", "include_gated", "publishers", - "runtime", "fast", "measure", - } - if set(request) != required: - raise ValueError("adoption request does not match the persisted schema") - if not isinstance(request["roles"], list) or not request["roles"]: - raise ValueError("request.roles must be a non-empty array") - if len(request["roles"]) > 6 or len(set(request["roles"])) != len(request["roles"]): - raise ValueError("request.roles must be bounded and unique") - if any(not isinstance(role, str) or not role for role in request["roles"]): - raise TypeError("request.roles must contain non-empty strings") - if not isinstance(request["shortlist_limit"], int) or isinstance(request["shortlist_limit"], bool): - raise TypeError("request.shortlist_limit must be an integer") - if not 1 <= request["shortlist_limit"] <= 20: - raise ValueError("request.shortlist_limit is out of range") - for key in ("allow_network", "offline", "refresh", "include_gated", "fast"): - if not isinstance(request[key], bool): - raise TypeError("request.{0} must be a boolean".format(key)) - if request["memory_gb"] is not None and ( - not isinstance(request["memory_gb"], (int, float)) or isinstance(request["memory_gb"], bool) - ): - raise TypeError("request.memory_gb must be a number or null") - for key in ("quantization", "runtime"): - if request[key] is not None and not isinstance(request[key], str): - raise TypeError("request.{0} must be a string or null".format(key)) - for key in ("licenses", "publishers"): - if not isinstance(request[key], list) or len(request[key]) > 20: - raise ValueError("request.{0} must be a bounded array".format(key)) - if any(not isinstance(item, str) for item in request[key]): - raise TypeError("request.{0} must contain strings".format(key)) - - -def _validate_timestamp(timestamp, field_name): - if not isinstance(timestamp, str): - raise TypeError("{0} must be an ISO-8601 timestamp".format(field_name)) - normalized = timestamp[:-1] + "+00:00" if timestamp.endswith("Z") else timestamp - try: - parsed = datetime.fromisoformat(normalized) - except ValueError: - raise ValueError("{0} must be an ISO-8601 timestamp".format(field_name)) - if parsed.tzinfo is None: - raise ValueError("{0} must include a timezone".format(field_name)) - - -def _validate_evidence(evidence): - required = { - "repo", "role", "strength", "status", "available_locally", "loads", - "reasoning_confirmed", "runtime", "note", "details", - } - strengths = set(EVIDENCE_SCORES) - statuses = {status.value for status in VerificationStatus} - for index, item in enumerate(evidence): - prefix = "evidence[{0}]".format(index) - if not isinstance(item, dict): - raise TypeError("{0} must be an object".format(prefix)) - if set(item) != required: - raise ValueError("{0} does not match the evidence schema".format(prefix)) - for key in ("repo", "role", "note"): - if not isinstance(item[key], str) or not item[key]: - raise ValueError("{0}.{1} must be a non-empty string".format(prefix, key)) - if len(item["note"]) > 300: - raise ValueError("{0}.note exceeds its maximum length".format(prefix)) - if item["strength"] not in strengths: - raise ValueError("{0}.strength is invalid".format(prefix)) - if item["status"] not in statuses: - raise ValueError("{0}.status is invalid".format(prefix)) - if not isinstance(item["available_locally"], bool): - raise TypeError("{0}.available_locally must be a boolean".format(prefix)) - for key in ("loads", "reasoning_confirmed"): - if item[key] is not None and not isinstance(item[key], bool): - raise TypeError("{0}.{1} must be a boolean or null".format(prefix, key)) - if item["runtime"] is not None and not isinstance(item["runtime"], str): - raise TypeError("{0}.runtime must be a string or null".format(prefix)) - if not isinstance(item["details"], dict): - raise TypeError("{0}.details must be an object".format(prefix)) - - -def _validate_phase_artifacts(value): - completed = value["completed_phases"] - verify_complete = "verify" in completed - compare_complete = "compare" in completed - recommend_complete = "recommend" in completed - shortlist = value["shortlist"] - evidence = value["evidence"] - comparisons = value["comparisons"] - recommendations = value["recommendations"] - if not verify_complete and evidence: - raise ValueError("evidence is not valid before verification completes") - if verify_complete: - _validate_records_match(shortlist, evidence, "evidence") - if not compare_complete and comparisons: - raise ValueError("comparisons are not valid before comparison completes") - if compare_complete: - _validate_records_match(shortlist, comparisons, "comparisons") - if not recommend_complete and recommendations: - raise ValueError("recommendations are not valid before recommendation completes") - - -def _validate_records_match(shortlist, records, field_name): - if len(shortlist) != len(records): - raise ValueError("{0} does not cover the current shortlist".format(field_name)) - for index, (candidate, record) in enumerate(zip(shortlist, records)): - if not isinstance(candidate, dict) or not isinstance(record, dict): - raise TypeError("{0}[{1}] must be an object".format(field_name, index)) - candidate_repo = candidate.get("repo") or candidate.get("repository") - if not isinstance(candidate_repo, str) or not candidate_repo: - raise ValueError("shortlist[{0}] has no repository".format(index)) - if not isinstance(candidate.get("role"), str) or not candidate["role"]: - raise ValueError("shortlist[{0}] has no role".format(index)) - if record.get("repo") != candidate_repo or record.get("role") != candidate["role"]: - raise ValueError("{0}[{1}] does not match the current shortlist".format(field_name, index)) - - -def _string_list(value): - if value is None: - return [] - if isinstance(value, str): - return [value] - return [str(item) for item in value] - - -def _bounded(value, limit=300): - text = " ".join(str(value).split()) - if len(text) <= limit: - return text - return text[: max(0, limit - 3)] + "..." diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/blueprint.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/blueprint.py deleted file mode 100644 index 483d7cf..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/blueprint.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Deterministic dataset-creation blueprint from a DomainIntent. - -Guidance only: no download, no training, no execution. Templates are fixed so -the same intent always yields the same blueprint text and structure. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Dict, Tuple - -from .interview import DomainIntent - - -@dataclass(frozen=True) -class DatasetBlueprint: - goal: str - schema_fields: Tuple[str, ...] - labeling_notes: str - split_guidance: str - license_privacy: str - mlx_next_steps: str - - def to_dict(self) -> Dict[str, object]: - return { - "goal": self.goal, - "schema_fields": list(self.schema_fields), - "labeling_notes": self.labeling_notes, - "split_guidance": self.split_guidance, - "license_privacy": self.license_privacy, - "mlx_next_steps": self.mlx_next_steps, - } - - def to_markdown(self) -> str: - lines = [ - "### Goal", - "", - self.goal, - "", - "### Suggested schema", - "", - ] - for field_name in self.schema_fields: - lines.append("- `{0}`".format(field_name)) - lines.extend([ - "", - "### Labeling notes", - "", - self.labeling_notes, - "", - "### Train / val split", - "", - self.split_guidance, - "", - "### License and privacy", - "", - self.license_privacy, - "", - "### MLX fine-tune next steps", - "", - self.mlx_next_steps, - "", - ]) - return "\n".join(lines) - - -_DEFAULT_FIELDS = ( - "id", - "input_text", - "label", - "source", - "license", - "notes", -) - - -def build_dataset_blueprint(intent: DomainIntent) -> DatasetBlueprint: - """Build a deterministic dataset blueprint for an empty catalog result.""" - roles = ", ".join(intent.roles) or "general" - keywords = ", ".join(intent.keywords) or "none" - licenses = ", ".join(intent.license_allow) or "document and respect upstream licenses" - notes = intent.notes.strip() or "none recorded" - - schema = list(_DEFAULT_FIELDS) - if "vision" in intent.roles: - schema.insert(1, "image_path") - if "embedding" in intent.roles: - schema.insert(-2, "embedding_text") - - goal = ( - "Create a small, domain-specific dataset for **{0}** covering roles " - "({1}) and keywords ({2}). Prefer quality over volume; keep examples " - "representative of real production inputs." - ).format(intent.domain, roles, keywords) - - labeling = ( - "Define clear positive/negative or multi-class labels tied to the domain. " - "Double-check borderline cases. Keywords to cover: {0}. Extra notes: {1}." - ).format(keywords, notes) - - split = ( - "Hold out 10–20% for validation with no leakage from near-duplicates. " - "Stratify by label when classes are imbalanced. Keep a tiny smoke-test " - "slice for local MLX dry-runs." - ) - - license_privacy = ( - "Allowed licenses for this intent: {0}. Do not include secrets, credentials, " - "or personal data without consent and redaction. Record provenance per row." - ).format(licenses) - - next_steps = ( - "1. Draft the schema above as JSONL or Parquet locally (no Hub upload required).\n" - "2. Re-run `mlx-agent research` after datasets appear on the Hub, or keep this " - "blueprint as the creation guide.\n" - "3. When a base MLX model is chosen, use `mlx-agent adopt start` to verify and " - "wire it — research/blueprint never downloads or trains.\n" - "4. Fine-tune offline with your preferred MLX tooling against the local dataset." - ) - - return DatasetBlueprint( - goal=goal, - schema_fields=tuple(schema), - labeling_notes=labeling, - split_guidance=split, - license_privacy=license_privacy, - mlx_next_steps=next_steps, - ) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/cli.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/cli.py deleted file mode 100644 index bbe2888..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/cli.py +++ /dev/null @@ -1,1924 +0,0 @@ -"""Command-line entry points for the dependency-free MLX agent core.""" - -import argparse -import json -import os -import sys -import urllib.parse -from datetime import datetime, timezone -from pathlib import Path - -from .adoption import ADOPTION_SCHEMA_VERSION, AdoptionRequest, AdoptionWorkflow -from .bench import ( - BENCH_PROBE_ID, - GEN_TOKENS_DEFAULT, - RUNS_DEFAULT, - TIMEOUT_DEFAULT, - BenchError, - measure_runtime, -) -from .contracts import ErrorDetail, ResultEnvelope -from .convert import ( - Q_BITS_CHOICES, - ConvertError, - plan_convert, - start_convert, - status_convert, -) -from .discovery import DiscoveryRequest, DiscoveryService -from .fleet import ( - FleetConfigAdapter, - FleetError, - assignments_from_adoption, - parse_assignments, - parse_runtime_map, -) -from .fuse import ( - FuseError, - plan_fuse, - start_fuse, - status_fuse, -) -from .host import HostInventory -from .huggingface import HuggingFaceClient -from .installer import Installer, InstallerConflictError -from .interview import build_intent, run_interview -from .lora import ( - BATCH_DEFAULT, - ITERS_DEFAULT, - LAYERS_DEFAULT, - LR_DEFAULT, - LoraError, - plan_lora, - start_lora, - status_lora, -) -from .modality import ALL_FACET_IDS, FOUNDATION_IDS, resolve_facets, resolve_modalities -from .models import DISCOVERY_ROLES, render_md, wire -from .project_blueprint import ( - build_brief, - generate_design_pack, - render_design_pack, - write_design_pack, -) -from .providers import ProviderRegistry -from .research import generate_pack, render_pack, write_pack -from .serve import ( - MAX_TOKENS_DEFAULT, - ServeError, - default_model_present, - load_recipes, - plan_start, - receipts_root, - start_serve, - status_serve, - stop_serve, - wired_port_claim, -) -from .transactions import ( - COOPERATIVE_CONCURRENCY_NOTE, - ConcurrentTransactionError, - Receipt, - Transaction, - _assert_safe_target, - _read_regular, - preview_rollback, - rollback, -) -from .verification import ( - LMStudioRuntimeClient, - MLXVLMRuntimeClient, - OllamaRuntimeClient, - OpenAICompatibleRuntimeClient, - Verifier, -) -from .watch import ( - WatchError, - build_snapshot, - collect_owned, - diff_snapshots, - read_baseline, - snapshot_candidates, - write_snapshot, -) -from .wiring import ConfigAdapter - - -FIXTURE_WARNING = { - "code": "synthetic_fixture", - "message": "Fixture-backed discovery; this is not live Hugging Face evidence.", -} - - -def _fixture_http_get(payload): - - def get(url, timeout=10.0): - del timeout - if "/api/datasets" in url: - return payload.get("datasets", []) - if "/tree/main" in url: - repo = urllib.parse.unquote(url.split("/api/models/", 1)[1].split("/tree/main", 1)[0]) - return payload["trees"].get(repo, []) - if "/api/models/" in url: - repo = urllib.parse.unquote(url.split("/api/models/", 1)[1]) - return payload["details"].get(repo, {}) - if url.startswith("http://127.0.0.1") or url.startswith("http://localhost"): - raise OSError("fixture does not emulate local runtime endpoints") - query = urllib.parse.urlsplit(url).query - params = urllib.parse.parse_qs(query) - filters = params.get("filter", []) - if "peft" in filters: - return payload.get("adapters", []) - return payload["models"] - - return get - - -def _discovery_service_from_environment(state_dir=None): - fixture = os.environ.get("MLX_AGENT_FIXTURE") - if not fixture: - return DiscoveryService(state_dir=state_dir), None, None - try: - payload = json.loads(Path(fixture).read_text()) - _validate_fixture(payload) - except (OSError, ValueError, TypeError, json.JSONDecodeError) as error: - return None, None, ResultEnvelope.fail( - "discover", "invalid_fixture", "MLX_AGENT_FIXTURE is invalid: {0}".format(error), - "Use a valid test fixture or unset MLX_AGENT_FIXTURE to run live discovery.", - ) - service = DiscoveryService( - host=HostInventory(**payload["host"]), - huggingface=HuggingFaceClient( - http_get=_fixture_http_get(payload), - card_get=lambda url, timeout=8: None, - ), - state_dir=state_dir, - cache_enabled=False, - ) - return service, FIXTURE_WARNING, None - - -def _validate_fixture(payload): - if not isinstance(payload, dict): - raise ValueError("fixture root must be an object") - if not isinstance(payload.get("models"), list): - raise ValueError("fixture.models must be a list") - for index, model in enumerate(payload["models"]): - if not isinstance(model, dict): - raise ValueError("fixture.models[{0}] must be an object".format(index)) - identifiers = [model[key] for key in ("id", "modelId") if key in model] - if not identifiers: - raise ValueError("fixture.models[{0}] must contain id or modelId".format(index)) - if any(not isinstance(identifier, str) or not identifier for identifier in identifiers): - raise ValueError("fixture.models[{0}].id and modelId must be non-empty strings".format(index)) - for counter in ("downloads", "likes"): - if counter in model and (not isinstance(model[counter], int) or isinstance(model[counter], bool)): - raise ValueError("fixture.models[{0}].{1} must be an integer".format(index, counter)) - if not isinstance(payload.get("details"), dict): - raise ValueError("fixture.details must be an object") - if not isinstance(payload.get("trees"), dict): - raise ValueError("fixture.trees must be an object") - host = payload.get("host") - if not isinstance(host, dict): - raise ValueError("fixture.host must be an object") - if set(host) != {"ram_gb", "chip", "ollama", "lmstudio"}: - raise ValueError("fixture.host must contain only ram_gb, chip, ollama, and lmstudio") - if host["ram_gb"] is not None and (not isinstance(host["ram_gb"], int) or isinstance(host["ram_gb"], bool)): - raise ValueError("fixture.host.ram_gb must be an integer or null") - if host["chip"] is not None and not isinstance(host["chip"], str): - raise ValueError("fixture.host.chip must be a string or null") - if not isinstance(host["ollama"], bool) or not isinstance(host["lmstudio"], bool): - raise ValueError("fixture.host runtime flags must be booleans") - - -def _add_discovery_arguments(parser): - parser.add_argument("--role", choices=DISCOVERY_ROLES) - parser.add_argument("--limit", type=int, default=6) - parser.add_argument("--memory-gb", type=float, help="maximum host memory budget in GB (keeps 20%% runtime headroom)") - parser.add_argument("--quantization", help="normalized quantization such as 4bit or q8") - parser.add_argument("--license", dest="licenses", action="append", help="allow only this license (repeatable)") - parser.add_argument("--publisher", dest="publishers", action="append", help="allow only this publisher (repeatable)") - parser.add_argument("--runtime", choices=["ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"], help="require a runtime compatible with the model role") - parser.add_argument("--context", dest="context_tokens", type=int, default=None, help="tighten fit checks to weights plus KV cache at this context length") - parser.add_argument("--exclude-gated", dest="include_gated", action="store_false", default=True, help="exclude gated repositories") - parser.add_argument("--include-gated", dest="include_gated", action="store_true", help="include gated repositories (the legacy default)") - cache_group = parser.add_mutually_exclusive_group() - cache_group.add_argument("--refresh", action="store_true", help="bypass a fresh cache and fetch live evidence") - cache_group.add_argument("--offline", action="store_true", help="use only a matching local cache entry") - parser.add_argument("--state-dir", help="directory for versioned discovery cache entries") - parser.add_argument("--new", action="store_true", help="sort by most-recently-updated") - parser.add_argument("--fast", action="store_true", help="skip per-model enrichment (name heuristics only)") - parser.add_argument("--json", action="store_true") - parser.add_argument("--wire", metavar="REPO", help="emit setup + config for a model, instead of discovering") - parser.add_argument("--target", choices=["ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"], default="mlx_lm") - parser.add_argument("--port", type=int, default=8080) - - -def _run_discovery(arguments, legacy): - if arguments.wire: - print(wire(arguments.wire, arguments.target, arguments.port)) - return 0 - if arguments.context_tokens is not None and not 1024 <= arguments.context_tokens <= 1048576: - result = ResultEnvelope.fail( - "discover", - "invalid_arguments", - "--context must be between 1024 and 1048576 tokens.", - "Pass a bounded context length, or omit --context for weights-only fit.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - error = result.to_dict()["error"] - print("discover failed [{0}]: {1}\nremediation: {2}".format(error["code"], error["message"], error["remediation"])) - return 2 - service, fixture_warning, fixture_error = _discovery_service_from_environment(arguments.state_dir) - result = fixture_error or service.discover(DiscoveryRequest( - role=arguments.role, - memory_gb=arguments.memory_gb, - quantization=arguments.quantization, - licenses=arguments.licenses, - include_gated=arguments.include_gated, - publishers=arguments.publishers, - runtime=arguments.runtime, - refresh=arguments.refresh, - offline=arguments.offline, - limit=arguments.limit, - new=arguments.new, - fast=arguments.fast, - context_tokens=arguments.context_tokens, - )) - if fixture_warning: - result = ResultEnvelope.ok("discover", result.data, warnings=[fixture_warning]) - print("warning: synthetic fixture-backed discovery; not live Hugging Face evidence.", file=sys.stderr) - value = result.to_dict() - if legacy: - report = value["data"] if result.status == "ok" else {"host": service.host.to_dict() if service else HostInventory().to_dict(), "error": value["error"]["message"], "roles": {}} - print(json.dumps(report, indent=2) if arguments.json else render_md(report)) - return 0 if result.status == "ok" else 2 - if arguments.json: - print(json.dumps(value, indent=2)) - elif result.status == "ok": - print(render_md(value["data"])) - else: - error = value["error"] - print("discover failed [{0}]: {1}\nremediation: {2}".format(error["code"], error["message"], error["remediation"])) - return 0 if result.status == "ok" else 2 - - -def _run_inspect_host(arguments): - host, warnings = HostInventory.inspect(HuggingFaceClient()._http_get) - result = ResultEnvelope.ok("inspect-host", host.to_dict(), warnings=warnings) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Host inventory: {0}".format(json.dumps(host.to_dict(), sort_keys=True))) - for warning in warnings: - print("warning [{0}/{1}]: {2}".format(warning["code"], warning["probe"], warning["message"])) - return 0 - - -def _adoption_state_path(arguments): - return arguments.state or os.environ.get("MLX_AGENT_ADOPTION_STATE") - - -def _emit_adoption_result(result, as_json): - value = result.to_dict() - if as_json: - print(json.dumps(value, indent=2)) - elif result.status == "ok": - state = result.data["state"] - print("Adoption {0}: {1}".format(state["status"], state["phase"])) - if state["recommendations"]: - for item in state["recommendations"]: - print("{0}: {1} [{2}]".format(item["role"], item["repo"], item["evidence_strength"])) - requested_roles = state.get("request", {}).get("roles", []) - recommended_roles = { - item.get("role") for item in state.get("recommendations", []) - } - if ( - state.get("status") == "complete" - and "tool-use" in requested_roles - and "tool-use" not in recommended_roles - ): - print("No verified tool-use model was found.") - print( - "No model was downloaded. Install a shortlisted candidate in a " - "supported local runtime and start adoption again." - ) - else: - error = value["error"] - print("{0} failed [{1}]: {2}\nremediation: {3}".format( - result.operation, error["code"], error["message"], error["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _run_adoption(arguments): - operation = "adopt-{0}".format(arguments.adopt_command) - state_path = _adoption_state_path(arguments) - if not state_path: - return _emit_adoption_result(ResultEnvelope.fail( - operation, - "state_path_required", - "No adoption state path was supplied.", - "Pass --state PATH or set MLX_AGENT_ADOPTION_STATE.", - ), arguments.json) - - if arguments.adopt_command == "status": - workflow = AdoptionWorkflow( - discovery_service=DiscoveryService(), verifier=Verifier(), state_path=state_path - ) - try: - state = workflow.status(state_path) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - return _emit_adoption_result(ResultEnvelope.fail( - operation, - "adoption_state_invalid", - "Adoption state could not be read: {0}".format(error), - "Check --state PATH and restore a schema-version {0} adoption handoff.".format( - ADOPTION_SCHEMA_VERSION - ), - ), arguments.json) - return _emit_adoption_result( - ResultEnvelope.ok(operation, {"state": state.to_dict()}), arguments.json - ) - - service, fixture_warning, fixture_error = _discovery_service_from_environment() - if fixture_error: - error = fixture_error.to_dict()["error"] - return _emit_adoption_result(ResultEnvelope.fail( - operation, - error["code"], - error["message"], - error["remediation"], - retryable=error["retryable"], - ), arguments.json) - workflow = AdoptionWorkflow( - discovery_service=service, - verifier=Verifier(metadata_client=getattr(service, "_huggingface", None)), - state_path=state_path, - ) - try: - if arguments.adopt_command == "start": - state = workflow.start(AdoptionRequest( - roles=tuple(arguments.roles or ("general",)), - state_path=state_path, - shortlist_limit=arguments.shortlist_limit, - allow_network=arguments.allow_network and not arguments.offline, - offline=arguments.offline, - refresh=arguments.refresh, - fast=arguments.fast, - )) - else: - state = workflow.resume(state_path) - while state.phase != "complete": - state = workflow.advance(state) - except (OSError, TypeError, ValueError, RuntimeError, json.JSONDecodeError) as error: - return _emit_adoption_result(ResultEnvelope.fail( - operation, - "adoption_failed", - "Adoption workflow could not continue: {0}".format(error), - "Inspect the saved state with 'adopt status', resolve the reported issue, and resume.", - ), arguments.json) - warnings = [fixture_warning] if fixture_warning else [] - return _emit_adoption_result( - ResultEnvelope.ok(operation, {"state": state.to_dict()}, warnings=warnings), - arguments.json, - ) - - -def _add_adoption_arguments(parser): - actions = parser.add_subparsers(dest="adopt_command", required=True) - start = actions.add_parser("start", help="start and durably run a model adoption workflow") - start.add_argument("--state", help="adoption handoff path") - start.add_argument("--role", dest="roles", action="append", choices=DISCOVERY_ROLES) - start.add_argument("--shortlist-limit", type=int, default=4) - start.add_argument("--offline", action="store_true", help="use cached discovery and no metadata network requests") - start.add_argument("--refresh", action="store_true", help="refresh model discovery") - start.add_argument("--fast", action="store_true", help="use heuristic-only discovery enrichment") - start.add_argument("--no-network", dest="allow_network", action="store_false", default=True, help="do not inspect missing-model metadata") - start.add_argument("--measure", action="store_true", help="measure verified shortlist candidates (bench) before ranking") - start.add_argument("--json", action="store_true") - for name in ("resume", "status"): - action = actions.add_parser(name, help="{0} an adoption handoff".format(name)) - action.add_argument("--state", help="adoption handoff path") - action.add_argument("--json", action="store_true") - - -def _add_research_arguments(parser): - parser.add_argument("--domain", help="one-line domain description (required unless --interview)") - parser.add_argument("--role", dest="roles", action="append", choices=DISCOVERY_ROLES, help="model role to research (repeatable)") - parser.add_argument("--keyword", dest="keywords", action="append", help="domain keyword to prioritize (repeatable)") - parser.add_argument( - "--modality", - dest="modalities", - action="append", - choices=list(FOUNDATION_IDS), - help="foundational modality to seed (repeatable): audio, video, document-vision", - ) - parser.add_argument( - "--facet", - dest="facets", - action="append", - choices=list(ALL_FACET_IDS), - help="modality facet to prioritize (repeatable)", - ) - parser.add_argument("--license", dest="licenses", action="append", help="allow only this license (repeatable)") - parser.add_argument("--memory-gb", type=float, help="host memory budget in GB") - parser.add_argument("--notes", default="", help="free-form constraints") - parser.add_argument("--project", default=str(Path.cwd()), help="project root; the pack is written under /mlx-research") - parser.add_argument("--limit", type=int, default=6, help="candidates fetched per role during discovery; the pack is also capped at this many total") - parser.add_argument("--interview", action="store_true", help="ask questions interactively on stdin") - parser.add_argument("--no-write", dest="write", action="store_false", default=True, help="render the pack without writing a file") - parser.add_argument("--json", action="store_true") - - -def _stdin_reader(question): - prompt = question["prompt"] - if question.get("kind") == "multi": - prompt += " [{0}]".format(", ".join(question.get("choices", ()))) - return input("{0}\n> ".format(prompt)) - - -def _emit_research(result, arguments, markdown=None): - value = result.to_dict() - if arguments.json: - print(json.dumps(value, indent=2)) - elif result.status == "ok": - if "path" in result.data: - print("Research pack written to {0}".format(result.data["path"])) - else: - print(markdown if markdown is not None else "Research pack generated.") - else: - error = value["error"] - print("research failed [{0}]: {1}\nremediation: {2}".format( - error["code"], error["message"], error["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _run_research(arguments): - operation = "research" - try: - cli_modalities = tuple(arguments.modalities or ()) - cli_facets = tuple(arguments.facets or ()) - if arguments.interview: - intent = run_interview( - _stdin_reader, - preset_modalities=cli_modalities, - preset_facets=cli_facets, - ) - elif not arguments.domain: - return _emit_research(ResultEnvelope.fail( - operation, "domain_required", - "No domain description was supplied.", - "Pass --domain \"...\" or use --interview.", - ), arguments) - else: - detect_text = " ".join( - [arguments.domain] + list(arguments.keywords or []) - ) - modalities = resolve_modalities(cli=cli_modalities, text=detect_text) - if not modalities: - return _emit_research(ResultEnvelope.fail( - operation, "modality_required", - "No foundational modality was supplied or detected from the domain text.", - "Pass --modality audio|video|document-vision (repeatable), " - "or use --interview to choose interactively.", - ), arguments) - facets = resolve_facets(modalities, cli=cli_facets, text=detect_text) - intent = build_intent({ - "domain": arguments.domain, - "roles": arguments.roles or [], - "keywords": ",".join(arguments.keywords or []), - "license": ",".join(arguments.licenses or []), - "memory_gb": arguments.memory_gb, - "notes": arguments.notes, - "modalities": list(modalities), - "facets": list(facets), - }) - except (ValueError, EOFError) as error: - return _emit_research(ResultEnvelope.fail( - operation, "invalid_intent", str(error), - "Correct the interview answers or flags and retry.", - ), arguments) - - service, fixture_warning, fixture_error = _discovery_service_from_environment() - if fixture_error: - error = fixture_error.to_dict()["error"] - return _emit_research(ResultEnvelope.fail( - operation, error["code"], error["message"], error["remediation"], - ), arguments) - hf_client = getattr(service, "_huggingface", None) or HuggingFaceClient() - moment = datetime.now(timezone.utc) - try: - pack = generate_pack(intent, service, hf_client, limit=arguments.limit, now=moment) - except (OSError, TypeError, ValueError) as error: - return _emit_research(ResultEnvelope.fail( - operation, "research_failed", - "Research could not complete: {0}".format(error), - "Check network access to huggingface.co, or unset MLX_AGENT_FIXTURE.", - ), arguments) - markdown = render_pack(pack) - data = {"pack": pack.to_dict()} - if arguments.write: - try: - data["path"] = str(write_pack( - markdown, intent, root=arguments.project, now=moment, pack=pack, - )) - except (OSError, ValueError) as error: - return _emit_research(ResultEnvelope.fail( - operation, "write_failed", - "Research pack could not be written: {0}".format(error), - "Choose a writable --project directory without a symlinked mlx-research folder.", - ), arguments) - warnings = [fixture_warning] if fixture_warning else [] - return _emit_research( - ResultEnvelope.ok(operation, data, warnings=warnings), arguments, markdown - ) - - -def _add_blueprint_arguments(parser): - parser.add_argument("--goal", help="one-line project goal (required)") - parser.add_argument( - "--modality", - dest="modalities", - action="append", - choices=list(FOUNDATION_IDS), - help="foundational modality to include (repeatable)", - ) - parser.add_argument("--memory-gb", type=float, help="host memory budget in GB") - parser.add_argument("--notes", default="", help="free-form constraints") - parser.add_argument( - "--project", - default=str(Path.cwd()), - help="project root; the pack is written under /mlx-blueprints", - ) - parser.add_argument( - "--no-write", - dest="write", - action="store_false", - default=True, - help="render the pack without writing a file", - ) - parser.add_argument("--json", action="store_true") - - -def _emit_blueprint(result, arguments, markdown=None): - value = result.to_dict() - if arguments.json: - print(json.dumps(value, indent=2)) - elif result.status == "ok": - if "path" in result.data: - print("Project design pack written to {0}".format(result.data["path"])) - else: - print(markdown if markdown is not None else "Project design pack generated.") - else: - error = value["error"] - print("blueprint failed [{0}]: {1}\nremediation: {2}".format( - error["code"], error["message"], error["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _run_blueprint(arguments): - operation = "blueprint" - try: - if not arguments.goal: - return _emit_blueprint(ResultEnvelope.fail( - operation, "goal_required", - "No project goal was supplied.", - "Pass --goal \"...\".", - ), arguments) - brief = build_brief({ - "goal": arguments.goal, - "modalities": arguments.modalities or [], - "memory_gb": arguments.memory_gb, - "notes": arguments.notes, - }) - except (ValueError, TypeError) as error: - return _emit_blueprint(ResultEnvelope.fail( - operation, "invalid_brief", str(error), - "Correct the flags and retry.", - ), arguments) - - moment = datetime.now(timezone.utc) - pack = generate_design_pack(brief, now=moment) - markdown = render_design_pack(pack) - data = {"pack": pack.to_dict()} - if arguments.write: - try: - data["path"] = str(write_design_pack( - markdown, brief, root=arguments.project, now=moment, pack=pack, - )) - except (OSError, ValueError) as error: - return _emit_blueprint(ResultEnvelope.fail( - operation, "write_failed", - "Design pack could not be written: {0}".format(error), - "Choose a writable --project directory without a symlinked mlx-blueprints folder.", - ), arguments) - return _emit_blueprint(ResultEnvelope.ok(operation, data), arguments, markdown) - - -def _emit_wire_result(result, as_json, human=None): - if as_json: - print(json.dumps(result.to_dict(), indent=2)) - elif human is not None: - print(human) - elif result.status == "ok": - print(json.dumps(result.data, indent=2)) - else: - error = result.to_dict()["error"] - print("{0} failed [{1}]: {2}\nremediation: {3}".format( - result.operation, error["code"], error["message"], error["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _receipt_data(receipt): - value = receipt.to_dict() - value["receipt_path"] = receipt.receipt_path - return value - - -def _wire_failure(operation, code, message, remediation, data=None): - return ResultEnvelope( - operation=operation, status="error", data=data or {}, - warnings=[{"code": "cooperative_concurrency", "message": COOPERATIVE_CONCURRENCY_NOTE}], - error=ErrorDetail(code, message, remediation), - ) - - -def _wire_ok(operation, data): - return ResultEnvelope.ok(operation, data, warnings=[{ - "code": "cooperative_concurrency", "message": COOPERATIVE_CONCURRENCY_NOTE, - }]) - - -def _wire_render(arguments): - path = _assert_safe_target(arguments.path) - existing = _read_regular(path).decode("utf-8") - adapter = ConfigAdapter.detect(path, runtime=arguments.target) - content = adapter.render(arguments.model, arguments.target, existing) - adapter.validate(content) - return path, adapter, content - - -def _run_wire(arguments): - operation = "wire-{0}".format(arguments.wire_command) - try: - if arguments.wire_command == "status": - location = _assert_safe_target(arguments.receipt) - receipt = Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) - return _emit_wire_result(_wire_ok(operation, {"receipt": _receipt_data(receipt)}), arguments.json) - if arguments.wire_command == "rollback": - if not arguments.confirm: - preview = preview_rollback(arguments.receipt) - result = _wire_ok( - operation, {"preview": preview, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print(json.dumps(preview, indent=2)) - print( - "Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH." - ) - return 2 - if not arguments.preview_hash: - return _emit_wire_result(_wire_failure( - operation, - "preview_hash_required", - "--confirm requires the hash from a reviewed rollback preview.", - "Run wire rollback without --confirm, inspect the current-state preview, then pass --preview-hash.", - ), arguments.json) - receipt = rollback(arguments.receipt, preview_hash=arguments.preview_hash) - result = _wire_ok(operation, {"receipt": _receipt_data(receipt)}) if receipt.status == "rolled_back" else _wire_failure( - operation, receipt.status, "Rollback did not complete; receipt status is {0}.".format(receipt.status), - "Inspect the receipt validations and restore the verified backup manually.", {"receipt": _receipt_data(receipt)}, - ) - return _emit_wire_result(result, arguments.json) - - path, adapter, content = _wire_render(arguments) - if arguments.wire_command == "render": - return _emit_wire_result(_wire_ok(operation, { - "path": str(path), "runtime": arguments.target, "config": content, - "validation": {"parse": True}, - }), arguments.json, human=content) - transaction = Transaction(receipts_dir=arguments.receipts_dir) - preview = transaction.preview([{ - "path": str(path), "content": content, "runtime": arguments.target, - "adapter": adapter, "endpoint": arguments.endpoint, - }]) - if not arguments.confirm: - result = _wire_ok(operation, {"preview": preview, "requires_confirmation": True}) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print(preview["diff"]) - print("Confirmation required: rerun with --confirm to apply this transaction.") - return 2 - if not arguments.preview_hash: - return _emit_wire_result(_wire_failure( - operation, "preview_hash_required", "--confirm requires the preview hash from a prior preview.", - "Run wire apply without --confirm, inspect the preview, then pass its --preview-hash value.", - {"preview": preview}, - ), arguments.json) - if arguments.preview_hash != preview["preview_hash"]: - return _emit_wire_result(_wire_failure( - operation, "preview_stale", "The supplied preview hash does not match the current preview.", - "Generate and inspect a new preview before confirming this mutation.", - {"preview": preview}, - ), arguments.json) - if not arguments.json: - print(preview["diff"]) - receipt = transaction.apply(arguments.preview_hash) - data = {"preview": preview, "receipt": _receipt_data(receipt)} - result = _wire_ok(operation, data) if receipt.status == "applied" else _wire_failure( - operation, receipt.status, "Wire did not apply; receipt status is {0}.".format(receipt.status), - "Inspect the receipt validation results and use the recorded recovery path before retrying.", data, - ) - return _emit_wire_result(result, arguments.json) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - code = "cooperative_lock_busy" if isinstance(error, ConcurrentTransactionError) else ("preview_stale" if str(error).startswith("preview is stale") else "wire_failed") - return _emit_wire_result(_wire_failure( - operation, code, "Wire could not complete: {0}".format(error), - "Correct the target configuration or receipt, then render a new preview.", - ), arguments.json) - - -_BENCH_RUNTIMES = ("ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm") - - -def _bench_runtime_client(name): - if name == "ollama": - return OllamaRuntimeClient() - if name == "lmstudio": - return LMStudioRuntimeClient() - if name == "mlx-vlm": - return MLXVLMRuntimeClient() - if name == "mlx_lm": - return OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080") - if name == "litellm": - return OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000") - raise ValueError("unsupported bench runtime: {0}".format(name)) - - -def _add_bench_arguments(parser): - actions = parser.add_subparsers(dest="bench_command", required=True) - run = actions.add_parser( - "run", - help="measure a model already served by a local runtime (never downloads)", - ) - run.add_argument("--repo", required=True, help="model identifier exactly as the runtime serves it") - run.add_argument("--runtime", required=True, choices=_BENCH_RUNTIMES) - run.add_argument("--role", default="general", choices=DISCOVERY_ROLES) - run.add_argument("--runs", type=int, default=RUNS_DEFAULT, help="timed runs after one warm-up (1-10)") - run.add_argument("--gen-tokens", type=int, default=GEN_TOKENS_DEFAULT, help="tokens generated per run (16-2048)") - run.add_argument("--timeout", type=float, default=TIMEOUT_DEFAULT, help="total measurement deadline in seconds") - run.add_argument("--export", default=None, help="append an anonymized result line to this JSONL file") - run.add_argument("--json", action="store_true") - aggregate = actions.add_parser( - "aggregate", - help="deduplicate and median-aggregate bench export files", - ) - aggregate.add_argument("--exports", required=True, help="directory of .jsonl export files") - aggregate.add_argument("--out", default=None, help="aggregate output path (default: print only)") - aggregate.add_argument("--json", action="store_true") - - -def _run_bench(arguments): - operation = "bench-{0}".format(arguments.bench_command) - try: - if arguments.bench_command == "aggregate": - from .bench import aggregate_exports - - exports_dir = Path(arguments.exports) - if not exports_dir.is_dir(): - raise BenchError( - "invalid_arguments", - "The exports directory does not exist: {0}".format(arguments.exports), - "Point --exports at a directory of bench export .jsonl files.", - ) - files = sorted(exports_dir.glob("*.jsonl")) - aggregate = aggregate_exports(files) - if arguments.out: - destination = Path(arguments.out) - content = (json.dumps(aggregate, indent=2, sort_keys=True) + "\n").encode("utf-8") - destination.write_bytes(content) - data = { - "entries": aggregate["entries"], - "sources": len(files), - "written": str(arguments.out) if arguments.out else None, - } - result = ResultEnvelope.ok(operation, data) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Aggregate: {0} entries from {1} export file(s){2}".format( - len(aggregate["entries"]), len(files), - " -> {0}".format(arguments.out) if arguments.out else "", - )) - return 0 - chip = None - try: - chip = HostInventory.inspect().chip - except Exception: - chip = None - measurement = measure_runtime( - arguments.repo, - _bench_runtime_client(arguments.runtime), - runs=arguments.runs, - gen_tokens=arguments.gen_tokens, - timeout=arguments.timeout, - chip=chip, - ) - evidence = measurement.to_evidence(role=arguments.role).to_dict() - data = {"measurement": measurement.to_dict(), "evidence": evidence} - if arguments.export: - from . import __version__ - from .bench import append_export, export_record - - destination = append_export( - arguments.export, export_record(measurement, __version__) - ) - data["exported"] = str(destination) - result = ResultEnvelope.ok(operation, data) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Bench: {0} on {1} ({2} runs)".format( - measurement.repo, measurement.runtime, measurement.runs - )) - ttft = "{0} ms".format(measurement.ttft_ms) if measurement.ttft_ms is not None else "n/a" - prefill = ( - "{0} tok/s".format(measurement.prefill_toks) - if measurement.prefill_toks is not None - else "n/a" - ) - print(" decode: {0} tok/s (spread {1}%)".format(measurement.decode_toks, measurement.spread_pct)) - print(" ttft: {0}".format(ttft)) - print(" prefill: {0}".format(prefill)) - return 0 - except BenchError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (TypeError, ValueError) as error: - result = ResultEnvelope.fail( - operation, - "invalid_arguments", - str(error), - "Correct the bench arguments and retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("bench failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _add_serve_arguments(parser): - actions = parser.add_subparsers(dest="serve_command", required=True) - start = actions.add_parser( - "start", - help="preview, then confirmation-gated launch of a local MLX server", - ) - start.add_argument("--repo", required=True, help="publisher/model present in the local Hugging Face cache") - start.add_argument("--runtime", required=True, choices=["mlx_lm", "mlx-vlm"]) - start.add_argument("--port", type=int, default=None, help="loopback port (defaults per runtime recipe)") - start.add_argument("--max-tokens", type=int, default=MAX_TOKENS_DEFAULT) - start.add_argument("--adapter-path", default=None, help="LoRA adapter directory (mlx_lm only)") - start.add_argument("--launchd", action="store_true", help="install a launchd agent plist instead of spawning now") - start.add_argument("--launchd-dir", default=None, help="launchd target directory (defaults to ~/Library/LaunchAgents)") - start.add_argument("--confirm", action="store_true", help="authorize this reviewed server launch") - start.add_argument("--preview-hash", help="hash returned by the separately reviewed serve start preview") - start.add_argument("--receipts-dir", default=None, help="receipt root (defaults to the current directory)") - start.add_argument("--hf-cache", default=None, help="Hugging Face cache root for the local-model gate") - start.add_argument("--json", action="store_true") - stop = actions.add_parser("stop", help="stop exactly the server a serve receipt owns") - stop.add_argument("--port", type=int, required=True) - stop.add_argument("--receipts-dir", default=None) - stop.add_argument("--json", action="store_true") - status = actions.add_parser("status", help="cross-check serve receipts against live processes") - status.add_argument("--receipts-dir", default=None) - status.add_argument("--json", action="store_true") - - -def _run_serve(arguments): - operation = "serve-{0}".format(arguments.serve_command) - try: - if arguments.serve_command == "status": - entries = status_serve(arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, {"servers": entries}), arguments.json, - human=_serve_status_human(entries), - ) - if arguments.serve_command == "stop": - outcome = stop_serve(arguments.port, arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="serve {0}: port {1} (pid {2})".format( - outcome["status"], outcome["port"], outcome["pid"] - ), - ) - recipes = load_recipes() - plan = plan_start( - arguments.repo, - arguments.runtime, - recipes, - port=arguments.port, - max_tokens=arguments.max_tokens, - adapter_path=arguments.adapter_path, - ) - if arguments.launchd: - return _run_serve_launchd(arguments, plan, operation) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Serve plan: {0} on 127.0.0.1:{1}".format(plan["repo"], plan["port"])) - print(" argv: {0}".format(" ".join(plan["argv"]))) - print(" readiness: {0}".format(plan["readiness"])) - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - wired_claims = wired_port_claim([ - arguments.receipts_dir or str(Path.cwd()) - ]) - outcome = start_serve( - plan, - receipts_dir=arguments.receipts_dir, - confirm=True, - preview_hash=arguments.preview_hash, - model_present=default_model_present(arguments.hf_cache), - wired_claims=wired_claims, - ) - receipt = outcome["receipt"] - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="serve started: {0} on 127.0.0.1:{1} (pid {2})\n log: {3}".format( - receipt["repo"], receipt["port"], receipt["pid"], receipt["log_path"] - ), - ) - except ServeError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "serve_failed", - str(error), - "Correct the serve arguments or receipt state, then retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("serve failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _run_serve_launchd(arguments, plan, operation): - """Preview-confirm installation of a launchd agent for a serve plan.""" - from .serve import LaunchdPlistAdapter, launchd_label, render_launchd_plist - - launchd_dir = Path(arguments.launchd_dir) if arguments.launchd_dir else ( - Path.home() / "Library" / "LaunchAgents" - ) - target = launchd_dir / "{0}.plist".format(launchd_label(plan["port"])) - log_path = str(receipts_root(arguments.receipts_dir) / "{0}.log".format(plan["port"])) - content = render_launchd_plist(plan, log_path=log_path) - adapter = LaunchdPlistAdapter(target) - adapter.validate(content) - if target.exists(): - result = ResultEnvelope.fail( - operation, - "output_exists", - "A launchd plist already exists at {0}.".format(target), - "Unload and remove it yourself; serve never overwrites launchd agents.", - ) - return _emit_serve_result(result, arguments.json) - if not launchd_dir.is_dir(): - result = ResultEnvelope.fail( - operation, - "invalid_arguments", - "The launchd directory does not exist: {0}".format(launchd_dir), - "Create it yourself, or pass --launchd-dir.", - ) - return _emit_serve_result(result, arguments.json) - if not default_model_present(arguments.hf_cache)(plan["repo"]): - result = ResultEnvelope.fail( - operation, - "model_not_local", - "The model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; serve never downloads.", - ) - return _emit_serve_result(result, arguments.json) - transaction = Transaction(receipts_dir=arguments.receipts_dir) - preview = transaction.preview([{ - "path": str(target), "content": content, "runtime": "launchd", - "adapter": adapter, "endpoint": None, - }]) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"preview": preview, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print(preview["diff"]) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - if not arguments.preview_hash: - result = ResultEnvelope.fail( - operation, - "preview_hash_required", - "--confirm requires the preview hash from a prior preview.", - "Run serve start --launchd without --confirm, inspect the preview, then pass --preview-hash.", - ) - return _emit_serve_result(result, arguments.json) - if arguments.preview_hash != preview["preview_hash"]: - result = ResultEnvelope.fail( - operation, - "preview_stale", - "The supplied preview hash does not match the current preview.", - "Generate and inspect a new preview before confirming this mutation.", - ) - return _emit_serve_result(result, arguments.json) - receipt = transaction.apply(arguments.preview_hash) - data = {"receipt": _receipt_data(receipt), "plist": str(target)} - result = ResultEnvelope.ok(operation, data) - return _emit_serve_result( - result, arguments.json, - human="launchd agent installed: {0}\n load it with: launchctl bootstrap gui/$(id -u) {1}".format( - target, target - ), - ) - - -def _emit_serve_result(result, as_json, human=None): - if as_json: - print(json.dumps(result.to_dict(), indent=2)) - elif human: - print(human) - elif result.status == "ok": - print(json.dumps(result.to_dict()["data"], indent=2)) - else: - payload = result.to_dict()["error"] - print("{0} failed [{1}]: {2}\nremediation: {3}".format( - result.operation, payload["code"], payload["message"], payload["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _serve_status_human(entries): - if not entries: - return "No serve receipts." - lines = [] - for entry in entries: - state = "alive" if entry["alive"] else "dead" - if entry["alive"] and not entry["argv_match"]: - state = "alive (argv mismatch)" - lines.append(" 127.0.0.1:{0} {1} ({2}) pid {3} - {4}".format( - entry["port"], entry["repo"], entry["runtime"], entry["pid"], state - )) - return "Serve receipts:\n" + "\n".join(lines) - - -def _add_fleet_arguments(parser): - actions = parser.add_subparsers(dest="fleet_command", required=True) - for name in ("render", "apply"): - action = actions.add_parser( - name, - help="{0} a one-shot per-role router configuration".format(name), - ) - action.add_argument("--path", required=True, help="target router configuration file") - action.add_argument("--assign", action="append", default=None, metavar="ROLE=REPO", - help="per-role model assignment (repeatable)") - action.add_argument("--from-adoption", default=None, metavar="STATE", - help="read role assignments from a completed adopt handoff") - action.add_argument("--runtime-map", action="append", default=None, metavar="ROLE=RUNTIME", - help="per-role runtime override: mlx_lm or mlx-vlm (repeatable)") - action.add_argument("--allow-missing", action="store_true", - help="warn instead of failing when a model is not in a local inventory") - action.add_argument("--json", action="store_true") - if name == "apply": - action.add_argument("--confirm", action="store_true", help="explicitly authorize this reviewed mutation") - action.add_argument("--preview-hash", help="hash returned by the separately reviewed fleet apply preview") - action.add_argument("--receipts-dir", help="directory for non-secret transaction receipts") - action.add_argument("--endpoint", help="optional local runtime health endpoint") - - -def _fleet_inputs(arguments): - if arguments.assign and arguments.from_adoption: - raise FleetError( - "invalid_arguments", - "--assign and --from-adoption are mutually exclusive.", - "Pick explicit assignments or an adoption handoff, not both.", - ) - if arguments.from_adoption: - assignments = assignments_from_adoption(arguments.from_adoption) - else: - assignments = parse_assignments(arguments.assign) - runtime_map = parse_runtime_map(arguments.runtime_map) - return assignments, runtime_map - - -def _fleet_missing_models(assignments): - from .model_doctor import ( - collect_runtime_inventory, - default_hf_cache, - inspect_hf_cache, - ) - - known = {item["id"].casefold() for item in inspect_hf_cache(default_hf_cache())} - runtime_clients = [ - OllamaRuntimeClient(), - LMStudioRuntimeClient(), - OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080"), - MLXVLMRuntimeClient(), - OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000"), - ] - runtime_inventory, _errors = collect_runtime_inventory(runtime_clients) - known.update(item["id"].casefold() for item in runtime_inventory) - return sorted( - repo for repo in set(assignments.values()) if repo.casefold() not in known - ) - - -def _run_fleet(arguments): - operation = "fleet-{0}".format(arguments.fleet_command) - try: - assignments, runtime_map = _fleet_inputs(arguments) - path = _assert_safe_target(arguments.path) - existing = _read_regular(path).decode("utf-8") - adapter = FleetConfigAdapter(path) - content = adapter.render(assignments, runtime_map, existing=existing) - missing = _fleet_missing_models(assignments) - warnings = [] - if missing and not arguments.allow_missing: - return _emit_wire_result(_wire_failure( - operation, - "model_not_local", - "Models are not present in a local inventory: {0}".format(", ".join(missing)), - "Download them first, or pass --allow-missing to record a reviewed warning.", - ), arguments.json) - if missing: - warnings.append({ - "code": "model_not_local", - "message": "Not in a local inventory: {0}".format(", ".join(missing)), - }) - if arguments.fleet_command == "render": - result = _wire_ok(operation, { - "path": str(path), "assignments": assignments, "config": content, - "validation": {"parse": True}, - }) - result.warnings.extend(warnings) - return _emit_wire_result(result, arguments.json, human=content) - transaction = Transaction(receipts_dir=arguments.receipts_dir) - preview = transaction.preview([{ - "path": str(path), "content": content, "runtime": "litellm", - "adapter": adapter, "endpoint": arguments.endpoint, - }]) - if not arguments.confirm: - result = _wire_ok(operation, {"preview": preview, "requires_confirmation": True}) - result.warnings.extend(warnings) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print(preview["diff"]) - print("Confirmation required: rerun with --confirm to apply this transaction.") - return 2 - if not arguments.preview_hash: - return _emit_wire_result(_wire_failure( - operation, "preview_hash_required", "--confirm requires the preview hash from a prior preview.", - "Run fleet apply without --confirm, inspect the preview, then pass its --preview-hash value.", - {"preview": preview}, - ), arguments.json) - if arguments.preview_hash != preview["preview_hash"]: - return _emit_wire_result(_wire_failure( - operation, "preview_stale", "The supplied preview hash does not match the current preview.", - "Generate and inspect a new preview before confirming this mutation.", - {"preview": preview}, - ), arguments.json) - if not arguments.json: - print(preview["diff"]) - receipt = transaction.apply(arguments.preview_hash) - data = {"preview": preview, "receipt": _receipt_data(receipt)} - result = _wire_ok(operation, data) if receipt.status == "applied" else _wire_failure( - operation, receipt.status, "Fleet did not apply; receipt status is {0}.".format(receipt.status), - "Inspect the receipt validation results and use the recorded recovery path before retrying.", data, - ) - result.warnings.extend(warnings) - return _emit_wire_result(result, arguments.json) - except FleetError as error: - return _emit_wire_result(_wire_failure( - operation, error.code, str(error), error.remediation, - ), arguments.json) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - code = "cooperative_lock_busy" if isinstance(error, ConcurrentTransactionError) else ("preview_stale" if str(error).startswith("preview is stale") else "fleet_failed") - return _emit_wire_result(_wire_failure( - operation, code, "Fleet could not complete: {0}".format(error), - "Correct the assignments or target configuration, then render a new preview.", - ), arguments.json) - - -def _watch_runtime_clients(): - return [ - OllamaRuntimeClient(), - LMStudioRuntimeClient(), - OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080"), - MLXVLMRuntimeClient(), - OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000"), - ] - - -def _add_watch_arguments(parser): - actions = parser.add_subparsers(dest="watch_command", required=True) - for name in ("snapshot", "diff"): - action = actions.add_parser( - name, - help="{0} the owned-model Hugging Face watch state".format(name), - ) - action.add_argument("--state-dir", default="./.mlx-agent-watch", - help="watch state root (default ./.mlx-agent-watch)") - action.add_argument("--limit", type=int, default=6, - help="discovery results per role (default 6)") - action.add_argument("--json", action="store_true") - - -def _run_watch(arguments): - operation = "watch-{0}".format(arguments.watch_command) - try: - service, fixture_warning, fixture_error = _discovery_service_from_environment() - if fixture_error is not None: - return _emit_wire_result(fixture_error, arguments.json) - result = service.discover(DiscoveryRequest(limit=arguments.limit)) - if result.status != "ok": - error = result.to_dict()["error"] - return _emit_wire_result(ResultEnvelope.fail( - operation, error["code"], error["message"], error["remediation"] - ), arguments.json) - owned = collect_owned(_watch_runtime_clients()) - candidates = snapshot_candidates(result.data) - if arguments.watch_command == "snapshot": - snapshot = build_snapshot(owned, candidates) - destination = write_snapshot(arguments.state_dir, snapshot) - data = { - "state": str(destination), - "owned": len(snapshot["owned"]), - "candidates": len(snapshot["candidates"]), - "rotated_previous": snapshot["previous"] is not None, - } - result = ResultEnvelope.ok(operation, data) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Watch snapshot: {0} owned, {1} tracked repos -> {2}".format( - data["owned"], data["candidates"], data["state"] - )) - return 0 - baseline = read_baseline(arguments.state_dir) - current = build_snapshot(owned, candidates) - findings = diff_snapshots(baseline, current) - data = { - "baseline_created_at": baseline["created_at"], - "findings": findings, - } - warnings = [ - {"code": finding["code"], "message": "{0}: {1}".format(finding["repo"], finding["detail"])} - for finding in findings - ] - result = ResultEnvelope.ok(operation, data, warnings=warnings) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Watch diff against {0}: {1} finding(s)".format( - baseline["created_at"], len(findings) - )) - for finding in findings: - print(" [{0}] {1} - {2}".format(finding["code"], finding["repo"], finding["detail"])) - return 0 - except WatchError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "watch_failed", - str(error), - "Correct the state directory and retry; watch only writes its own state file.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("watch failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _add_convert_arguments(parser): - actions = parser.add_subparsers(dest="convert_command", required=True) - start = actions.add_parser( - "start", - help="preview, then confirmation-gated quantization of a cached model", - ) - start.add_argument("--repo", required=True, help="publisher/model present in the local Hugging Face cache") - start.add_argument("--q-bits", type=int, default=4, choices=Q_BITS_CHOICES) - start.add_argument("--out", default=None, help="output directory (default -MLX-bit)") - start.add_argument("--confirm", action="store_true", help="authorize this reviewed conversion") - start.add_argument("--preview-hash", help="hash returned by the separately reviewed convert preview") - start.add_argument("--receipts-dir", default=None) - start.add_argument("--hf-cache", default=None) - start.add_argument("--json", action="store_true") - status = actions.add_parser("status", help="cross-check convert receipts against live processes") - status.add_argument("--receipts-dir", default=None) - status.add_argument("--json", action="store_true") - - -def _run_convert(arguments): - operation = "convert-{0}".format(arguments.convert_command) - try: - if arguments.convert_command == "status": - entries = status_convert(arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, {"jobs": entries}), arguments.json, - human=_convert_status_human(entries), - ) - plan = plan_convert(arguments.repo, q_bits=arguments.q_bits, out=arguments.out) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Convert plan: {0} -> {1} ({2}bit)".format( - plan["repo"], plan["out"], plan["q_bits"] - )) - print(" argv: {0}".format(" ".join(plan["argv"]))) - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - outcome = start_convert( - plan, - receipts_dir=arguments.receipts_dir, - confirm=True, - preview_hash=arguments.preview_hash, - model_present=default_model_present(arguments.hf_cache), - ) - receipt = outcome["receipt"] - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="convert started: {0} -> {1} (pid {2})\n log: {3}".format( - receipt["repo"], receipt["out"], receipt["pid"], receipt["log_path"] - ), - ) - except ConvertError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "convert_failed", - str(error), - "Correct the convert arguments or receipt state, then retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("convert failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _convert_status_human(entries): - if not entries: - return "No convert receipts." - lines = [] - for entry in entries: - lines.append(" {0} ({1}bit) -> {2}: {3}".format( - entry["repo"], entry["q_bits"], entry["out"], entry["state"] - )) - return "Convert jobs:\n" + "\n".join(lines) - - -def _add_lora_arguments(parser): - actions = parser.add_subparsers(dest="lora_command", required=True) - start = actions.add_parser( - "start", - help="preview, then confirmation-gated LoRA training on a cached model", - ) - start.add_argument("--repo", required=True, help="publisher/model base present in the local Hugging Face cache") - start.add_argument("--data", required=True, help="dataset directory with train.jsonl") - start.add_argument("--iters", type=int, default=ITERS_DEFAULT) - start.add_argument("--batch-size", type=int, default=BATCH_DEFAULT) - start.add_argument("--learning-rate", type=float, default=LR_DEFAULT) - start.add_argument("--num-layers", type=int, default=LAYERS_DEFAULT, help="-1 trains all layers") - start.add_argument("--out", default=None, help="adapter output directory (default -lora)") - start.add_argument("--confirm", action="store_true", help="authorize this reviewed training run") - start.add_argument("--preview-hash", help="hash returned by the separately reviewed lora preview") - start.add_argument("--receipts-dir", default=None) - start.add_argument("--hf-cache", default=None) - start.add_argument("--json", action="store_true") - status = actions.add_parser("status", help="cross-check lora receipts against live processes") - status.add_argument("--receipts-dir", default=None) - status.add_argument("--json", action="store_true") - - -def _run_lora(arguments): - operation = "lora-{0}".format(arguments.lora_command) - try: - if arguments.lora_command == "status": - entries = status_lora(arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, {"jobs": entries}), arguments.json, - human=_lora_status_human(entries), - ) - plan = plan_lora( - arguments.repo, - arguments.data, - iters=arguments.iters, - batch_size=arguments.batch_size, - learning_rate=arguments.learning_rate, - num_layers=arguments.num_layers, - out=arguments.out, - ) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("LoRA plan: {0} + {1} train lines -> {2}".format( - plan["repo"], plan["dataset"]["train_lines"], plan["out"] - )) - print(" argv: {0}".format(" ".join(plan["argv"]))) - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - outcome = start_lora( - plan, - receipts_dir=arguments.receipts_dir, - confirm=True, - preview_hash=arguments.preview_hash, - model_present=default_model_present(arguments.hf_cache), - ) - receipt = outcome["receipt"] - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="lora started: {0} -> {1} (pid {2})\n log: {3}".format( - receipt["repo"], receipt["out"], receipt["pid"], receipt["log_path"] - ), - ) - except LoraError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "lora_failed", - str(error), - "Correct the lora arguments or receipt state, then retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("lora failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _lora_status_human(entries): - if not entries: - return "No lora receipts." - lines = [] - for entry in entries: - lines.append(" {0} -> {1}: {2}".format( - entry["repo"], entry["out"], entry["state"] - )) - return "LoRA jobs:\n" + "\n".join(lines) - - -def _add_fuse_arguments(parser): - actions = parser.add_subparsers(dest="fuse_command", required=True) - start = actions.add_parser( - "start", - help="preview, then confirmation-gated LoRA fuse into the base model", - ) - start.add_argument("--repo", required=True, help="publisher/model base present in the local Hugging Face cache") - start.add_argument("--adapter", required=True, help="completed lora adapter directory") - start.add_argument("--out", default=None, help="fused output directory (default -fused)") - start.add_argument("--confirm", action="store_true", help="authorize this reviewed fuse") - start.add_argument("--preview-hash", help="hash returned by the separately reviewed fuse preview") - start.add_argument("--receipts-dir", default=None) - start.add_argument("--hf-cache", default=None) - start.add_argument("--json", action="store_true") - status = actions.add_parser("status", help="cross-check fuse receipts against live processes") - status.add_argument("--receipts-dir", default=None) - status.add_argument("--json", action="store_true") - - -def _run_fuse(arguments): - operation = "fuse-{0}".format(arguments.fuse_command) - try: - if arguments.fuse_command == "status": - entries = status_fuse(arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, {"jobs": entries}), arguments.json, - human=_fuse_status_human(entries), - ) - plan = plan_fuse(arguments.repo, arguments.adapter, out=arguments.out) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Fuse plan: {0} + {1} -> {2}".format( - plan["repo"], plan["adapter"], plan["out"] - )) - print(" argv: {0}".format(" ".join(plan["argv"]))) - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - outcome = start_fuse( - plan, - receipts_dir=arguments.receipts_dir, - confirm=True, - preview_hash=arguments.preview_hash, - model_present=default_model_present(arguments.hf_cache), - ) - receipt = outcome["receipt"] - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="fuse started: {0} -> {1} (pid {2})\n log: {3}".format( - receipt["repo"], receipt["out"], receipt["pid"], receipt["log_path"] - ), - ) - except FuseError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "fuse_failed", - str(error), - "Correct the fuse arguments or receipt state, then retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("fuse failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _fuse_status_human(entries): - if not entries: - return "No fuse receipts." - lines = [] - for entry in entries: - lines.append(" {0} -> {1}: {2}".format( - entry["repo"], entry["out"], entry["state"] - )) - return "Fuse jobs:\n" + "\n".join(lines) - - -def _add_wire_arguments(parser): - actions = parser.add_subparsers(dest="wire_command", required=True) - for name in ("render", "apply"): - action = actions.add_parser(name, help="{0} a deterministic runtime configuration ({1})".format(name, "advisory lock protects cooperative writers" if name == "apply" else "safe render")) - action.add_argument("model", help="Hugging Face model repository") - action.add_argument("--target", choices=["ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"], default="mlx_lm") - action.add_argument("--path", required=True, help="target configuration file") - action.add_argument("--json", action="store_true") - if name == "apply": - action.add_argument("--confirm", action="store_true", help="explicitly authorize this reviewed mutation") - action.add_argument("--preview-hash", help="hash returned by the separately reviewed wire apply preview") - action.add_argument("--receipts-dir", help="directory for non-secret transaction receipts") - action.add_argument("--endpoint", help="optional local runtime health endpoint") - status = actions.add_parser("status", help="inspect a Wire receipt") - status.add_argument("receipt") - status.add_argument("--json", action="store_true") - restore = actions.add_parser("rollback", help="restore a Wire receipt's exact backup") - restore.add_argument("receipt") - restore.add_argument("--confirm", action="store_true", help="explicitly authorize this rollback") - restore.add_argument("--preview-hash", help="hash returned by the separately reviewed rollback preview") - restore.add_argument("--json", action="store_true") - - -def _installer_registry(): - root = Path(__file__).resolve().parents[2] - home = os.environ.get("MLX_AGENT_HOME") - config_root = os.environ.get("MLX_AGENT_CONFIG_ROOT") or os.environ.get("XDG_STATE_HOME") - xdg_config_home = os.environ.get("XDG_CONFIG_HOME") - return ProviderRegistry( - root / "plugin.json", home=home, config_root=config_root, - xdg_config_home=xdg_config_home, - ) - - -def _add_installer_arguments(parser, include_providers=True): - if include_providers: - parser.add_argument("providers", nargs="*", help="explicit provider IDs; omit to inspect detected choices") - parser.add_argument("--scope", choices=["user", "project"], default="user") - parser.add_argument("--project", default=str(Path.cwd()), help="project root for project-scoped installation") - parser.add_argument("--dry-run", action="store_true", help="show the reviewed plan without mutating files") - parser.add_argument("--confirm", action="store_true", help="authorize a separately reviewed installer preview") - parser.add_argument("--preview-hash", help="preview hash returned by the installer dry run") - parser.add_argument("--json", action="store_true") - - -def _add_doctor_arguments(parser): - _add_installer_arguments(parser) - parser.add_argument( - "--wired-root", - action="append", - default=None, - help="root scanned for Wire-managed configs (repeatable; defaults to the project root)", - ) - parser.add_argument( - "--hf-cache", - default=None, - help="Hugging Face cache root (defaults to ~/.cache/huggingface/hub)", - ) - parser.add_argument( - "--prune", - action="store_true", - help="preview deletion of incomplete HF cache snapshots (requires --confirm --preview-hash to execute)", - ) - - -def _run_model_doctor(arguments): - from .model_doctor import ( - PruneError, - default_hf_cache, - execute_prune, - plan_prune, - run_model_doctor, - ) - - operation = "doctor-models" - wired_roots = arguments.wired_root or [arguments.project] - hf_cache = arguments.hf_cache or default_hf_cache() - if arguments.prune: - try: - plan = plan_prune(hf_cache) - outcome = execute_prune( - plan, - confirm=arguments.confirm, - preview_hash=arguments.preview_hash, - ) - except PruneError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("doctor failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - if outcome["status"] == "preview": - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - if not plan["candidates"]: - print("Nothing to prune: no incomplete cache snapshots.") - for candidate in plan["candidates"]: - print(" prune: {0} ({1:.1f} GB) - {2}".format( - candidate["repo"], candidate["bytes"] / 1e9, candidate["path"] - )) - if plan["candidates"]: - print("IRREVERSIBLE: these cache directories will be deleted permanently.") - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --prune --confirm --preview-hash PREVIEW_HASH.") - return 0 if not plan["candidates"] else 2 - result = ResultEnvelope.ok(operation, outcome) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - removed = [item for item in outcome["removed"] if item.get("removed")] - print("Pruned {0} incomplete snapshot(s).".format(len(removed))) - for item in removed: - print(" removed: {0}".format(item["repo"])) - return 0 - runtime_clients = [ - OllamaRuntimeClient(), - LMStudioRuntimeClient(), - OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080"), - MLXVLMRuntimeClient(), - OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000"), - ] - try: - report = run_model_doctor(wired_roots, hf_cache, runtime_clients) - except (OSError, TypeError, ValueError) as error: - result = ResultEnvelope.fail( - operation, - "doctor_failed", - str(error), - "Correct the scanned roots and retry; doctor never mutates anything.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("doctor failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - warnings = [ - {"code": finding["code"], "message": "{0}: {1}".format( - finding.get("path") or finding.get("model") or finding["code"], - finding["remediation"], - )} - for finding in report["findings"] - ] - result = ResultEnvelope.ok(operation, report, warnings=warnings) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - summary = report["summary"] - print("Doctor: {0} model(s), {1} wired config(s), {2} finding(s)".format( - summary["models"], summary["wired_configs"], summary["findings"] - )) - print(" hf-cache disk: {0:.1f} GB".format(summary["hf_cache_bytes"] / 1e9)) - for finding in report["findings"]: - location = finding.get("path") or finding.get("model") or "" - print(" [{0}] {1} - {2}".format(finding["code"], location, finding["remediation"])) - for endpoint in report["endpoints"]: - print(" endpoint {0}: {1}".format(endpoint["endpoint"], endpoint["status"])) - for runtime_error in report["runtime_errors"]: - print(" runtime unreachable: {0}".format(runtime_error)) - return 0 - - -def _installer_result(operation, data, as_json, status="ok", error=None): - result = ResultEnvelope.ok(operation, data) if status == "ok" else ResultEnvelope.fail( - operation, error[0], error[1], error[2] - ) - if as_json: - print(json.dumps(result.to_dict(), indent=2)) - elif status == "ok": - if data.get("selection_required"): - available = [item["id"] for item in data["providers"] if item["available"]] - print("No provider was selected. Detected providers: {0}".format(", ".join(available) if available else "none")) - elif "preview" in data: - compatibility = data.get("plan", {}).get("compatibility", []) - if compatibility: - print("Provider compatibility:") - for item in compatibility: - details = "{0}: {1}".format(item["id"], item["state"]) - if item.get("version"): - details += " (version {0})".format(item["version"]) - if item.get("error"): - details += " - {0}".format(item["error"]) - print(details) - print(data["preview"]["diff"]) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - else: - print(json.dumps(data, indent=2)) - else: - print("{0} failed [{1}]: {2}\nremediation: {3}".format(operation, error[0], error[1], error[2])) - return 0 if status == "ok" else 2 - - -def _run_installer(arguments): - operation = arguments.command - if operation == "doctor" and "models" in (getattr(arguments, "providers", None) or []): - return _run_model_doctor(arguments) - try: - installer = Installer(_installer_registry(), project_root=arguments.project) - detections = [item.to_dict() for item in installer.detected()] - selected = tuple(getattr(arguments, "providers", ()) or ()) - if operation == "providers" or not selected: - return _installer_result(operation, { - "providers": detections, - "selection_required": operation != "providers", - "mutated": False, - }, arguments.json) - plan = installer.plan(operation, selected, arguments.scope, arguments.project) - if arguments.dry_run or (operation != "doctor" and not arguments.confirm): - return _installer_result(operation, {"plan": plan.to_dict(), "preview": plan.preview, "mutated": False}, arguments.json) - if operation != "doctor" and not arguments.preview_hash: - return _installer_result(operation, {"plan": plan.to_dict(), "preview": plan.preview}, arguments.json, "error", ( - "preview_hash_required", "--confirm requires the hash from an inspected preview.", - "Run the command with --dry-run, inspect its preview, then pass --preview-hash.", - )) - if operation != "doctor" and arguments.preview_hash != plan.preview["preview_hash"]: - return _installer_result(operation, {"plan": plan.to_dict(), "preview": plan.preview}, arguments.json, "error", ( - "preview_stale", "The supplied preview hash does not match the current plan.", - "Generate and inspect a new preview before confirming this mutation.", - )) - result = installer.execute(plan, arguments.preview_hash if operation != "doctor" else False) - data = result if isinstance(result, dict) else result.to_dict() - return _installer_result(operation, {"result": data, "mutated": operation != "doctor"}, arguments.json) - except InstallerConflictError as error: - lock_code = "legacy_lock_busy" if str(error).startswith("legacy_lock_busy") else ( - "legacy_lock_recreated" if str(error).startswith("legacy_lock_recreated") else "artifact_conflict" - ) - return _installer_result(operation, {}, arguments.json, "error", ( - lock_code, str(error), - "Stop older mlx-agent processes and remove only recreated legacy locks." if lock_code != "artifact_conflict" else "Preserve the modified file or restore it to the receipt hash before retrying.", - )) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - return _installer_result(operation, {}, arguments.json, "error", ( - "installer_failed", str(error), "Inspect the provider selection and preview, then correct the reported path or manifest issue.", - )) - - -def legacy_scout_main(argv=None): - parser = argparse.ArgumentParser(description="Discover MLX models on HuggingFace for this host.") - _add_discovery_arguments(parser) - return _run_discovery(parser.parse_args(argv), legacy=True) - - -def build_parser(): - parser = argparse.ArgumentParser(description="MLX agent command-line core.") - subcommands = parser.add_subparsers(dest="command", required=True) - discover = subcommands.add_parser("discover", help="discover MLX models for this host") - _add_discovery_arguments(discover) - inspect_host = subcommands.add_parser("inspect-host", help="inspect local Apple Silicon and runtime inventory without model discovery") - inspect_host.add_argument("--json", action="store_true") - adopt = subcommands.add_parser("adopt", help="run or inspect resumable model adoption") - _add_adoption_arguments(adopt) - research = subcommands.add_parser("research", help="build a read-only domain research pack (markdown)") - _add_research_arguments(research) - blueprint = subcommands.add_parser( - "blueprint", - help="emit a read-only MLX project design pack (quant/train guidance; no scaffolding)", - ) - _add_blueprint_arguments(blueprint) - wire_command = subcommands.add_parser("wire", help="render, apply, inspect, or roll back runtime wiring") - _add_wire_arguments(wire_command) - bench_command = subcommands.add_parser("bench", help="measure a locally served model without downloading it") - _add_bench_arguments(bench_command) - serve_command = subcommands.add_parser("serve", help="preview and launch a local MLX server (confirmation-gated)") - _add_serve_arguments(serve_command) - fleet_command = subcommands.add_parser("fleet", help="render or apply a one-shot per-role router configuration") - _add_fleet_arguments(fleet_command) - watch_command = subcommands.add_parser("watch", help="snapshot and diff owned models against Hugging Face") - _add_watch_arguments(watch_command) - convert_command = subcommands.add_parser("convert", help="preview and quantize a cached model (confirmation-gated)") - _add_convert_arguments(convert_command) - lora_command = subcommands.add_parser("lora", help="preview and run LoRA training on a cached model (confirmation-gated)") - _add_lora_arguments(lora_command) - fuse_command = subcommands.add_parser("fuse", help="preview and fuse a LoRA adapter into its base (confirmation-gated)") - _add_fuse_arguments(fuse_command) - providers_command = subcommands.add_parser("providers", help="list detected supported provider CLIs") - _add_installer_arguments(providers_command, include_providers=False) - for name in ("install", "update", "uninstall", "doctor"): - installer_command = subcommands.add_parser(name, help="{0} declared provider artifacts safely".format(name)) - if name == "doctor": - _add_doctor_arguments(installer_command) - else: - _add_installer_arguments(installer_command) - return parser - - -def main(argv=None): - parser = build_parser() - arguments = parser.parse_args(argv) - if arguments.command == "adopt": - return _run_adoption(arguments) - if arguments.command == "wire": - return _run_wire(arguments) - if arguments.command == "bench": - return _run_bench(arguments) - if arguments.command == "serve": - return _run_serve(arguments) - if arguments.command == "fleet": - return _run_fleet(arguments) - if arguments.command == "watch": - return _run_watch(arguments) - if arguments.command == "convert": - return _run_convert(arguments) - if arguments.command == "lora": - return _run_lora(arguments) - if arguments.command == "fuse": - return _run_fuse(arguments) - if arguments.command in {"providers", "install", "update", "uninstall", "doctor"}: - return _run_installer(arguments) - if arguments.command == "inspect-host": - return _run_inspect_host(arguments) - if arguments.command == "research": - return _run_research(arguments) - if arguments.command == "blueprint": - return _run_blueprint(arguments) - return _run_discovery(arguments, legacy=False) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/command_executor.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/command_executor.py deleted file mode 100644 index 737ecaf..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/command_executor.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Bounded stdin executor for native provider custom tools.""" - -from __future__ import annotations - -import argparse -import io -import json -import sys - -from .gemini_args import GeminiArgumentError, parse_gemini_arguments - - -MAX_ARGUMENT_BYTES = 4096 -_PROVIDERS = ("opencode",) -_CAPABILITIES = ("scout", "adopt", "wire", "bench") - - -class CommandExecutorError(ValueError): - """Custom-tool input or execution failed without exposing raw arguments.""" - - -def read_command_arguments(stream): - """Read one small UTF-8 argument payload from a byte stream.""" - if not hasattr(stream, "read"): - raise CommandExecutorError("custom tool input was rejected") - try: - value = stream.read(MAX_ARGUMENT_BYTES + 1) - except (OSError, ValueError, TypeError) as error: - raise CommandExecutorError("custom tool input was rejected") from error - if not isinstance(value, bytes) or len(value) > MAX_ARGUMENT_BYTES: - raise CommandExecutorError("custom tool input was rejected") - try: - return value.decode("utf-8") - except UnicodeDecodeError as error: - raise CommandExecutorError("custom tool input was rejected") from error - - -def execute_command(provider, capability, stream, core=None): - """Validate stdin and execute only the capability's allowlisted core argv.""" - if provider not in _PROVIDERS or capability not in _CAPABILITIES: - raise CommandExecutorError("custom tool request was rejected") - try: - arguments = parse_gemini_arguments(capability, read_command_arguments(stream)) - except (GeminiArgumentError, CommandExecutorError) as error: - raise CommandExecutorError("custom tool request was rejected") from error - if core is None: - from .cli import main as core - try: - result = core(arguments) - except Exception as error: - raise CommandExecutorError("custom tool execution failed") from error - try: - exit_code = 0 if result is None else int(result) - except (TypeError, ValueError) as error: - raise CommandExecutorError("custom tool execution failed") from error - return {"status": "ok", "provider": provider, "capability": capability, "exit_code": exit_code} - - -def main(argv=None, stdin=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--provider", required=True, choices=_PROVIDERS) - parser.add_argument("--capability", required=True, choices=_CAPABILITIES) - arguments = parser.parse_args(argv) - try: - result = execute_command(arguments.provider, arguments.capability, stdin or sys.stdin.buffer) - except CommandExecutorError: - print(json.dumps({"status": "error", "provider": arguments.provider, "capability": arguments.capability, "error": "command rejected"}, sort_keys=True)) - return 2 - print(json.dumps(result, sort_keys=True)) - return result["exit_code"] - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/contextfit.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/contextfit.py deleted file mode 100644 index b355316..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/contextfit.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Bounded KV-cache arithmetic for context-aware memory fit.""" - -from __future__ import annotations - - -_MAX_INT = 2 ** 63 - 1 -MAX_ARCH_VALUE = 1000000 -DEFAULT_DTYPE_BYTES = 2 -CONTEXT_MIN, CONTEXT_MAX = 1024, 1048576 - - -def extract_architecture(config): - """Return bounded architecture facts from a Hugging Face config object.""" - if not isinstance(config, dict): - return None - - def _positive_int(key): - value = config.get(key) - if ( - not isinstance(value, int) - or isinstance(value, bool) - or not 0 < value <= MAX_ARCH_VALUE - ): - return None - return value - - layers = _positive_int("num_hidden_layers") - attention_heads = _positive_int("num_attention_heads") - kv_heads = _positive_int("num_key_value_heads") or attention_heads - head_dim = _positive_int("head_dim") - if head_dim is None: - hidden_size = _positive_int("hidden_size") - if hidden_size is not None and attention_heads: - if hidden_size % attention_heads != 0: - return None - head_dim = hidden_size // attention_heads - if layers is None or kv_heads is None or head_dim is None: - return None - return { - "layers": layers, - "kv_heads": kv_heads, - "head_dim": head_dim, - "max_position_embeddings": _positive_int("max_position_embeddings"), - } - - -def kv_bytes_per_token(architecture, dtype_bytes=DEFAULT_DTYPE_BYTES): - if not isinstance(architecture, dict): - return None - layers = architecture.get("layers") - kv_heads = architecture.get("kv_heads") - head_dim = architecture.get("head_dim") - if not all(isinstance(value, int) and value > 0 for value in (layers, kv_heads, head_dim)): - return None - return min(2 * layers * kv_heads * head_dim * dtype_bytes, _MAX_INT) - - -def kv_cache_bytes(architecture, context_tokens, dtype_bytes=DEFAULT_DTYPE_BYTES): - per_token = kv_bytes_per_token(architecture, dtype_bytes) - if per_token is None: - return None - if not isinstance(context_tokens, int) or isinstance(context_tokens, bool): - raise TypeError("context_tokens must be an integer") - if not CONTEXT_MIN <= context_tokens <= CONTEXT_MAX: - raise ValueError( - "context_tokens must be between {0} and {1}".format(CONTEXT_MIN, CONTEXT_MAX) - ) - return min(per_token * context_tokens, _MAX_INT) - - -def max_context_tokens(architecture, weight_bytes, budget_bytes, dtype_bytes=DEFAULT_DTYPE_BYTES): - """Largest context whose weights + KV fit the budget; None when unknown.""" - per_token = kv_bytes_per_token(architecture, dtype_bytes) - if per_token is None: - return None - if not isinstance(weight_bytes, (int, float)) or not isinstance(budget_bytes, (int, float)): - return None - if weight_bytes < 0 or budget_bytes <= 0: - return None - headroom = budget_bytes - weight_bytes - if headroom <= 0: - return 0 - maximum = int(headroom // per_token) - cap = architecture.get("max_position_embeddings") - if isinstance(cap, int) and cap > 0: - maximum = min(maximum, cap) - return min(maximum, CONTEXT_MAX) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/contracts.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/contracts.py deleted file mode 100644 index 7f1accf..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/contracts.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Versioned result contracts shared by every provider adapter.""" - -from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional - - -SCHEMA_VERSION = "1.0" - - -def _require_non_empty_string(value: Any, name: str) -> None: - if not isinstance(value, str): - raise TypeError("{0} must be a string".format(name)) - if not value: - raise ValueError("{0} must not be empty".format(name)) - - -def _validate_warnings(warnings: Any) -> List[Dict[str, str]]: - if warnings is None: - return [] - if not isinstance(warnings, list): - raise TypeError("warnings must be a list of objects of strings") - - validated: List[Dict[str, str]] = [] - for index, warning in enumerate(warnings): - if not isinstance(warning, dict) or not all( - isinstance(item, str) for item in warning.values() - ): - raise TypeError("warnings[{0}] must be an object of strings".format(index)) - validated.append(dict(warning)) - return validated - - -@dataclass(frozen=True) -class ErrorDetail: - """A stable, actionable description of an unsuccessful operation.""" - - code: str - message: str - remediation: str - retryable: bool = False - - -@dataclass(frozen=True) -class ResultEnvelope: - """The provider-neutral result returned by MLX agent operations.""" - - operation: str - status: str - data: Dict[str, Any] = field(default_factory=dict) - warnings: List[Dict[str, str]] = field(default_factory=list) - error: Optional[ErrorDetail] = None - schema_version: str = SCHEMA_VERSION - generated_at: str = field( - default_factory=lambda: datetime.now(timezone.utc).isoformat() - ) - - @classmethod - def ok(cls, operation: str, data: Dict[str, Any], warnings=None): - """Create a successful versioned result.""" - _require_non_empty_string(operation, "operation") - if not isinstance(data, dict): - raise TypeError("data must be an object") - return cls( - operation=operation, - status="ok", - data=dict(data), - warnings=_validate_warnings(warnings), - ) - - @classmethod - def fail( - cls, - operation: str, - code: str, - message: str, - remediation: str, - retryable=False, - ): - """Create an error result with a machine-readable recovery path.""" - _require_non_empty_string(operation, "operation") - _require_non_empty_string(code, "code") - _require_non_empty_string(message, "message") - _require_non_empty_string(remediation, "remediation") - if not isinstance(retryable, bool): - raise TypeError("retryable must be a boolean") - return cls( - operation=operation, - status="error", - error=ErrorDetail(code, message, remediation, retryable), - ) - - def to_dict(self) -> Dict[str, Any]: - """Serialize the envelope without an empty error object on success.""" - value = asdict(self) - if self.error is None: - value.pop("error") - return value diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/convert.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/convert.py deleted file mode 100644 index 908d885..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/convert.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Confirmation-gated local MLX quantization jobs.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from pathlib import Path - -from .serve import ( - _argv_matches, - _pid_alive, - _pid_command, - _terminate_pid, -) -from .transactions import _atomic_in_directory -from .wiring import _MODEL - - -CONVERT_RECEIPT_SCHEMA_VERSION = "1.0" -CONVERT_RECEIPT_KIND = "convert" -Q_BITS_CHOICES = (4, 8) -EXECUTABLE = "mlx_lm.convert" -MAX_LOG_TAIL_BYTES = 64 * 1024 - - -class ConvertError(RuntimeError): - """Classified convert failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def receipts_root(root=None, kind="convert"): - base = Path(root) if root is not None else Path.cwd() - return base / ".mlx-agent-receipts" / kind - - -def plan_convert(repo, q_bits=4, out=None): - """Render the exact conversion plan; pure and side-effect free.""" - if not isinstance(repo, str) or not _MODEL.fullmatch(repo): - raise ConvertError( - "invalid_repo", - "convert requires a safe publisher/model identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - if not isinstance(q_bits, int) or isinstance(q_bits, bool): - raise ConvertError( - "invalid_arguments", "q_bits must be an integer.", - "Pass --q-bits 4 or --q-bits 8.", - ) - if q_bits not in Q_BITS_CHOICES: - raise ConvertError( - "invalid_arguments", - "q_bits must be one of {0}.".format(list(Q_BITS_CHOICES)), - "Only bounded 4bit and 8bit recipes are supported in v1.", - ) - if out is None: - name = repo.split("/", 1)[1] - out = "{0}-MLX-{1}bit".format(name, q_bits) - plan = { - "repo": repo, - "q_bits": q_bits, - "out": str(out), - "argv": [ - EXECUTABLE, - "--hf-path", repo, - "--mlx-path", str(out), - "--q-bits", str(q_bits), - ], - } - canonical = json.dumps(plan, sort_keys=True, separators=(",", ":")) - plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() - return plan - - -def _receipt_path(root, plan): - return root / "{0}-{1}bit.json".format( - plan["repo"].split("/", 1)[1], plan["q_bits"] - ) - - -def _read_receipt(path, kind=CONVERT_RECEIPT_KIND): - try: - value = json.loads(Path(path).read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - return None - if not isinstance(value, dict) or value.get("kind") != kind: - return None - if not isinstance(value.get("pid"), int) or isinstance(value.get("pid"), bool): - return None - if not isinstance(value.get("argv"), list) or not isinstance(value.get("repo"), str): - return None - return value - - -def _write_receipt(root, receipt, filename): - content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") - _atomic_in_directory(root, filename, content, 0o600) - - -def start_convert(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, spawn=None, now=_utc_now, - pid_alive=None): - """Execute a reviewed conversion plan; the only mutating entry point.""" - from .serve import _default_spawn, _default_which - - which = which or _default_which - spawn = spawn or _default_spawn - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise ConvertError( - "preview_hash_required", - "--confirm requires the hash from a reviewed convert preview.", - "Run convert start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise ConvertError( - "preview_stale", - "The supplied preview hash does not match this convert plan.", - "Re-run convert start without --confirm and review the fresh plan.", - ) - if which(EXECUTABLE) is None: - raise ConvertError( - "runtime_not_installed", - "The {0} executable is not installed.".format(EXECUTABLE), - "Install mlx-lm yourself (pip install mlx-lm); convert never installs runtimes.", - ) - if model_present is not None and not model_present(plan["repo"]): - raise ConvertError( - "model_not_local", - "The source model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; convert never downloads.", - ) - if Path(plan["out"]).exists(): - raise ConvertError( - "output_exists", - "The output path already exists: {0}".format(plan["out"]), - "Pick a fresh --out path; convert never overwrites.", - ) - for receipt_path in sorted(root.glob("*.json")) if root.is_dir() else []: - existing = _read_receipt(receipt_path) - if existing is None or existing.get("completed_at"): - continue - if pid_alive(existing["pid"]): - raise ConvertError( - "job_in_progress", - "A convert job is still running (pid {0}).".format(existing["pid"]), - "Wait for it to finish (convert status) before starting another.", - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}-{1}bit.log".format(plan["repo"].split("/", 1)[1], plan["q_bits"]) - pid = spawn(plan["argv"], str(log_path)) - receipt = { - "schema_version": CONVERT_RECEIPT_SCHEMA_VERSION, - "kind": CONVERT_RECEIPT_KIND, - "repo": plan["repo"], - "q_bits": plan["q_bits"], - "out": plan["out"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "started_at": now(), - "preview_hash": plan["preview_hash"], - "completed_at": None, - "exit_status": None, - } - _write_receipt( - root, receipt, - "{0}-{1}bit.json".format(receipt["repo"].split("/", 1)[1], receipt["q_bits"]), - ) - return {"status": "started", "receipt": receipt} - - -def status_convert(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - now=_utc_now): - """Cross-check convert receipts against live processes; marks exits once.""" - root = receipts_root(receipts_dir) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - receipt = _read_receipt(path) - if receipt is None: - continue - entry = { - "receipt": str(path), - "repo": receipt["repo"], - "q_bits": receipt["q_bits"], - "out": receipt["out"], - "pid": receipt["pid"], - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - "completed_at": receipt.get("completed_at"), - } - if receipt.get("completed_at"): - entry["state"] = receipt.get("exit_status") or "done" - entries.append(entry) - continue - if pid_alive(receipt["pid"]): - command = pid_command(receipt["pid"]) - entry["state"] = "running" if _argv_matches( - {"argv": receipt["argv"], "port": None, "repo": receipt["repo"]}, - command, - require_port=False, - ) else "unknown" - entries.append(entry) - continue - exit_status = "done" if Path(receipt["out"]).exists() else "failed" - receipt["completed_at"] = now() - receipt["exit_status"] = exit_status - _write_receipt( - root, receipt, - "{0}-{1}bit.json".format(receipt["repo"].split("/", 1)[1], receipt["q_bits"]), - ) - entry["state"] = exit_status - entry["completed_at"] = receipt["completed_at"] - entries.append(entry) - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/discovery.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/discovery.py deleted file mode 100644 index 889e34d..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/discovery.py +++ /dev/null @@ -1,528 +0,0 @@ -"""Model discovery, filtering, ranking, and cached structured Scout results.""" - -from __future__ import annotations - -import hashlib -import json -import os -import tempfile -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path - -from .contracts import ResultEnvelope -from .host import HostInventory -from .huggingface import HuggingFaceClient -from .models import ( - DISCOVERY_ROLES, - REPUTABLE, - base_name, - classify, - classify_roles, - infer_quantization, - quant_rank, - resolve_ram, - resolve_reasoning, - wiring, -) - - -CACHE_SCHEMA_VERSION = "2.0" -CACHE_TTL_SECONDS = 24 * 60 * 60 - - -@dataclass(frozen=True) -class DiscoveryRequest: - """Bounded, explicit discovery policy. Legacy ``new`` and ``fast`` remain supported.""" - - role: object = None - memory_gb: object = None - quantization: object = None - licenses: object = None - include_gated: bool = True - publishers: object = None - runtime: object = None - refresh: bool = False - offline: bool = False - limit: int = 6 - new: bool = False - fast: bool = False - context_tokens: object = None - - def cache_request(self): - """The reproducible request shape, excluding transport-only cache switches.""" - return { - "role": self.role, - "memory_gb": self.memory_gb, - "quantization": _canonical_quantization(self.quantization), - "licenses": _normalise_values(self.licenses), - "include_gated": self.include_gated, - "publishers": _normalise_values(self.publishers), - "runtime": self.runtime, - "limit": self.limit, - "new": self.new, - "fast": self.fast, - "context_tokens": self.context_tokens, - } - - -def _normalise_values(value): - if value is None: - return [] - if isinstance(value, str): - return [value.lower()] - return sorted(str(item).lower() for item in value) - - -def _canonical_quantization(value): - if value is None: - return None - normalized = infer_quantization(str(value)) - return normalized or str(value).strip().lower() - - -def _utc_now(): - return datetime.now(timezone.utc) - - -def _parse_timestamp(value): - try: - return datetime.fromisoformat(value.replace("Z", "+00:00")) - except (AttributeError, TypeError, ValueError): - return None - - -def _community_bench_for(repo, chip): - """Return the community bench entry matching repo and host chip, if any.""" - if not isinstance(repo, str) or not isinstance(chip, str) or not chip: - return None - try: - from .bench import load_community_bench - - entry = load_community_bench().get((repo, chip)) - except Exception: - return None - if not isinstance(entry, dict) or entry.get("decode_toks") is None: - return None - result = dict(entry) - result["chip"] = chip - return result - - -def _kv_estimate(architecture, ram_gb, budget_gb, context_tokens): - """Attach KV-cache math when architecture, weights, and budget are known.""" - if not isinstance(architecture, dict) or ram_gb is None or budget_gb is None: - return None - from .contextfit import ( - DEFAULT_DTYPE_BYTES, - kv_bytes_per_token, - kv_cache_bytes, - max_context_tokens, - ) - - if kv_bytes_per_token(architecture) is None: - return None - weight_bytes = float(ram_gb) * 1e9 - budget_bytes = float(budget_gb) * 0.8 * 1e9 - block = { - "max_context_tokens": max_context_tokens(architecture, weight_bytes, budget_bytes), - "dtype_bytes": DEFAULT_DTYPE_BYTES, - "src": "huggingface_model_metadata", - } - if context_tokens is not None: - kv_bytes = kv_cache_bytes(architecture, context_tokens) - block["context_tokens"] = context_tokens - block["kv_gb"] = round(kv_bytes / 1e9, 1) if kv_bytes is not None else None - return block - - -def _estimates(ram, budget, kv_block): - estimates = {"ram_gb": ram, "memory_budget_gb": budget, "headroom_fraction": 0.2} - if kv_block is not None: - estimates["kv"] = kv_block - return estimates - - -class DiscoveryService: - def __init__(self, host=None, huggingface=None, state_dir=None, cache_enabled=None, cache_ttl_seconds=CACHE_TTL_SECONDS): - self._host = host - self._huggingface = huggingface or HuggingFaceClient() - injected_state_dir = state_dir or os.environ.get("MLX_AGENT_STATE_DIR") - self._state_dir = Path(injected_state_dir) if injected_state_dir else None - self._cache_enabled = self._state_dir is not None if cache_enabled is None else cache_enabled - self._cache_ttl_seconds = cache_ttl_seconds - - @property - def host(self): - if self._host is None: - self._host = HostInventory.detect(self._huggingface.http_get) - return self._host - - def discover(self, request): - cache = self._read_cache(request) - if request.offline: - if cache is None: - return ResultEnvelope.fail( - "discover", - "offline_cache_missing", - "No cached discovery response matches this request.", - "Run discovery while online first, or remove --offline.", - ) - data, stale = cache - warnings = [_warning("stale_cache")] if stale else [] - return ResultEnvelope.ok("discover", self._response_data(request, data, "stale" if stale else "fresh"), warnings=warnings) - if cache is not None and not cache[1] and not request.refresh: - return ResultEnvelope.ok("discover", self._response_data(request, cache[0], "fresh")) - - host = self.host - list_sort = "lastModified" if request.new else "trendingScore" - list_url_builder = getattr(self._huggingface, "list_models_url", None) - list_url = ( - list_url_builder(sort=list_sort, limit_fetch=300) - if callable(list_url_builder) else None - ) - try: - raw = self._huggingface.list_models(sort=list_sort) - except Exception as error: - return ResultEnvelope.fail( - "discover", - "network_unavailable", - "HuggingFace query failed: {0}".format(error), - "Check network access to huggingface.co and retry the discovery command.", - retryable=True, - ) - - data = self._build_response(request, raw, host.to_dict(), list_url=list_url) - self._write_cache(request, data) - return ResultEnvelope.ok("discover", self._response_data(request, data, "refreshed" if request.refresh else "miss")) - - def _build_response(self, request, raw, host_data, list_url=None): - buckets = {role: [] for role in DISCOVERY_ROLES} - rejected = {} - seen_repo, seen_base = set(), {} - rejection_limit = max(1, int(request.limit or 1)) * 3 - for model in raw: - repo = model.get("id") or model.get("modelId") - if not repo or repo in seen_repo: - continue - seen_repo.add(repo) - if ( - request.role - and request.role != "tool-use" - and classify(repo) != request.role - ): - continue - enrichment = {} if request.fast else self._huggingface.inspect_model(repo) - roles, tool_use = classify_roles(repo, enrichment) - selected_roles = ( - (request.role,) if request.role in roles else () - ) if request.role else roles - if not selected_roles: - continue - accepted = False - rejected_candidate = None - for role in selected_roles: - candidate = self._candidate( - repo, - model, - role, - roles, - tool_use, - host_data, - enrichment, - request, - list_url, - ) - reasons = self._rejection_reasons(candidate, request) - candidate["rejection_reasons"] = reasons - if reasons: - rejected_candidate = rejected_candidate or candidate - continue - accepted = True - key = (role, candidate["base"]) - current = seen_base.get(key) - if current is not None: - if self._selection_key(candidate) > self._selection_key(current): - buckets[role].remove(current) - buckets[role].append(candidate) - seen_base[key] = candidate - continue - seen_base[key] = candidate - buckets[role].append(candidate) - if not accepted and rejected_candidate is not None: - if len(rejected) < rejection_limit: - rejected[repo] = rejected_candidate - - for role in buckets: - buckets[role].sort(key=self._selection_key, reverse=True) - buckets[role] = buckets[role][: max(0, int(request.limit))] - - return { - "host": host_data, - "fast": request.fast, - "roles": {role: items for role, items in buckets.items() if items}, - "rejected": rejected, - "request": request.cache_request(), - } - - def _candidate( - self, - repo, - model, - role, - roles, - tool_use, - host_data, - enrichment, - request, - list_url=None, - ): - ram, ram_source = resolve_ram(repo, enrichment) - reasoning, reason_source = resolve_reasoning(repo, enrichment) - publisher = repo.split("/", 1)[0].lower() - quantization = infer_quantization(repo) - downloads = model.get("downloads", 0) - likes = model.get("likes", 0) - trusted = publisher in REPUTABLE - budget = request.memory_gb if request.memory_gb is not None else host_data.get("ram_gb") - architecture = enrichment.get("architecture") - kv_block = _kv_estimate(architecture, ram, budget, request.context_tokens) - fits = ram is None or budget is None or ram < float(budget) * 0.8 - if request.context_tokens is not None and kv_block is not None: - fits = (ram + kv_block["kv_gb"]) < float(budget) * 0.8 - community_bench = _community_bench_for(repo, host_data.get("chip")) - rank_score = (1000000 if trusted else 0) + (quant_rank(repo) * 10000) + min(downloads, 9999) + min(likes, 999) - metadata_available = enrichment.get("metadata_available") is True - gated_status = "unknown" - if metadata_available: - gated_status = "gated" if enrichment.get("gated") else "public" - facts = { - "repository": repo, - "downloads": downloads, - "likes": likes, - "gated": gated_status, - "license": enrichment.get("license") if metadata_available else None, - } - if enrichment.get("tree_available") is True and enrichment.get("weight_bytes"): - facts["weight_bytes"] = enrichment["weight_bytes"] - candidate = { - # Legacy report keys. - "repo": repo, - "role": role, - "roles": list(roles), - "downloads": downloads, - "likes": likes, - "trusted": trusted, - "base": base_name(repo), - "qrank": quant_rank(repo), - "est_ram_gb": ram, - "ram_src": ram_source, - "fits": fits, - "reasoning": reasoning, - "reason_src": reason_source, - "gated": enrichment.get("gated") if metadata_available else None, - "license": facts["license"], - "wiring": wiring(repo, role, host_data), - # Explainable structured fields. Memory remains an estimate, even when - # the weights came from a measured file-size listing. - "facts": facts, - "estimates": _estimates(ram, budget, kv_block), - "tool_use": tool_use, - "heuristics": { - "role": role, - "primary_role": roles[0], - "roles": list(roles), - "quantization": quantization, - "trusted_publisher": trusted, - }, - "provenance": self._provenance( - repo, - role, - tool_use, - enrichment, - request.fast, - ram_source, - reason_source, - list_url, - ), - "rank_score": rank_score, - "selection_reasons": self._selection_reasons(role, quantization, trusted, fits), - "rejection_reasons": [], - } - if community_bench is not None: - candidate["community_bench"] = community_bench - return candidate - - @staticmethod - def _provenance( - repo, - role, - tool_use, - enrichment, - fast, - ram_source, - reason_source, - list_url=None, - ): - encoded = repo.replace("/", "%2F") - records = [{ - "source": "huggingface_model_list", - "url": list_url or "https://huggingface.co/api/models", - "fields": ["repository", "downloads", "likes"], - }] - if not fast and enrichment.get("metadata_available") is True: - metadata_fields = ["gated", "license"] - if reason_source in ("chat_template", "tags", "checked"): - metadata_fields.append("reasoning") - if tool_use["source"] in ("chat_template", "tags", "checked"): - metadata_fields.append("tool_use") - records.append({ - "source": "huggingface_model_metadata", - "url": enrichment.get("metadata_url") or "https://huggingface.co/api/models/{0}".format(encoded), - "fields": metadata_fields, - }) - if not fast and enrichment.get("tree_available") is True and enrichment.get("weight_bytes"): - records.append({ - "source": "huggingface_repository_tree", - "url": enrichment.get("tree_url") or "https://huggingface.co/api/models/{0}/tree/main?recursive=true".format(encoded), - "fields": ["weight_bytes"], - }) - if tool_use["source"] in ("chat_template", "tags", "checked"): - role_basis = ( - "primary role from repository name; tool-use signal from " - "Hugging Face metadata ({0})" - ).format(tool_use["source"]) - elif tool_use["source"] == "name": - role_basis = "primary role and tool-use signal from repository name" - else: - role_basis = "primary role from repository name; no tool-use signal" - records.append({ - "source": "local_role_derivation", - "fields": ["role", "roles", "primary_role"], - "basis": role_basis, - }) - name_fields = ["quantization", "trusted_publisher"] - if tool_use["source"] == "name": - name_fields.append("tool_use") - if reason_source == "name": - name_fields.append("reasoning") - records.append({ - "source": "local_name_derivation", - "fields": name_fields, - }) - records.append({ - "source": "local_memory_estimate", - "fields": ["ram_gb", "fits"], - "basis": ram_source or "unavailable", - }) - records.append({ - "source": "local_ranking_derivation", - "fields": ["rank_score", "selection_reasons", "rejection_reasons"], - }) - return records - - @staticmethod - def _selection_reasons(role, quantization, trusted, fits): - reasons = ["role membership: {0}".format(role)] - if quantization: - reasons.append("quantization inferred from repository name: {0}".format(quantization)) - if trusted: - reasons.append("publisher is in the configured reputable set") - if fits: - reasons.append("estimated weights fit within the memory budget with headroom") - return reasons - - @staticmethod - def _selection_key(candidate): - # The final repository ID makes ranking stable across equal evidence. - return (candidate["trusted"], candidate["qrank"], candidate["downloads"], candidate["likes"], candidate["repo"]) - - @staticmethod - def _rejection_reasons(candidate, request): - reasons = [] - licenses = set(_normalise_values(request.licenses)) - publishers = set(_normalise_values(request.publishers)) - wanted_quantization = _canonical_quantization(request.quantization) - if not request.include_gated and candidate["facts"]["gated"] == "gated": - reasons.append("gated") - if not request.include_gated and candidate["facts"]["gated"] == "unknown": - reasons.append("gated_status_unknown") - if licenses and (candidate["license"] or "").lower() not in licenses: - reasons.append("license") - if publishers and candidate["repo"].split("/", 1)[0].lower() not in publishers: - reasons.append("publisher") - if wanted_quantization and candidate["heuristics"]["quantization"] != wanted_quantization: - reasons.append("quantization") - if request.memory_gb is not None and not candidate["fits"]: - reasons.append("memory_budget") - if request.runtime and not HostInventory.runtime_supports(request.runtime, candidate["heuristics"]["role"]): - reasons.append("runtime") - return reasons - - def _cache_path(self, request): - encoded = json.dumps(request.cache_request(), sort_keys=True, separators=(",", ":")).encode("utf-8") - return self._state_dir / "discovery-{0}.json".format(hashlib.sha256(encoded).hexdigest()[:20]) - - def _read_cache(self, request): - if not self._cache_enabled: - return None - try: - payload = json.loads(self._cache_path(request).read_text()) - if payload.get("schema_version") != CACHE_SCHEMA_VERSION or payload.get("request") != request.cache_request(): - return None - fetched_at = _parse_timestamp(payload.get("fetched_at")) - if fetched_at is None or not isinstance(payload.get("response"), dict): - return None - stale = (_utc_now() - fetched_at).total_seconds() > self._cache_ttl_seconds - return payload["response"], stale - except (OSError, TypeError, ValueError, json.JSONDecodeError): - return None - - def _write_cache(self, request, response): - if not self._cache_enabled: - return - try: - self._state_dir.mkdir(parents=True, exist_ok=True) - payload = { - "schema_version": CACHE_SCHEMA_VERSION, - "fetched_at": _utc_now().isoformat(), - "request": request.cache_request(), - "response": response, - } - destination = self._cache_path(request) - with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=str(destination.parent), delete=False) as temporary: - json.dump(payload, temporary, sort_keys=True) - temporary.flush() - os.fsync(temporary.fileno()) - temporary_name = temporary.name - os.replace(temporary_name, destination) - except OSError: - # Discovery evidence is still useful if the local cache cannot be written. - return - - def _cached_data(self, response, status): - data = dict(response) - data["cache"] = {"status": status, "ttl_seconds": self._cache_ttl_seconds} - return data - - def _response_data(self, request, response, status): - if not self._cache_enabled and self._legacy_compatible(request): - return {key: response[key] for key in ("host", "fast", "roles")} - return self._cached_data(response, status) - - @staticmethod - def _legacy_compatible(request): - return ( - request.memory_gb is None - and request.quantization is None - and request.licenses is None - and request.include_gated - and request.publishers is None - and request.runtime is None - and not request.refresh - and not request.offline - ) - - -def _warning(code): - return {"code": code, "message": "Cached discovery response is older than the 24 hour TTL."} diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/fuse.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/fuse.py deleted file mode 100644 index b35f13a..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/fuse.py +++ /dev/null @@ -1,213 +0,0 @@ -"""Confirmation-gated LoRA fuse jobs on the convert machinery.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from pathlib import Path - -from .convert import ( - _read_receipt, - _write_receipt, - receipts_root, -) -from .serve import _argv_matches, _pid_alive, _pid_command -from .wiring import _MODEL - - -FUSE_RECEIPT_SCHEMA_VERSION = "1.0" -FUSE_RECEIPT_KIND = "fuse" -EXECUTABLE = "mlx_lm.fuse" - - -class FuseError(RuntimeError): - """Classified fuse failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def validate_adapter(adapter_path): - """Confirm the adapter directory looks like an mlx_lm LoRA output.""" - root = Path(adapter_path) - if not root.is_dir(): - raise FuseError( - "adapter_invalid", - "The adapter path is not a directory: {0}".format(adapter_path), - "Point --adapter at a completed lora output directory.", - ) - config = root / "adapter_config.json" - if not config.is_file(): - raise FuseError( - "adapter_invalid", - "The adapter directory lacks adapter_config.json.", - "Point --adapter at a completed lora output directory.", - ) - try: - value = json.loads(config.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError) as error: - raise FuseError( - "adapter_invalid", - "adapter_config.json could not be parsed: {0}".format(error), - "Repair or regenerate the adapter before fusing.", - ) - if not isinstance(value, dict): - raise FuseError( - "adapter_invalid", - "adapter_config.json must contain a JSON object.", - "Repair or regenerate the adapter before fusing.", - ) - return {"adapter_config": True} - - -def plan_fuse(repo, adapter, out=None): - """Validate inputs and render the exact fuse plan; side-effect free.""" - if not isinstance(repo, str) or not _MODEL.fullmatch(repo): - raise FuseError( - "invalid_repo", - "fuse requires a safe publisher/model base identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - validate_adapter(adapter) - if out is None: - out = "{0}-fused".format(repo.split("/", 1)[1]) - plan = { - "repo": repo, - "adapter": str(adapter), - "out": str(out), - "argv": [ - EXECUTABLE, - "--model", repo, - "--adapter-path", str(adapter), - "--save-path", str(out), - ], - } - canonical = json.dumps(plan, sort_keys=True, separators=(",", ":")) - plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() - return plan - - -def start_fuse(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, spawn=None, now=_utc_now, - pid_alive=None): - """Execute a reviewed fuse plan; the only mutating entry point.""" - from .serve import _default_spawn, _default_which - - which = which or _default_which - spawn = spawn or _default_spawn - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir, FUSE_RECEIPT_KIND) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise FuseError( - "preview_hash_required", - "--confirm requires the hash from a reviewed fuse preview.", - "Run fuse start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise FuseError( - "preview_stale", - "The supplied preview hash does not match this fuse plan.", - "Re-run fuse start without --confirm and review the fresh plan.", - ) - if which(EXECUTABLE) is None: - raise FuseError( - "runtime_not_installed", - "The {0} executable is not installed.".format(EXECUTABLE), - "Install mlx-lm yourself (pip install mlx-lm); fuse never installs runtimes.", - ) - if model_present is not None and not model_present(plan["repo"]): - raise FuseError( - "model_not_local", - "The base model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; fuse never downloads.", - ) - if Path(plan["out"]).exists(): - raise FuseError( - "output_exists", - "The fused output path already exists: {0}".format(plan["out"]), - "Pick a fresh --out path; fuse never overwrites.", - ) - for receipt_path in sorted(root.glob("*.json")) if root.is_dir() else []: - existing = _read_receipt(receipt_path, FUSE_RECEIPT_KIND) - if existing is None or existing.get("completed_at"): - continue - if pid_alive(existing["pid"]): - raise FuseError( - "job_in_progress", - "A fuse job is still running (pid {0}).".format(existing["pid"]), - "Wait for it to finish (fuse status) before starting another.", - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}-fuse.log".format(plan["repo"].split("/", 1)[1]) - pid = spawn(plan["argv"], str(log_path)) - receipt = { - "schema_version": FUSE_RECEIPT_SCHEMA_VERSION, - "kind": FUSE_RECEIPT_KIND, - "repo": plan["repo"], - "adapter": plan["adapter"], - "out": plan["out"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "started_at": now(), - "preview_hash": plan["preview_hash"], - "completed_at": None, - "exit_status": None, - } - _write_receipt(root, receipt, "{0}-fuse.json".format(plan["repo"].split("/", 1)[1])) - return {"status": "started", "receipt": receipt} - - -def status_fuse(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - now=_utc_now): - """Cross-check fuse receipts against live processes; marks exits once.""" - root = receipts_root(receipts_dir, FUSE_RECEIPT_KIND) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - receipt = _read_receipt(path, FUSE_RECEIPT_KIND) - if receipt is None: - continue - entry = { - "receipt": str(path), - "repo": receipt["repo"], - "adapter": receipt.get("adapter"), - "out": receipt["out"], - "pid": receipt["pid"], - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - "completed_at": receipt.get("completed_at"), - } - if receipt.get("completed_at"): - entry["state"] = receipt.get("exit_status") or "done" - entries.append(entry) - continue - if pid_alive(receipt["pid"]): - command = pid_command(receipt["pid"]) - entry["state"] = "running" if _argv_matches( - {"argv": receipt["argv"], "port": None, "repo": receipt["repo"]}, - command, - require_port=False, - ) else "unknown" - entries.append(entry) - continue - exit_status = "done" if Path(receipt["out"]).exists() else "failed" - receipt["completed_at"] = now() - receipt["exit_status"] = exit_status - _write_receipt(root, receipt, path.name) - entry["state"] = exit_status - entry["completed_at"] = receipt["completed_at"] - entries.append(entry) - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/gemini_args.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/gemini_args.py deleted file mode 100644 index 886acaf..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/gemini_args.py +++ /dev/null @@ -1,340 +0,0 @@ -"""Strict argument grammar for Gemini custom-command input. - -Gemini TOML commands pass user text to the model, not directly to a process. -This module converts that opaque text into an allowlisted MLX-agent argv list -before a bundled skill may execute the structured CLI. -""" - -from __future__ import annotations - -import argparse -import json -import re -import shlex -import urllib.parse -from pathlib import Path - -from .models import ROLES - - -class GeminiArgumentError(ValueError): - """Gemini command input is outside the documented capability grammar.""" - - -_RUNTIMES = {"ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"} -_ROLES = {role for role, _keywords, _label in ROLES} -_MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,191}$") -_SAFE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+:-]{0,127}$") -_SAFE_QUANTIZATION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") -_METACHARACTERS = set(";|&`$<>") - - -def _tokens(raw): - if not isinstance(raw, str): - raise GeminiArgumentError("arguments must be text") - if len(raw) > 4096 or any(ord(character) < 32 or ord(character) == 127 for character in raw): - raise GeminiArgumentError("arguments must not contain control characters") - if any(character in _METACHARACTERS for character in raw): - raise GeminiArgumentError("shell metacharacters are not allowed") - try: - tokens = shlex.split(raw, posix=True) - except ValueError as error: - raise GeminiArgumentError("arguments must use balanced shell-style quotes: {0}".format(error)) - if len(tokens) > 32 or any(not token or len(token) > 512 for token in tokens): - raise GeminiArgumentError("arguments exceed the supported command grammar") - return tokens - - -def _value(tokens, index, flag): - if index + 1 >= len(tokens) or tokens[index + 1].startswith("--"): - raise GeminiArgumentError("{0} requires a value".format(flag)) - return tokens[index + 1] - - -def _path(value, label): - if not isinstance(value, str) or not value or value.startswith("~") or "\\" in value: - raise GeminiArgumentError("{0} must be a portable non-empty path".format(label)) - path = Path(value) - if any(part in {"", ".", ".."} for part in path.parts): - raise GeminiArgumentError("{0} must not traverse parent directories".format(label)) - return value - - -def _model(value): - if not _MODEL.fullmatch(value): - raise GeminiArgumentError("model must be a publisher/model identifier") - return value - - -def _integer(value, label, minimum, maximum): - if not re.fullmatch(r"[0-9]+", value): - raise GeminiArgumentError("{0} must be an integer".format(label)) - number = int(value) - if number < minimum or number > maximum: - raise GeminiArgumentError("{0} must be between {1} and {2}".format(label, minimum, maximum)) - return str(number) - - -def _decimal(value, label, minimum, maximum): - if not re.fullmatch(r"(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+)", value): - raise GeminiArgumentError("{0} must be a decimal number".format(label)) - number = float(value) - if number < minimum or number > maximum: - raise GeminiArgumentError("{0} must be between {1} and {2}".format(label, minimum, maximum)) - return value - - -def _endpoint(value): - try: - parsed = urllib.parse.urlsplit(value) - hostname = parsed.hostname - username = parsed.username - password = parsed.password - port = parsed.port - except (UnicodeError, ValueError) as error: - raise GeminiArgumentError("endpoint is not a valid local HTTP URL") from error - if ( - parsed.scheme not in {"http", "https"} - or hostname not in {"127.0.0.1", "localhost", "::1"} - or username is not None - or password is not None - or parsed.query - or parsed.fragment - ): - raise GeminiArgumentError("endpoint must be a credential-free local HTTP URL") - if port is not None and not 1 <= port <= 65535: - raise GeminiArgumentError("endpoint port must be between 1 and 65535") - return value - - -def _parse_scout(tokens): - argv, index = ["discover"], 0 - seen, cache = set(), None - values = { - "--role": (lambda value: value if value in _ROLES else None, "role"), - "--limit": (lambda value: _integer(value, "limit", 1, 50), "limit"), - "--memory-gb": (lambda value: _decimal(value, "memory-gb", 0.25, 1024), "memory-gb"), - "--quantization": (lambda value: value if _SAFE_QUANTIZATION.fullmatch(value) else None, "quantization"), - "--runtime": (lambda value: value if value in _RUNTIMES else None, "runtime"), - "--context": (lambda value: _integer(value, "context", 1024, 1048576), "context"), - "--state-dir": (lambda value: _path(value, "state-dir"), "state-dir"), - "--wire": (lambda value: _model(value), "wire"), - "--target": (lambda value: value if value in _RUNTIMES else None, "target"), - "--port": (lambda value: _integer(value, "port", 1, 65535), "port"), - } - repeatable = {"--license", "--publisher"} - booleans = {"--exclude-gated", "--include-gated", "--new", "--fast", "--json"} - while index < len(tokens): - flag = tokens[index] - if flag in booleans: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - if flag in {"--exclude-gated", "--include-gated"} and {"--exclude-gated", "--include-gated"} & seen: - raise GeminiArgumentError("gated-repository flags are mutually exclusive") - seen.add(flag) - argv.append(flag) - index += 1 - elif flag in {"--refresh", "--offline"}: - if cache is not None: - raise GeminiArgumentError("--refresh and --offline are mutually exclusive") - cache = flag - argv.append(flag) - index += 1 - elif flag in repeatable: - value = _value(tokens, index, flag) - if not _SAFE_NAME.fullmatch(value): - raise GeminiArgumentError("{0} must be a safe identifier".format(flag)) - argv.extend([flag, value]) - index += 2 - elif flag in values: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - value = values[flag][0](_value(tokens, index, flag)) - if value is None: - raise GeminiArgumentError("invalid {0} value".format(values[flag][1])) - seen.add(flag) - argv.extend([flag, value]) - index += 2 - else: - raise GeminiArgumentError("unsupported Scout argument: {0}".format(flag)) - return argv - - -def _parse_adopt(tokens): - if not tokens or tokens[0] not in {"start", "resume", "status"}: - raise GeminiArgumentError("Adopt requires start, resume, or status") - action, argv, index = tokens[0], ["adopt", tokens[0]], 1 - state = None - allowed_booleans = {"--json"} if action != "start" else {"--offline", "--refresh", "--fast", "--no-network", "--measure", "--json"} - while index < len(tokens): - flag = tokens[index] - if flag == "--state": - if state is not None: - raise GeminiArgumentError("duplicate flag: --state") - state = _path(_value(tokens, index, flag), "state") - argv.extend([flag, state]) - index += 2 - elif action == "start" and flag == "--role": - role = _value(tokens, index, flag) - if role not in _ROLES: - raise GeminiArgumentError("unknown adoption role") - argv.extend([flag, role]) - index += 2 - elif action == "start" and flag == "--shortlist-limit": - argv.extend([flag, _integer(_value(tokens, index, flag), "shortlist-limit", 1, 20)]) - index += 2 - elif flag in allowed_booleans and flag not in argv: - argv.append(flag) - index += 1 - else: - raise GeminiArgumentError("unsupported Adopt argument: {0}".format(flag)) - if state is None: - raise GeminiArgumentError("Adopt requires an explicit --state path") - return argv - - -def _parse_wire(tokens): - if not tokens or tokens[0] not in {"render", "apply", "status", "rollback"}: - raise GeminiArgumentError("Wire requires render, apply, status, or rollback") - action, argv, index = tokens[0], ["wire", tokens[0]], 1 - if action in {"status", "rollback"}: - if index >= len(tokens): - raise GeminiArgumentError("Wire {0} requires a receipt path".format(action)) - argv.append(_path(tokens[index], "receipt")) - index += 1 - allowed = ( - {"--json"} - if action == "status" - else {"--confirm", "--preview-hash", "--json"} - ) - else: - if index >= len(tokens): - raise GeminiArgumentError("Wire {0} requires a model".format(action)) - argv.append(_model(tokens[index])) - index += 1 - allowed = {"--target", "--path", "--json"} - if action == "apply": - allowed.update({"--confirm", "--preview-hash", "--receipts-dir", "--endpoint"}) - path_seen = False - while index < len(tokens): - flag = tokens[index] - if flag not in allowed: - raise GeminiArgumentError("unsupported Wire argument: {0}".format(flag)) - if flag in argv: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - if flag in {"--json", "--confirm"}: - argv.append(flag) - index += 1 - else: - value = _value(tokens, index, flag) - if flag == "--target": - if value not in _RUNTIMES: - raise GeminiArgumentError("unsupported Wire target") - elif flag in {"--path", "--receipts-dir"}: - value = _path(value, flag[2:]) - path_seen = path_seen or flag == "--path" - elif flag == "--preview-hash": - if not re.fullmatch(r"[0-9a-f]{64}", value): - raise GeminiArgumentError("preview-hash must be a SHA-256 digest") - elif flag == "--endpoint": - value = _endpoint(value) - argv.extend([flag, value]) - index += 2 - if action in {"render", "apply"} and not path_seen: - raise GeminiArgumentError("Wire {0} requires --path".format(action)) - return argv - - -def _parse_bench(tokens): - if not tokens or tokens[0] not in {"run", "aggregate"}: - raise GeminiArgumentError("Bench requires the run or aggregate action") - if tokens[0] == "aggregate": - return _parse_bench_aggregate(tokens) - argv, index = ["bench", "run"], 1 - seen = set() - values = { - "--repo": (lambda value: _model(value), "repo"), - "--runtime": (lambda value: value if value in _RUNTIMES else None, "runtime"), - "--role": (lambda value: value if value in _ROLES else None, "role"), - "--runs": (lambda value: _integer(value, "runs", 1, 10), "runs"), - "--gen-tokens": (lambda value: _integer(value, "gen-tokens", 16, 2048), "gen-tokens"), - "--timeout": (lambda value: _decimal(value, "timeout", 1, 600), "timeout"), - "--export": (lambda value: _path(value, "export"), "export"), - } - while index < len(tokens): - flag = tokens[index] - if flag == "--json": - if flag in seen: - raise GeminiArgumentError("duplicate flag: --json") - seen.add(flag) - argv.append(flag) - index += 1 - elif flag in values: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - value = values[flag][0](_value(tokens, index, flag)) - if value is None: - raise GeminiArgumentError("invalid {0} value".format(values[flag][1])) - seen.add(flag) - argv.extend([flag, value]) - index += 2 - else: - raise GeminiArgumentError("unsupported Bench argument: {0}".format(flag)) - for required in ("--repo", "--runtime"): - if required not in seen: - raise GeminiArgumentError("Bench run requires {0}".format(required)) - return argv - - -def _parse_bench_aggregate(tokens): - argv, index = ["bench", "aggregate"], 1 - seen = set() - while index < len(tokens): - flag = tokens[index] - if flag == "--json": - if flag in seen: - raise GeminiArgumentError("duplicate flag: --json") - seen.add(flag) - argv.append(flag) - index += 1 - elif flag in {"--exports", "--out"}: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - seen.add(flag) - argv.extend([flag, _path(_value(tokens, index, flag), flag[2:])]) - index += 2 - else: - raise GeminiArgumentError("unsupported Bench argument: {0}".format(flag)) - if "--exports" not in seen: - raise GeminiArgumentError("Bench aggregate requires --exports") - return argv - - -def parse_gemini_arguments(capability, raw): - """Return a validated argv list; this function never runs a process.""" - tokens = _tokens(raw) - if capability == "scout": - return _parse_scout(tokens) - if capability == "adopt": - return _parse_adopt(tokens) - if capability == "wire": - return _parse_wire(tokens) - if capability == "bench": - return _parse_bench(tokens) - raise GeminiArgumentError("unknown Gemini capability") - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("capability", choices=("scout", "adopt", "wire", "bench")) - parser.add_argument("raw", help="one opaque custom-command argument string") - arguments = parser.parse_args(argv) - try: - print(json.dumps({"argv": parse_gemini_arguments(arguments.capability, arguments.raw)})) - except GeminiArgumentError as error: - parser.error(str(error)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/gemini_executor.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/gemini_executor.py deleted file mode 100644 index 3a0fc12..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/gemini_executor.py +++ /dev/null @@ -1,140 +0,0 @@ -"""No-shell executor for Gemini custom-command argument files.""" - -from __future__ import annotations - -import argparse -import json -import os -import stat -import tempfile -from pathlib import Path - -from .gemini_args import GeminiArgumentError, parse_gemini_arguments - - -MAX_ARGS_BYTES = 4096 -_ARGS_DIRECTORY = "mlx-agent-gemini-command-args" - - -class GeminiCommandError(ValueError): - """A custom-command argument file is unsafe or invalid.""" - - -def command_args_root(): - """Return the private OS-temporary root reserved for Gemini argument files.""" - root = Path(tempfile.gettempdir()) / _ARGS_DIRECTORY - try: - root.mkdir(mode=0o700, parents=True, exist_ok=True) - details = os.lstat(str(root)) - except OSError as error: - raise GeminiCommandError("private argument storage is unavailable") from error - if ( - stat.S_ISLNK(details.st_mode) - or not stat.S_ISDIR(details.st_mode) - or details.st_uid != os.getuid() - or stat.S_IMODE(details.st_mode) != 0o700 - ): - raise GeminiCommandError("private argument storage is unsafe") - return root - - -def _safe_args_path(value): - root = command_args_root() - path = Path(value) - if not path.is_absolute() or path.parent != root or ".." in path.parts or not path.name: - raise GeminiCommandError("argument file path is outside private storage") - return path - - -def _discard_if_safe(path, opened=None): - """Remove only the named regular file we opened, or a rejected symlink.""" - try: - current = os.lstat(str(path)) - if stat.S_ISLNK(current.st_mode): - os.unlink(str(path)) - elif stat.S_ISREG(current.st_mode) and (opened is None or (current.st_dev, current.st_ino) == (opened.st_dev, opened.st_ino)): - os.unlink(str(path)) - except OSError: - pass - - -def _read_args_file(value): - path = _safe_args_path(value) - opened = None - descriptor = None - try: - try: - before = os.lstat(str(path)) - except OSError as error: - raise GeminiCommandError("argument file is unavailable") from error - if ( - stat.S_ISLNK(before.st_mode) - or not stat.S_ISREG(before.st_mode) - or before.st_uid != os.getuid() - or stat.S_IMODE(before.st_mode) != 0o600 - ): - raise GeminiCommandError("argument file must be a regular file") - if before.st_size > MAX_ARGS_BYTES: - raise GeminiCommandError("argument file exceeds the maximum size") - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - try: - descriptor = os.open(str(path), flags) - opened = os.fstat(descriptor) - except OSError as error: - raise GeminiCommandError("argument file cannot be opened safely") from error - if ( - not stat.S_ISREG(opened.st_mode) - or opened.st_uid != os.getuid() - or stat.S_IMODE(opened.st_mode) != 0o600 - or (opened.st_dev, opened.st_ino) != (before.st_dev, before.st_ino) - ): - raise GeminiCommandError("argument file changed before it could be read") - data = os.read(descriptor, MAX_ARGS_BYTES + 1) - if len(data) > MAX_ARGS_BYTES: - raise GeminiCommandError("argument file exceeds the maximum size") - try: - return data.decode("utf-8") - except UnicodeDecodeError as error: - raise GeminiCommandError("argument file must be UTF-8") from error - finally: - if descriptor is not None: - os.close(descriptor) - _discard_if_safe(path, opened) - - -def execute_gemini_command(capability, args_file, core=None): - """Read, validate, execute, and discard one trusted Gemini argument file.""" - try: - raw = _read_args_file(args_file) - argv = parse_gemini_arguments(capability, raw) - except (GeminiArgumentError, GeminiCommandError) as error: - raise GeminiCommandError("Gemini command arguments were rejected") from error - if core is None: - from .cli import main as core - try: - result = core(argv) - except Exception as error: - raise GeminiCommandError("Gemini command execution failed") from error - exit_code = 0 if result is None else int(result) - return { - "status": "ok" if exit_code == 0 else "error", - "capability": capability, - "exit_code": exit_code, - } - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--capability", required=True, choices=("scout", "adopt", "wire")) - parser.add_argument("--args-file", required=True) - arguments = parser.parse_args(argv) - try: - result = execute_gemini_command(arguments.capability, arguments.args_file) - print(json.dumps(result, sort_keys=True)) - except GeminiCommandError: - parser.error("Gemini command arguments were rejected") - return result["exit_code"] - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/gemini_transport.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/gemini_transport.py deleted file mode 100644 index bf7a648..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/gemini_transport.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Deterministic contract simulation for Gemini custom-command TOML transport. - -This module simulates documented ``{{args}}`` prompt substitution for tests and -local adapters. It is not a claim about Gemini model or tool behavior. -""" - -from __future__ import annotations - -import os -import subprocess -import tempfile -from pathlib import Path - -from .gemini_executor import MAX_ARGS_BYTES, GeminiCommandError, command_args_root, execute_gemini_command - - -_OPEN = "\n" -_CLOSE = "\n" - - -class GeminiTransportError(ValueError): - """The simulated TOML transport could not safely carry command data.""" - - -def substitute_toml_args(prompt, raw_args): - """Simulate one documented TOML placeholder substitution without parsing it.""" - if not isinstance(prompt, str) or not isinstance(raw_args, str) or prompt.count("{{args}}") != 1: - raise GeminiTransportError("TOML transport template is invalid") - return prompt.replace("{{args}}", raw_args) - - -def extract_untrusted_payload(rendered_prompt): - """Extract the delimited payload exactly, preserving embedded delimiter text.""" - if not isinstance(rendered_prompt, str): - raise GeminiTransportError("TOML transport payload is invalid") - start = rendered_prompt.find(_OPEN) - end = rendered_prompt.rfind(_CLOSE) - if start < 0 or end < start + len(_OPEN): - raise GeminiTransportError("TOML transport delimiters are invalid") - return rendered_prompt[start + len(_OPEN):end] - - -def write_private_args_file(payload): - """Write opaque transport data to a 0600 private file without a shell.""" - if not isinstance(payload, str): - raise GeminiTransportError("TOML transport payload is invalid") - encoded = payload.encode("utf-8") - if len(encoded) > MAX_ARGS_BYTES: - raise GeminiTransportError("TOML transport payload exceeds the maximum size") - root = command_args_root() - descriptor = None - name = None - complete = False - try: - descriptor, name = tempfile.mkstemp(prefix="transport-", suffix=".args", dir=str(root)) - os.fchmod(descriptor, 0o600) - os.write(descriptor, encoded) - complete = True - return name - except OSError as error: - raise GeminiTransportError("private transport storage is unavailable") from error - finally: - if descriptor is not None: - os.close(descriptor) - if name is not None and not complete: - try: - os.unlink(name) - except OSError: - pass - - -def build_structured_executor_call(skill_dir, capability, args_file): - """Build the only permitted non-shell Gemini executor process shape.""" - skill_path = Path(skill_dir) - argument_path = Path(args_file) - root = command_args_root() - if ( - capability not in {"scout", "adopt", "wire"} - or not skill_path.is_absolute() - or not argument_path.is_absolute() - or argument_path.parent != root - or not argument_path.name - ): - raise GeminiTransportError("structured executor fields are invalid") - return { - "program": "python3", - "argv": ["-m", "mlx_agent.gemini_executor", "--capability", capability, "--args-file", str(argument_path)], - "env": {"PYTHONPATH": str(skill_path / "src")}, - "shell": False, - } - - -def run_structured_executor_call(call, extra_env=None): - """Run a validated structured executor fixture through subprocess without a shell.""" - if not isinstance(call, dict) or set(call) != {"program", "argv", "env", "shell"}: - raise GeminiTransportError("structured executor fields are invalid") - program, argv, env, shell = call["program"], call["argv"], call["env"], call["shell"] - if ( - program != "python3" - or not isinstance(argv, list) - or len(argv) != 6 - or argv[:3] != ["-m", "mlx_agent.gemini_executor", "--capability"] - or argv[3] not in {"scout", "adopt", "wire"} - or argv[4] != "--args-file" - or not isinstance(argv[5], str) - or not isinstance(env, dict) - or set(env) != {"PYTHONPATH"} - or not isinstance(env["PYTHONPATH"], str) - ): - raise GeminiTransportError("structured executor fields are invalid") - if shell is not False or not env["PYTHONPATH"]: - raise GeminiTransportError("structured executor fields are invalid") - environment = dict(os.environ) - if extra_env is not None: - environment.update(extra_env) - environment.update(env) - return subprocess.run([program] + argv, env=environment, shell=False, text=True, capture_output=True, check=False) - - -def simulate_toml_transport(prompt, raw_args, capability, core=None): - """Simulate TOML prompt transport and dispatch only through the executor.""" - rendered = substitute_toml_args(prompt, raw_args) - payload = extract_untrusted_payload(rendered) - args_file = write_private_args_file(payload) - try: - return execute_gemini_command(capability, args_file, core=core) - except GeminiCommandError as error: - raise GeminiTransportError("TOML transport payload was rejected") from error diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/host.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/host.py deleted file mode 100644 index a7ee4c7..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/host.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Local host inventory probes used by Scout.""" - -import subprocess -from dataclasses import asdict, dataclass - - -@dataclass(frozen=True) -class HostInventory: - ram_gb: object = None - chip: object = None - ollama: bool = False - lmstudio: bool = False - - @staticmethod - def runtime_supports(runtime, role): - """Whether a runtime can serve the requested model class, independent of install state.""" - if runtime == "mlx-vlm": - return role == "vision" - if role == "vision": - return False - return runtime in ("mlx_lm", "ollama", "lmstudio", "litellm") - - @classmethod - def detect(cls, http_get, check_output=None): - return cls.inspect(http_get, check_output=check_output)[0] - - @classmethod - def inspect(cls, http_get, check_output=None): - """Return bounded local inventory plus classified unavailable-probe warnings.""" - check_output = check_output or subprocess.check_output - values = {"ram_gb": None, "chip": None, "ollama": False, "lmstudio": False} - warnings = [] - try: - values["ram_gb"] = round(int(check_output(["sysctl", "-n", "hw.memsize"]).strip()) / 1073741824) - except Exception: - warnings.append({"code": "host_probe_unavailable", "probe": "memory", "message": "Host memory probe unavailable."}) - try: - values["chip"] = check_output(["sysctl", "-n", "machdep.cpu.brand_string"], text=True).strip() - except Exception: - warnings.append({"code": "host_probe_unavailable", "probe": "chip", "message": "Host chip probe unavailable."}) - try: - http_get("http://127.0.0.1:11434/api/tags", timeout=3) - values["ollama"] = True - except Exception: - warnings.append({"code": "runtime_probe_unavailable", "probe": "ollama", "message": "Local runtime probe unavailable."}) - try: - http_get("http://localhost:1234/v1/models", timeout=3) - values["lmstudio"] = True - except Exception: - warnings.append({"code": "runtime_probe_unavailable", "probe": "lmstudio", "message": "Local runtime probe unavailable."}) - return cls(**values), warnings - - def to_dict(self): - return asdict(self) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/huggingface.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/huggingface.py deleted file mode 100644 index 3a84ed5..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/huggingface.py +++ /dev/null @@ -1,457 +0,0 @@ -"""Dependency-free Hugging Face Hub access for Scout.""" - -import http.client -import json -import queue -import re -import threading -import time -import urllib.parse - -from .contextfit import extract_architecture -from .models import ( - REASONER_HINTS, - TEMPLATE_REASON, - TEMPLATE_TOOL_USE, - TOOL_USE_HINTS, - TOOL_USE_TAGS, -) - - -HF_API = "https://huggingface.co/api/models" -HF_DATASETS_API = "https://huggingface.co/api/datasets" -HF_API_HOST = "huggingface.co" -UA = {"User-Agent": "mlx-scout/0.2 (+https://github.com/cavi-ai/mlx-agent)"} -HF_RESPONSE_MAX_BYTES = 8 * 1024 * 1024 -_HTTP_READ_CHUNK_BYTES = 64 * 1024 -HF_CARD_HOST = "huggingface.co" -MODEL_CARD_MAX_BYTES = 512 * 1024 -_CARD_PATH_SUFFIX = "/raw/main/README.md" - - -def _is_allowed_api_path(path): - if path == "/api/models" or path.startswith("/api/models/"): - return True - if path == "/api/datasets" or path.startswith("/api/datasets/"): - return True - return False - - -def _is_valid_card_path(path): - """Accept model or dataset README raw paths on the fixed card host.""" - if not path.endswith(_CARD_PATH_SUFFIX): - return False - slash_count = path.count("/") - if slash_count == 5 and not path.startswith("/datasets/"): - return True - if slash_count == 6 and path.startswith("/datasets/"): - return True - return False - - -def http_json( - url, - timeout=10.0, - connection_factory=None, - clock=time.monotonic, - completion_wait=None, -): - """Read bounded JSON from the fixed Hugging Face API under one deadline.""" - parsed = urllib.parse.urlsplit(url) - if parsed.scheme != "https" or parsed.hostname != HF_API_HOST: - raise ValueError("Hugging Face URL must use the fixed HTTPS API host") - if parsed.port not in (None, 443): - raise ValueError("Hugging Face URL must use the default HTTPS port") - if parsed.username is not None or parsed.password is not None: - raise ValueError("Hugging Face URL must not contain credentials") - if parsed.fragment: - raise ValueError("Hugging Face URL must not contain a fragment") - if not _is_allowed_api_path(parsed.path): - raise ValueError("Hugging Face URL must target the models or datasets API") - - deadline = clock() + timeout - remaining = _deadline_remaining(deadline, clock) - if connection_factory is None: - connection = http.client.HTTPSConnection( - HF_API_HOST, - 443, - timeout=remaining, - ) - else: - connection = connection_factory(HF_API_HOST, 443, remaining) - target = parsed.path or "/" - if parsed.query: - target = "{0}?{1}".format(target, parsed.query) - return _run_http_worker( - connection, - lambda: _http_json_operation(connection, target, deadline, clock), - deadline, - clock, - completion_wait, - ) - - -def _run_http_worker(connection, operation, deadline, clock, completion_wait): - results = queue.Queue(maxsize=1) - completion = threading.Event() - - def run(): - try: - result = (True, operation()) - except BaseException as error: # noqa: BLE001 - propagated to caller - result = (False, error) - results.put_nowait(result) - completion.set() - - worker = threading.Thread( - target=run, - name="mlx-agent-huggingface-request", - daemon=True, - ) - try: - worker.start() - except BaseException: - connection.close() - raise - - try: - remaining = _deadline_remaining(deadline, clock) - except TimeoutError: - connection.close() - raise - try: - if completion_wait is None: - completed = completion.wait(remaining) - else: - completed = completion_wait(completion, remaining) - except BaseException: - connection.close() - raise - if not completed: - connection.close() - raise TimeoutError("Hugging Face request exceeded its overall deadline") - - try: - remaining = _deadline_remaining(deadline, clock) - except TimeoutError: - connection.close() - raise - worker.join(timeout=remaining) - if worker.is_alive(): - connection.close() - raise TimeoutError("Hugging Face request exceeded its overall deadline") - - succeeded, value = results.get_nowait() - if succeeded: - return value - raise value - - -def _http_json_operation(connection, target, deadline, clock): - try: - connection.request("GET", target, headers=UA) - _set_connection_timeout( - connection, - _deadline_remaining(deadline, clock), - ) - response = connection.getresponse() - _deadline_remaining(deadline, clock) - if 300 <= response.status < 400: - raise http.client.HTTPException( - "redirect responses are not allowed for Hugging Face requests" - ) - if not 200 <= response.status < 300: - raise http.client.HTTPException( - "Hugging Face returned HTTP status {0}".format(response.status) - ) - content_length = response.headers.get("Content-Length") - if content_length is not None: - try: - declared_length = int(content_length) - except (TypeError, ValueError) as error: - raise ValueError("invalid Hugging Face Content-Length") from error - if declared_length < 0: - raise ValueError("invalid Hugging Face Content-Length") - if declared_length > HF_RESPONSE_MAX_BYTES: - raise ValueError("Hugging Face response exceeds size limit") - body = _read_bounded_body( - response, - connection, - deadline, - clock, - ) - return json.loads(body.decode("utf-8")) - finally: - connection.close() - - -def http_card_text( - url, - timeout=8.0, - connection_factory=None, - clock=time.monotonic, - completion_wait=None, -): - """Read bounded README/model-card text from the fixed card host.""" - parsed = urllib.parse.urlsplit(url) - if parsed.scheme != "https" or parsed.hostname != HF_CARD_HOST: - raise ValueError("card URL must use the fixed HTTPS card host") - if parsed.port not in (None, 443): - raise ValueError("card URL must use the default HTTPS port") - if parsed.username is not None or parsed.password is not None: - raise ValueError("card URL must not contain credentials") - if parsed.fragment: - raise ValueError("card URL must not contain a fragment") - if not _is_valid_card_path(parsed.path): - raise ValueError( - "card URL must target //raw/main/README.md " - "or datasets///raw/main/README.md" - ) - - deadline = clock() + timeout - remaining = _deadline_remaining(deadline, clock) - if connection_factory is None: - connection = http.client.HTTPSConnection(HF_CARD_HOST, 443, timeout=remaining) - else: - connection = connection_factory(HF_CARD_HOST, 443, remaining) - target = parsed.path - if parsed.query: - target = "{0}?{1}".format(target, parsed.query) - return _run_http_worker( - connection, - lambda: _http_text_operation(connection, target, deadline, clock), - deadline, - clock, - completion_wait, - ) - - -def _http_text_operation(connection, target, deadline, clock): - try: - connection.request("GET", target, headers=UA) - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - response = connection.getresponse() - _deadline_remaining(deadline, clock) - if 300 <= response.status < 400: - raise http.client.HTTPException( - "redirect responses are not allowed for card requests" - ) - if not 200 <= response.status < 300: - raise http.client.HTTPException( - "card host returned HTTP status {0}".format(response.status) - ) - content_length = response.headers.get("Content-Length") - if content_length is not None: - try: - declared_length = int(content_length) - except (TypeError, ValueError) as error: - raise ValueError("invalid card Content-Length") from error - if declared_length < 0 or declared_length > MODEL_CARD_MAX_BYTES: - raise ValueError("card response exceeds size limit") - body = _read_bounded_body( - response, - connection, - deadline, - clock, - max_bytes=MODEL_CARD_MAX_BYTES, - ) - return body.decode("utf-8", errors="replace") - finally: - connection.close() - - -def _read_bounded_body(response, connection, deadline, clock, max_bytes=HF_RESPONSE_MAX_BYTES): - chunks = [] - total = 0 - while True: - _set_connection_timeout( - connection, - _deadline_remaining(deadline, clock), - ) - chunk = response.read( - min(_HTTP_READ_CHUNK_BYTES, max_bytes - total + 1) - ) - _deadline_remaining(deadline, clock) - if not chunk: - break - total += len(chunk) - if total > max_bytes: - raise ValueError("Hugging Face response exceeds size limit") - chunks.append(chunk) - return b"".join(chunks) - - -def _deadline_remaining(deadline, clock): - remaining = deadline - clock() - if remaining <= 0: - raise TimeoutError("Hugging Face request exceeded its overall deadline") - return remaining - - -def _set_connection_timeout(connection, timeout): - sock = getattr(connection, "sock", None) - if sock is not None: - sock.settimeout(timeout) - - -class HuggingFaceClient: - def __init__(self, http_get=http_json, card_get=http_card_text): - self._http_get = http_get - self._card_get = card_get - - @property - def http_get(self): - return self._http_get - - def fetch_model_card(self, repo, timeout=8): - """Return bounded README/model-card text, or None on any failure.""" - quoted = "/".join(urllib.parse.quote(part) for part in repo.split("/")) - url = "https://{0}/{1}/raw/main/README.md".format(HF_CARD_HOST, quoted) - try: - return self._card_get(url, timeout=timeout) - except Exception: - return None - - def fetch_dataset_card(self, repo, timeout=8): - """Return bounded dataset README text, or None on any failure.""" - quoted = "/".join(urllib.parse.quote(part) for part in repo.split("/")) - url = "https://{0}/datasets/{1}/raw/main/README.md".format(HF_CARD_HOST, quoted) - try: - return self._card_get(url, timeout=timeout) - except Exception: - return None - - @staticmethod - def list_models_url(sort="trendingScore", limit_fetch=300): - query = urllib.parse.urlencode({"filter": "mlx", "sort": sort, "direction": "-1", "limit": limit_fetch}) - return "{0}?{1}".format(HF_API, query) - - @staticmethod - def list_adapters_url(search="", limit_fetch=20): - params = { - "filter": "peft", - "sort": "downloads", - "direction": "-1", - "limit": limit_fetch, - } - if search: - params["search"] = search - return "{0}?{1}".format(HF_API, urllib.parse.urlencode(params)) - - @staticmethod - def list_datasets_url(search="", limit_fetch=20): - params = { - "sort": "downloads", - "direction": "-1", - "limit": limit_fetch, - } - if search: - params["search"] = search - return "{0}?{1}".format(HF_DATASETS_API, urllib.parse.urlencode(params)) - - def list_models(self, sort="trendingScore", limit_fetch=300): - return self._http_get(self.list_models_url(sort=sort, limit_fetch=limit_fetch)) - - def list_adapters(self, search="", limit_fetch=20, timeout=10): - """List PEFT/LoRA adapter model rows from the Hub (read-only).""" - rows = self._http_get( - self.list_adapters_url(search=search, limit_fetch=limit_fetch), - timeout=timeout, - ) - return rows if isinstance(rows, list) else [] - - def list_datasets(self, search="", limit_fetch=20, timeout=10): - """List dataset rows from the Hub (read-only).""" - rows = self._http_get( - self.list_datasets_url(search=search, limit_fetch=limit_fetch), - timeout=timeout, - ) - return rows if isinstance(rows, list) else [] - - def inspect_model_metadata(self, repo, timeout=8): - """Inspect model metadata without fetching the recursive repository tree.""" - quoted = urllib.parse.quote(repo) - model_url = "{0}/{1}".format(HF_API, quoted) - tree_url = "{0}/{1}/tree/main?recursive=true".format(HF_API, quoted) - output = { - "weight_bytes": None, - "tags": [], - "gated": None, - "license": None, - "reasoning": None, - "reason_src": None, - "tool_use": None, - "tool_use_src": None, - "tool_use_confidence": "none", - "params_total": None, - "architecture": None, - "metadata_available": False, - "tree_available": False, - "metadata_url": model_url, - "tree_url": tree_url, - "repository_url": "https://huggingface.co/{0}".format(repo), - } - try: - metadata = self._http_get(model_url, timeout=timeout) - output["metadata_available"] = True - tags = metadata.get("tags", []) or [] - output["tags"] = tags - output["gated"] = bool(metadata.get("gated")) - config = metadata.get("config") or {} - card_data = metadata.get("cardData") or {} - output["license"] = card_data.get("license") or next((tag.split("license:", 1)[1] for tag in tags if tag.startswith("license:")), None) - output["params_total"] = (metadata.get("safetensors") or {}).get("total") - output["architecture"] = extract_architecture(config) - template = ((config.get("tokenizer_config") or {}).get("chat_template") or "") - lower_tags = [tag.lower() for tag in tags] - normalized_tags = { - re.sub(r"\s+", "-", str(tag).strip().lower()) - for tag in tags - } - if TEMPLATE_REASON.search(template): - output["reasoning"], output["reason_src"] = True, "chat_template" - elif any(tag in ("reasoning", "thinking", "chain-of-thought") for tag in lower_tags): - output["reasoning"], output["reason_src"] = True, "tags" - elif REASONER_HINTS.search(repo): - output["reasoning"], output["reason_src"] = True, "name" - else: - output["reasoning"], output["reason_src"] = False, "checked" - if TEMPLATE_TOOL_USE.search(template): - output.update({ - "tool_use": True, - "tool_use_src": "chat_template", - "tool_use_confidence": "explicit", - }) - elif normalized_tags & TOOL_USE_TAGS: - output.update({ - "tool_use": True, - "tool_use_src": "tags", - "tool_use_confidence": "explicit", - }) - elif TOOL_USE_HINTS.search(repo): - output.update({ - "tool_use": True, - "tool_use_src": "name", - "tool_use_confidence": "weak", - }) - else: - output.update({ - "tool_use": False, - "tool_use_src": "checked", - "tool_use_confidence": "explicit", - }) - except Exception: - pass - return output - - def inspect_model(self, repo): - output = self.inspect_model_metadata(repo, timeout=8) - tree_url = output["tree_url"] - try: - tree = self._http_get(tree_url, timeout=8) - output["tree_available"] = True - weights = sum(item.get("size", 0) for item in tree if item.get("path", "").endswith((".safetensors", ".gguf", ".bin"))) - output["weight_bytes"] = weights or None - except Exception: - pass - return output diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/installer.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/installer.py deleted file mode 100644 index 06819e5..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/installer.py +++ /dev/null @@ -1,696 +0,0 @@ -"""Scoped, receipt-owned installer built on the Transaction safety boundary.""" - -from __future__ import annotations - -import hashlib -import json -import os -import uuid -from dataclasses import dataclass, field -from pathlib import Path - -from .providers import detect_providers -from .transactions import ( - LegacyLockError, Receipt, Transaction, _assert_safe_directory, _assert_safe_target, - _atomic_in_directory, _read_regular, _read_target, _walk_directory, - _rollback_receipt_hash, legacy_lock_problem, rollback, -) -from .wiring import redact_secrets - - -class InstallerConflictError(ValueError): - """A requested operation would touch an unowned or modified artifact.""" - - -class _ArtifactAdapter: - version = "installer-1.0" - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("installer artifacts must be UTF-8 text") - redacted = redact_secrets(content) - try: - contains_persisted_secret = json.loads(redacted) != json.loads(content) - except (TypeError, ValueError, json.JSONDecodeError): - contains_persisted_secret = redacted != content - if contains_persisted_secret: - raise ValueError("installer artifacts must not contain persisted secrets") - return True - - -class _SourceCodeArtifactAdapter: - """Source modules may contain secret-pattern examples but never persisted values.""" - - version = "installer-source-1.0" - secret_scan_exempt = True - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("installer artifacts must be UTF-8 text") - return True - - -@dataclass(frozen=True) -class _PlannedTransaction: - provider_id: str - changes: tuple - preview_hash: str - destinations: tuple - transaction_id: str = None - expected_receipt_path: str = None - child_index: int = None - - -@dataclass -class InstallerReceipt: - status: str - receipts: list = field(default_factory=list) - receipt_path: str = "" - targets: list = field(default_factory=list) - batch_path: str = "" - - def to_dict(self): - return { - "status": self.status, - "receipt_path": self.receipt_path or None, - "batch_path": self.batch_path or None, - "targets": list(self.targets), - "receipts": [_receipt_data(item) for item in self.receipts], - } - - -@dataclass -class InstallPlan: - action: str - provider_ids: tuple - scope: str - project_root: Path - preview: dict - transactions: list = field(default_factory=list) - rollback_receipts: list = field(default_factory=list) - uninstall_hashes: dict = field(default_factory=dict) - restore_changes: tuple = field(default_factory=tuple) - binding: dict = field(default_factory=dict) - compatibility: tuple = field(default_factory=tuple) - noop: bool = False - - def to_dict(self): - return { - "action": self.action, - "providers": list(self.provider_ids), - "scope": self.scope, - "project_root": str(self.project_root), - "preview": self.preview, - "compatibility": list(self.compatibility), - "noop": self.noop, - } - - -def _sha256(content): - return hashlib.sha256(content).hexdigest() - - -def _receipt_data(receipt): - value = receipt.to_dict() - value["receipt_path"] = receipt.receipt_path - return value - - -def _digest(value): - return _sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")) - - -def _read_optional_target(path): - """Read an artifact without following links; missing parent paths mean absent.""" - try: - content, exists, _mode = _read_target(path) - return content if exists else None - except ValueError as error: - if str(error).startswith("path component does not exist:"): - return None - raise - - -def _adapter_for_artifact(artifact): - is_runtime_source = "src" in artifact.source.parts and "mlx_agent" in artifact.source.parts - return _SourceCodeArtifactAdapter() if is_runtime_source else _ArtifactAdapter() - - -class Installer: - """Plan first; execute only the exact reviewed transaction previews.""" - - def __init__(self, registry, project_root=None, executable_lookup=None, env=None, - transaction_factory=None, rollback_func=None, probe_runner=None): - self.registry = registry - self.project_root = Path(project_root).resolve() if project_root else Path.cwd().resolve() - self.executable_lookup = executable_lookup - self.env = env - self.transaction_factory = transaction_factory or Transaction - self.rollback_func = rollback_func or rollback - self.probe_runner = probe_runner - - def detected(self): - return detect_providers( - self.registry.definitions().values(), self.env, - executable_lookup=self.executable_lookup, - probe_runner=self.probe_runner, - ) - - def _compatibility(self, provider_ids): - definitions = self.registry.definitions() - return tuple( - item.to_dict() for item in detect_providers( - [definitions[provider_id] for provider_id in provider_ids], - self.env, - executable_lookup=self.executable_lookup, - probe_runner=self.probe_runner, - ) - ) - - def plan(self, action, providers, scope="user", project_root=None): - if action not in {"install", "update", "uninstall", "doctor"}: - raise ValueError("unsupported installer action: {0}".format(action)) - if scope not in {"user", "project"}: - raise ValueError("scope must be 'user' or 'project'") - project = Path(project_root or self.project_root).resolve() - if scope == "project" and not project.is_dir(): - raise ValueError("project root must be an existing directory") - definitions = self.registry.definitions() - selected = tuple(providers or ()) - if not selected: - raise ValueError("provider selection is required; inspect detected providers first") - if len(selected) != len(set(selected)) or any(item not in definitions for item in selected): - raise ValueError("providers must be unique supported provider IDs") - if action == "doctor": - return self._doctor_plan(selected, definitions, scope, project) - if action == "uninstall": - return self._uninstall_plan(selected, definitions, scope, project) - return self._install_plan(action, selected, definitions, scope, project) - - def execute(self, plan, confirmed=False): - if plan.action == "doctor": - return self._doctor(plan) - if plan.noop: - return InstallerReceipt("noop") - if confirmed is not True and confirmed != plan.preview["preview_hash"]: - raise PermissionError("explicit confirmation for this installer preview is required") - self._verify_plan_binding(plan) - batch_path = self._create_batch(plan) - if plan.action == "uninstall": - return self._execute_uninstall(plan, batch_path) - return self._execute_install(plan, batch_path, self._batch_operations(plan, batch_path)) - - def _execute_install(self, plan, batch_path, operations): - successful, all_receipts = [], [] - try: - prepared = [(operation, *self._repreview(operation, plan.scope, plan.project_root)) for operation in operations] - except PermissionError as error: - self._batch_update(batch_path, "rolled_back", successful, error=str(error)) - raise - try: - for operation, transaction, preview in prepared: - self._batch_update(batch_path, "pending", all_receipts, active=self._active_operation(operation)) - receipt = transaction.apply(operation.preview_hash) - all_receipts.append(receipt) - self._batch_update(batch_path, "pending", all_receipts) - if receipt.status != "applied": - raise InstallerConflictError("provider transaction did not apply: {0}".format(receipt.status)) - successful.append(receipt) - self._batch_update(batch_path, "complete", successful) - return InstallerReceipt("applied", all_receipts, successful[-1].receipt_path, [target for item in successful for target in item.targets], str(batch_path)) - except LegacyLockError as error: - self._batch_update(batch_path, "rolled_back", all_receipts, error=str(error)) - raise InstallerConflictError(str(error)) - except Exception as error: - recovered, unresolved = self._compensate_install(all_receipts) - child_requires_recovery = any(item.status in {"recovery_required", "rollback_failed"} for item in all_receipts) - status = "rolled_back" if recovered and not child_requires_recovery else "recovery_required" - remediation = unresolved or ([{"paths": list(all_receipts[-1].targets), "message": "Child receipt requires manual recovery: {0}".format(all_receipts[-1].receipt_path)}] if child_requires_recovery and all_receipts else []) - self._batch_update(batch_path, status, all_receipts, error=str(error), remediation=remediation) - return InstallerReceipt(status, all_receipts, all_receipts[-1].receipt_path if all_receipts else "", [target for item in all_receipts for target in item.targets], str(batch_path)) - - def _execute_uninstall(self, plan, batch_path): - completed, attempted = [], [] - try: - for receipt in plan.rollback_receipts: - self._batch_update(batch_path, "pending", attempted, active={"provider": "uninstall", "transaction_id": receipt.transaction_id, "expected_receipt_path": receipt.receipt_path}) - result = self.rollback_func( - receipt.receipt_path, - expected_after_hashes=receipt.after_hashes, - expected_receipt_hash=_rollback_receipt_hash(receipt), - ) - recorded = self._rollback_result(result, receipt, plan.scope, plan.project_root) - attempted.append(recorded) - self._batch_update(batch_path, "pending", attempted) - if recorded.status != "rolled_back": - raise InstallerConflictError("uninstall rollback did not complete: {0}".format(recorded.status)) - completed.append(recorded) - self._cleanup_empty_artifact_dirs(plan) - self._batch_update(batch_path, "complete", attempted) - return InstallerReceipt("rolled_back", attempted, attempted[-1].receipt_path if attempted else "", [target for item in attempted for target in item.targets], str(batch_path)) - except Exception as error: - unresolved_children = [item for item in attempted if item.status != "rolled_back"] - if not completed: - status = "recovery_required" if unresolved_children else "rolled_back" - remediation = self._unresolved_child_remediation(unresolved_children) - self._batch_update(batch_path, status, attempted, error=str(error), remediation=remediation) - message = "uninstall recovery is required" if unresolved_children else "uninstall aborted before mutation" - raise InstallerConflictError("{0}: {1}".format(message, error)) - recovered, unresolved = self._compensate_uninstall(plan.restore_changes, completed, plan.scope, plan.project_root) - status = "rolled_back" if recovered and not unresolved_children else "recovery_required" - remediation = self._unresolved_child_remediation(unresolved_children) + unresolved - self._batch_update(batch_path, status, attempted, error=str(error), remediation=remediation) - if status == "recovery_required": - raise InstallerConflictError("uninstall recovery is required: {0}".format(error)) - return InstallerReceipt(status, attempted, attempted[-1].receipt_path if attempted else "", [target for item in attempted for target in item.targets], str(batch_path)) - - def _install_plan(self, action, selected, definitions, scope, project): - transactions, summary = [], [] - for provider_id in selected: - definition = definitions[provider_id] - changes = [] - for artifact in definition.artifacts: - if not definition.applies_to(scope, artifact): - continue - target = definition.artifact_destination(scope, project, artifact) - desired_bytes, source_exists, _source_mode = _read_target(artifact.source) - if not source_exists: - raise ValueError("provider artifact source disappeared: {0}".format(artifact.source)) - try: - desired = desired_bytes.decode("utf-8") - except UnicodeDecodeError as error: - raise ValueError("provider artifact source must be UTF-8") from error - current = _read_optional_target(target) - if current is not None: - history = self._artifact_history(definition, target, scope, project) - if not history or _sha256(current) != history[-1].after_hashes[str(target)]: - raise InstallerConflictError("refusing to overwrite unowned or modified artifact: {0}".format(target)) - if current == desired.encode("utf-8"): - continue - adapter = _adapter_for_artifact(artifact) - changes.append({"path": str(target), "content": desired, "adapter": adapter}) - if changes: - transaction = self._new_transaction(scope, project) - preview = transaction.preview(changes) - transactions.append(_PlannedTransaction(provider_id, tuple(changes), preview["preview_hash"], tuple(str(item["path"]) for item in changes))) - summary.append({"provider": provider_id, "changes": preview["changes"], "diff": preview["diff"]}) - return self._make_plan(action, selected, scope, project, transactions=transactions, summary=summary, definitions=definitions) - - def _uninstall_plan(self, selected, definitions, scope, project): - steps, summary, uninstall_hashes, restore_changes = [], [], {}, {} - seen = set() - for provider_id in selected: - definition = definitions[provider_id] - provider_steps = [] - for artifact in definition.artifacts: - if not definition.applies_to(scope, artifact): - continue - target = definition.artifact_destination(scope, project, artifact) - history = self._artifact_history(definition, target, scope, project) - if not history: - continue - oldest, latest = history[0], history[-1] - target_name = str(target) - if oldest.backup_paths.get(target_name) is not None: - raise InstallerConflictError("artifact was not receipt-owned at installation: {0}".format(target)) - location = _assert_safe_target(target) - content = _read_regular(location) - if _sha256(content) != latest.after_hashes[target_name]: - raise InstallerConflictError("refusing to remove user-modified artifact: {0}".format(target)) - uninstall_hashes[target_name] = latest.after_hashes[target_name] - restore_changes.setdefault(provider_id, []).append({ - "path": target_name, - "content": content.decode("utf-8"), - "adapter": _adapter_for_artifact(artifact), - }) - provider_steps.extend(history) - for receipt in sorted({item.receipt_path: item for item in provider_steps}.values(), key=lambda item: item.timestamp, reverse=True): - if receipt.receipt_path not in seen: - seen.add(receipt.receipt_path) - steps.append(receipt) - summary.append({"provider": provider_id, "changes": len(provider_steps), "diff": "remove receipt-owned artifacts"}) - return self._make_plan("uninstall", selected, scope, project, rollback_receipts=steps, - uninstall_hashes=uninstall_hashes, restore_changes=tuple((provider_id, tuple(changes)) for provider_id, changes in restore_changes.items()), - summary=summary, definitions=definitions) - - def _doctor_plan(self, selected, definitions, scope, project): - return self._make_plan("doctor", selected, scope, project, definitions=definitions) - - def _cleanup_empty_artifact_dirs(self, plan): - """Remove only emptied receipt-owned descendants; retain host-owned roots.""" - definitions = self.registry.definitions() - for provider_id in plan.provider_ids: - definition = definitions[provider_id] - provider_root = definition.destination(plan.scope, plan.project_root) - for artifact in definition.artifacts: - if not definition.applies_to(plan.scope, artifact): - continue - destination = definition.artifact_destination(plan.scope, plan.project_root, artifact) - try: - relative = destination.relative_to(provider_root) - except ValueError: - # Some providers own companion artifacts (for example, - # Gemini command files) beneath a host-wide directory - # rather than their package directory. Removing empty - # parents there could remove host-owned structure. - continue - stop = provider_root / relative.parts[0] - current = destination.parent - while current != stop: - try: - _parent, descriptor = _walk_directory(current.parent) - try: - os.rmdir(current.name, dir_fd=descriptor) - os.fsync(descriptor) - finally: - os.close(descriptor) - except (FileNotFoundError, OSError, ValueError): - break - current = current.parent - - def _doctor(self, plan): - problems, checked, provider_states = [], [], [] - definitions = self.registry.definitions() - detections = { - item.id: item for item in detect_providers( - [definitions[provider_id] for provider_id in plan.provider_ids], - self.env, - executable_lookup=self.executable_lookup, - probe_runner=self.probe_runner, - ) - } - for provider_id in plan.provider_ids: - definition = definitions[provider_id] - artifact_problems = [] - for artifact in definition.artifacts: - if not definition.applies_to(plan.scope, artifact): - continue - target = definition.artifact_destination(plan.scope, plan.project_root, artifact) - checked.append(str(target)) - try: - safe_root = _assert_safe_directory(plan.project_root if plan.scope == "project" else definition.user_root) - location = _assert_safe_target(target) - location.relative_to(safe_root) - history = self._artifact_history(definition, target, plan.scope, plan.project_root) - if not history or _sha256(_read_regular(location)) != history[-1].after_hashes[str(target)]: - raise ValueError("artifact hash is not receipt-owned") - except (OSError, ValueError): - problem = {"provider": provider_id, "code": "artifact_invalid", "path": str(target)} - problems.append(problem) - artifact_problems.append(problem) - detection = detections[provider_id] - if definition.install_mode == "portable": - integration_state = "portable" - elif definition.install_mode == "staged": - integration_state = "staged" - problems.append({ - "provider": provider_id, - "code": "provider_staged", - "message": "artifacts are staged but native host visibility is not proven", - }) - elif detection.state == "native-visible": - integration_state = "native-visible" - else: - integration_state = "staged" - code = "provider_unsupported" if detection.state == "unsupported" else "provider_absent" - problems.append({ - "provider": provider_id, - "code": code, - "message": detection.error or "provider compatibility could not be proven", - }) - provider_states.append({ - "provider": provider_id, - "artifact_state": "invalid" if artifact_problems else "installed", - "integration_state": integration_state, - "compatibility": detection.to_dict(), - }) - problems.extend(self._batch_problems(plan.scope, plan.project_root)) - lock_root = self._receipts_dir(plan.scope, plan.project_root) / "locks" - for provider_id in plan.provider_ids: - definition = definitions[provider_id] - targets = [ - str(definition.artifact_destination(plan.scope, plan.project_root, artifact)) - for artifact in definition.artifacts - if definition.applies_to(plan.scope, artifact) - ] - try: - legacy_problem = legacy_lock_problem(targets, lock_root) - except ValueError as error: - problems.append({"provider": provider_id, "code": "legacy_lock_invalid", "message": str(error)}) - continue - problems.extend({"provider": provider_id, **item} for item in legacy_problem) - return {"healthy": not problems, "problems": problems, "checked": checked, - "providers": provider_states, - "cooperative_concurrency": "Transaction advisory locks protect cooperative writers."} - - def _batch_problems(self, scope, project): - batches = self._receipts_dir(scope, project) / "batches" - try: - safe_batches = _assert_safe_directory(batches) - except ValueError: - return [] - problems = [] - for path in safe_batches.glob("*/batch.json"): - try: - batch = self._read_batch(path) - except (OSError, ValueError, json.JSONDecodeError): - problems.append({"code": "batch_recovery_required", "path": str(path)}) - continue - if batch["status"] in {"pending", "recovery_required"}: - active = batch.get("active") - if isinstance(active, dict) and active.get("expected_receipt_path"): - expected = active["expected_receipt_path"] - try: - child = _assert_safe_target(expected) - child_value = json.loads(_read_regular(child).decode("utf-8")) - if child_value.get("transaction_id") != active.get("transaction_id"): - raise ValueError("active child receipt does not match batch transaction") - except (OSError, ValueError, json.JSONDecodeError): - expected = "missing or untrusted active receipt: {0}".format(expected) - problems.append({"code": "batch_recovery_required", "path": str(path), "active_receipt": expected}) - else: - problems.append({"code": "batch_recovery_required", "path": str(path)}) - return problems - - def _make_plan(self, action, selected, scope, project, transactions=(), rollback_receipts=(), - uninstall_hashes=None, restore_changes=(), summary=(), definitions=None): - definitions = definitions or self.registry.definitions() - compatibility = self._compatibility(selected) - destinations = [{"provider": provider_id, "targets": [str(definitions[provider_id].artifact_destination(scope, project, artifact)) for artifact in definitions[provider_id].artifacts if definitions[provider_id].applies_to(scope, artifact)]} for provider_id in selected] - binding = { - "action": action, "providers": list(selected), "scope": scope, "project": str(project), - "destinations": destinations, - "inner_previews": [{"provider": item.provider_id, "hash": item.preview_hash, "targets": list(item.destinations)} for item in transactions], - "rollback_chain": [self._receipt_identity(item) for item in rollback_receipts], - "compatibility": list(compatibility), - } - preview = {"changes": sum(item["changes"] for item in summary), "providers": list(selected), - "diff": "\n".join(item["diff"] for item in summary), "preview_hash": _digest(binding), - "requires_confirmation": action != "doctor"} - return InstallPlan(action, selected, scope, project, preview, list(transactions), list(rollback_receipts), - dict(uninstall_hashes or {}), tuple(restore_changes), binding, compatibility, - noop=not transactions and not rollback_receipts) - - def _verify_plan_binding(self, plan): - definitions = self.registry.definitions() - binding = { - "action": plan.action, "providers": list(plan.provider_ids), "scope": plan.scope, - "project": str(plan.project_root), - "destinations": [{"provider": provider_id, "targets": [str(definitions[provider_id].artifact_destination(plan.scope, plan.project_root, artifact)) for artifact in definitions[provider_id].artifacts if definitions[provider_id].applies_to(plan.scope, artifact)]} for provider_id in plan.provider_ids], - "inner_previews": [{"provider": item.provider_id, "hash": item.preview_hash, "targets": list(item.destinations)} for item in plan.transactions], - "rollback_chain": [self._receipt_identity(self._load_receipt(item.receipt_path, plan.scope, plan.project_root)) for item in plan.rollback_receipts], - "compatibility": list(self._compatibility(plan.provider_ids)), - } - if binding != plan.binding or _digest(binding) != plan.preview["preview_hash"]: - raise PermissionError("installer preview is stale; regenerate and review it before confirmation") - - def _repreview(self, operation, scope, project): - transaction = self._new_transaction(scope, project, operation.transaction_id) - preview = transaction.preview(list(operation.changes)) - if preview["preview_hash"] != operation.preview_hash: - raise PermissionError("installer preview is stale; an inner preview changed") - return transaction, preview - - def _new_transaction(self, scope, project, transaction_id=None): - receipts_dir = self._receipts_dir(scope, project) - return self.transaction_factory( - receipts_dir=receipts_dir, - lock_root=receipts_dir / "locks", - create_target_parents=True, - transaction_id=transaction_id, - ) - - @staticmethod - def _operation_transaction_id(batch_id, child_index, provider_id): - return str(uuid.uuid5(uuid.UUID(batch_id), "{0}:{1}".format(child_index, provider_id))) - - def _artifact_history(self, definition, target, scope, project): - target_name = str(target) - allowed = {str(definition.artifact_destination(scope, project, artifact)) for artifact in definition.artifacts if definition.applies_to(scope, artifact)} - return [item for item in self._all_receipts(scope, project) if target_name in item.targets and set(item.targets).issubset(allowed) and item.status == "applied"] - - def _all_receipts(self, scope, project): - receipts_dir = self._receipts_dir(scope, project) - try: - safe_dir = _assert_safe_directory(receipts_dir) - except ValueError: - return [] - result = [] - for candidate in safe_dir.glob("*/receipt.json"): - try: - result.append(self._load_receipt(candidate, scope, project)) - except (OSError, ValueError, json.JSONDecodeError): - continue - return sorted(result, key=lambda item: item.timestamp) - - def _load_receipt(self, path, scope, project): - receipts_dir = _assert_safe_directory(self._receipts_dir(scope, project)) - location = _assert_safe_target(path) - if location.parent.parent != receipts_dir: - raise ValueError("installer receipt escapes its scoped receipt directory") - return Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) - - @staticmethod - def _receipt_identity(receipt): - return {"transaction_id": receipt.transaction_id, "receipt_path": receipt.receipt_path, - "targets": list(receipt.targets), "before_hashes": receipt.before_hashes, - "after_hashes": receipt.after_hashes, "backup_paths": receipt.backup_paths, - "rollback_receipt_hash": _rollback_receipt_hash(receipt)} - - def _rollback_result(self, result, planned, scope, project): - if all(hasattr(result, field) for field in ("transaction_id", "receipt_path", "targets", "before_hashes", "after_hashes", "backup_paths")): - return result - recorded = self._load_receipt(planned.receipt_path, scope, project) - recorded.status = result.status - return recorded - - @staticmethod - def _unresolved_child_remediation(receipts): - return [{"paths": list(receipt.targets), "message": "Rollback child requires manual recovery: {0}".format(receipt.receipt_path)} for receipt in receipts] - - def _compensate_install(self, receipts): - unresolved = [] - for receipt in reversed(receipts): - if receipt.status == "rolled_back": - continue - try: - expected = { - "expected_after_hashes": receipt.after_hashes, - "expected_receipt_hash": _rollback_receipt_hash(receipt), - } if receipt.status == "applied" else {} - result = self.rollback_func(receipt.receipt_path, **expected) - if result.status != "rolled_back": - unresolved.append({"paths": list(receipt.targets), "message": "Could not recover child receipt: {0}".format(receipt.receipt_path)}) - except Exception as error: - unresolved.append({"paths": list(receipt.targets), "message": redact_secrets(str(error))}) - return not unresolved, unresolved - - def _compensate_uninstall(self, changes, completed, scope, project): - if not changes: - return True, [] - expected = {} - for receipt in completed: - for target in receipt.targets: - expected[target] = {"hash": receipt.before_hashes[target], - "exists": receipt.backup_paths[target] is not None, - "mode": receipt.target_modes[target]} - unresolved = [] - try: - affected = {target for receipt in completed for target in receipt.targets} - for _provider_id, provider_changes in changes: - provider_changes = [change for change in provider_changes if change["path"] in affected] - if not provider_changes: - continue - expected_current = {change["path"]: expected[change["path"]] for change in provider_changes} - transaction = self._new_transaction(scope, project) - preview = transaction.preview(provider_changes, expected_current=expected_current) - if transaction.apply(preview["preview_hash"], expected_current=expected_current).status != "applied": - unresolved.append({"paths": [change["path"] for change in provider_changes], "message": "Compensation transaction did not apply."}) - return not unresolved, unresolved - except Exception as error: - return False, [{"paths": sorted(expected), "message": redact_secrets(str(error))}] - - def _receipts_dir(self, scope, project): - return (self.registry.config_root / "mlx-agent" / "installer-receipts") if scope == "user" else (project / ".mlx-agent" / "installer-receipts") - - def _create_batch(self, plan): - batches = self._receipts_dir(plan.scope, plan.project_root) / "batches" - directory, descriptor = _walk_directory(batches, create=True) - batch_id = str(uuid.uuid4()) - try: - os.mkdir(batch_id, 0o700, dir_fd=descriptor) - os.fsync(descriptor) - finally: - os.close(descriptor) - path = directory / batch_id / "batch.json" - operations = [] - for index, item in enumerate(plan.transactions): - operation = self._active_operation(item, batch_id, index) - operation["expected_receipt_path"] = str(self._receipts_dir(plan.scope, plan.project_root) / operation["transaction_id"] / "receipt.json") - operations.append(operation) - immutable = { - "schema_version": "1.0", "batch_id": batch_id, "action": plan.action, "scope": plan.scope, - "project_root": str(plan.project_root), "preview_hash": plan.preview["preview_hash"], - "binding": plan.binding, - "operations": operations, - } - self._write_batch(path, dict(immutable, status="pending", children=[], active=None, remediation=[])) - return path - - def _batch_update(self, path, status, receipts, active=None, error=None, remediation=None): - value = self._read_batch(path) - children = {item["receipt_path"]: item for item in value.get("children", [])} - for receipt in receipts: - children[receipt.receipt_path] = dict(self._receipt_identity(receipt), status=receipt.status) - value["children"] = list(children.values()) - value["status"] = status - value["active"] = active - if error: - value.setdefault("remediation", []).append({"paths": [], "message": redact_secrets(str(error))}) - if remediation: - value.setdefault("remediation", []).extend(remediation) - self._write_batch(path, value) - - def _read_batch(self, path): - location = _assert_safe_target(path) - value = json.loads(_read_regular(location).decode("utf-8")) - required = {"schema_version", "batch_id", "action", "scope", "project_root", "preview_hash", "binding", "operations", "status", "children", "active", "remediation"} - if not isinstance(value, dict) or set(value) != required: - raise ValueError("installer batch journal is malformed") - return value - - def _write_batch(self, path, value): - root = _assert_safe_directory(Path(path).parent) - _atomic_in_directory(root, "batch.json", (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600) - - def _batch_operations(self, plan, path): - batch = self._read_batch(path) - operations = batch["operations"] - if len(operations) != len(plan.transactions): - raise ValueError("installer batch operations do not match the confirmed plan") - result = [] - for index, (planned, operation) in enumerate(zip(plan.transactions, operations)): - expected = self._active_operation(planned, batch["batch_id"], index) - expected["expected_receipt_path"] = str(self._receipts_dir(plan.scope, plan.project_root) / expected["transaction_id"] / "receipt.json") - if operation != expected: - raise ValueError("installer batch operation binding is stale") - result.append(_PlannedTransaction(planned.provider_id, planned.changes, planned.preview_hash, - planned.destinations, operation["transaction_id"], - operation["expected_receipt_path"], operation["child_index"])) - return result - - @staticmethod - def _active_operation(operation, batch_id=None, child_index=None): - if operation.transaction_id is None: - if batch_id is None or child_index is None: - raise ValueError("batch operation identity is required") - transaction_id = Installer._operation_transaction_id(batch_id, child_index, operation.provider_id) - expected_receipt_path = None - else: - transaction_id = operation.transaction_id - expected_receipt_path = operation.expected_receipt_path - return {"provider": operation.provider_id, - "child_index": operation.child_index if operation.child_index is not None else child_index, - "transaction_id": transaction_id, - "expected_receipt_path": expected_receipt_path, - "preview_hash": operation.preview_hash, "targets": list(operation.destinations)} diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/interview.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/interview.py deleted file mode 100644 index d4711cb..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/interview.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Domain-agnostic interview engine producing a validated DomainIntent. - -The question set is a deterministic, config-driven backbone. An optional -``assist`` callable may refine answers, but every assist result is re-validated -through ``build_intent`` and can never bypass validation or inject unknown -roles. Modality foundations seed roles/keywords via ``modality`` profiles. -No network access, no LLM dependency for the deterministic path. -""" - -from __future__ import annotations - -import math - -from dataclasses import dataclass -from typing import Callable, Dict, List, Mapping, Optional, Sequence, Tuple - -from .modality import ( - FACET_CHOICES, - MODALITY_CHOICES, - apply_modality_profile, - detect_facets, - detect_modalities, - resolve_facets, - validate_facets, - validate_modalities, -) -from .models import DISCOVERY_ROLES - - -ROLE_CHOICES: Dict[str, str] = { - "Vision / OCR": "vision", - "Embeddings": "embedding", - "Coding": "coding", - "Reasoning": "reasoning", - "General chat": "general", - "Tool use / function calling": "tool-use", -} - - -QUESTIONS: Tuple[Dict[str, object], ...] = ( - { - "id": "domain", - "prompt": "What are you building? Describe the domain in a sentence.", - "kind": "text", - }, - { - "id": "roles", - "prompt": "Which model roles do you need? (choose one or more)", - "kind": "multi", - "choices": tuple(ROLE_CHOICES.keys()), - }, - { - "id": "keywords", - "prompt": "Any domain keywords to prioritize? (comma-separated, optional)", - "kind": "text", - }, - { - "id": "license", - "prompt": "Restrict to specific licenses? (comma-separated, optional)", - "kind": "text", - }, - { - "id": "memory_gb", - "prompt": "Memory budget in GB? (optional, e.g. 32)", - "kind": "text", - }, - { - "id": "notes", - "prompt": "Any other constraints or notes? (optional)", - "kind": "text", - }, -) - -_FOLLOW_ON_QUESTIONS = tuple(question for question in QUESTIONS if question["id"] != "domain") - - -@dataclass(frozen=True) -class DomainIntent: - """A validated, deterministic description of what the user wants.""" - - domain: str - roles: Tuple[str, ...] - keywords: Tuple[str, ...] = () - license_allow: Tuple[str, ...] = () - memory_gb: Optional[float] = None - notes: str = "" - modalities: Tuple[str, ...] = () - facets: Tuple[str, ...] = () - - def to_dict(self) -> Dict[str, object]: - return { - "domain": self.domain, - "roles": list(self.roles), - "keywords": list(self.keywords), - "license_allow": list(self.license_allow), - "memory_gb": self.memory_gb, - "notes": self.notes, - "modalities": list(self.modalities), - "facets": list(self.facets), - } - - -def _dedupe(values: Sequence[str]) -> Tuple[str, ...]: - seen = [] - for value in values: - if value and value not in seen: - seen.append(value) - return tuple(seen) - - -def _split_csv(raw: object) -> Tuple[str, ...]: - if raw is None: - return () - if isinstance(raw, (list, tuple)): - parts = [str(item) for item in raw] - else: - parts = str(raw).split(",") - return _dedupe([part.strip().lower() for part in parts if part.strip()]) - - -def _split_labels(raw: object) -> Tuple[str, ...]: - if raw is None or raw == "": - return () - if isinstance(raw, (list, tuple)): - return _dedupe([str(item).strip() for item in raw if str(item).strip()]) - return _dedupe([part.strip() for part in str(raw).split(",") if part.strip()]) - - -def _resolve_roles(raw: object) -> Tuple[str, ...]: - if raw is None or raw == "": - labels: Sequence[str] = () - elif isinstance(raw, (list, tuple)): - labels = [str(item) for item in raw] - else: - labels = [part.strip() for part in str(raw).split(",")] - roles = [] - for label in labels: - cleaned = label.strip() - if not cleaned: - continue - if cleaned in ROLE_CHOICES: - roles.append(ROLE_CHOICES[cleaned]) - elif cleaned in DISCOVERY_ROLES: - roles.append(cleaned) - else: - raise ValueError("unknown role: {0}".format(cleaned)) - return _dedupe(roles) - - -def _resolve_memory(raw: object) -> Optional[float]: - if raw is None: - return None - text = str(raw).strip() - if not text: - return None - try: - value = float(text) - except ValueError as error: - raise ValueError("memory_gb must be a number") from error - if not math.isfinite(value) or value <= 0: - raise ValueError("memory_gb must be a positive number") - return value - - -def _resolve_modality_ids(raw: object) -> Tuple[str, ...]: - labels = _split_labels(raw) - ids = [] - for label in labels: - if label in MODALITY_CHOICES: - ids.append(MODALITY_CHOICES[label]) - else: - ids.append(label) - return validate_modalities(ids) - - -def _resolve_facet_ids(raw: object, modalities: Sequence[str]) -> Tuple[str, ...]: - labels = _split_labels(raw) - if not labels: - return () - label_to_id = {} - for modality in modalities: - label_to_id.update(FACET_CHOICES.get(modality, {})) - ids = [] - for label in labels: - if label in label_to_id: - ids.append(label_to_id[label]) - else: - ids.append(label) - return validate_facets(ids, modalities=modalities) - - -def build_intent( - answers: Mapping[str, object], - assist: Optional[Callable[["DomainIntent"], Mapping[str, object]]] = None, -) -> DomainIntent: - """Validate raw answers into a DomainIntent. Deterministic given answers. - - If ``assist`` is provided it receives the validated intent and may return a - mapping of overrides; those overrides are merged and the intent is rebuilt - through this same validator (assist can never bypass validation). - """ - domain = str(answers.get("domain", "")).strip() - if not domain: - raise ValueError("domain is required") - - modalities_raw = answers.get("modalities") - if modalities_raw is None or modalities_raw == "": - modalities = detect_modalities(domain) - keyword_text = " ".join(_split_csv(answers.get("keywords"))) - if keyword_text: - modalities = _dedupe(list(modalities) + list(detect_modalities(keyword_text))) - else: - modalities = _resolve_modality_ids(modalities_raw) - - facets_raw = answers.get("facets") - detect_text = "{0} {1}".format(domain, " ".join(_split_csv(answers.get("keywords")))) - if facets_raw is None or facets_raw == "": - facets = resolve_facets(modalities, cli=(), text=detect_text) if modalities else () - else: - facets = _resolve_facet_ids(facets_raw, modalities) - - roles = _resolve_roles(answers.get("roles")) - keywords = _split_csv(answers.get("keywords")) - if modalities: - roles, keywords = apply_modality_profile(roles, keywords, modalities, facets) - if not roles: - roles = ("general",) - - intent = DomainIntent( - domain=domain, - roles=roles, - keywords=keywords, - license_allow=_split_csv( - answers["license"] if "license" in answers else answers.get("license_allow") - ), - memory_gb=_resolve_memory(answers.get("memory_gb")), - notes=str(answers.get("notes", "")).strip(), - modalities=modalities, - facets=facets, - ) - if assist is None: - return intent - overrides = assist(intent) - if not overrides: - return intent - merged = intent.to_dict() - merged.update(dict(overrides)) - return build_intent(merged, assist=None) - - -def run_interview( - reader: Callable[[Mapping[str, object]], object], - assist: Optional[Callable[["DomainIntent"], Mapping[str, object]]] = None, - preset_modalities: Sequence[str] = (), - preset_facets: Sequence[str] = (), -) -> DomainIntent: - """Ask questions via ``reader`` with detect-then-ask modality resolution.""" - answers: Dict[str, object] = {} - domain_question = QUESTIONS[0] - answers["domain"] = reader(domain_question) - domain = str(answers["domain"]).strip() - - modalities = validate_modalities(preset_modalities) if preset_modalities else detect_modalities(domain) - if not modalities: - modality_question = { - "id": "modalities", - "prompt": ( - "Which foundational modalities apply? " - "(audio, video, document extraction / vision)" - ), - "kind": "multi", - "choices": tuple(MODALITY_CHOICES.keys()), - } - modalities = _resolve_modality_ids(reader(modality_question)) - if not modalities: - raise ValueError("at least one foundational modality is required") - answers["modalities"] = list(modalities) - - facets: List[str] = list(validate_facets(preset_facets, modalities=modalities)) if preset_facets else [] - if not facets: - for modality in modalities: - detected = detect_facets(modality, domain) - if detected: - facets.extend(detected) - else: - facet_question = { - "id": "facets:{0}".format(modality), - "prompt": "Which {0} facets matter? (optional multi-select)".format(modality), - "kind": "multi", - "choices": tuple(FACET_CHOICES[modality].keys()), - } - response = reader(facet_question) - if response is None or response == "": - continue - facets.extend(_resolve_facet_ids(response, (modality,))) - if not facets and modalities: - facets = list(resolve_facets(modalities, cli=(), text=domain)) - answers["facets"] = list(_dedupe(facets)) - - for question in _FOLLOW_ON_QUESTIONS: - answers[str(question["id"])] = reader(question) - return build_intent(answers, assist=assist) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/lora.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/lora.py deleted file mode 100644 index 336b859..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/lora.py +++ /dev/null @@ -1,326 +0,0 @@ -"""Confirmation-gated LoRA training jobs on the convert machinery.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from pathlib import Path - -from .convert import ( - _read_receipt, - _write_receipt, - receipts_root, -) -from .serve import _argv_matches, _pid_alive, _pid_command -from .wiring import _MODEL - - -LORA_RECEIPT_SCHEMA_VERSION = "1.0" -LORA_RECEIPT_KIND = "lora" -EXECUTABLE = "mlx_lm.lora" -ITERS_MIN, ITERS_MAX, ITERS_DEFAULT = 1, 100000, 1000 -BATCH_MIN, BATCH_MAX, BATCH_DEFAULT = 1, 64, 4 -LR_MIN, LR_MAX, LR_DEFAULT = 1e-6, 1e-2, 1e-5 -LAYERS_MIN, LAYERS_MAX, LAYERS_DEFAULT = -1, 128, 16 -MAX_DATASET_LINES = 10000 -DATASET_SAMPLE_LINES = 50 - - -class LoraError(RuntimeError): - """Classified lora failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def validate_dataset(data_dir): - """Validate an mlx_lm training dataset directory; bounded and read-only.""" - root = Path(data_dir) - train = root / "train.jsonl" - if not root.is_dir() or not train.is_file(): - raise LoraError( - "dataset_invalid", - "The dataset directory must contain train.jsonl: {0}".format(data_dir), - "Provide a directory with train.jsonl (text or messages per line).", - ) - summary = {"train_lines": _validate_jsonl(train), "valid_lines": 0} - valid = root / "valid.jsonl" - if valid.is_file(): - summary["valid_lines"] = _validate_jsonl(valid) - return summary - - -def _validate_jsonl(path): - count = 0 - try: - with open(path, "r", encoding="utf-8") as handle: - for line in handle: - count += 1 - if count > MAX_DATASET_LINES: - raise LoraError( - "dataset_invalid", - "{0} exceeds {1} lines.".format(path.name, MAX_DATASET_LINES), - "Split the dataset; lora validates bounded datasets only.", - ) - if count > DATASET_SAMPLE_LINES: - continue - stripped = line.strip() - if not stripped: - continue - try: - record = json.loads(stripped) - except json.JSONDecodeError: - raise LoraError( - "dataset_invalid", - "{0} line {1} is not valid JSON.".format(path.name, count), - "Fix the dataset line; every line must be one JSON object.", - ) - _validate_record(record, path.name, count) - except OSError as error: - raise LoraError( - "dataset_invalid", - "The dataset could not be read: {0}".format(error), - "Pass a readable dataset directory.", - ) - if count == 0: - raise LoraError( - "dataset_invalid", - "{0} is empty.".format(path.name), - "Provide at least one training example.", - ) - return count - - -def _validate_record(record, name, line): - if not isinstance(record, dict): - raise LoraError( - "dataset_invalid", - "{0} line {1} must be a JSON object.".format(name, line), - "Use {\"text\": ...} or {\"messages\": [...]} per line.", - ) - text = record.get("text") - if isinstance(text, str) and text.strip(): - return - messages = record.get("messages") - if isinstance(messages, list) and messages: - for message in messages: - if ( - not isinstance(message, dict) - or not isinstance(message.get("role"), str) - or not isinstance(message.get("content"), str) - ): - raise LoraError( - "dataset_invalid", - "{0} line {1} has a malformed message.".format(name, line), - "Each message needs string role and content fields.", - ) - return - raise LoraError( - "dataset_invalid", - "{0} line {1} needs a non-empty text field or a messages array.".format(name, line), - "Use {\"text\": ...} or {\"messages\": [{\"role\": ..., \"content\": ...}]}.", - ) - - -def _validate_hyperparameters(iters, batch_size, learning_rate, num_layers): - if not isinstance(iters, int) or isinstance(iters, bool) or not ITERS_MIN <= iters <= ITERS_MAX: - raise LoraError( - "invalid_arguments", - "iters must be between {0} and {1}.".format(ITERS_MIN, ITERS_MAX), - "Pass a bounded --iters value.", - ) - if not isinstance(batch_size, int) or isinstance(batch_size, bool) or not BATCH_MIN <= batch_size <= BATCH_MAX: - raise LoraError( - "invalid_arguments", - "batch-size must be between {0} and {1}.".format(BATCH_MIN, BATCH_MAX), - "Pass a bounded --batch-size value.", - ) - if ( - not isinstance(learning_rate, (int, float)) - or isinstance(learning_rate, bool) - or not LR_MIN <= float(learning_rate) <= LR_MAX - ): - raise LoraError( - "invalid_arguments", - "learning-rate must be between {0} and {1}.".format(LR_MIN, LR_MAX), - "Pass a bounded --learning-rate value.", - ) - if not isinstance(num_layers, int) or isinstance(num_layers, bool) or not LAYERS_MIN <= num_layers <= LAYERS_MAX: - raise LoraError( - "invalid_arguments", - "num-layers must be between {0} and {1}.".format(LAYERS_MIN, LAYERS_MAX), - "Pass a bounded --num-layers value (-1 trains all layers).", - ) - - -def plan_lora(repo, data, iters=ITERS_DEFAULT, batch_size=BATCH_DEFAULT, - learning_rate=LR_DEFAULT, num_layers=LAYERS_DEFAULT, out=None): - """Validate inputs and render the exact training plan; side-effect free.""" - if not isinstance(repo, str) or not _MODEL.fullmatch(repo): - raise LoraError( - "invalid_repo", - "lora requires a safe publisher/model base identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - _validate_hyperparameters(iters, batch_size, learning_rate, num_layers) - dataset = validate_dataset(data) - if out is None: - out = "{0}-lora".format(repo.split("/", 1)[1]) - plan = { - "repo": repo, - "data": str(data), - "iters": iters, - "batch_size": batch_size, - "learning_rate": float(learning_rate), - "num_layers": num_layers, - "out": str(out), - "dataset": dataset, - "argv": [ - EXECUTABLE, - "--model", repo, - "--train", - "--data", str(data), - "--iters", str(iters), - "--batch-size", str(batch_size), - "--learning-rate", repr(float(learning_rate)), - "--num-layers", str(num_layers), - "--adapter-path", str(out), - ], - } - canonical = json.dumps( - {key: plan[key] for key in sorted(plan) if key != "preview_hash"}, - sort_keys=True, separators=(",", ":"), - ) - plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() - return plan - - -def _receipt_filename(plan): - return "{0}-lora.json".format(plan["repo"].split("/", 1)[1]) - - -def start_lora(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, spawn=None, now=_utc_now, - pid_alive=None): - """Execute a reviewed training plan; the only mutating entry point.""" - from .serve import _default_spawn, _default_which - - which = which or _default_which - spawn = spawn or _default_spawn - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir, LORA_RECEIPT_KIND) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise LoraError( - "preview_hash_required", - "--confirm requires the hash from a reviewed lora preview.", - "Run lora start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise LoraError( - "preview_stale", - "The supplied preview hash does not match this lora plan.", - "Re-run lora start without --confirm and review the fresh plan.", - ) - if which(EXECUTABLE) is None: - raise LoraError( - "runtime_not_installed", - "The {0} executable is not installed.".format(EXECUTABLE), - "Install mlx-lm yourself (pip install mlx-lm); lora never installs runtimes.", - ) - if model_present is not None and not model_present(plan["repo"]): - raise LoraError( - "model_not_local", - "The base model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; lora never downloads.", - ) - if Path(plan["out"]).exists(): - raise LoraError( - "output_exists", - "The adapter output path already exists: {0}".format(plan["out"]), - "Pick a fresh --out path; lora never overwrites.", - ) - for receipt_path in sorted(root.glob("*.json")) if root.is_dir() else []: - existing = _read_receipt(receipt_path, LORA_RECEIPT_KIND) - if existing is None or existing.get("completed_at"): - continue - if pid_alive(existing["pid"]): - raise LoraError( - "job_in_progress", - "A lora job is still running (pid {0}).".format(existing["pid"]), - "Wait for it to finish (lora status) before starting another.", - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}-lora.log".format(plan["repo"].split("/", 1)[1]) - pid = spawn(plan["argv"], str(log_path)) - receipt = { - "schema_version": LORA_RECEIPT_SCHEMA_VERSION, - "kind": LORA_RECEIPT_KIND, - "repo": plan["repo"], - "data": plan["data"], - "iters": plan["iters"], - "out": plan["out"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "started_at": now(), - "preview_hash": plan["preview_hash"], - "completed_at": None, - "exit_status": None, - } - _write_receipt(root, receipt, _receipt_filename(plan)) - return {"status": "started", "receipt": receipt} - - -def status_lora(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - now=_utc_now): - """Cross-check lora receipts against live processes; marks exits once.""" - root = receipts_root(receipts_dir, LORA_RECEIPT_KIND) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - receipt = _read_receipt(path, LORA_RECEIPT_KIND) - if receipt is None: - continue - entry = { - "receipt": str(path), - "repo": receipt["repo"], - "data": receipt.get("data"), - "out": receipt["out"], - "pid": receipt["pid"], - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - "completed_at": receipt.get("completed_at"), - } - if receipt.get("completed_at"): - entry["state"] = receipt.get("exit_status") or "done" - entries.append(entry) - continue - if pid_alive(receipt["pid"]): - command = pid_command(receipt["pid"]) - entry["state"] = "running" if _argv_matches( - {"argv": receipt["argv"], "port": None, "repo": receipt["repo"]}, - command, - require_port=False, - ) else "unknown" - entries.append(entry) - continue - exit_status = "done" if Path(receipt["out"]).exists() else "failed" - receipt["completed_at"] = now() - receipt["exit_status"] = exit_status - _write_receipt(root, receipt, path.name) - entry["state"] = exit_status - entry["completed_at"] = receipt["completed_at"] - entries.append(entry) - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/modality.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/modality.py deleted file mode 100644 index 03205fb..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/modality.py +++ /dev/null @@ -1,283 +0,0 @@ -"""Foundational modality layers for research intent seeding. - -Three foundations (audio, video, document-vision) seed existing discovery roles -and keywords. No new discovery roles or runtimes. Detection is deterministic -lexicon matching only — no LLM classification. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Dict, Iterable, Mapping, Optional, Sequence, Tuple - - -@dataclass(frozen=True) -class FacetProfile: - id: str - label: str - lexicon: Tuple[str, ...] - seed_keywords: Tuple[str, ...] - - -@dataclass(frozen=True) -class ModalityProfile: - id: str - label: str - lexicon: Tuple[str, ...] - default_roles: Tuple[str, ...] - seed_keywords: Tuple[str, ...] - facets: Tuple[FacetProfile, ...] - - -MODALITY_PROFILES: Tuple[ModalityProfile, ...] = ( - ModalityProfile( - id="audio", - label="Audio (ASR / TTS / music)", - lexicon=( - "audio", "speech", "whisper", "asr", "tts", "voice", "music", - "sound", "transcri", - ), - default_roles=("general",), - seed_keywords=("audio", "speech", "whisper"), - facets=( - FacetProfile( - id="asr", - label="ASR / speech-to-text", - lexicon=("asr", "speech-to-text", "speech to text", "transcri", "whisper", "stt"), - seed_keywords=("asr", "whisper", "transcription", "speech-to-text"), - ), - FacetProfile( - id="tts", - label="TTS / text-to-speech", - lexicon=("tts", "text-to-speech", "text to speech", "narrat", "voice synthes"), - seed_keywords=("tts", "text-to-speech", "speech synthesis"), - ), - FacetProfile( - id="music", - label="Music", - lexicon=("music", "melody", "song", "soundtrack", "audio generation"), - seed_keywords=("music", "melody", "song"), - ), - ), - ), - ModalityProfile( - id="video", - label="Video", - lexicon=("video", "footage", "clip", "mp4", "frames"), - default_roles=("vision", "general"), - seed_keywords=("video",), - facets=( - FacetProfile( - id="understanding", - label="Understanding / captioning", - lexicon=("understanding", "caption", "describe video", "video-llm", "vlm video"), - seed_keywords=("video", "caption", "understanding"), - ), - FacetProfile( - id="generation", - label="Generation", - lexicon=("generation", "generate video", "diffusion", "text-to-video", "text to video"), - seed_keywords=("video", "generation", "diffusion", "text-to-video"), - ), - FacetProfile( - id="action", - label="Action / tracking", - lexicon=("action recognition", "action", "tracking", "pose", "motion"), - seed_keywords=("video", "action", "tracking"), - ), - ), - ), - ModalityProfile( - id="document-vision", - label="Document extraction / vision", - lexicon=( - "ocr", "document", "pdf", "scanned", "invoice", "layout", - "handwriting", "vision", "vlm", "image understanding", - ), - default_roles=("vision",), - seed_keywords=("ocr", "document", "vision"), - facets=( - FacetProfile( - id="ocr", - label="OCR / text extraction", - lexicon=("ocr", "optical character", "scanned", "handwriting", "text extraction"), - seed_keywords=("ocr", "handwriting", "scanned"), - ), - FacetProfile( - id="layout", - label="Layout / structure", - lexicon=("layout", "form parsing", "table extraction", "document structure"), - seed_keywords=("layout", "document", "pdf"), - ), - FacetProfile( - id="general-vision", - label="General vision / VLM", - lexicon=("general vision", "vlm", "image caption", "visual qa", "vision-language"), - seed_keywords=("vision", "vlm", "image"), - ), - ), - ), -) - -FOUNDATION_IDS: Tuple[str, ...] = tuple(profile.id for profile in MODALITY_PROFILES) -ALL_FACET_IDS: Tuple[str, ...] = tuple( - facet.id for profile in MODALITY_PROFILES for facet in profile.facets -) -MODALITY_CHOICES: Dict[str, str] = {profile.label: profile.id for profile in MODALITY_PROFILES} -FACET_CHOICES: Dict[str, Dict[str, str]] = { - profile.id: {facet.label: facet.id for facet in profile.facets} - for profile in MODALITY_PROFILES -} - -_PROFILES_BY_ID: Mapping[str, ModalityProfile] = {profile.id: profile for profile in MODALITY_PROFILES} -_FACETS_BY_ID: Mapping[str, FacetProfile] = { - facet.id: facet for profile in MODALITY_PROFILES for facet in profile.facets -} -_FACET_TO_MODALITY: Mapping[str, str] = { - facet.id: profile.id for profile in MODALITY_PROFILES for facet in profile.facets -} - - -def _dedupe(values: Iterable[str]) -> Tuple[str, ...]: - seen = [] - for value in values: - if value and value not in seen: - seen.append(value) - return tuple(seen) - - -def _haystack(text: str) -> str: - return " ".join(str(text or "").lower().split()) - - -def _lexicon_hit(haystack: str, lexicon: Sequence[str]) -> bool: - return any(token in haystack for token in lexicon) - - -def detect_modalities(text: str) -> Tuple[str, ...]: - """Return foundation ids matched by deterministic lexicon hits.""" - haystack = _haystack(text) - if not haystack: - return () - hits = [] - for profile in MODALITY_PROFILES: - if _lexicon_hit(haystack, profile.lexicon): - hits.append(profile.id) - return tuple(hits) - - -def detect_facets(modality: str, text: str) -> Tuple[str, ...]: - """Return facet ids for one foundation matched in text.""" - profile = _PROFILES_BY_ID.get(modality) - if profile is None: - raise ValueError("unknown modality: {0}".format(modality)) - haystack = _haystack(text) - if not haystack: - return () - hits = [] - for facet in profile.facets: - if _lexicon_hit(haystack, facet.lexicon): - hits.append(facet.id) - return tuple(hits) - - -def validate_modalities(modalities: Sequence[str]) -> Tuple[str, ...]: - resolved = _dedupe([str(item).strip() for item in modalities if str(item).strip()]) - for modality in resolved: - if modality not in _PROFILES_BY_ID: - raise ValueError("unknown modality: {0}".format(modality)) - return resolved - - -def validate_facets(facets: Sequence[str], modalities: Optional[Sequence[str]] = None) -> Tuple[str, ...]: - resolved = _dedupe([str(item).strip() for item in facets if str(item).strip()]) - allowed = None - if modalities is not None: - allowed = set() - for modality in validate_modalities(modalities): - allowed.update(facet.id for facet in _PROFILES_BY_ID[modality].facets) - for facet in resolved: - if facet not in _FACETS_BY_ID: - raise ValueError("unknown facet: {0}".format(facet)) - if allowed is not None and facet not in allowed: - raise ValueError( - "facet {0} is not valid for modalities {1}".format( - facet, ",".join(modalities) or "none" - ) - ) - return resolved - - -def resolve_modalities(cli: Sequence[str] = (), text: str = "") -> Tuple[str, ...]: - """CLI modalities win; otherwise detect from text.""" - if cli: - return validate_modalities(cli) - return detect_modalities(text) - - -def resolve_facets( - modalities: Sequence[str], - cli: Sequence[str] = (), - text: str = "", -) -> Tuple[str, ...]: - """Resolve facets: CLI wins; else detect; else seed all facets for each modality.""" - modalities = validate_modalities(modalities) - if not modalities: - return () - if cli: - return validate_facets(cli, modalities=modalities) - detected = [] - for modality in modalities: - detected.extend(detect_facets(modality, text)) - detected = _dedupe(detected) - if detected: - return validate_facets(detected, modalities=modalities) - # Lock: seed all facet ids for selected foundations when none detected. - all_facets = [] - for modality in modalities: - all_facets.extend(facet.id for facet in _PROFILES_BY_ID[modality].facets) - return tuple(all_facets) - - -def apply_modality_profile( - roles: Sequence[str], - keywords: Sequence[str], - modalities: Sequence[str], - facets: Sequence[str] = (), -) -> Tuple[Tuple[str, ...], Tuple[str, ...]]: - """Union user roles/keywords with profile seeds. User values are preserved.""" - modalities = validate_modalities(modalities) - facets = validate_facets(facets, modalities=modalities) if facets else () - seeded_roles = list(roles) - seeded_keywords = list(keywords) - for modality in modalities: - profile = _PROFILES_BY_ID[modality] - for role in profile.default_roles: - if role not in seeded_roles: - seeded_roles.append(role) - for keyword in profile.seed_keywords: - if keyword not in seeded_keywords: - seeded_keywords.append(keyword) - facet_ids = facets - if modalities and not facet_ids: - facet_ids = resolve_facets(modalities, cli=(), text="") - for facet_id in facet_ids: - facet = _FACETS_BY_ID[facet_id] - for keyword in facet.seed_keywords: - if keyword not in seeded_keywords: - seeded_keywords.append(keyword) - return tuple(seeded_roles), tuple(seeded_keywords) - - -def modality_label(modality_id: str) -> str: - profile = _PROFILES_BY_ID.get(modality_id) - if profile is None: - raise ValueError("unknown modality: {0}".format(modality_id)) - return profile.label - - -def facet_label(facet_id: str) -> str: - facet = _FACETS_BY_ID.get(facet_id) - if facet is None: - raise ValueError("unknown facet: {0}".format(facet_id)) - return facet.label diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/models.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/models.py deleted file mode 100644 index ef170de..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/models.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Pure model classification, sizing, wiring, and rendering helpers.""" - -from __future__ import annotations - -import json -import re - -from .runtime_preference import prefer_runtime_for_role, wiring_for_preference - - -ROLES = [ - ("vision", ["-vl", "vision", "vlm", "ocr", "-omni"], "Vision / OCR (needs mlx-vlm)"), - ("embedding", ["embed", "embedding", "bge", "nomic", "gte"], "Embeddings"), - ("coding", ["coder", "-code", "devstral", "codestral", "starcoder"], "Coding"), - ("reasoning", ["gpt-oss", "-a3b", "thinking", "reason", "qwq", "-r1", "deepseek-r"], "Reasoning"), - ("general", ["instruct", "-it", "chat", "mistral", "gemma", "qwen", "llama", "phi"], "General"), -] -REASONER_HINTS = re.compile(r"(thinking|reason|-a3b\b|qwq|-r1\b|deepseek-r|gpt-oss)", re.I) -TEMPLATE_REASON = re.compile(r"(reasoning_effort||<\|think|channel\|>analysis)", re.I) -TOOL_USE_HINTS = re.compile( - r"\b(?:tool[-_]?use|(?:function|tool)(?:[-_]call|[ -]calling))\b", - re.I, -) -TEMPLATE_TOOL_USE = re.compile( - r"(?:|\btool_calls\b|\bfunction_call\b|\bavailable_tools\b|" - r"(?:\{\{|\{%)[^}]*\btools\b[^}]*(?:\}\}|%\}))", - re.I, -) -TOOL_USE_TAGS = { - "tool-use", - "tool_use", - "function-calling", - "function_calling", - "tools", -} -DISCOVERY_ROLES = tuple(role for role, _keywords, _label in ROLES) + ("tool-use",) -REPUTABLE = {"mlx-community", "lmstudio-community", "unsloth", "mlx-omni", "mlxvlm", "qwen", "google", "mistralai", "meta-llama", "nvidia"} -QUANT_TOK = re.compile(r"[-_.]?(mlx|mxfp4|nvfp4|dwq|optiq|turboquant|rotorquant|oq4e?|mtp|q\d(_k(_[ms])?)?|int[48]|fp16|fp8|bf16|\d+\.?\d*bpw|\d+bit|e[24]b|4bit|8bit)$", re.I) -PARAM_RE = re.compile(r"(\d+(?:\.\d+)?)\s*b(?![a-z])", re.I) -QUANT_GB_PER_B = [ - (re.compile(r"(fp16|bf16|-16bit|f16)", re.I), 2.0), - (re.compile(r"(8bit|-q8|int8|fp8|mxfp8)", re.I), 1.0), - (re.compile(r"(6bit|-q6)", re.I), 0.75), - (re.compile(r"(4bit|-q4|mxfp4|nvfp4|int4)", re.I), 0.55), - (re.compile(r"(3bit|-q3)", re.I), 0.45), - (re.compile(r"(2bit|-q2)", re.I), 0.35), -] -QUANT_RANK = [("bf16", 6), ("fp16", 6), ("8bit", 5), ("q8", 5), ("6bit", 4), ("q6", 4), ("4bit", 3), ("q4", 3), ("mxfp4", 3), ("3bit", 2), ("2bit", 1)] - - -def infer_quantization(value): - """Return a normalized name-derived quantization, never a measured property.""" - low = value.lower() - for name, patterns in [ - ("fp16", ("fp16", "bf16", "f16")), - ("8bit", ("8bit", "q8", "int8", "fp8")), - ("6bit", ("6bit", "q6")), - ("4bit", ("4bit", "q4", "int4", "mxfp4", "nvfp4")), - ("3bit", ("3bit", "q3")), - ("2bit", ("2bit", "q2")), - ]: - if any(pattern in low for pattern in patterns): - return name - return None - - -def name_ram_gb(repo): - match = PARAM_RE.search(repo) - if not match: - return None - for pattern, value in QUANT_GB_PER_B: - if pattern.search(repo): - return round(float(match.group(1)) * value, 1) - return round(float(match.group(1)) * 0.55, 1) - - -def classify(repo): - low = repo.lower() - for role, keywords, _label in ROLES: - if any(keyword in low for keyword in keywords): - return role - return "general" - - -def classify_roles(repo, enrichment): - """Return ordered role memberships and the canonical tool-use signal.""" - primary = classify(repo) - explicit = enrichment.get("tool_use") - if explicit is not None: - source = enrichment.get("tool_use_src") or "checked" - signal = { - "supported": bool(explicit), - "source": source, - "confidence": "weak" if source == "name" else "explicit", - } - elif TOOL_USE_HINTS.search(repo): - signal = { - "supported": True, - "source": "name", - "confidence": "weak", - } - else: - signal = { - "supported": False, - "source": "none", - "confidence": "none", - } - roles = (primary, "tool-use") if signal["supported"] else (primary,) - return roles, signal - - -def base_name(repo): - name = repo.split("/")[-1] - previous = None - while name != previous: - previous = name - name = QUANT_TOK.sub("", name).rstrip("-_.") - return name.lower() - - -def quant_rank(repo): - low = repo.lower() - for token, rank in QUANT_RANK: - if token in low: - return rank - return 0 - - -def resolve_ram(repo, enrichment): - if enrichment.get("weight_bytes"): - return round(enrichment["weight_bytes"] / 1e9, 1), "estimated_from_weight_bytes" - if enrichment.get("params_total"): - for pattern, value in QUANT_GB_PER_B: - if pattern.search(repo): - return round(enrichment["params_total"] * value / 1e9, 1), "estimated_from_parameter_count" - return round(enrichment["params_total"] * 0.55 / 1e9, 1), "estimated_from_parameter_count" - estimate = name_ram_gb(repo) - return (estimate, "estimated_from_name") if estimate is not None else (None, None) - - -def resolve_reasoning(repo, enrichment): - if enrichment.get("reasoning") is not None: - return enrichment["reasoning"], enrichment.get("reason_src") - return bool(REASONER_HINTS.search(repo)), "name" - - -def wiring(repo, role, host): - preference = prefer_runtime_for_role(role, host or {}) - return wiring_for_preference(repo, role, preference) - - -def render_md(report): - host = report["host"] - out = ["# mlx-scout report", ""] - out.append("**Host:** {0} · {1}GB · Ollama {2} · LM Studio {3}".format( - host.get("chip") or "?", host.get("ram_gb") or "?", "✓" if host["ollama"] else "✗", "✓" if host["lmstudio"] else "✗") + (" _(fast mode: name heuristics)_" if report.get("fast") else "")) - if report.get("error"): - return "\n".join(out + ["\n> ⚠ {0}".format(report["error"])]) - labels = {role: label for role, _keywords, label in ROLES} - for role, items in report["roles"].items(): - out.extend(["\n## {0}".format(labels.get(role, role)), "| model | RAM | reasoning | fits | license | wiring |", "|---|---|---|---|---|---|"]) - for item in items: - ram = "{0}GB".format(item["est_ram_gb"]) if item.get("est_ram_gb") is not None else "?" - reason = "⚠ {0}".format(item.get("reason_src") or "") if item.get("reasoning") else "no" - license_name = (item.get("license") or "?") + (" 🔒" if item.get("gated") else "") - out.append("| `{0}`{1} | {2} | {3} | {4} | {5} | {6} |".format(item["repo"], " ⭐" if item.get("trusted") else "", ram, reason, "✓" if item["fits"] else "✗", license_name, item["wiring"])) - out.append("\n_RAM is always an estimate; measured model-file bytes, when available, are retained separately as evidence. Add KV-cache headroom (~1–4GB) beyond weights._") - out.append("_`reasoning ⚠` sources: chat_template > tags > name (weakest). Verify with a test-generate._") - return "\n".join(out) - - -def wire(repo, target, port): - short = repo.split("/")[-1] - if target == "ollama": - return "\n".join(["# Ollama runs curated tags / HF GGUF repos (not arbitrary mlx-community weights).", "ollama pull hf.co/{0} # GGUF repos; else use the matching library tag".format(repo), "# agent ref: ollama/"]) - if target == "lmstudio": - return "\n".join(["lms get {0} # download into LM Studio".format(repo), "lms server start # OpenAI-compatible API on http://localhost:1234/v1", "# agent ref: lmstudio/{0}".format(short.lower())]) - if target == "mlx_lm": - return "\n".join(["pip install mlx-lm", "mlx_lm.server --model {0} --port {1} --max-tokens 8192".format(repo, port), "# openai-compatible provider:", json.dumps({"id": "mlxlm", "type": "openai", "baseURL": "http://127.0.0.1:{0}/v1".format(port), "apiKey": "local"}, indent=2), "# agent ref: mlxlm/{0}".format(short)]) - if target == "mlx-vlm": - return "\n".join(["pip install mlx-vlm", "mlx_vlm.server --model {0} --port {1}".format(repo, port), json.dumps({"id": "mlxvlm", "type": "openai", "baseURL": "http://127.0.0.1:{0}/v1".format(port), "apiKey": "local"}, indent=2), "# agent ref: mlxvlm/{0}".format(short)]) - if target == "litellm": - return "\n".join(["# litellm config.yaml (run a native server first, e.g. mlx_lm.server):", "model_list:", " - model_name: {0}".format(short.lower()), " litellm_params:", " model: openai/{0}".format(repo), " api_base: http://127.0.0.1:{0}/v1".format(port), " api_key: local"]) - return "unknown target: {0}".format(target) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/probe_assets.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/probe_assets.py deleted file mode 100644 index 52e0c98..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/probe_assets.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Deterministic synthetic assets for role-fit verification probes.""" - -from __future__ import annotations - -import base64 -import struct -import zlib - - -VISION_PROBE_TEXT = "MLX42" - -_FONT_5X7 = { - "M": ( - "X...X", - "XX.XX", - "X.X.X", - "X.X.X", - "X...X", - "X...X", - "X...X", - ), - "L": ( - "X....", - "X....", - "X....", - "X....", - "X....", - "X....", - "XXXXX", - ), - "X": ( - "X...X", - "X...X", - ".X.X.", - "..X..", - ".X.X.", - "X...X", - "X...X", - ), - "4": ( - "...X.", - "..XX.", - ".X.X.", - "X..X.", - "XXXXX", - "...X.", - "...X.", - ), - "2": ( - ".XXX.", - "X...X", - "....X", - "...X.", - "..X..", - ".X...", - "XXXXX", - ), -} - -_cached_image_b64 = None - - -def _png_chunk(tag, data): - return ( - struct.pack(">I", len(data)) - + tag - + data - + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) - ) - - -def _png_gray(width, height, rows): - ihdr = struct.pack(">IIBBBBB", width, height, 8, 0, 0, 0, 0) - raw = b"".join(b"\x00" + bytes(row) for row in rows) - return ( - b"\x89PNG\r\n\x1a\n" - + _png_chunk(b"IHDR", ihdr) - + _png_chunk(b"IDAT", zlib.compress(raw)) - + _png_chunk(b"IEND", b"") - ) - - -def _render_text_bitmap(text, scale=6, padding=8): - glyphs = [_FONT_5X7[character] for character in text] - glyph_width = 5 - spacing = 1 - bitmap_width = len(glyphs) * (glyph_width + spacing) - spacing - bitmap_height = 7 - width = bitmap_width * scale + 2 * padding - height = bitmap_height * scale + 2 * padding - rows = [bytearray(width) for _ in range(height)] - for index, glyph in enumerate(glyphs): - origin_x = padding + index * (glyph_width + spacing) * scale - for gy, line in enumerate(glyph): - for gx, mark in enumerate(line): - if mark != "X": - continue - for dy in range(scale): - row = rows[padding + gy * scale + dy] - start = origin_x + gx * scale - for dx in range(scale): - row[start + dx] = 255 - return width, height, rows - - -def vision_probe_image_png(): - """Render the fixed probe word as a small deterministic grayscale PNG.""" - width, height, rows = _render_text_bitmap(VISION_PROBE_TEXT) - return _png_gray(width, height, rows) - - -def vision_probe_image_base64(): - global _cached_image_b64 - if _cached_image_b64 is None: - _cached_image_b64 = base64.b64encode(vision_probe_image_png()).decode("ascii") - return _cached_image_b64 diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/project_blueprint.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/project_blueprint.py deleted file mode 100644 index c0872fb..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/project_blueprint.py +++ /dev/null @@ -1,257 +0,0 @@ -"""MLX project design packs: guidance-only blueprints for downstream agents. - -No scaffolding, no downloads, no training. Deterministic templates from a short -project brief. Distinct from dataset blueprints in ``blueprint.py``. -""" - -from __future__ import annotations - -import json -import math -import os -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from typing import Dict, List, Mapping, Optional, Tuple - -from .modality import validate_modalities -from .research import slugify - - -@dataclass(frozen=True) -class ProjectBrief: - goal: str - modalities: Tuple[str, ...] = () - notes: str = "" - memory_gb: Optional[float] = None - - def to_dict(self) -> Dict[str, object]: - return { - "goal": self.goal, - "modalities": list(self.modalities), - "notes": self.notes, - "memory_gb": self.memory_gb, - } - - -@dataclass(frozen=True) -class ProjectDesignPack: - brief: ProjectBrief - generated_at: str - quantization_ideas: Tuple[str, ...] - training_loop: Tuple[str, ...] - lora_notes: Tuple[str, ...] - mtx_notes: Tuple[str, ...] - study_materials: Tuple[str, ...] - stack_path: Tuple[str, ...] - next_steps: Tuple[str, ...] - - def to_dict(self) -> Dict[str, object]: - return { - "brief": self.brief.to_dict(), - "generated_at": self.generated_at, - "quantization_ideas": list(self.quantization_ideas), - "training_loop": list(self.training_loop), - "lora_notes": list(self.lora_notes), - "mtx_notes": list(self.mtx_notes), - "study_materials": list(self.study_materials), - "stack_path": list(self.stack_path), - "next_steps": list(self.next_steps), - } - - -def _resolve_memory(raw: object) -> Optional[float]: - if raw is None: - return None - text = str(raw).strip() - if not text: - return None - try: - value = float(text) - except ValueError as error: - raise ValueError("memory_gb must be a number") from error - if not math.isfinite(value) or value <= 0: - raise ValueError("memory_gb must be a positive number") - return value - - -def build_brief(answers: Mapping[str, object]) -> ProjectBrief: - goal = str(answers.get("goal") or answers.get("domain") or "").strip() - if not goal: - raise ValueError("goal is required") - modalities_raw = answers.get("modalities") or () - if isinstance(modalities_raw, str): - modalities_raw = [part.strip() for part in modalities_raw.split(",") if part.strip()] - modalities = validate_modalities(list(modalities_raw)) if modalities_raw else () - return ProjectBrief( - goal=goal, - modalities=modalities, - notes=str(answers.get("notes", "")).strip(), - memory_gb=_resolve_memory(answers.get("memory_gb")), - ) - - -def generate_design_pack( - brief: ProjectBrief, - now: Optional[datetime] = None, -) -> ProjectDesignPack: - """Build a deterministic project design pack. Guidance only.""" - moment = now or datetime.now(timezone.utc) - visionish = any(item in brief.modalities for item in ("document-vision", "video")) - audio = "audio" in brief.modalities - memory = brief.memory_gb - - quant: List[str] = [ - "Start from published MLX community 4-bit / 8-bit weights when available.", - "Prefer 4-bit for chat/tooling under tight unified memory; keep 8-bit for quality-sensitive OCR or reasoning.", - "Record the exact Hub repo + quant tag in the research pack before fine-tuning.", - ] - if memory is not None: - quant.append( - "Respect the {0:g} GB budget with ~20% runtime headroom beyond weight size.".format(memory) - ) - if visionish: - quant.append( - "Vision stacks need mlx-vlm-compatible VLM quants; do not plan Ollama-only vision paths." - ) - if audio: - quant.append( - "Audio ASR/TTS often ships as specialty checkpoints — verify native MLX serve path before training plans." - ) - - training = ( - "Define a tiny offline JSONL/Parquet dataset before any train loop.", - "Dry-run one batch on Apple Silicon with mlx-lm / project tooling; abort on OOM.", - "Use LoRA/QLoRA-style adapters before full fine-tunes; keep base weights frozen.", - "Hold out 10–20% validation with no near-duplicate leakage.", - "This blueprint does not train or download — execution stays outside mlx-agent.", - ) - - lora = ( - "Search adapters in `mlx-agent research` (PEFT/LoRA section) before training new ones.", - "Serve adapters with mlx_lm `--adapter-path` after verify; keep base + adapter versions pinned.", - "If no Hub adapter fits, use the research dataset blueprint to draft labels locally.", - ) - - mtx = ( - "Treat experimental MTX / mixed-precision experiments as optional research notes, not production defaults.", - "Document any experimental kernel or quant recipe separately from the shipping wire config.", - "Prefer proven mlx-community quants for the first vertical slice; revisit MTX only after a working baseline.", - ) - - study = ( - "Apple MLX documentation: model convert / quantize / generate flows.", - "mlx-lm and mlx-vlm server READMEs for OpenAI-compatible local serving.", - "mlx-scout runtimes reference: LM Studio vs mlx_lm vs Ollama trade-offs.", - "PEFT/LoRA primers focused on low-rank adapters for domain dial-in.", - ) - - stack = ( - "1. Run `mlx-agent research` with the same goal/modalities to rank models, adapters, and datasets.", - "2. Follow the pack's Runtime preference (mlx-vlm / LM Studio / mlx_lm; Ollama only as curated alternate).", - "3. `mlx-agent adopt start` to verify a candidate on a local runtime.", - "4. `mlx-agent wire` to apply a confirmation-gated config — never skip preview hashes.", - ) - - next_steps = ( - "Freeze the goal and modalities above; do not expand scope mid-slice.", - "Generate a research pack and pick one base model + optional adapter.", - "Create or obtain a small labeled dataset offline.", - "Implement training outside mlx-agent; keep this design pack as the agent-ingestible plan.", - ) - if brief.notes: - next_steps = next_steps + ("Honor project notes: {0}".format(brief.notes),) - - return ProjectDesignPack( - brief=brief, - generated_at=moment.isoformat(), - quantization_ideas=tuple(quant), - training_loop=training, - lora_notes=lora, - mtx_notes=mtx, - study_materials=study, - stack_path=stack, - next_steps=next_steps, - ) - - -def render_design_pack(pack: ProjectDesignPack) -> str: - brief = pack.brief - lines = [ - "# MLX Project Design Pack: {0}".format(brief.goal), - "", - "- Generated: {0}".format(pack.generated_at), - "- Modalities: {0}".format(", ".join(brief.modalities) or "none"), - "- Memory budget (GB): {0}".format( - brief.memory_gb if brief.memory_gb is not None else "none" - ), - ] - if brief.notes: - lines.append("- Notes: {0}".format(brief.notes)) - lines.extend([ - "", - "## Goal", - "", - brief.goal, - "", - "## Recommended stack path", - "", - ]) - for item in pack.stack_path: - lines.append("- {0}".format(item)) - lines.extend(["", "## Quantization ideas", ""]) - for item in pack.quantization_ideas: - lines.append("- {0}".format(item)) - lines.extend(["", "## Training loop sketch", ""]) - for item in pack.training_loop: - lines.append("- {0}".format(item)) - lines.extend(["", "## LoRA / adapter notes", ""]) - for item in pack.lora_notes: - lines.append("- {0}".format(item)) - lines.extend(["", "## Experimental MTX notes", ""]) - for item in pack.mtx_notes: - lines.append("- {0}".format(item)) - lines.extend(["", "## Study materials", ""]) - for item in pack.study_materials: - lines.append("- {0}".format(item)) - lines.extend(["", "## Next steps", ""]) - for index, item in enumerate(pack.next_steps, start=1): - lines.append("{0}. {1}".format(index, item)) - lines.extend([ - "", - "## Notes", - "", - "This pack is guidance only for downstream agents. " - "mlx-agent blueprint does not scaffold repositories, download weights, or run training.", - "", - ]) - return "\n".join(lines) - - -def write_design_pack( - markdown: str, - brief: ProjectBrief, - root: Optional[object] = None, - now: Optional[datetime] = None, - pack: Optional[ProjectDesignPack] = None, -) -> Path: - """Write under ``/mlx-blueprints`` and never outside it.""" - moment = now or datetime.now(timezone.utc) - base = Path(root) if root is not None else Path.cwd() - out_dir = (base / "mlx-blueprints").resolve() - unresolved_dir = base / "mlx-blueprints" - if unresolved_dir.is_symlink(): - raise ValueError("refusing to write design pack through a symlinked folder") - timestamp = moment.strftime("%Y%m%dT%H%M%SZ") - filename = "{0}-{1}.md".format(slugify(brief.goal), timestamp) - path = (out_dir / filename).resolve() - if os.path.commonpath([str(out_dir), str(path)]) != str(out_dir): - raise ValueError("refusing to write design pack outside the project folder") - out_dir.mkdir(parents=True, exist_ok=True) - path.write_text(markdown, encoding="utf-8") - if pack is not None: - sidecar = path.with_suffix(".json") - if os.path.commonpath([str(out_dir), str(sidecar.resolve())]) != str(out_dir): - raise ValueError("refusing to write design pack outside the project folder") - sidecar.write_text(json.dumps(pack.to_dict(), indent=2) + "\n", encoding="utf-8") - return path diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/providers.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/providers.py deleted file mode 100644 index 8e5fad1..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/providers.py +++ /dev/null @@ -1,436 +0,0 @@ -"""Manifest-backed provider definitions and non-mutating provider detection.""" - -from __future__ import annotations - -import json -import os -import re -import selectors -import shutil -import subprocess -import time -from dataclasses import dataclass -from pathlib import Path - - -_GEMINI_EXTENSION_SUFFIX = Path(".gemini") / "extensions" / "mlx-agent" - - -@dataclass(frozen=True) -class ProviderArtifact: - source: Path - destination: Path - project_destination: Path = None - scopes: tuple = ("user", "project") - - -@dataclass(frozen=True) -class ProviderDefinition: - id: str - detect_commands: tuple - user_root: Path - project_root: Path - invocation_kind: str - invocation_prefix: str - minimum_version: str - last_tested_version: str - version_probe_args: tuple - version_probe_timeout: float - install_mode: str - artifacts: tuple - config_paths: tuple - - def destination(self, scope, project_root=None): - if scope == "user": - return self.user_root - if scope != "project": - raise ValueError("scope must be 'user' or 'project'") - if project_root is None: - raise ValueError("project scope requires a project root") - return Path(project_root).resolve() / self.project_root - - def artifact_destination(self, scope, project_root, artifact): - """Return the receipt-owned target for one declared artifact.""" - if scope == "project" and artifact.project_destination is not None: - return Path(project_root).resolve() / artifact.project_destination - return self.destination(scope, project_root) / artifact.destination - - @staticmethod - def applies_to(scope, artifact): - return scope in artifact.scopes - - -@dataclass(frozen=True) -class ProviderDetection: - id: str - available: bool - command: str = "" - command_path: str = None - state: str = "absent" - version: str = None - minimum_version: str = None - last_tested_version: str = None - error: str = None - install_mode: str = "direct" - - def to_dict(self): - return { - "id": self.id, - "available": self.available, - "command": self.command or None, - "command_path": self.command_path, - "state": self.state, - "version": self.version, - "minimum_version": self.minimum_version, - "last_tested_version": self.last_tested_version, - "error": self.error, - "install_mode": self.install_mode, - } - - -def _safe_relative(value, field, allow_current=False): - path = Path(value) - if allow_current and str(value) == ".": - return path - if path.is_absolute() or ".." in path.parts or not path.parts: - raise ValueError("{0} must be a safe relative path".format(field)) - return path - - -class ProviderRegistry: - """Load immutable installation data from the canonical plugin manifest.""" - - def __init__(self, manifest_path, home=None, config_root=None, xdg_config_home=None): - self.manifest_path = Path(manifest_path).resolve() - self.home = Path(home).expanduser().resolve() if home else Path.home().resolve() - self.config_root = ( - Path(config_root).expanduser().resolve() if config_root else self.home - ) - self.xdg_config_home = ( - Path(xdg_config_home).expanduser().resolve() - if xdg_config_home else (self.home / ".config").resolve() - ) - - def definitions(self): - try: - manifest = json.loads(self.manifest_path.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError) as error: - raise ValueError("provider manifest could not be read: {0}".format(error)) - providers = manifest.get("providers") - if not isinstance(providers, dict): - raise ValueError("provider manifest has no providers object") - definitions = {} - for provider_id, value in providers.items(): - definitions[provider_id] = self._definition(provider_id, value) - return definitions - - def _definition(self, provider_id, value): - if not isinstance(value, dict): - raise ValueError("provider {0} must be an object".format(provider_id)) - required = ( - "detect_commands", "user_root", "project_root", "artifacts", "config_paths", - "invocation", "minimum_version", "last_tested_version", "version_probe", - "install_mode", - ) - if any(key not in value for key in required): - raise ValueError("provider {0} has no installer definition".format(provider_id)) - commands = value["detect_commands"] - if not isinstance(commands, list) or not all(isinstance(item, str) and item for item in commands): - raise ValueError("provider {0}.detect_commands is invalid".format(provider_id)) - user_root = self._user_root(value["user_root"], provider_id) - project_root = self._project_root(value["project_root"], provider_id) - invocation = value["invocation"] - if not isinstance(invocation, dict) or set(invocation) != {"kind", "prefix"}: - raise ValueError("provider {0}.invocation is invalid".format(provider_id)) - invocation_kind, invocation_prefix = invocation["kind"], invocation["prefix"] - if invocation_kind not in {"command", "skill"} or not isinstance(invocation_prefix, str): - raise ValueError("provider {0}.invocation is invalid".format(provider_id)) - if invocation_kind == "command" and invocation_prefix != "/": - raise ValueError("provider {0}.invocation is invalid".format(provider_id)) - if invocation_kind == "skill" and invocation_prefix not in {"", "$"}: - raise ValueError("provider {0}.invocation is invalid".format(provider_id)) - minimum_version = value["minimum_version"] - last_tested_version = value["last_tested_version"] - for label, version in ( - ("minimum_version", minimum_version), - ("last_tested_version", last_tested_version), - ): - if version is not None and not _SEMVER.fullmatch(version): - raise ValueError("provider {0}.{1} is invalid".format(provider_id, label)) - probe = value["version_probe"] - if probe is None: - version_probe_args, version_probe_timeout = (), 0.0 - elif ( - not isinstance(probe, dict) - or set(probe) != {"args", "timeout_seconds"} - or not isinstance(probe["args"], list) - or not 1 <= len(probe["args"]) <= 4 - or not all(isinstance(item, str) and item and len(item) <= 64 for item in probe["args"]) - or not isinstance(probe["timeout_seconds"], (int, float)) - or isinstance(probe["timeout_seconds"], bool) - or not 0 < probe["timeout_seconds"] <= 5 - ): - raise ValueError("provider {0}.version_probe is invalid".format(provider_id)) - else: - version_probe_args = tuple(probe["args"]) - version_probe_timeout = float(probe["timeout_seconds"]) - install_mode = value["install_mode"] - if install_mode not in {"direct", "staged", "portable"}: - raise ValueError("provider {0}.install_mode is invalid".format(provider_id)) - if install_mode == "portable" and (commands or probe is not None or minimum_version is not None or last_tested_version is not None): - raise ValueError("portable providers must not declare CLI compatibility probes") - if install_mode != "portable" and (not commands or probe is None or minimum_version is None): - raise ValueError("native providers require a minimum version and bounded probe") - artifacts = [] - if not isinstance(value["artifacts"], list) or not value["artifacts"]: - raise ValueError("provider {0}.artifacts is invalid".format(provider_id)) - for index, artifact in enumerate(value["artifacts"]): - if not isinstance(artifact, dict) or set(artifact) not in ({"source", "destination"}, {"source", "destination", "scope"}, {"source", "destination", "project_destination"}, {"source", "destination", "project_destination", "scope"}): - raise ValueError("provider {0}.artifacts[{1}] is invalid".format(provider_id, index)) - source = _safe_relative(artifact["source"], "artifact source") - unresolved = self.manifest_path.parent / source - cursor = self.manifest_path.parent - for component in source.parts: - cursor = cursor / component - if cursor.is_symlink(): - raise ValueError( - "provider artifact source must not contain symlinks: {0}".format(source) - ) - location = unresolved.resolve() - if self.manifest_path.parent not in location.parents or not (location.is_file() or location.is_dir()): - raise ValueError("provider artifact source is outside the plugin or missing: {0}".format(source)) - destination = _safe_relative( - artifact["destination"], "artifact destination", allow_current=True - ) - project_destination = ( - _safe_relative(artifact["project_destination"], "artifact project_destination") - if "project_destination" in artifact else None - ) - scopes = (artifact.get("scope", "user"),) if "scope" in artifact else ("user", "project") - if scopes not in (("user",), ("project",)) and scopes != ("user", "project"): - raise ValueError("provider {0}.artifacts[{1}].scope is invalid".format(provider_id, index)) - if location.is_file(): - artifacts.append(ProviderArtifact(location, destination, project_destination, scopes)) - else: - for child in sorted(location.rglob("*")): - if child.is_symlink(): - raise ValueError( - "provider artifact source must not contain symlinks: {0}".format(child) - ) - resolved = child.resolve() - relative = child.relative_to(location) - if "__pycache__" in relative.parts or child.suffix == ".pyc": - continue - if child.is_file() and location in resolved.parents: - project_child = project_destination / child.relative_to(location) if project_destination else None - artifacts.append(ProviderArtifact(resolved, destination / child.relative_to(location), project_child, scopes)) - config_paths = value["config_paths"] - if not isinstance(config_paths, list) or not all(isinstance(item, str) for item in config_paths): - raise ValueError("provider {0}.config_paths is invalid".format(provider_id)) - if provider_id == "gemini": - self._validate_gemini_extension_layout(user_root, project_root, artifacts) - if provider_id == "opencode": - self._validate_opencode_layout(user_root, project_root, artifacts) - return ProviderDefinition( - id=provider_id, - detect_commands=tuple(commands), - user_root=user_root, - project_root=project_root, - invocation_kind=invocation_kind, - invocation_prefix=invocation_prefix, - minimum_version=minimum_version, - last_tested_version=last_tested_version, - version_probe_args=version_probe_args, - version_probe_timeout=version_probe_timeout, - install_mode=install_mode, - artifacts=tuple(artifacts), - config_paths=tuple(config_paths), - ) - - def _validate_gemini_extension_layout(self, user_root, project_root, artifacts): - """Keep Gemini's install roots aligned with its extension discovery layout.""" - if user_root != (self.home / _GEMINI_EXTENSION_SUFFIX).resolve(): - raise ValueError("provider gemini.user_root must target home/.gemini/extensions/mlx-agent") - if project_root != _GEMINI_EXTENSION_SUFFIX: - raise ValueError("provider gemini.project_root must target .gemini/extensions/mlx-agent") - if not any(item.destination == Path("gemini-extension.json") for item in artifacts): - raise ValueError("provider gemini must install gemini-extension.json") - for item in artifacts: - if item.project_destination is not None and item.project_destination.parts[:1] != (".gemini",): - raise ValueError("provider gemini project artifacts must stay under .gemini") - - def _validate_opencode_layout(self, user_root, project_root, artifacts): - """Keep OpenCode's documented global and project discovery locations exact.""" - if user_root != (self.xdg_config_home / "opencode").resolve(): - raise ValueError("provider opencode.user_root must target the OpenCode XDG config directory") - if project_root != Path(".opencode"): - raise ValueError("provider opencode.project_root must target .opencode") - if not any(item.destination == Path("commands/mlx-scout.md") for item in artifacts): - raise ValueError("provider opencode must install native command files") - if not any(item.destination == Path("agents/mlx-advisor.md") for item in artifacts): - raise ValueError("provider opencode must install its advisor agent") - if not any(item.destination == Path("skills/mlx-scout/SKILL.md") for item in artifacts): - raise ValueError("provider opencode must install native skills") - if any(item.destination == Path("opencode.json") for item in artifacts): - raise ValueError("provider opencode must not mutate global or project config") - if not any(item.destination == Path("plugins/mlx-agent-command.ts") for item in artifacts): - raise ValueError("provider opencode must install its native custom tool") - if not any(item.destination == Path("src/mlx_agent/command_executor.py") for item in artifacts): - raise ValueError("provider opencode must install its custom-tool runtime") - - def _user_root(self, template, provider_id): - if not isinstance(template, str) or "{project}" in template: - raise ValueError("provider {0}.user_root is invalid".format(provider_id)) - try: - path = Path(template.format( - home=str(self.home), config_root=str(self.config_root), - xdg_config_home=str(self.xdg_config_home), - )).resolve() - except (KeyError, ValueError) as error: - raise ValueError("provider {0}.user_root is invalid: {1}".format(provider_id, error)) - allowed_roots = (self.home, self.config_root, self.xdg_config_home) - if not any(root == path or root in path.parents for root in allowed_roots): - raise ValueError("provider {0}.user_root must be under an approved user configuration root".format(provider_id)) - return path - - def _project_root(self, template, provider_id): - if not isinstance(template, str) or "{project}" not in template: - raise ValueError("provider {0}.project_root is invalid".format(provider_id)) - marker = Path("/__mlx_agent_project_marker__") - try: - rendered = Path(template.format( - project=str(marker), home=str(self.home), config_root=str(self.config_root), - xdg_config_home=str(self.xdg_config_home), - )).resolve() - return rendered.relative_to(marker) - except (KeyError, ValueError): - raise ValueError("provider {0}.project_root is invalid".format(provider_id)) - - -_SEMVER = re.compile(r"(? _MAX_VERSION_OUTPUT: - raise ValueError("version probe output exceeded the bounded limit") - returncode = process.wait(timeout=max(0.01, deadline - time.monotonic())) - return subprocess.CompletedProcess(argv, returncode, bytes(output)) - finally: - selector.close() - if process.poll() is None: - process.kill() - process.wait() - if process.stdout is not None: - process.stdout.close() - - -def detect_providers(definitions, env=None, path=None, executable_lookup=None, probe_runner=None): - """Report installed provider CLIs only; this function never installs anything.""" - environment = dict(os.environ if env is None else env) - search_path = path if path is not None else environment.get("PATH", "") - lookup = executable_lookup or shutil.which - detections = [] - for definition in definitions: - if definition.install_mode == "portable": - detections.append(ProviderDetection( - definition.id, True, state="portable", install_mode="portable" - )) - continue - command_path = None - command = "" - for candidate in definition.detect_commands: - found = lookup(candidate, path=search_path) - if found: - command, command_path = candidate, str(found) - break - if not command_path: - detections.append(ProviderDetection( - definition.id, False, command, command_path, state="absent", - minimum_version=definition.minimum_version, - last_tested_version=definition.last_tested_version, - error="provider executable was not found", - install_mode=definition.install_mode, - )) - continue - try: - argv = [command_path] + list(definition.version_probe_args) - if probe_runner is None: - completed = _run_bounded_probe( - argv, - timeout=definition.version_probe_timeout, - env=environment, - ) - else: - completed = probe_runner( - argv, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=definition.version_probe_timeout, - check=False, - shell=False, - env=environment, - ) - raw = completed.stdout or b"" - if isinstance(raw, bytes): - output = raw[:_MAX_VERSION_OUTPUT].decode("utf-8", errors="replace") - else: - output = str(raw)[:_MAX_VERSION_OUTPUT] - match = _SEMVER.search(output) - if completed.returncode != 0 or match is None: - raise ValueError("version probe did not return a SemVer core") - version = match.group(1) - supported = definition.last_tested_version is not None and ( - _version_tuple(definition.minimum_version) - <= _version_tuple(version) - <= _version_tuple(definition.last_tested_version) - ) - state = "native-visible" if supported else "unsupported" - detections.append(ProviderDetection( - definition.id, supported, command, command_path, state=state, - version=version, minimum_version=definition.minimum_version, - last_tested_version=definition.last_tested_version, - error=None if supported else "provider version is outside the validated range", - install_mode=definition.install_mode, - )) - except (OSError, subprocess.SubprocessError, ValueError) as error: - detections.append(ProviderDetection( - definition.id, False, command, command_path, state="unsupported", - minimum_version=definition.minimum_version, - last_tested_version=definition.last_tested_version, - error=str(error), install_mode=definition.install_mode, - )) - return detections diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/research.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/research.py deleted file mode 100644 index c817b79..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/research.py +++ /dev/null @@ -1,502 +0,0 @@ -"""Read-only domain research packs: discovery -> scoring -> project-local markdown. - -No verification, no wiring, no downloads. Discovery and the Hugging Face client -are injected so this module is fully testable offline. Output is written only -inside the project-local ``mlx-research`` folder. -""" - -from __future__ import annotations - -import json -import os -import re -from dataclasses import dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -from .blueprint import DatasetBlueprint, build_dataset_blueprint -from .discovery import DiscoveryRequest -from .interview import DomainIntent -from .modality import FACET_CHOICES, facet_label, modality_label -from .runtime_preference import RuntimePreference, prefer_runtime, wiring_for_preference -from .scoring import ScoreResult, rank_scored, score_candidate - - -_SLUG_RE = re.compile(r"[^a-z0-9]+") -_CARD_EXCERPT_CHARS = 240 -_DEFAULT_CATALOG_LIMIT = 5 - - -@dataclass(frozen=True) -class ResearchCandidate: - repo: str - role: str - score: float - wiring: str - signals: Tuple[dict, ...] - provenance: Tuple[dict, ...] - card_present: bool - card_excerpt: str - - def to_dict(self): - return { - "repo": self.repo, - "role": self.role, - "score": self.score, - "wiring": self.wiring, - "signals": [dict(signal) for signal in self.signals], - "provenance": [dict(record) for record in self.provenance], - "card_present": self.card_present, - "card_excerpt": self.card_excerpt, - } - - -@dataclass(frozen=True) -class CatalogItem: - repo: str - kind: str # adapter | dataset - score: float - signals: Tuple[dict, ...] - provenance: Tuple[dict, ...] - card_excerpt: str - record: Dict[str, object] - - def to_dict(self): - return { - "repo": self.repo, - "kind": self.kind, - "score": self.score, - "signals": [dict(signal) for signal in self.signals], - "provenance": [dict(record) for record in self.provenance], - "card_excerpt": self.card_excerpt, - "record": dict(self.record), - } - - -@dataclass(frozen=True) -class ResearchPack: - intent: DomainIntent - candidates: Tuple[ResearchCandidate, ...] - generated_at: str - warnings: Tuple[dict, ...] = field(default_factory=tuple) - adapters: Tuple[CatalogItem, ...] = field(default_factory=tuple) - datasets: Tuple[CatalogItem, ...] = field(default_factory=tuple) - dataset_blueprint: Optional[DatasetBlueprint] = None - runtime_preference: Optional[RuntimePreference] = None - - def to_dict(self): - return { - "intent": self.intent.to_dict(), - "candidates": [candidate.to_dict() for candidate in self.candidates], - "generated_at": self.generated_at, - "warnings": [dict(warning) for warning in self.warnings], - "adapters": [item.to_dict() for item in self.adapters], - "datasets": [item.to_dict() for item in self.datasets], - "dataset_blueprint": ( - self.dataset_blueprint.to_dict() if self.dataset_blueprint else None - ), - "runtime_preference": ( - self.runtime_preference.to_dict() if self.runtime_preference else None - ), - } - - -def slugify(domain: str) -> str: - slug = _SLUG_RE.sub("-", domain.strip().lower()).strip("-") - return slug or "domain" - - -def candidate_metadata(candidate: Dict[str, object]) -> Dict[str, object]: - """Map a discovery candidate to the scoring core's metadata shape.""" - roles = candidate.get("roles") or ([candidate["role"]] if candidate.get("role") else []) - return { - "roles": list(roles), - "license": candidate.get("license"), - "downloads": candidate.get("downloads", 0), - "likes": candidate.get("likes", 0), - "est_ram_gb": candidate.get("est_ram_gb"), - "tags": candidate.get("tags", []), - } - - -def hub_row_metadata(row: Dict[str, object], intent: DomainIntent) -> Dict[str, object]: - """Map a Hub adapter/dataset row into the scoring metadata shape.""" - tags = list(row.get("tags") or []) - license_name = row.get("license") - if not license_name: - for tag in tags: - text = str(tag) - if text.startswith("license:"): - license_name = text.split(":", 1)[1] - break - return { - "roles": list(intent.roles), - "license": license_name, - "downloads": row.get("downloads", 0), - "likes": row.get("likes", 0), - "est_ram_gb": None, - "tags": tags, - } - - -def catalog_search_query(intent: DomainIntent) -> str: - parts = list(intent.keywords) - if intent.domain and intent.domain.lower() not in {p.lower() for p in parts}: - parts.append(intent.domain) - return " ".join(parts).strip() - - -def _excerpt(card_text: Optional[str]) -> str: - if not card_text: - return "" - collapsed = " ".join(card_text.split()) - return collapsed[:_CARD_EXCERPT_CHARS] - - -def _repo_id(row: Dict[str, object]) -> Optional[str]: - repo = row.get("id") or row.get("modelId") or row.get("repo") - if not repo: - return None - return str(repo) - - -def _score_catalog_rows( - intent: DomainIntent, - rows: List[Dict[str, object]], - kind: str, - fetch_card, - catalog_limit: int, -) -> Tuple[CatalogItem, ...]: - scored: List[Tuple[str, ScoreResult]] = [] - detail: Dict[str, Dict[str, object]] = {} - for row in rows: - repo = _repo_id(row) - if not repo or repo in detail: - continue - try: - card_text = fetch_card(repo) - except Exception: - card_text = None - result = score_candidate(intent, hub_row_metadata(row, intent), card_text) - scored.append((repo, result)) - detail[repo] = {"row": row, "card_text": card_text} - - ranked = rank_scored(scored)[:max(0, catalog_limit)] - items: List[CatalogItem] = [] - for repo, result in ranked: - row = detail[repo]["row"] - card_text = detail[repo]["card_text"] - items.append(CatalogItem( - repo=repo, - kind=kind, - score=result.score, - signals=tuple(signal.to_dict() for signal in result.signals), - provenance=result.provenance, - card_excerpt=_excerpt(card_text), - record=dict(row), - )) - return tuple(items) - - -def generate_pack( - intent: DomainIntent, - discovery_service, - hf_client, - limit: int = 6, - catalog_limit: int = _DEFAULT_CATALOG_LIMIT, - now: Optional[datetime] = None, -) -> ResearchPack: - """Build a research pack. Read-only: never verifies, wires, or downloads.""" - moment = now or datetime.now(timezone.utc) - seen: Dict[str, Dict[str, object]] = {} - warnings: List[dict] = [] - host: Dict[str, object] = {} - for role in intent.roles: - request = DiscoveryRequest( - role=role, - memory_gb=intent.memory_gb, - licenses=list(intent.license_allow) or None, - limit=limit, - ) - envelope = discovery_service.discover(request) - if envelope.status != "ok": - detail = envelope.error - warnings.append({ - "code": detail.code if detail else "discovery_failed", - "message": "{0}: {1}".format(role, detail.message if detail else "discovery failed"), - }) - continue - if not host and isinstance(envelope.data.get("host"), dict): - host = dict(envelope.data["host"]) - for bucket in envelope.data.get("roles", {}).values(): - for candidate in bucket: - repo = candidate.get("repo") - if repo and repo not in seen: - seen[repo] = candidate - - preference = prefer_runtime( - host, - roles=intent.roles, - modalities=intent.modalities, - ) - - scored: List[Tuple[str, ScoreResult]] = [] - detail_by_repo: Dict[str, Dict[str, object]] = {} - for repo, candidate in seen.items(): - try: - card_text = hf_client.fetch_model_card(repo) - except Exception: - card_text = None - result = score_candidate(intent, candidate_metadata(candidate), card_text) - scored.append((repo, result)) - detail_by_repo[repo] = {"candidate": candidate, "card_text": card_text} - - ranked = rank_scored(scored)[:max(0, limit)] - candidates = [] - for repo, result in ranked: - candidate = detail_by_repo[repo]["candidate"] - card_text = detail_by_repo[repo]["card_text"] - role = candidate.get("role", "") - candidates.append(ResearchCandidate( - repo=repo, - role=role, - score=result.score, - wiring=wiring_for_preference(repo, role, preference), - signals=tuple(signal.to_dict() for signal in result.signals), - provenance=result.provenance, - card_present=bool(card_text), - card_excerpt=_excerpt(card_text), - )) - - search = catalog_search_query(intent) - try: - adapter_rows = list(hf_client.list_adapters(search=search, limit_fetch=max(catalog_limit * 4, 20))) - except Exception as error: - warnings.append({ - "code": "adapter_search_failed", - "message": "adapter search failed: {0}".format(error), - }) - adapter_rows = [] - try: - dataset_rows = list(hf_client.list_datasets(search=search, limit_fetch=max(catalog_limit * 4, 20))) - except Exception as error: - warnings.append({ - "code": "dataset_search_failed", - "message": "dataset search failed: {0}".format(error), - }) - dataset_rows = [] - - adapters = _score_catalog_rows( - intent, - adapter_rows, - "adapter", - hf_client.fetch_model_card, - catalog_limit, - ) - datasets = _score_catalog_rows( - intent, - dataset_rows, - "dataset", - hf_client.fetch_dataset_card, - catalog_limit, - ) - blueprint = build_dataset_blueprint(intent) if not datasets else None - - return ResearchPack( - intent=intent, - candidates=tuple(candidates), - generated_at=moment.isoformat(), - warnings=tuple(warnings), - adapters=adapters, - datasets=datasets, - dataset_blueprint=blueprint, - runtime_preference=preference, - ) - - -def _render_catalog_section(title: str, items: Tuple[CatalogItem, ...], empty_message: str) -> List[str]: - lines = ["## {0}".format(title), ""] - if not items: - lines.append(empty_message) - lines.append("") - return lines - for index, item in enumerate(items, start=1): - lines.append("### {0}. `{1}` — score {2}/100".format(index, item.repo, item.score)) - lines.append("") - lines.append("- Kind: {0}".format(item.kind)) - matched = [s for s in item.signals if s["applicable"] and s["matched"]] - if matched: - lines.append("- Why it ranked:") - for signal in matched: - lines.append(" - {0}: {1}".format(signal["id"], signal["detail"])) - if item.card_excerpt: - lines.append("- Card excerpt: {0}".format(item.card_excerpt)) - lines.append("") - return lines - - -def render_pack(pack: ResearchPack) -> str: - intent = pack.intent - lines = [ - "# MLX Research Pack: {0}".format(intent.domain), - "", - "- Generated: {0}".format(pack.generated_at), - "- Roles: {0}".format(", ".join(intent.roles) or "none"), - "- Keywords: {0}".format(", ".join(intent.keywords) or "none"), - "- License filter: {0}".format(", ".join(intent.license_allow) or "none"), - "- Memory budget (GB): {0}".format(intent.memory_gb if intent.memory_gb is not None else "none"), - "- Modalities: {0}".format(", ".join(intent.modalities) or "none"), - "- Facets: {0}".format(", ".join(intent.facets) or "none"), - ] - if intent.notes: - lines.append("- Notes: {0}".format(intent.notes)) - lines.extend(["", "## Summary", ""]) - if pack.candidates: - lines.append( - "{0} candidate(s) ranked by transparent scoring. Scores are read-only " - "estimates from metadata and model-card text; no model was downloaded, " - "verified, or wired.".format(len(pack.candidates)) - ) - else: - lines.append("No candidates were found for this intent. See warnings below.") - summary_bits = [] - if pack.adapters: - summary_bits.append("{0} adapter(s)".format(len(pack.adapters))) - if pack.datasets: - summary_bits.append("{0} dataset(s)".format(len(pack.datasets))) - if pack.dataset_blueprint is not None: - summary_bits.append("dataset blueprint (no Hub datasets matched)") - if summary_bits: - lines.append("Catalog enrichment: {0}.".format("; ".join(summary_bits))) - if pack.warnings: - lines.extend(["", "## Warnings", ""]) - for warning in pack.warnings: - lines.append("- [{0}] {1}".format(warning.get("code", "warning"), warning.get("message", ""))) - lines.extend(["", "## Modality foundations", ""]) - if intent.modalities: - for modality in intent.modalities: - allowed = set(FACET_CHOICES.get(modality, {}).values()) - facet_ids = [facet for facet in intent.facets if facet in allowed] - lines.append("- `{0}` — {1}".format(modality, modality_label(modality))) - if facet_ids: - lines.append(" - Facets: {0}".format( - ", ".join( - "`{0}` ({1})".format(facet, facet_label(facet)) - for facet in facet_ids - ) - )) - lines.append("") - else: - lines.append("No foundational modalities were selected for this pack.") - lines.append("") - lines.extend(["", "## Runtime preference", ""]) - preference = pack.runtime_preference - if preference is not None: - lines.append("- Preferred: `{0}`".format(preference.preferred)) - lines.append( - "- Alternates: {0}".format( - ", ".join("`{0}`".format(item) for item in preference.alternates) - if preference.alternates - else "none" - ) - ) - lines.append("- Rationale: {0}".format(preference.rationale)) - snapshot = preference.host_snapshot - lines.append( - "- Host inventory: Ollama {0}; LM Studio {1}".format( - "up" if snapshot.get("ollama") else "down", - "up" if snapshot.get("lmstudio") else "down", - ) - ) - lines.append("") - else: - lines.append("No runtime preference was computed for this pack.") - lines.append("") - lines.extend(["", "## Candidates", ""]) - for index, candidate in enumerate(pack.candidates, start=1): - lines.append("### {0}. `{1}` — score {2}/100".format(index, candidate.repo, candidate.score)) - lines.append("") - lines.append("- Role: {0}".format(candidate.role)) - lines.append("- Suggested wiring: {0}".format(candidate.wiring)) - lines.append("- Model card: {0}".format("present" if candidate.card_present else "not found")) - matched = [s for s in candidate.signals if s["applicable"] and s["matched"]] - if matched: - lines.append("- Why it ranked:") - for signal in matched: - lines.append(" - {0}: {1}".format(signal["id"], signal["detail"])) - if candidate.card_excerpt: - lines.append("- Card excerpt: {0}".format(candidate.card_excerpt)) - lines.append("") - lines.extend(_render_catalog_section( - "Adapters / LoRAs", - pack.adapters, - "No PEFT/LoRA adapters matched this intent on the Hub (read-only search).", - )) - lines.extend(_render_catalog_section( - "Datasets", - pack.datasets, - "No datasets matched this intent on the Hub. See the dataset blueprint below.", - )) - if pack.dataset_blueprint is not None: - lines.extend([ - "## Dataset blueprint", - "", - "No Hub datasets ranked for this intent. Use this guidance-only blueprint " - "to create a local dataset — nothing is downloaded or trained by research.", - "", - pack.dataset_blueprint.to_markdown().rstrip(), - "", - ]) - lines.extend([ - "## Next steps", - "", - "1. Review candidates, adapters, and datasets above and pick a local stack.", - "2. Prefer the runtime in **Runtime preference** above" - + ( - " (`{0}`)".format(pack.runtime_preference.preferred) - if pack.runtime_preference is not None - else "" - ) - + "; Ollama remains a valid alternate for curated tags when listed.", - "3. Install a base model in that runtime (LM Studio, mlx_lm/mlx-vlm, or Ollama).", - "4. Run `mlx-agent adopt start` to verify and wire it — research does not verify or download.", - "5. If no datasets matched, follow the dataset blueprint to create labeled data locally.", - "", - "## Notes", - "", - "This pack is a read-only research artifact intended for ingestion by other agents. " - "All scores are estimates with per-signal provenance; verify capability before relying on any model. " - "Runtime preference is guidance from host inventory and modality/role rules — it does not change scores.", - "", - ]) - return "\n".join(lines) - - -def write_pack( - markdown: str, - intent: DomainIntent, - root: Optional[object] = None, - now: Optional[datetime] = None, - pack: Optional[ResearchPack] = None, -) -> Path: - """Write the pack under ``/mlx-research`` and never outside it.""" - moment = now or datetime.now(timezone.utc) - base = Path(root) if root is not None else Path.cwd() - out_dir = (base / "mlx-research").resolve() - unresolved_dir = base / "mlx-research" - if unresolved_dir.is_symlink(): - raise ValueError("refusing to write research pack through a symlinked folder") - timestamp = moment.strftime("%Y%m%dT%H%M%SZ") - filename = "{0}-{1}.md".format(slugify(intent.domain), timestamp) - path = (out_dir / filename).resolve() - if os.path.commonpath([str(out_dir), str(path)]) != str(out_dir): - raise ValueError("refusing to write research pack outside the project folder") - out_dir.mkdir(parents=True, exist_ok=True) - path.write_text(markdown, encoding="utf-8") - if pack is not None: - sidecar = path.with_suffix(".json") - if os.path.commonpath([str(out_dir), str(sidecar.resolve())]) != str(out_dir): - raise ValueError("refusing to write research pack outside the project folder") - sidecar.write_text(json.dumps(pack.to_dict(), indent=2) + "\n", encoding="utf-8") - return path diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/adapter-runtime.json b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/adapter-runtime.json deleted file mode 100644 index b0e6f55..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/adapter-runtime.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "schema_version": "1.0", - "purpose": "Bundled adapter runtime resource contract" -} diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/bench-community.json b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/bench-community.json deleted file mode 100644 index 4c3573d..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/bench-community.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "schema_version": "1.0", - "entries": [] -} diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/references/model-families.md b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/references/model-families.md deleted file mode 100644 index e331847..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/references/model-families.md +++ /dev/null @@ -1,35 +0,0 @@ -# Model family quirks (chat templates, tool calling, reasoning) - -Family-specific behaviors that change how you wire, prompt, and verify. Read this before debugging a "broken" model — it is usually a template mismatch. - -## Qwen (Qwen3 and later) - -- Ships **hybrid reasoning**: `enable_thinking` in the chat template toggles hidden `` blocks. `mlx-agent` detects this from the template and tags reasoning models so they stay out of fast/utility roles. -- Tool calling uses the Hermes-style `` JSON blocks; works well through `mlx_lm.server` and LM Studio. Verified tool-use is still per-model — template support is not behavior proof. -- MoE variants (`A3B`, `A35B`) are the speed pick on Apple Silicon; dense variants are the consistency pick. - -## Gemma - -- **No system role.** A system prompt must be folded into the first user turn; runtimes that auto-inject a system role produce degraded or refused outputs. -- Google ships **QAT (quantization-aware trained)** builds — their 4bit is unusually strong; prefer publisher QAT over generic 4bit re-quants. -- Tool calling is template-level and inconsistent across sizes; verify before assigning a tool-use role. - -## gpt-oss (OpenAI open weights) - -- **MXFP4 native** — use the MXFP4 builds; do not re-quantize to generic 4bit. -- Uses the **harmony** response format with `reasoning_effort` levels (low/medium/high). These are reasoning models by design: expect hidden analysis channels; never put them in a fast/cheap slot. -- Strong tool calling when served through runtimes that parse harmony (recent `mlx-lm`, LM Studio). - -## Llama (Meta, MLX ports) - -- Solid generic instruction following; tool calling depends heavily on the exact instruct template version — verify, don't assume. -- Community MLX ports vary in template fidelity; check the repo's `chat_template` presence before wiring (missing template → runtime default, often wrong). - -## Vision (Qwen-VL and friends) - -- Vision models need `mlx-vlm`; Ollama's engine does not run them. Wire them on their own port (`:8083` by convention) so they never collide with the text server. -- OCR quality varies more with input resolution handling than with quant; keep 4bit unless the documents are dense. - -## Embedding models - -- Served via `/v1/embeddings` (OpenAI-compatible) or `/api/embed` (Ollama). They cannot chat; a "ready" generation probe is not meaningful for them — rely on the embedding probe, not text generation. diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/references/quantization.md b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/references/quantization.md deleted file mode 100644 index 7600943..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/references/quantization.md +++ /dev/null @@ -1,25 +0,0 @@ -# MLX quantization guide - -How to pick a quant on Apple Silicon. Read this before recommending a specific quantized repo. - -## The tradeoff ladder - -| Quant | Size vs bf16 | Quality | When to pick | -| --- | --- | --- | --- | -| `bf16` / `fp16` | 1.0× | reference | Fine-tuning source, quality ceiling checks. Rarely for serving. | -| `8bit` | ~0.5× | near-reference | RAM is comfortable and the task is quality-sensitive (reasoning, code review). | -| `4bit` | ~0.25× | good | Default serving choice. Best size/quality point for most models. | -| `MXFP4` | ~0.25× | good | Native format for gpt-oss; use the publisher's own MXFP4 builds. | -| `3bit` and below | ≤0.2× | noticeably worse | Only when nothing else fits; expect degradation on reasoning and tool calling. | - -Rules of thumb: - -- **Spend RAM on a bigger model at 4bit, not a smaller model at 8bit.** A 32B at 4bit (~18 GB) usually beats a 14B at 8bit (~15 GB) on hard tasks. -- **Mixed MoE quants matter.** MoE models (e.g. Qwen3-A3B) activate few parameters per token, so at the same RAM they decode much faster than dense. On memory-bandwidth-bound Apple Silicon, active parameters set decode speed. -- **Weights are not the whole budget.** KV cache grows linearly with context: `2 × layers × kv_heads × head_dim × context × 2 bytes` (fp16). A model that "fits" can still die at long context. `mlx-agent discover --context N` computes this for you; the candidate's `estimates.kv.max_context_tokens` is the honest ceiling. -- **Publisher quants beat community re-quants** for quality-sensitive roles: mlx-community and the original publisher calibrate with different data and care. When both exist, prefer the publisher or `mlx-community` (trusted), then unsloth. -- **LoRA preserves quant.** `mlx_lm.lora` trains on the quantized base and `mlx_lm.fuse` keeps the quant, so you do not need bf16 weights to fine-tune. - -## Reasoning models and quant - -Reasoning models emit long hidden chains, so they are more sensitive to aggressive quant than chat models (errors compound over thousands of generated tokens). Prefer 8bit for a reasoning role when RAM allows; 4bit is acceptable for MXFP4-native families. diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/references/troubleshooting.md b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/references/troubleshooting.md deleted file mode 100644 index 4ae6b5c..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/references/troubleshooting.md +++ /dev/null @@ -1,33 +0,0 @@ -# Troubleshooting local MLX serving - -Symptom-first playbook. Each entry: cause, then the fix. Run `mlx-agent doctor models` first — half of these are visible there. - -## "It was working yesterday" - -1. `mlx-agent doctor models` — look for `drift_missing_model` (deleted weights), `drift_hash_mismatch` (hand-edited config), `endpoint_down`. -2. `mlx-agent serve status` — a serve receipt whose pid died, or an argv mismatch after a manual restart. - -## Metal / memory allocation failures at load or mid-generation - -- Cause: weights + KV cache exceed unified memory under pressure. macOS will swap-thrash before it errors cleanly. -- Fix: drop context length (KV is linear in context — check `estimates.kv.max_context_tokens` from discovery), close other GPU consumers (browsers, other servers), or drop a quant tier. Two loaded 30B+ models do not coexist on ≤64 GB. - -## Slow tokens/sec - -- Expected ceiling: decode tok/s ≈ memory bandwidth ÷ active bytes per token. Dense 32B 4bit on a high-end M-series is tens of tok/s, not hundreds. -- Checklist: MoE instead of dense (2–5× at the same RAM); native `mlx_lm` on native MLX weights instead of Ollama for MoE; shorter context (KV reads grow); don't bench while another server is loaded — measurements contend. -- Prompt processing (prefill) is compute-bound, not bandwidth-bound; long prompts are slow on every runtime. Measure with `mlx-agent bench run` before blaming the model. - -## Server answers but agent calls fail - -- Port collision: two configs on one port (`doctor models` → `drift_endpoint_conflict`). Convention: mlx_lm `:8080`, mlx-vlm `:8083`, LiteLLM `:4000`, Ollama `:11434`, LM Studio `:1234`. -- Auth header: wired configs reference `MLX_AGENT_LOCAL_API_KEY` by design; export any value, but the variable must exist for OpenAI-compatible clients that send a key. -- Chat template missing/wrong: outputs look like raw completion text, tool calls never parse. Check the repo ships a `chat_template`; see `references/model-families.md` for family quirks (Gemma has no system role). - -## Tool calling fails verification - -- Metadata ≠ behavior. Only a schema-valid synthetic call counts. If the probe fails with `malformed_arguments` or `missing_tool_call`, the model (or its template at that quant) does not do reliable tool use — pick a different model for the tool-use role; do not lower the bar. - -## Gated repositories - -- `gated: true` means HF auth is required before any runtime can pull. Accept the terms on the Hub page with the account that owns the local `HF_TOKEN`; nothing in this pack downloads, so nothing here can accept terms for you. diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/serve-recipes.json b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/serve-recipes.json deleted file mode 100644 index d297d5e..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/resources/serve-recipes.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "schema_version": "1.0", - "recipes": { - "mlx_lm": { - "executable": "mlx_lm.server", - "argv": ["{executable}", "--model", "{repo}", "--port", "{port}", "--max-tokens", "{max_tokens}"], - "adapter_argv": ["--adapter-path", "{adapter_path}"], - "default_port": 8080, - "readiness": "http://127.0.0.1:{port}/v1/models", - "install_hint": "pip install mlx-lm" - }, - "mlx-vlm": { - "executable": "mlx_vlm.server", - "argv": ["{executable}", "--model", "{repo}", "--port", "{port}"], - "adapter_argv": null, - "default_port": 8083, - "readiness": "http://127.0.0.1:{port}/v1/models", - "install_hint": "pip install mlx-vlm" - } - } -} diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/runtime_preference.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/runtime_preference.py deleted file mode 100644 index d458cc1..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/runtime_preference.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Justified MLX-native runtime preference for research packs and discovery wiring. - -Reporting only: does not change scoring. Uses existing HostInventory flags -(ollama, lmstudio) — no new runtime probes. Ollama remains a valid alternate -for text workloads and is never removed from wire/CLI surfaces. - -This module stays free of interview/models imports to avoid cycles. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Mapping, Sequence, Tuple - - -_VISION_MODALITIES = frozenset({"document-vision", "video"}) - - -@dataclass(frozen=True) -class RuntimePreference: - preferred: str - alternates: Tuple[str, ...] - rationale: str - host_snapshot: Mapping[str, object] - - def to_dict(self): - return { - "preferred": self.preferred, - "alternates": list(self.alternates), - "rationale": self.rationale, - "host_snapshot": { - "ollama": bool(self.host_snapshot.get("ollama")), - "lmstudio": bool(self.host_snapshot.get("lmstudio")), - "chip": self.host_snapshot.get("chip"), - "ram_gb": self.host_snapshot.get("ram_gb"), - }, - } - - -def _needs_vision_path(roles: Sequence[str], modalities: Sequence[str]) -> bool: - if "vision" in roles: - return True - return any(modality in _VISION_MODALITIES for modality in modalities) - - -def prefer_runtime( - host: Mapping[str, object], - roles: Sequence[str] = (), - modalities: Sequence[str] = (), -) -> RuntimePreference: - """Return a deterministic preferred runtime with alternates and rationale.""" - ollama = bool(host.get("ollama")) - lmstudio = bool(host.get("lmstudio")) - snapshot = { - "ollama": ollama, - "lmstudio": lmstudio, - "chip": host.get("chip"), - "ram_gb": host.get("ram_gb"), - } - - if _needs_vision_path(roles, modalities): - alternates = [] - if lmstudio: - alternates.append("lmstudio") - return RuntimePreference( - preferred="mlx-vlm", - alternates=tuple(alternates), - rationale=( - "Vision / document / video workloads need native mlx-vlm; " - "Ollama does not run VLMs. LM Studio can help download weights " - "when its server is up." - ), - host_snapshot=snapshot, - ) - - audio = "audio" in modalities - if lmstudio: - alternates = ["mlx_lm"] - if ollama: - alternates.append("ollama") - rationale = ( - "LM Studio is the lowest-friction native MLX path for arbitrary Hub " - "repos when its local server is up." - ) - if audio: - rationale += ( - " Audio packs still prefer native MLX weights; Ollama is limited " - "to curated tags." - ) - return RuntimePreference( - preferred="lmstudio", - alternates=tuple(alternates), - rationale=rationale, - host_snapshot=snapshot, - ) - - alternates = [] - if ollama: - alternates.append("ollama") - rationale = ( - "Prefer headless mlx_lm for native MLX quants and Hub repos when LM Studio " - "is not detected." - ) - if audio: - rationale += ( - " Audio packs benefit from native MLX serving; Ollama remains a valid " - "alternate for curated tags only." - ) - elif ollama: - rationale += ( - " Ollama remains a valid first choice for curated library tags." - ) - else: - rationale += " Start LM Studio or install mlx-lm to serve local models." - - return RuntimePreference( - preferred="mlx_lm", - alternates=tuple(alternates), - rationale=rationale, - host_snapshot=snapshot, - ) - - -def wiring_for_preference(repo: str, role: str, preference: RuntimePreference) -> str: - """Render a suggested wiring string for the preferred runtime.""" - del role - short = repo.split("/")[-1].lower() - name = repo.split("/")[-1] - preferred = preference.preferred - if preferred == "mlx-vlm": - return "mlx-vlm server → `mlxvlm/{0}`".format(name) - if preferred == "lmstudio": - return "LM Studio → `lmstudio/{0}`".format(short) - if preferred == "ollama": - return "Ollama → `ollama/` (curated / GGUF; not arbitrary mlx-community)" - return "`mlx_lm.server --model {0}` → custom provider".format(repo) - - -def prefer_runtime_for_role(role: str, host: Mapping[str, object]) -> RuntimePreference: - """Scout/discovery helper: preference from a single role + host inventory.""" - modalities = ("document-vision",) if role == "vision" else () - return prefer_runtime(host, roles=(role or "general",), modalities=modalities) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/scoring.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/scoring.py deleted file mode 100644 index 2bad358..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/scoring.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Deterministic, transparent scoring of a model against a DomainIntent. - -Pure and side-effect free. Signals that do not apply to the intent are excluded -from both the numerator and denominator so a candidate is never penalized for a -constraint the user did not request. No network access. -""" - -from __future__ import annotations - -import math -from dataclasses import dataclass -from typing import List, Mapping, Optional, Sequence, Tuple - -from .interview import DomainIntent - - -SIGNAL_WEIGHTS = { - "role_match": 30.0, - "keyword_match": 25.0, - "popularity": 15.0, - "license_ok": 10.0, - "memory_fit": 10.0, - "card_quality": 10.0, -} - -_SIGNAL_SOURCES = { - "role_match": "local_role_derivation", - "keyword_match": "card_text_and_tags", - "popularity": "huggingface_model_list", - "license_ok": "huggingface_model_metadata", - "memory_fit": "local_memory_estimate", - "card_quality": "card_text", -} - -_POPULARITY_LOG_CEILING = 6.0 # ~1e6 downloads saturates the signal. -_MEMORY_HEADROOM_FRACTION = 0.8 - - -@dataclass(frozen=True) -class Signal: - id: str - weight: float - applicable: bool - matched: Optional[bool] - contribution: float - source: str - detail: str - - def to_dict(self): - return { - "id": self.id, - "weight": self.weight, - "applicable": self.applicable, - "matched": self.matched, - "contribution": round(self.contribution, 3), - "source": self.source, - "detail": self.detail, - } - - -@dataclass(frozen=True) -class ScoreResult: - score: float - signals: Tuple[Signal, ...] - provenance: Tuple[dict, ...] - - def to_dict(self): - return { - "score": self.score, - "signals": [signal.to_dict() for signal in self.signals], - "provenance": [dict(record) for record in self.provenance], - } - - -def _signal(signal_id, applicable, matched, fraction, detail): - weight = SIGNAL_WEIGHTS[signal_id] - contribution = weight * fraction if applicable else 0.0 - return Signal( - id=signal_id, - weight=weight, - applicable=applicable, - matched=matched if applicable else None, - contribution=contribution, - source=_SIGNAL_SOURCES[signal_id], - detail=detail, - ) - - -def _role_signal(intent: DomainIntent, metadata: Mapping[str, object]) -> Signal: - model_roles = set(metadata.get("roles") or []) - matched = [role for role in intent.roles if role in model_roles] - fraction = len(matched) / len(intent.roles) if intent.roles else 0.0 - return _signal( - "role_match", True, bool(matched), fraction, - "matched roles: {0}".format(", ".join(matched) or "none"), - ) - - -def _keyword_signal( - intent: DomainIntent, metadata: Mapping[str, object], card_text: Optional[str] -) -> Signal: - if not intent.keywords: - return _signal("keyword_match", False, None, 0.0, "no keywords requested") - tags = " ".join(str(tag).lower() for tag in (metadata.get("tags") or [])) - haystack = "{0} {1}".format((card_text or "").lower(), tags) - matched = [keyword for keyword in intent.keywords if keyword in haystack] - fraction = len(matched) / len(intent.keywords) - return _signal( - "keyword_match", True, bool(matched), fraction, - "matched keywords: {0}".format(", ".join(matched) or "none"), - ) - - -def _popularity_signal(metadata: Mapping[str, object]) -> Signal: - downloads = metadata.get("downloads") or 0 - try: - downloads = max(0, int(downloads)) - except (TypeError, ValueError): - downloads = 0 - fraction = min(1.0, math.log10(downloads + 1) / _POPULARITY_LOG_CEILING) - return _signal( - "popularity", True, downloads > 0, fraction, - "downloads: {0}".format(downloads), - ) - - -def _license_signal(intent: DomainIntent, metadata: Mapping[str, object]) -> Signal: - if not intent.license_allow: - return _signal("license_ok", False, None, 0.0, "no license restriction") - license_name = (metadata.get("license") or "").lower() - matched = license_name in set(intent.license_allow) - return _signal( - "license_ok", True, matched, 1.0 if matched else 0.0, - "license: {0}".format(license_name or "unknown"), - ) - - -def _memory_signal(intent: DomainIntent, metadata: Mapping[str, object]) -> Signal: - est_ram = metadata.get("est_ram_gb") - if intent.memory_gb is None or est_ram is None: - return _signal("memory_fit", False, None, 0.0, "no budget or estimate") - fits = float(est_ram) < intent.memory_gb * _MEMORY_HEADROOM_FRACTION - return _signal( - "memory_fit", True, fits, 1.0 if fits else 0.0, - "est_ram_gb {0} vs budget {1}".format(est_ram, intent.memory_gb), - ) - - -def _card_quality_signal(card_text: Optional[str]) -> Signal: - if not card_text: - return _signal("card_quality", True, False, 0.0, "no model card text") - quality = 0.5 - lowered = card_text.lower() - if "```" in card_text or "usage" in lowered or "example" in lowered or "how to" in lowered: - quality += 0.5 - return _signal( - "card_quality", True, True, quality, - "card present; usage/example: {0}".format(quality > 0.5), - ) - - -def score_candidate( - intent: DomainIntent, - metadata: Mapping[str, object], - card_text: Optional[str], -) -> ScoreResult: - signals = ( - _role_signal(intent, metadata), - _keyword_signal(intent, metadata, card_text), - _popularity_signal(metadata), - _license_signal(intent, metadata), - _memory_signal(intent, metadata), - _card_quality_signal(card_text), - ) - applicable = [signal for signal in signals if signal.applicable] - denominator = sum(signal.weight for signal in applicable) - numerator = sum(signal.contribution for signal in applicable) - score = round(100.0 * numerator / denominator, 1) if denominator else 0.0 - provenance = tuple( - { - "source": signal.source, - "fields": [signal.id], - "basis": signal.detail, - } - for signal in applicable - ) - return ScoreResult(score=score, signals=signals, provenance=provenance) - - -def rank_scored( - entries: Sequence[Tuple[str, ScoreResult]] -) -> List[Tuple[str, ScoreResult]]: - """Sort (repo, ScoreResult) entries by score desc, then repo asc.""" - return sorted(entries, key=lambda entry: (-entry[1].score, entry[0])) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/serve.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/serve.py deleted file mode 100644 index 264cfca..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/serve.py +++ /dev/null @@ -1,528 +0,0 @@ -"""Confirmation-gated launcher for local MLX serving runtimes. - -Serve is the only component that spawns processes, so every mutation goes -through the wire-style preview -> confirm -> receipt flow. Serve never -installs runtimes, never downloads models, binds only to 127.0.0.1, and stops -only processes that it started itself, verified against their receipts. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import shutil -import signal -import subprocess -import time -import urllib.error -import urllib.request -from datetime import datetime, timezone -from pathlib import Path - -from .model_doctor import scan_wired_configs -from .transactions import _atomic_in_directory -from .wiring import require_secret_free_config, validate_health_endpoint - - -SERVE_RECEIPT_SCHEMA_VERSION = "1.0" -SERVE_RECEIPT_KIND = "serve" -MAX_TOKENS_MIN, MAX_TOKENS_MAX, MAX_TOKENS_DEFAULT = 256, 65536, 8192 -PORT_MIN, PORT_MAX = 1, 65535 -READINESS_DEADLINE_DEFAULT = 60.0 -STOP_DEADLINE_SECONDS = 10.0 -MAX_RECEIPTS = 50 - -_RECIPES_PATH = Path(__file__).resolve().parent / "resources" / "serve-recipes.json" - - -class ServeError(RuntimeError): - """Classified serve failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def load_recipes(path=None): - location = Path(path) if path is not None else _RECIPES_PATH - value = json.loads(location.read_text(encoding="utf-8")) - recipes = value.get("recipes") - if not isinstance(recipes, dict) or not recipes: - raise ValueError("serve recipe table is missing its recipes") - for name, recipe in recipes.items(): - for key in ("executable", "argv", "default_port", "readiness", "install_hint"): - if key not in recipe: - raise ValueError("serve recipe {0} is missing {1}".format(name, key)) - return recipes - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def _preview_hash(plan): - canonical = json.dumps( - {key: plan[key] for key in sorted(plan) if key != "preview_hash"}, - sort_keys=True, separators=(",", ":"), - ) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest() - - -def receipts_root(root=None): - base = Path(root) if root is not None else Path.cwd() - return base / ".mlx-agent-receipts" / "serve" - - -def plan_start(repo, runtime, recipes, port=None, max_tokens=MAX_TOKENS_DEFAULT, - adapter_path=None): - """Render the exact start plan; pure and side-effect free.""" - if not isinstance(repo, str) or not repo.strip() or "/" not in repo: - raise ServeError( - "invalid_repo", - "serve requires a publisher/model repository identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - recipe = recipes.get(runtime) - if recipe is None: - raise ServeError( - "unsupported_runtime", - "serve does not launch the {0} runtime.".format(runtime), - "Ollama and LM Studio manage their own servers; serve supports: {0}.".format( - ", ".join(sorted(recipes)) - ), - ) - if not isinstance(max_tokens, int) or isinstance(max_tokens, bool): - raise ServeError( - "invalid_arguments", "max_tokens must be an integer.", - "Pass --max-tokens between {0} and {1}.".format(MAX_TOKENS_MIN, MAX_TOKENS_MAX), - ) - if not MAX_TOKENS_MIN <= max_tokens <= MAX_TOKENS_MAX: - raise ServeError( - "invalid_arguments", - "max_tokens is outside {0}-{1}.".format(MAX_TOKENS_MIN, MAX_TOKENS_MAX), - "Pass a bounded --max-tokens value.", - ) - selected_port = recipe["default_port"] if port is None else port - if not isinstance(selected_port, int) or isinstance(selected_port, bool): - raise ServeError( - "invalid_arguments", "port must be an integer.", - "Pass --port between {0} and {1}.".format(PORT_MIN, PORT_MAX), - ) - if not PORT_MIN <= selected_port <= PORT_MAX: - raise ServeError( - "invalid_arguments", - "port is outside {0}-{1}.".format(PORT_MIN, PORT_MAX), - "Pass a valid loopback port.", - ) - if adapter_path is not None and not recipe.get("adapter_argv"): - raise ServeError( - "unsupported_runtime", - "The {0} recipe does not support adapter serving.".format(runtime), - "Serve adapters with a runtime whose recipe declares adapter support.", - ) - values = { - "executable": recipe["executable"], - "repo": repo, - "port": str(selected_port), - "max_tokens": str(max_tokens), - } - argv = [part.format(**values) for part in recipe["argv"]] - if adapter_path is not None: - argv.extend( - part.format(adapter_path=str(adapter_path)) for part in recipe["adapter_argv"] - ) - plan = { - "repo": repo, - "runtime": runtime, - "port": selected_port, - "max_tokens": max_tokens, - "adapter_path": str(adapter_path) if adapter_path is not None else None, - "argv": argv, - "readiness": recipe["readiness"].format(port=selected_port), - "bind": "127.0.0.1", - } - plan["preview_hash"] = _preview_hash(plan) - return plan - - -def _default_which(executable): - return shutil.which(executable) - - -def _default_port_free(port): - import socket - - probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - probe.settimeout(0.5) - return probe.connect_ex(("127.0.0.1", port)) != 0 - finally: - probe.close() - - -def _default_spawn(argv, log_path): - handle = open(log_path, "ab", buffering=0) - process = subprocess.Popen( - argv, - stdin=subprocess.DEVNULL, - stdout=handle, - stderr=subprocess.STDOUT, - start_new_session=True, - ) - return process.pid - - -def _default_readiness(url, deadline_seconds, clock=time.monotonic, sleep=time.sleep): - validate_health_endpoint(url) - deadline = clock() + deadline_seconds - while clock() < deadline: - try: - request = urllib.request.Request(url, method="GET") - opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - with opener.open(request, timeout=2.0) as response: - if 200 <= response.status < 400: - return True - except (OSError, ValueError, urllib.error.URLError): - pass - sleep(0.5) - return False - - -def _pid_alive(pid): - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True - except OSError: - return False - return True - - -def _pid_command(pid): - try: - result = subprocess.run( - ["ps", "-p", str(pid), "-o", "command="], - capture_output=True, text=True, timeout=5, - ) - except (OSError, subprocess.SubprocessError): - return None - if result.returncode != 0: - return None - return result.stdout.strip() or None - - -def start_serve(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, port_free=None, - wired_claims=None, spawn=None, readiness=None, - readiness_deadline=READINESS_DEADLINE_DEFAULT, now=_utc_now, - pid_alive=None): - """Execute a reviewed start plan; the only mutating serve entry point.""" - which = which or _default_which - port_free = port_free or _default_port_free - spawn = spawn or _default_spawn - readiness = readiness or _default_readiness - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise ServeError( - "preview_hash_required", - "--confirm requires the hash from a reviewed serve preview.", - "Run serve start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise ServeError( - "preview_stale", - "The supplied preview hash does not match this start plan.", - "Re-run serve start without --confirm and review the fresh plan.", - ) - - if which(plan["argv"][0]) is None: - recipe_hint = load_recipes()[plan["runtime"]]["install_hint"] - raise ServeError( - "runtime_not_installed", - "The {0} executable is not installed.".format(plan["argv"][0]), - "Install it yourself ({0}); serve never installs runtimes.".format(recipe_hint), - ) - if model_present is not None and not model_present(plan["repo"]): - raise ServeError( - "model_not_local", - "The model is not present in a local inventory.", - "Download it with the runtime's own pull command first; serve never downloads models.", - ) - if not port_free(plan["port"]): - raise ServeError( - "port_in_use", - "Port {0} is already bound on 127.0.0.1.".format(plan["port"]), - "Pick a free --port, or stop the process that owns this one.", - ) - if wired_claims is not None: - claimed = wired_claims(plan["port"], plan["runtime"]) - if claimed: - raise ServeError( - "port_in_use", - "A wired config already claims port {0}.".format(plan["port"]), - "Move this server to its own --port.", - ) - if plan["adapter_path"] is not None and not Path(plan["adapter_path"]).is_dir(): - raise ServeError( - "invalid_arguments", - "The adapter path does not exist: {0}".format(plan["adapter_path"]), - "Pass an existing --adapter-path directory produced by mlx_lm.lora.", - ) - - receipt_path = root / "{0}.json".format(plan["port"]) - if receipt_path.exists(): - existing = _read_receipt(receipt_path) - if existing is not None and pid_alive(existing.get("pid", -1)): - raise ServeError( - "port_in_use", - "A serve receipt for port {0} is still live.".format(plan["port"]), - "Run serve stop --port {0} first.".format(plan["port"]), - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}.log".format(plan["port"]) - pid = spawn(plan["argv"], str(log_path)) - if not readiness(plan["readiness"], readiness_deadline): - _terminate_pid(pid) - raise ServeError( - "readiness_timeout", - "The server did not answer its readiness endpoint in time.", - "Inspect the serve log at {0}.".format(log_path), - ) - - receipt = { - "schema_version": SERVE_RECEIPT_SCHEMA_VERSION, - "kind": SERVE_RECEIPT_KIND, - "repo": plan["repo"], - "runtime": plan["runtime"], - "port": plan["port"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "readiness": plan["readiness"], - "started_at": now(), - "preview_hash": plan["preview_hash"], - } - content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") - _atomic_in_directory(root, receipt_path.name, content, 0o600) - return {"status": "started", "receipt": receipt} - - -def _read_receipt(path): - try: - value = json.loads(Path(path).read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - return None - if not isinstance(value, dict) or value.get("kind") != SERVE_RECEIPT_KIND: - return None - if not isinstance(value.get("pid"), int) or isinstance(value.get("pid"), bool): - return None - if not isinstance(value.get("argv"), list) or not isinstance(value.get("port"), int): - return None - return value - - -def status_serve(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command): - """Cross-check serve receipts against live processes; read-only.""" - root = receipts_root(receipts_dir) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - if len(entries) >= MAX_RECEIPTS: - break - receipt = _read_receipt(path) - if receipt is None: - continue - alive = pid_alive(receipt["pid"]) - command = pid_command(receipt["pid"]) if alive else None - entries.append({ - "receipt": str(path), - "repo": receipt["repo"], - "runtime": receipt["runtime"], - "port": receipt["port"], - "pid": receipt["pid"], - "alive": alive, - "argv_match": _argv_matches(receipt, command) if alive else False, - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - }) - return entries - - -def _argv_matches(receipt, command, require_port=True): - if not command: - return False - argv = receipt.get("argv") or [] - if not argv: - return False - executable = Path(str(argv[0])).name - if executable not in command or str(receipt.get("repo")) not in command: - return False - if require_port: - return "--port {0}".format(receipt.get("port")) in command - return True - - -def _terminate_pid(pid, sig=signal.SIGTERM): - try: - os.kill(pid, sig) - except OSError: - return False - return True - - -def stop_serve(port, receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - terminate=_terminate_pid, clock=time.monotonic, sleep=time.sleep): - """Stop exactly the process a serve receipt owns; never kills by port scan.""" - root = receipts_root(receipts_dir) - receipt_path = root / "{0}.json".format(port) - receipt = _read_receipt(receipt_path) if receipt_path.is_file() else None - if receipt is None: - raise ServeError( - "receipt_not_found", - "No serve receipt exists for port {0}.".format(port), - "serve stop only stops processes that serve started; stop foreign processes yourself.", - ) - pid = receipt["pid"] - if not pid_alive(pid): - receipt_path.unlink() - return {"status": "already_stopped", "port": port, "pid": pid} - command = pid_command(pid) - if not _argv_matches(receipt, command): - raise ServeError( - "pid_argv_mismatch", - "The live pid {0} does not match the serve receipt; refusing to signal it.".format(pid), - "Inspect the process yourself; the receipt is retained at {0}.".format(receipt_path), - ) - terminate(pid, signal.SIGTERM) - deadline = clock() + STOP_DEADLINE_SECONDS - while clock() < deadline: - if not pid_alive(pid): - receipt_path.unlink() - return {"status": "stopped", "port": port, "pid": pid} - sleep(0.2) - terminate(pid, signal.SIGKILL) - if pid_alive(pid): - raise ServeError( - "stop_failed", - "The process did not exit after SIGTERM and SIGKILL.", - "Inspect pid {0} yourself; the receipt is retained at {1}.".format(pid, receipt_path), - ) - receipt_path.unlink() - return {"status": "stopped", "port": port, "pid": pid, "forced": True} - - -def wired_port_claim(roots): - """Build a wired_claims(port, runtime) gate from Wire-managed configs.""" - wired = scan_wired_configs(roots) - claims = {} - for config in wired: - endpoint = config.get("endpoint") - if not endpoint: - continue - try: - parsed = validate_health_endpoint(endpoint) - from urllib.parse import urlsplit - - parts = urlsplit(parsed) - port = parts.port or (443 if parts.scheme == "https" else 80) - except ValueError: - continue - claims.setdefault(port, config["runtime"]) - - def check(port, runtime): - owner = claims.get(port) - return owner is not None and owner != runtime - - return check - - -def default_model_present(hf_cache=None): - from .model_doctor import default_hf_cache, inspect_hf_cache - - cache = Path(hf_cache) if hf_cache is not None else default_hf_cache() - known = {item["id"].casefold() for item in inspect_hf_cache(cache)} - - def check(repo): - return repo.casefold() in known - - return check - - -LAUNCHD_LABEL_PREFIX = "com.mlx-agent.serve." -_LAUNCHD_LABEL = re.compile(r"\A" + LAUNCHD_LABEL_PREFIX.replace(".", r"\.") + r"(\d{1,5})\Z") - - -def launchd_label(port): - return "{0}{1}".format(LAUNCHD_LABEL_PREFIX, port) - - -def render_launchd_plist(plan, log_path=None): - """Render a deterministic launchd plist for a reviewed serve plan.""" - from xml.sax.saxutils import escape - - label = launchd_label(plan["port"]) - log = log_path or str(receipts_root() / "{0}.log".format(plan["port"])) - arguments = "\n".join( - " {0}".format(escape(str(part))) for part in plan["argv"] - ) - return ( - '\n' - '\n' - '\n' - '\n' - ' Label\n' - ' {label}\n' - ' ProgramArguments\n' - ' \n' - '{arguments}\n' - ' \n' - ' RunAtLoad\n' - ' \n' - ' KeepAlive\n' - ' \n' - ' StandardOutPath\n' - ' {log}\n' - ' StandardErrorPath\n' - ' {log}\n' - '\n' - '\n' - ).format(label=escape(label), arguments=arguments, log=escape(str(log))) - - -class LaunchdPlistAdapter: - """Validate the exact launchd plist subset serve renders.""" - - version = "1.0" - runtime = "launchd" - - def __init__(self, path=None): - self.path = Path(path) if path is not None else None - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("plist content must be text") - require_secret_free_config(content) - lines = content.splitlines() - if len(lines) < 8 or not lines[0].startswith("': - raise ValueError("launchd plist must use the managed XML subset") - if lines[-1] != "" or lines[-2] != "": - raise ValueError("launchd plist must close its dict and plist elements") - text = content - for required in ("Label", "ProgramArguments", "RunAtLoad"): - if required not in text: - raise ValueError("launchd plist is missing {0}".format(required)) - label_match = re.search(r"Label\s*\n\s*([^<]+)", text) - if label_match is None or _LAUNCHD_LABEL.fullmatch(label_match.group(1)) is None: - raise ValueError("launchd plist label must use the managed serve prefix") - return True diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/verification.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/verification.py deleted file mode 100644 index 65b5e75..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/verification.py +++ /dev/null @@ -1,1163 +0,0 @@ -"""Safe, bounded verification of discovered local-model candidates.""" - -from __future__ import annotations - -import ast -import http.client -import ipaddress -import json -import math -import threading -import time -from copy import deepcopy -from dataclasses import asdict, dataclass, field -from enum import Enum -from types import MappingProxyType -from typing import Any, Dict, Optional -from urllib.parse import urlsplit - -from .probe_assets import VISION_PROBE_TEXT, vision_probe_image_base64 -from .wiring import redact_secrets - - -TOOL_USE_PROBE_ID = "tool-use-v1" -TOOL_USE_TOOL_NAME = "lookup_widget" -TOOL_USE_ARGUMENTS = MappingProxyType({"widget_id": "widget-42"}) -TOOL_USE_PROMPT = ( - "Call lookup_widget exactly once for widget-42. " - "Do not answer directly and do not call any other tool." -) -TOOL_USE_TOOLS = [ - { - "type": "function", - "function": { - "name": TOOL_USE_TOOL_NAME, - "description": "Look up one synthetic widget by its identifier.", - "parameters": { - "type": "object", - "properties": { - "widget_id": { - "type": "string", - "description": "Synthetic widget identifier.", - } - }, - "required": ["widget_id"], - "additionalProperties": False, - }, - }, - } -] -_TOOL_USE_ARGUMENTS_MAX_CHARS = 512 -INVENTORY_RESPONSE_MAX_BYTES = 1024 * 1024 -PROBE_RESPONSE_MAX_BYTES = 256 * 1024 -_HTTP_READ_CHUNK_BYTES = 16 * 1024 - -CODING_PROBE_ID = "coding-v1" -REASONING_PROBE_ID = "reasoning-v1" -VISION_PROBE_ID = "vision-v1" -EMBEDDING_PROBE_ID = "embedding-v1" -ROLE_PROBE_IDS = MappingProxyType({ - "coding": CODING_PROBE_ID, - "reasoning": REASONING_PROBE_ID, - "vision": VISION_PROBE_ID, - "embedding": EMBEDDING_PROBE_ID, -}) - -CODING_PROBE_PROMPT = ( - "Write a Python function named add that takes two arguments a and b " - "and returns their sum. Reply with only the function definition." -) -CODING_PROBE_MAX_TOKENS = 256 -_CODING_PROBE_MAX_CODE_CHARS = 2000 -_CODING_PROBE_BLOCKED_TOKENS = ( - "import", "__", "while", "for", "open", "exec", "eval", "input", - "globals", "locals", "compile", "breakpoint", "exit", "quit", - "getattr", "setattr", "delattr", "vars", "help", "print", -) -REASONING_PROBE_PROMPT = ( - "A farmer has 17 sheep. All but 9 run away. " - "How many sheep does the farmer have left? Reply with only the number." -) -REASONING_PROBE_ANSWER = 9 -REASONING_PROBE_MAX_TOKENS = 64 -VISION_PROBE_PROMPT = ( - "What text is shown in this image? Reply with only the text in the image." -) -VISION_PROBE_MAX_TOKENS = 64 -EMBEDDING_PROBE_SENTENCES = ( - "The cat sat on the warm windowsill.", - "A kitten rested on the sunny ledge.", - "Quantum chromodynamics describes the strong nuclear force.", -) -EMBEDDING_PROBE_MAX_VECTORS = 8 -_EMBEDDING_PROBE_MAX_DIMENSION = 8192 - - -def _role_probe_outcome(reason): - return {"valid": reason == "valid", "reason": reason} - - -def validate_coding_response(response): - """Validate the fixed coding probe without retaining raw model output.""" - try: - content, _hidden = _generation_text(response) - code = _extract_probe_code(content) - if code is None: - return _role_probe_outcome("missing_content") - if len(code) > _CODING_PROBE_MAX_CODE_CHARS: - return _role_probe_outcome("unsafe_content") - lowered = code.casefold() - if any(token in lowered for token in _CODING_PROBE_BLOCKED_TOKENS): - return _role_probe_outcome("unsafe_content") - try: - tree = ast.parse(code) - except (SyntaxError, ValueError, RecursionError, MemoryError): - return _role_probe_outcome("parse_failure") - function_defs = [ - node for node in tree.body - if isinstance(node, ast.FunctionDef) and node.name == "add" - ] - if not function_defs: - return _role_probe_outcome("missing_function") - namespace = {"__builtins__": {}} - try: - exec(compile(tree, "", "exec"), namespace) - add = namespace.get("add") - if not callable(add): - return _role_probe_outcome("missing_function") - if add(2, 3) != 5 or add(-1, 1) != 0: - return _role_probe_outcome("wrong_answer") - except Exception: - return _role_probe_outcome("wrong_answer") - return _role_probe_outcome("valid") - except Exception: - return _role_probe_outcome("invalid_response") - - -def _extract_probe_code(content): - if not isinstance(content, str) or not content.strip(): - return None - text = content.strip() - if "```" in text: - segments = text.split("```") - for segment in segments[1::2]: - body = segment.split("\n", 1) - candidate = body[1] if len(body) == 2 else "" - if candidate.strip(): - return candidate.strip() - return None - return text - - -def validate_reasoning_response(response): - """Validate the fixed arithmetic reasoning probe by exact answer match.""" - try: - content, _hidden = _generation_text(response) - if not isinstance(content, str) or not content.strip(): - return _role_probe_outcome("missing_content") - digits = [] - current = [] - for character in content: - if character.isdigit() or (character == "-" and not current): - current.append(character) - elif current: - digits.append("".join(current)) - current = [] - if current: - digits.append("".join(current)) - if not digits: - return _role_probe_outcome("missing_content") - try: - answer = int(digits[-1]) - except ValueError: - return _role_probe_outcome("missing_content") - if answer != REASONING_PROBE_ANSWER: - return _role_probe_outcome("wrong_answer") - return _role_probe_outcome("valid") - except Exception: - return _role_probe_outcome("invalid_response") - - -def validate_vision_response(response): - """Validate the fixed OCR probe by case-folded substring match.""" - try: - content, _hidden = _generation_text(response) - if not isinstance(content, str) or not content.strip(): - return _role_probe_outcome("missing_content") - normalized = "".join(content.split()).casefold() - if VISION_PROBE_TEXT.casefold() not in normalized: - return _role_probe_outcome("wrong_answer") - return _role_probe_outcome("valid") - except Exception: - return _role_probe_outcome("invalid_response") - - -def validate_embedding_response(response): - """Validate the fixed embedding probe by cosine ordering.""" - try: - vectors = _embedding_vectors(response) - if vectors is None or len(vectors) != len(EMBEDDING_PROBE_SENTENCES): - return _role_probe_outcome("invalid_response") - similar = _cosine_similarity(vectors[0], vectors[1]) - dissimilar = _cosine_similarity(vectors[0], vectors[2]) - if similar is None or dissimilar is None: - return _role_probe_outcome("invalid_response") - if not similar > dissimilar: - return _role_probe_outcome("order_inverted") - return _role_probe_outcome("valid") - except Exception: - return _role_probe_outcome("invalid_response") - - -def _embedding_vectors(response): - if not isinstance(response, dict): - return None - records = response.get("embeddings") - if records is None: - data = response.get("data") - if isinstance(data, list): - records = [ - item.get("embedding") if isinstance(item, dict) else None - for item in sorted( - data, - key=lambda item: item.get("index", 0) if isinstance(item, dict) else 0, - ) - ] - if not isinstance(records, list) or len(records) > EMBEDDING_PROBE_MAX_VECTORS: - return None - vectors = [] - for record in records: - if ( - not isinstance(record, list) - or not record - or len(record) > _EMBEDDING_PROBE_MAX_DIMENSION - or any(not isinstance(value, (int, float)) or isinstance(value, bool) for value in record) - ): - return None - vectors.append([float(value) for value in record]) - return vectors - - -def _cosine_similarity(left, right): - if len(left) != len(right): - return None - dot = sum(a * b for a, b in zip(left, right)) - norm_left = math.sqrt(sum(a * a for a in left)) - norm_right = math.sqrt(sum(b * b for b in right)) - if norm_left == 0 or norm_right == 0: - return None - return dot / (norm_left * norm_right) - - -def normalize_tool_call(response): - """Normalize one synthetic tool call without retaining raw model output.""" - try: - message = _tool_call_message(response) - if message is None: - return _tool_call_result("invalid_response") - - if "tool_calls" not in message or message["tool_calls"] is None: - return _tool_call_result("missing_tool_call") - tool_calls = message["tool_calls"] - if not isinstance(tool_calls, list): - return _tool_call_result("invalid_response") - if not tool_calls: - return _tool_call_result("missing_tool_call") - if len(tool_calls) != 1: - return _tool_call_result("multiple_tool_calls") - - tool_call = tool_calls[0] - if not isinstance(tool_call, dict): - return _tool_call_result("invalid_response") - function = tool_call.get("function") - if not isinstance(function, dict): - return _tool_call_result("invalid_response") - raw_name = function.get("name") - if not isinstance(raw_name, str): - return _tool_call_result("invalid_response") - if raw_name != TOOL_USE_TOOL_NAME: - return _tool_call_result("wrong_tool") - - if "arguments" not in function: - return _tool_call_result("malformed_arguments") - arguments = function["arguments"] - if isinstance(arguments, str): - if len(arguments) > _TOOL_USE_ARGUMENTS_MAX_CHARS: - return _tool_call_result("malformed_arguments") - try: - arguments = json.loads(arguments) - except (json.JSONDecodeError, RecursionError): - return _tool_call_result("malformed_arguments") - if not isinstance(arguments, dict): - return _tool_call_result("malformed_arguments") - if arguments != TOOL_USE_ARGUMENTS: - return _tool_call_result("schema_invalid") - return _tool_call_result("valid") - except Exception: - return _tool_call_result("invalid_response") - - -def _tool_call_message(response): - if not isinstance(response, dict): - return None - if "message" in response: - message = response["message"] - return message if isinstance(message, dict) else None - if "choices" not in response: - return None - choices = response["choices"] - if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict): - return None - message = choices[0].get("message") - return message if isinstance(message, dict) else None - - -def _tool_call_result(reason): - valid = reason == "valid" - return { - "valid": valid, - "tool_name": TOOL_USE_TOOL_NAME if valid else None, - "arguments": dict(TOOL_USE_ARGUMENTS) if valid else None, - "reason": reason, - } - - -class EvidenceStrength(str, Enum): - """Ordered labels describing how directly a candidate was verified.""" - - runtime_measured = "runtime_measured" - runtime_tested = "runtime_tested" - runtime_inventory = "runtime_inventory" - metadata_only = "metadata_only" - heuristic_only = "heuristic_only" - - # Conventional aliases preserve a readable internal style while the exact - # lowercase contract names remain available to API consumers. - RUNTIME_MEASURED = runtime_measured - RUNTIME_TESTED = runtime_tested - RUNTIME_INVENTORY = runtime_inventory - METADATA_ONLY = metadata_only - HEURISTIC_ONLY = heuristic_only - - -class VerificationStatus(str, Enum): - """Outcome labels independent from the source strength of the evidence.""" - - verified = "verified" - metadata_only = "metadata-only" - failed = "failed" - unsupported_runtime = "unsupported-runtime" - - VERIFIED = verified - METADATA_ONLY = metadata_only - FAILED = failed - UNSUPPORTED_RUNTIME = unsupported_runtime - - -@dataclass(frozen=True) -class VerificationEvidence: - """Portable evidence from one candidate verification attempt.""" - - repo: str - role: str - strength: EvidenceStrength - status: VerificationStatus - available_locally: bool - loads: Optional[bool] - reasoning_confirmed: Optional[bool] - runtime: Optional[str] - note: str - details: Dict[str, Any] = field(default_factory=dict) - - def to_dict(self) -> Dict[str, Any]: - value = asdict(self) - value["strength"] = self.strength.value - value["status"] = self.status.value - return value - - -def _http_json_get(url, timeout=3.0): - """Read a local runtime endpoint without installing or starting anything.""" - return _http_json_request( - url, - "GET", - timeout=timeout, - max_response_bytes=INVENTORY_RESPONSE_MAX_BYTES, - ) - - -def _http_json_post(url, payload, timeout=10.0): - """Send the single bounded generation probe used by verification.""" - return _http_json_request( - url, - "POST", - payload=payload, - timeout=timeout, - max_response_bytes=PROBE_RESPONSE_MAX_BYTES, - ) - - -def _http_event_stream(url, payload, timeout=120.0, max_response_bytes=INVENTORY_RESPONSE_MAX_BYTES, - connection_factory=None, clock=time.monotonic): - """Yield bounded JSON events from a validated loopback streaming endpoint. - - Handles both server-sent events (``data: {...}`` lines terminated by - ``data: [DONE]``) and newline-delimited JSON (Ollama-style). - """ - parsed = _validated_loopback_url(url) - deadline = clock() + timeout - port = parsed.port or (443 if parsed.scheme == "https" else 80) - if connection_factory is None: - connection_type = ( - http.client.HTTPSConnection - if parsed.scheme == "https" - else http.client.HTTPConnection - ) - connection = connection_type(parsed.hostname, port, timeout=_deadline_remaining(deadline, clock)) - else: - connection = connection_factory(parsed.scheme, parsed.hostname, port, _deadline_remaining(deadline, clock)) - try: - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - connection.request( - "POST", - parsed.path or "/", - body=json.dumps(payload).encode("utf-8"), - headers={"Content-Type": "application/json"}, - ) - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - response = connection.getresponse() - _deadline_remaining(deadline, clock) - if 300 <= response.status < 400: - raise http.client.HTTPException( - "redirect responses are not allowed for local runtime requests" - ) - if not 200 <= response.status < 300: - raise http.client.HTTPException( - "local runtime returned HTTP status {0}".format(response.status) - ) - buffer = b"" - total = 0 - done = False - while not done: - remaining = _deadline_remaining(deadline, clock) - _set_connection_timeout(connection, remaining) - chunk = response.read(_HTTP_READ_CHUNK_BYTES) - _deadline_remaining(deadline, clock) - if not chunk: - break - total += len(chunk) - if total > max_response_bytes: - raise ValueError("local runtime response exceeds size limit") - buffer += chunk - while b"\n" in buffer: - line, buffer = buffer.split(b"\n", 1) - event, done = _parse_stream_line(line) - if event is not None: - yield event - if done: - break - if not done and buffer.strip(): - event, _done = _parse_stream_line(buffer) - if event is not None: - yield event - finally: - connection.close() - - -def _parse_stream_line(line): - text = line.strip() - if not text: - return None, False - if text.startswith(b"data:"): - text = text[len(b"data:"):].strip() - if text == b"[DONE]": - return None, True - if not text: - return None, False - if not text.startswith(b"{"): - return None, False - try: - return json.loads(text.decode("utf-8")), False - except (UnicodeDecodeError, json.JSONDecodeError): - return None, False - - -def _http_json_request( - url, - method, - payload=None, - timeout=10.0, - max_response_bytes=PROBE_RESPONSE_MAX_BYTES, - connection_factory=None, - clock=time.monotonic, -): - """Exchange bounded JSON with one validated loopback endpoint.""" - parsed = _validated_loopback_url(url) - deadline = clock() + timeout - remaining = _deadline_remaining(deadline, clock) - port = parsed.port or (443 if parsed.scheme == "https" else 80) - if connection_factory is None: - connection_type = ( - http.client.HTTPSConnection - if parsed.scheme == "https" - else http.client.HTTPConnection - ) - connection = connection_type(parsed.hostname, port, timeout=remaining) - else: - connection = connection_factory( - parsed.scheme, - parsed.hostname, - port, - remaining, - ) - body = None - headers = {} - if payload is not None: - body = json.dumps(payload).encode("utf-8") - headers["Content-Type"] = "application/json" - try: - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - connection.request(method, parsed.path or "/", body=body, headers=headers) - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - response = connection.getresponse() - _deadline_remaining(deadline, clock) - if 300 <= response.status < 400: - raise http.client.HTTPException( - "redirect responses are not allowed for local runtime requests" - ) - if not 200 <= response.status < 300: - raise http.client.HTTPException( - "local runtime returned HTTP status {0}".format(response.status) - ) - response_body = _read_bounded_response( - response, - connection, - max_response_bytes, - deadline, - clock, - ) - return json.loads(response_body.decode("utf-8")) - finally: - connection.close() - - -def _read_bounded_response(response, connection, limit, deadline, clock): - if not isinstance(limit, int) or limit <= 0: - raise ValueError("response size limit must be a positive integer") - content_length = response.getheader("Content-Length") - if content_length is not None: - try: - declared_length = int(content_length) - except (TypeError, ValueError) as error: - raise ValueError("invalid Content-Length from local runtime") from error - if declared_length < 0: - raise ValueError("invalid Content-Length from local runtime") - if declared_length > limit: - raise ValueError("local runtime response exceeds size limit") - - chunks = [] - total = 0 - while True: - remaining = _deadline_remaining(deadline, clock) - _set_connection_timeout(connection, remaining) - chunk = response.read(min(_HTTP_READ_CHUNK_BYTES, limit - total + 1)) - _deadline_remaining(deadline, clock) - if not chunk: - break - total += len(chunk) - if total > limit: - raise ValueError("local runtime response exceeds size limit") - chunks.append(chunk) - return b"".join(chunks) - - -def _deadline_remaining(deadline, clock): - remaining = deadline - clock() - if remaining <= 0: - raise TimeoutError("local runtime request exceeded its overall deadline") - return remaining - - -def _set_connection_timeout(connection, timeout): - sock = getattr(connection, "sock", None) - if sock is not None: - sock.settimeout(timeout) - - -def _local_runtime_origin(base_url): - """Return a normalized credential-free HTTP(S) loopback origin.""" - parsed = _validated_loopback_url(base_url) - if parsed.path not in ("", "/"): - raise ValueError("runtime base URL must be an origin without a path") - return base_url.rstrip("/") - - -def _validated_loopback_url(url): - if not isinstance(url, str) or url != url.strip(): - raise ValueError("runtime URL must be a trimmed string") - parsed = urlsplit(url) - if parsed.scheme not in ("http", "https"): - raise ValueError("runtime URL must use HTTP(S)") - if parsed.username is not None or parsed.password is not None: - raise ValueError("runtime URL must not contain credentials") - if parsed.query or parsed.fragment: - raise ValueError("runtime URL must not contain a query or fragment") - hostname = parsed.hostname - if hostname is None: - raise ValueError("runtime URL must include a host") - try: - parsed.port - except ValueError as error: - raise ValueError("runtime URL has an invalid port") from error - if hostname.casefold() != "localhost": - try: - address = ipaddress.ip_address(hostname) - except ValueError as error: - raise ValueError("runtime URL must use a loopback host") from error - if not address.is_loopback: - raise ValueError("runtime URL must use a loopback host") - return parsed - - -def _tool_use_payload_tools(): - """Copy the fixed schema into ordinary JSON-serializable containers.""" - return deepcopy(TOOL_USE_TOOLS) - - -class OllamaRuntimeClient: - """Read-only inventory and bounded generation adapter for an existing Ollama.""" - - name = "ollama" - - def __init__(self, http_get=None, http_post=None, base_url="http://127.0.0.1:11434"): - self._http_get = http_get or _http_json_get - self._http_post = http_post or _http_json_post - self._base_url = _local_runtime_origin(base_url) - - def list_models(self): - return self._http_get("{0}/api/tags".format(self._base_url), timeout=3.0) - - def generate(self, model, prompt, max_tokens): - return self._http_post( - "{0}/api/chat".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "stream": False, - "options": {"num_predict": max_tokens}, - }, - timeout=10.0, - ) - - def probe_tool_use(self, model): - return self._http_post( - "{0}/api/chat".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": TOOL_USE_PROMPT}], - "tools": _tool_use_payload_tools(), - "stream": False, - "options": {"num_predict": 64}, - }, - timeout=10.0, - ) - - def embed(self, model, inputs): - return self._http_post( - "{0}/api/embed".format(self._base_url), - {"model": model, "input": list(inputs)}, - timeout=10.0, - ) - - def stream_generate(self, model, prompt, max_tokens, timeout=120.0): - return _http_event_stream( - "{0}/api/chat".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "stream": True, - "options": {"num_predict": max_tokens}, - }, - timeout=timeout, - ) - - -class OpenAICompatibleRuntimeClient: - """Bounded adapter for an existing local OpenAI-compatible runtime.""" - - def __init__(self, name, base_url, http_get=None, http_post=None): - self.name = name - self._http_get = http_get or _http_json_get - self._http_post = http_post or _http_json_post - self._base_url = _local_runtime_origin(base_url) - - def list_models(self): - return self._http_get("{0}/v1/models".format(self._base_url), timeout=3.0) - - def generate(self, model, prompt, max_tokens): - return self._http_post( - "{0}/v1/chat/completions".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - }, - timeout=10.0, - ) - - def probe_tool_use(self, model): - return self._http_post( - "{0}/v1/chat/completions".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": TOOL_USE_PROMPT}], - "tools": _tool_use_payload_tools(), - "tool_choice": "auto", - "max_tokens": 64, - }, - timeout=10.0, - ) - - def embed(self, model, inputs): - return self._http_post( - "{0}/v1/embeddings".format(self._base_url), - {"model": model, "input": list(inputs)}, - timeout=10.0, - ) - - def stream_generate(self, model, prompt, max_tokens, timeout=120.0): - return _http_event_stream( - "{0}/v1/chat/completions".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "stream": True, - "stream_options": {"include_usage": True}, - }, - timeout=timeout, - ) - - -class LMStudioRuntimeClient(OpenAICompatibleRuntimeClient): - """Backward-compatible adapter for an existing LM Studio runtime.""" - - name = "lmstudio" - - def __init__(self, http_get=None, http_post=None, base_url="http://127.0.0.1:1234"): - super().__init__( - "lmstudio", - base_url, - http_get=http_get, - http_post=http_post, - ) - - -class MLXVLMRuntimeClient(OpenAICompatibleRuntimeClient): - """Bounded adapter for an existing local mlx-vlm vision runtime.""" - - name = "mlx-vlm" - - def __init__(self, http_get=None, http_post=None, base_url="http://127.0.0.1:8083"): - super().__init__( - "mlx-vlm", - base_url, - http_get=http_get, - http_post=http_post, - ) - - def generate_vision(self, model, prompt, image_base64, max_tokens): - return self._http_post( - "{0}/v1/chat/completions".format(self._base_url), - { - "model": model, - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,{0}".format(image_base64) - }, - }, - ], - } - ], - "max_tokens": max_tokens, - }, - timeout=30.0, - ) - - -class Verifier: - """Verify installed candidates without ever installing or downloading them. - - Runtime clients implement ``list_models()`` and ``generate(model, prompt, - max_tokens)``; tool-capable clients also implement ``probe_tool_use(model)``. - Metadata clients implement ``inspect_model(repo)``. Keeping these protocols - implicit allows provider adapters and tests to supply clients without adding - runtime dependencies. - """ - - def __init__(self, runtime_clients=None, metadata_client=None, runtime_client=None): - if runtime_clients is None: - runtime_clients = ( - [] - if runtime_client is not None - else [ - OllamaRuntimeClient(), - LMStudioRuntimeClient(), - OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080"), - MLXVLMRuntimeClient(), - OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000"), - ] - ) - if isinstance(runtime_clients, dict): - runtime_clients = list(runtime_clients.values()) - self._runtime_clients = list(runtime_clients) - if runtime_client is not None: - self._runtime_clients.append(runtime_client) - self._metadata_client = metadata_client - self._inventory_cache = {} - self._inventory_cache_lock = threading.Lock() - - def clear_inventory_cache(self): - """Start a new verification phase with fresh runtime inventories.""" - with self._inventory_cache_lock: - self._inventory_cache.clear() - - def runtime_clients(self): - """Return the configured runtime clients without mutating them.""" - return tuple(self._runtime_clients) - - def _runtime_inventory(self, runtime): - cache_key = id(runtime) - with self._inventory_cache_lock: - if cache_key not in self._inventory_cache: - try: - result = (installed_model_ids(runtime.list_models()), None) - except Exception as error: - result = (None, _safe_error(error)) - self._inventory_cache[cache_key] = result - return self._inventory_cache[cache_key] - - def verify(self, candidate, host, allow_network=True) -> VerificationEvidence: - """Return the strongest available evidence without mutating runtimes.""" - del host # Reserved for runtime adapters whose probes depend on host facts. - repo = _candidate_repo(candidate) - role = str(candidate.get("role") or candidate.get("heuristics", {}).get("role") or "general") - inventory_errors = [] - - for runtime in self._runtime_clients: - installed, inventory_error = self._runtime_inventory(runtime) - if inventory_error is not None: - inventory_errors.append( - "{0}: {1}".format(_runtime_name(runtime), inventory_error) - ) - continue - if not _is_installed(repo, installed): - continue - runtime_name = _runtime_name(runtime) - if role == "tool-use": - return _verify_tool_use(repo, role, runtime_name, runtime) - try: - response = runtime.generate( - repo, - "Reply with the single word ready.", - max_tokens=24, - ) - content, hidden_reasoning = _generation_text(response) - reasoning_confirmed, reasoning_evidence = _reasoning_confirmation( - candidate, hidden_reasoning - ) - details = { - "visible_content": bool(content.strip()), - "hidden_reasoning": bool(hidden_reasoning), - "reasoning_evidence": reasoning_evidence, - } - if role in ROLE_PROBE_IDS: - details["probes"] = [_run_role_probe(role, repo, runtime)] - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_TESTED, - status=VerificationStatus.VERIFIED, - available_locally=True, - loads=True, - reasoning_confirmed=reasoning_confirmed, - runtime=runtime_name, - note="Runtime generation completed without downloading the model.", - details=details, - ) - except Exception as error: - safe_error = _safe_error(error) - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_INVENTORY, - status=VerificationStatus.FAILED, - available_locally=True, - loads=False, - reasoning_confirmed=None, - runtime=runtime_name, - note=_bounded( - "Installed runtime inventory found the model, but " - "generation failed: {0}".format(safe_error) - ), - details={"error": safe_error}, - ) - - if allow_network and self._metadata_client is not None: - try: - metadata = self._metadata_client.inspect_model(repo) - reasoning_confirmed, reasoning_evidence = _reasoning_confirmation( - candidate, metadata - ) - details = _bounded_metadata(metadata) - details["reasoning_evidence"] = reasoning_evidence - if role == "tool-use": - details["probe_id"] = TOOL_USE_PROBE_ID - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.METADATA_ONLY, - status=VerificationStatus.METADATA_ONLY, - available_locally=False, - loads=None, - reasoning_confirmed=reasoning_confirmed, - runtime=None, - note="Model is not installed; repository metadata was inspected without downloading it.", - details=details, - ) - except Exception as error: - inventory_errors.append("metadata: {0}".format(_safe_error(error))) - - note = "Model is not installed; only candidate heuristics were available." - if inventory_errors: - note += " Probe errors: {0}".format("; ".join(inventory_errors)) - details = { - "reasoning_heuristic": candidate.get("reasoning"), - "reasoning_source": candidate.get("reason_src"), - } - if role == "tool-use": - details["probe_id"] = TOOL_USE_PROBE_ID - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.HEURISTIC_ONLY, - status=VerificationStatus.METADATA_ONLY, - available_locally=False, - loads=None, - reasoning_confirmed=None, - runtime=None, - note=_bounded(note), - details=details, - ) - - -def _run_role_probe(role, repo, runtime): - """Run the role's deterministic probe; never raises into verification.""" - probe_id = ROLE_PROBE_IDS[role] - try: - if role == "coding": - response = runtime.generate(repo, CODING_PROBE_PROMPT, max_tokens=CODING_PROBE_MAX_TOKENS) - outcome = validate_coding_response(response) - elif role == "reasoning": - response = runtime.generate(repo, REASONING_PROBE_PROMPT, max_tokens=REASONING_PROBE_MAX_TOKENS) - outcome = validate_reasoning_response(response) - elif role == "embedding": - embed = getattr(runtime, "embed", None) - if not callable(embed): - outcome = _role_probe_outcome("unsupported_runtime") - else: - outcome = validate_embedding_response( - embed(repo, list(EMBEDDING_PROBE_SENTENCES)) - ) - elif role == "vision": - generate_vision = getattr(runtime, "generate_vision", None) - if not callable(generate_vision): - outcome = _role_probe_outcome("unsupported_runtime") - else: - response = generate_vision( - repo, - VISION_PROBE_PROMPT, - vision_probe_image_base64(), - VISION_PROBE_MAX_TOKENS, - ) - outcome = validate_vision_response(response) - else: - outcome = _role_probe_outcome("unsupported_runtime") - except Exception as error: - outcome = {"valid": False, "reason": "probe_error", "error": _bounded(_safe_error(error), 160)} - return {"probe_id": probe_id, "outcome": outcome} - - -def _verify_tool_use(repo, role, runtime_name, runtime): - if runtime_name == "mlx-vlm" or not callable(getattr(runtime, "probe_tool_use", None)): - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_INVENTORY, - status=VerificationStatus.UNSUPPORTED_RUNTIME, - available_locally=True, - loads=None, - reasoning_confirmed=None, - runtime=runtime_name, - note=_bounded( - "Installed runtime inventory found the model, but this runtime " - "does not support the bounded tool-use probe." - ), - details={ - "probe_id": TOOL_USE_PROBE_ID, - "outcome": {"reason": "unsupported_runtime"}, - }, - ) - try: - outcome = normalize_tool_call(runtime.probe_tool_use(repo)) - except Exception as error: - error_name = _redacted_error(error) - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_INVENTORY, - status=VerificationStatus.FAILED, - available_locally=True, - loads=False, - reasoning_confirmed=None, - runtime=runtime_name, - note=_bounded( - "Installed runtime inventory found the model, but the bounded " - "tool-use probe failed: {0}".format(error_name) - ), - details={ - "probe_id": TOOL_USE_PROBE_ID, - "error": _bounded(error_name, 160), - }, - ) - if outcome["valid"]: - note = "Runtime returned a schema-valid synthetic tool call." - else: - note = "Runtime responded, but the probe did not return a valid call." - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_TESTED, - status=( - VerificationStatus.VERIFIED - if outcome["valid"] - else VerificationStatus.FAILED - ), - available_locally=True, - loads=True, - reasoning_confirmed=None, - runtime=runtime_name, - note=note, - details={"probe_id": TOOL_USE_PROBE_ID, "outcome": outcome}, - ) - - -def _redacted_error(error): - name = error.__class__.__name__ - return _bounded(name if name else "runtime error", 160) - - -def _safe_error(error): - name = error.__class__.__name__ or "Error" - message = redact_secrets(str(error)) - value = "{0}: {1}".format(name, message) if message else name - return _bounded(value, 160) - - -def _candidate_repo(candidate): - if not isinstance(candidate, dict): - raise TypeError("candidate must be an object") - repo = candidate.get("repo") or candidate.get("repository") - if not isinstance(repo, str) or not repo: - raise ValueError("candidate.repo must be a non-empty string") - return repo - - -def _runtime_name(runtime): - return str(getattr(runtime, "name", runtime.__class__.__name__)) - - -def installed_model_ids(records) -> set: - """Return model IDs from a supported runtime's read-only inventory response.""" - if isinstance(records, dict): - records = records.get("models", records.get("data", [])) - models = set() - for record in records or []: - if isinstance(record, str): - models.add(record) - continue - if isinstance(record, dict): - value = record.get("repo") or record.get("id") or record.get("name") or record.get("model") - if isinstance(value, str) and value: - models.add(value) - return models - - -def _is_installed(repo, installed): - normalized = repo.casefold() - return any(item.casefold() == normalized for item in installed) - - -def _generation_text(response): - if isinstance(response, str): - return response, "" - if not isinstance(response, dict): - return "", "" - message = response.get("message") - if message is None: - choices = response.get("choices") or [] - if choices and isinstance(choices[0], dict): - message = choices[0].get("message") or choices[0] - if not isinstance(message, dict): - message = response - content = message.get("content") or message.get("response") or "" - hidden = ( - message.get("reasoning_content") - or message.get("thinking") - or message.get("reasoning") - or "" - ) - return str(content), str(hidden) - - -def _reasoning_confirmation(candidate, signal): - """Prefer any positive runtime or metadata signal over an inconclusive probe.""" - if isinstance(signal, str) and signal.strip(): - return True, "runtime_hidden" - if isinstance(signal, dict): - if signal.get("reasoning") is True: - source = signal.get("reason_src") or "field" - return True, "metadata_{0}".format(source) - tags = signal.get("tags") or [] - if any(str(tag).lower() in ("reasoning", "thinking", "chain-of-thought") for tag in tags): - return True, "metadata_tags" - if signal.get("chat_template"): - return True, "metadata_chat_template" - if candidate.get("reasoning") is True and candidate.get("reason_src") in ( - "chat_template", "tags", - ): - return True, "discovery_{0}".format(candidate["reason_src"]) - return False, "none" - - -def _bounded(value, limit=300): - text = " ".join(str(value).split()) - if len(text) <= limit: - return text - return text[: max(0, limit - 3)] + "..." - - -def _bounded_metadata(metadata): - if not isinstance(metadata, dict): - return {} - allowed = ("metadata_available", "gated", "license", "tags", "reasoning", "reason_src", "weight_bytes") - value = {key: metadata[key] for key in allowed if key in metadata} - if isinstance(value.get("tags"), list): - value["tags"] = value["tags"][:20] - return value diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/watch.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/watch.py deleted file mode 100644 index 63519d1..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/watch.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Stateful Hugging Face diff digest focused on models you own.""" - -from __future__ import annotations - -import json -from datetime import datetime, timezone -from pathlib import Path - -from .model_doctor import ( - collect_runtime_inventory, - default_hf_cache, - inspect_hf_cache, - scan_wired_configs, -) -from .models import base_name -from .transactions import _atomic_in_directory - - -WATCH_SCHEMA_VERSION = "1.0" -STATE_FILENAME = "watch-state.json" -MAX_OWNED = 500 -MAX_CANDIDATES = 2000 -MAX_FINDINGS = 100 - - -class WatchError(RuntimeError): - """Classified watch failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def state_file(state_dir): - return Path(state_dir) / "watch" / STATE_FILENAME - - -def collect_owned(runtime_clients, hf_cache=None, wired_roots=None): - """Collect owned model ids from local inventories and wired configs.""" - owned = [] - seen = set() - - def add(model_id, source): - if not isinstance(model_id, str) or not model_id: - return - key = model_id.casefold() - if key in seen or len(owned) >= MAX_OWNED: - return - seen.add(key) - owned.append({"id": model_id, "source": source}) - - cache = Path(hf_cache) if hf_cache is not None else default_hf_cache() - for item in inspect_hf_cache(cache): - add(item["id"], "hf-cache") - runtime_inventory, _errors = collect_runtime_inventory(runtime_clients) - for item in runtime_inventory: - add(item["id"], item["source"]) - for config in scan_wired_configs(wired_roots or [Path.cwd()]): - add(config.get("model"), "wired") - return owned - - -def snapshot_candidates(discovery_data): - """Flatten a discovery result into a bounded repo -> signal mapping.""" - candidates = {} - roles = discovery_data.get("roles") if isinstance(discovery_data, dict) else None - if not isinstance(roles, dict): - return candidates - for records in roles.values(): - if not isinstance(records, list): - continue - for record in records: - if not isinstance(record, dict): - continue - repo = record.get("repo") - if not isinstance(repo, str) or not repo or repo in candidates: - continue - facts = record.get("facts") if isinstance(record.get("facts"), dict) else {} - candidates[repo] = { - "base": record.get("base") or base_name(repo), - "weight_bytes": facts.get("weight_bytes"), - "gated": facts.get("gated"), - "license": record.get("license"), - } - if len(candidates) >= MAX_CANDIDATES: - return candidates - return candidates - - -def build_snapshot(owned, candidates, now=None): - return { - "schema_version": WATCH_SCHEMA_VERSION, - "created_at": now or _utc_now(), - "owned": list(owned)[:MAX_OWNED], - "candidates": dict(candidates), - "previous": None, - } - - -def write_snapshot(state_dir, snapshot): - """Persist the snapshot, rotating any existing one into `previous`.""" - destination = state_file(state_dir) - destination.parent.mkdir(parents=True, exist_ok=True) - existing = None - if destination.is_file(): - try: - existing = json.loads(destination.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - existing = None - if isinstance(existing, dict) and existing.get("schema_version") == WATCH_SCHEMA_VERSION: - snapshot["previous"] = { - key: existing.get(key) - for key in ("schema_version", "created_at", "owned", "candidates") - } - snapshot["previous"]["previous"] = None - content = (json.dumps(snapshot, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") - _atomic_in_directory(destination.parent, destination.name, content, 0o600) - return destination - - -def read_baseline(state_dir): - destination = state_file(state_dir) - if not destination.is_file(): - raise WatchError( - "missing_baseline", - "No watch snapshot exists yet.", - "Run mlx-agent watch snapshot first to record a baseline.", - ) - try: - value = json.loads(destination.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError) as error: - raise WatchError( - "invalid_state", - "The watch state file could not be read: {0}".format(error), - "Remove the corrupted file and run watch snapshot again.", - ) - if not isinstance(value, dict) or value.get("schema_version") != WATCH_SCHEMA_VERSION: - raise WatchError( - "invalid_state", - "The watch state file has an unsupported schema.", - "Run watch snapshot to record a fresh baseline.", - ) - return value - - -def diff_snapshots(baseline, current): - """Classify changes relevant to owned models between two snapshots.""" - findings = [] - owned_ids = {item["id"].casefold() for item in current.get("owned", [])} - owned_bases = {base_name(item["id"]) for item in current.get("owned", [])} - baseline_candidates = baseline.get("candidates") or {} - baseline_owned = {item["id"].casefold() for item in baseline.get("owned", [])} - - for repo, info in sorted((current.get("candidates") or {}).items()): - if len(findings) >= MAX_FINDINGS: - break - base = base_name(info.get("base") or repo) - if repo.casefold() not in owned_ids and base_name(repo) not in owned_bases and base not in owned_bases: - continue - previous = baseline_candidates.get(repo) - if previous is None: - findings.append({ - "code": "new_quant_of_owned", - "repo": repo, - "base": base, - "detail": "new tracked repository matching an owned model", - }) - continue - old_bytes = previous.get("weight_bytes") - new_bytes = info.get("weight_bytes") - if old_bytes is not None and new_bytes is not None and old_bytes != new_bytes: - findings.append({ - "code": "updated_tracked_repo", - "repo": repo, - "base": base, - "detail": "weight bytes changed from {0} to {1}".format(old_bytes, new_bytes), - }) - if previous.get("gated") != info.get("gated"): - findings.append({ - "code": "gated_changed", - "repo": repo, - "base": base, - "detail": "gated changed from {0} to {1}".format( - previous.get("gated"), info.get("gated") - ), - }) - - for item in baseline.get("owned", []): - if len(findings) >= MAX_FINDINGS: - break - if item["id"].casefold() not in owned_ids: - findings.append({ - "code": "owned_missing", - "repo": item["id"], - "base": base_name(item["id"]), - "detail": "previously owned via {0}; no longer present".format( - item.get("source", "unknown") - ), - }) - return findings diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/wiring.py b/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/wiring.py deleted file mode 100644 index a0e753d..0000000 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/wiring.py +++ /dev/null @@ -1,353 +0,0 @@ -"""Deterministic, injection-safe configuration renderers for Wire.""" - -from __future__ import annotations - -import ipaddress -import json -import re -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path - - -_RUNTIMES = {"ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"} -_MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,191}$") -_SECRET_NAMES = { - "accesskey", "accesstoken", "apikey", "auth", "authorization", "clientsecret", "credential", - "credentials", "idtoken", "passwd", "password", "privatekey", - "refreshtoken", "secret", "secretkey", "token", -} -_ASSIGNMENT_SECRET = re.compile( - r"(?im)(^|[\s,{&?;])([\"']?(?:" - r"[a-z0-9_-]*(?:token|secret|passw(?:or)?d|authorization|credentials?|auth)|" - r"[a-z0-9_-]*(?:api|access|private)[_-]?key" - r")[\"']?\s*[:=]\s*)" - r"([^\s,}\]\r\n]+|\"(?:\\.|[^\"])*\"|'(?:\\.|[^'])*')" -) -_URL_USERINFO = re.compile(r"(https?://)([^/@\s]+@)", re.I) -_OLLAMA = re.compile(r"\A# MLX_AGENT_WIRE v1\nFROM (" + _MODEL.pattern[1:-1] + r")\n\Z") -_ENV_REFERENCE = re.compile( - r"^(?:os\.environ(?:\[[\"'][A-Z][A-Z0-9_]{0,127}[\"']\]|/[A-Z][A-Z0-9_]{0,127})|" - r"\$\{[A-Z][A-Z0-9_]{0,127}\}|\$[A-Z][A-Z0-9_]{0,127}|env:[A-Z][A-Z0-9_]{0,127})$" -) -_STATE_CHANGING_PATH_PARTS = { - "admin", "create", "delete", "install", "kill", "mutate", "pull", "push", - "reload", "remove", "reset", "restart", "shutdown", "update", "write", -} - - -def _secret_name(name): - raw = str(name).lower() - collapsed = re.sub(r"[^a-z0-9]", "", raw) - if collapsed in _SECRET_NAMES: - return True - segments = [item for item in re.split(r"[^a-z0-9]+", raw) if item] - if any( - item in {"auth", "authorization", "credential", "credentials", "passwd", "password", "secret", "token"} - for item in segments - ): - return True - return collapsed.endswith( - ("accesskey", "apikey", "authorization", "credential", "credentials", "passwd", "password", "privatekey", "secret", "token") - ) - - -def _environment_reference(value): - return isinstance(value, str) and _ENV_REFERENCE.fullmatch(value.strip()) is not None - - -def _json_contains_resolved_secret(value): - if isinstance(value, dict): - for key, item in value.items(): - if _secret_name(key) and not _environment_reference(item): - return True - if _json_contains_resolved_secret(item): - return True - return False - if isinstance(value, list): - return any(_json_contains_resolved_secret(item) for item in value) - if isinstance(value, str): - try: - parsed = urllib.parse.urlsplit(value) - except ValueError: - return False - if parsed.scheme not in {"http", "https"}: - return False - if parsed.username is not None or parsed.password is not None: - return True - return any( - _secret_name(key) and not _environment_reference(item) - for key, item in urllib.parse.parse_qsl(parsed.query, keep_blank_values=True) - ) - return False - - -def contains_resolved_secrets(content): - """Return whether configuration text contains a resolved credential value.""" - if not isinstance(content, str): - raise TypeError("configuration content must be text") - try: - parsed = json.loads(content) - except (TypeError, ValueError, json.JSONDecodeError): - parsed = None - if parsed is not None and _json_contains_resolved_secret(parsed): - return True - if _URL_USERINFO.search(content): - return True - for match in _ASSIGNMENT_SECRET.finditer(content): - raw = match.group(3).strip() - if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {"\"", "'"}: - raw = raw[1:-1] - if not _environment_reference(raw): - return True - return False - - -def require_secret_free_config(content): - """Fail closed without echoing a resolved secret-bearing value.""" - if contains_resolved_secrets(content): - raise ValueError( - "existing configuration contains resolved secret-bearing fields; " - "move secrets to environment references or a secret-free managed fragment" - ) - return content - - -def _redact_json(value): - if isinstance(value, dict): - return {key: "" if _secret_name(key) else _redact_json(item) for key, item in value.items()} - if isinstance(value, list): - return [_redact_json(item) for item in value] - if isinstance(value, str): - return re.sub(r"https?://[^\s\"']+", lambda match: redact_endpoint(match.group(0)), value) - return value - - -def redact_endpoint(endpoint): - """Return only a non-credential origin/path representation of an endpoint.""" - try: - parsed = urllib.parse.urlsplit(str(endpoint)) - if parsed.scheme not in {"http", "https"} or not parsed.hostname: - return "" - host = parsed.hostname - if ":" in host and not host.startswith("["): - host = "[{0}]".format(host) - origin = "{0}://{1}".format(parsed.scheme, host) - if parsed.port is not None: - origin += ":{0}".format(parsed.port) - return origin + (parsed.path or "/") - except ValueError: - return "" - - -def redact_secrets(content): - """Redact common credential forms before they can enter previews or receipts.""" - text = str(content) - try: - parsed = json.loads(text) - except (TypeError, ValueError, json.JSONDecodeError): - parsed = None - if parsed is not None: - return json.dumps(_redact_json(parsed), sort_keys=True, separators=(",", ": ")) - text = _URL_USERINFO.sub(r"\1@", text) - def redact_assignment(match): - value = match.group(3) - replacement = value if "" in value else "" - return match.group(1) + match.group(2) + replacement - text = _ASSIGNMENT_SECRET.sub(redact_assignment, text) - def redact_url(match): - return redact_endpoint(match.group(0)) - text = re.sub(r"https?://[^\s\"']+", redact_url, text) - return text - - -def validate_health_endpoint(endpoint): - """Validate one credential-free, read-only loopback HTTP(S) health URL.""" - if not isinstance(endpoint, str) or not endpoint: - raise ValueError("health endpoint must be a non-empty URL") - try: - parsed = urllib.parse.urlsplit(endpoint) - port = parsed.port - except ValueError as error: - raise ValueError("health endpoint has an invalid port") from error - if parsed.scheme not in {"http", "https"} or not parsed.hostname: - raise ValueError("health endpoint must use HTTP(S)") - if parsed.username is not None or parsed.password is not None or parsed.query or parsed.fragment: - raise ValueError("health endpoint must not contain credentials, query, or fragment") - if port is not None and not 1 <= port <= 65535: - raise ValueError("health endpoint port is outside 1-65535") - hostname = parsed.hostname.lower().rstrip(".") - if hostname != "localhost": - try: - address = ipaddress.ip_address(hostname) - except ValueError as error: - raise ValueError("health endpoint must use localhost or a loopback IP literal") from error - if not address.is_loopback: - raise ValueError("health endpoint must resolve only to loopback") - decoded_path = urllib.parse.unquote(parsed.path or "/").lower() - parts = {part for part in decoded_path.split("/") if part} - if parts & _STATE_CHANGING_PATH_PARTS: - raise ValueError("health endpoint path appears state-changing") - return endpoint - - -def _endpoint_origin(endpoint): - parsed = urllib.parse.urlsplit(validate_health_endpoint(endpoint)) - port = parsed.port or (443 if parsed.scheme == "https" else 80) - return parsed.scheme.lower(), parsed.hostname.lower().rstrip("."), port - - -class _SameLoopbackOriginRedirect(urllib.request.HTTPRedirectHandler): - def __init__(self, origin): - super().__init__() - self.origin = origin - - def redirect_request(self, request, file_pointer, code, message, headers, new_url): - destination = urllib.parse.urljoin(request.full_url, new_url) - if _endpoint_origin(destination) != self.origin: - raise urllib.error.URLError("health redirect left the reviewed loopback origin") - return super().redirect_request( - request, file_pointer, code, message, headers, destination - ) - - -class ConfigAdapter: - """A small format adapter using only documented, exact stdlib-safe subsets.""" - - version = "2.0" - - def __init__(self, runtime, path=None): - if runtime not in _RUNTIMES: - raise ValueError("unsupported runtime: {0}".format(runtime)) - self.runtime = runtime - self.path = Path(path) if path is not None else None - - @classmethod - def detect(cls, path, runtime=None): - location = Path(path) - selected = runtime - if selected is None: - name = location.name.lower() - if name == "modelfile" or location.suffix.lower() == ".modelfile": - selected = "ollama" - elif location.suffix.lower() in {".yaml", ".yml"}: - selected = "litellm" - elif "lmstudio" in name: - selected = "lmstudio" - else: - selected = "mlx_lm" - return cls(selected, location) - - def render(self, model, runtime=None, existing=""): - selected = runtime or self.runtime - if selected != self.runtime: - return ConfigAdapter(selected, self.path).render(model, existing=existing) - self._validate_model(model) - if not isinstance(existing, str): - raise TypeError("existing config content must be text") - require_secret_free_config(existing) - if selected == "ollama": - if existing and existing != self._render_ollama(model): - raise ValueError("Ollama Wire accepts only its exact managed configuration") - return self._render_ollama(model) - if selected == "litellm": - if existing: - self._validate_litellm(existing) - return self._render_litellm(model) - return self._render_json(model, existing) - - @staticmethod - def _validate_model(model): - if not isinstance(model, str) or not _MODEL.fullmatch(model): - raise ValueError("model must be a safe publisher/model identifier") - - def _render_json(self, model, existing): - if existing.strip(): - try: - document = json.loads(existing) - except json.JSONDecodeError as error: - raise ValueError("existing JSON configuration is invalid: {0}".format(error)) - if not isinstance(document, dict): - raise ValueError("existing JSON configuration must be an object") - else: - document = {} - port = 8081 if self.runtime == "mlx-vlm" else 8080 - provider_id = "mlxvlm" if self.runtime == "mlx-vlm" else self.runtime.replace("_", "") - document["mlx_agent_wire"] = { - "marker": "MLX_AGENT_WIRE", - "version": self.version, - "runtime": self.runtime, - "model": model, - "provider": { - "id": provider_id, - "type": "openai", - "base_url": "http://127.0.0.1:{0}/v1".format(port), - "api_key_env": "MLX_AGENT_LOCAL_API_KEY", - }, - } - return json.dumps(document, indent=2, sort_keys=True) + "\n" - - @staticmethod - def _render_ollama(model): - return "# MLX_AGENT_WIRE v1\nFROM {0}\n".format(model) - - @staticmethod - def _render_litellm(model): - short = model.split("/", 1)[1].lower() - return ( - "# MLX_AGENT_WIRE v1\n" - "model_list:\n" - " - model_name: {0}\n" - " litellm_params:\n" - " model: openai/{1}\n" - " api_base: http://127.0.0.1:8080/v1\n" - " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY\n" - ).format(short, model) - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("configuration content must be text") - require_secret_free_config(content) - if self.runtime in {"lmstudio", "mlx_lm", "mlx-vlm"}: - value = json.loads(content) - if not isinstance(value, dict): - raise ValueError("JSON configuration must be an object") - return True - if self.runtime == "ollama": - match = _OLLAMA.fullmatch(content) - if match is None: - raise ValueError("Ollama configuration must match the exact Wire grammar") - self._validate_model(match.group(1)) - return True - self._validate_litellm(content) - return True - - @staticmethod - def _validate_litellm(content): - lines = content.splitlines() - if len(lines) != 7 or lines[0] != "# MLX_AGENT_WIRE v1" or lines[1] != "model_list:": - raise ValueError("LiteLLM configuration must use the supported Wire YAML subset") - model_name = re.fullmatch(r" - model_name: ([A-Za-z0-9][A-Za-z0-9._-]{0,191})", lines[2]) - model = re.fullmatch(r" model: openai/(" + _MODEL.pattern[1:-1] + r")", lines[4]) - if lines[3] != " litellm_params:" or lines[5] != " api_base: http://127.0.0.1:8080/v1" or lines[6] != " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY" or not model_name or not model: - raise ValueError("LiteLLM configuration must use the supported Wire YAML subset") - ConfigAdapter._validate_model(model.group(1)) - if model_name.group(1) != model.group(1).split("/", 1)[1].lower(): - raise ValueError("LiteLLM model_name does not match the model identifier") - - @staticmethod - def health_check(endpoint, timeout=2.0): - if not endpoint: - return True - try: - endpoint = validate_health_endpoint(endpoint) - request = urllib.request.Request(endpoint, method="GET") - opener = urllib.request.build_opener( - urllib.request.ProxyHandler({}), - _SameLoopbackOriginRedirect(_endpoint_origin(endpoint)) - ) - with opener.open(request, timeout=timeout) as response: - return 200 <= response.status < 400 - except (OSError, ValueError, urllib.error.URLError): - return False diff --git a/packages/codex/mlx-agent/skills/mlx-watch/SKILL.md b/packages/codex/mlx-agent/skills/mlx-watch/SKILL.md new file mode 100644 index 0000000..7fb2ee4 --- /dev/null +++ b/packages/codex/mlx-agent/skills/mlx-watch/SKILL.md @@ -0,0 +1,19 @@ +--- +name: "mlx-watch" +description: "Digest Hugging Face changes for owned models against a stored baseline." +--- + +Use `$mlx-agent:mlx-watch` to invoke this installed Codex skill explicitly. Codex does not support custom `/mlx-*` slash commands. + +# MLX Watch + +Resolve `` as the absolute directory two levels above the directory containing this SKILL.md; it holds `scripts/` and `src/`. Never resolve the bundled executable from the shell working directory. + +canonical capability ID: mlx-agent.watch + +Record or compare the owned-model baseline: + +`python3 /scripts/mlx-agent watch snapshot --json` +`python3 /scripts/mlx-agent watch diff --json` + +Present the classified findings as returned. Watch writes only its own state file and never downloads model weights or changes configuration. diff --git a/packages/codex/mlx-agent/skills/mlx-wire/SKILL.md b/packages/codex/mlx-agent/skills/mlx-wire/SKILL.md index 07264fa..00fc49a 100644 --- a/packages/codex/mlx-agent/skills/mlx-wire/SKILL.md +++ b/packages/codex/mlx-agent/skills/mlx-wire/SKILL.md @@ -7,20 +7,20 @@ Use `$mlx-agent:mlx-wire` to invoke this installed Codex skill explicitly. Codex # MLX Wire -Resolve `` as the absolute directory containing this SKILL.md. Never resolve the bundled executable from the shell working directory. +Resolve `` as the absolute directory two levels above the directory containing this SKILL.md; it holds `scripts/` and `src/`. Never resolve the bundled executable from the shell working directory. canonical capability ID: mlx-agent.wire Use the structured CLI to inspect the target configuration without mutation: -`python3 /scripts/mlx-agent wire render --target --path --json` +`python3 /scripts/mlx-agent wire render --target --path --json` Then request the exact transaction diff and preview hash without confirmation. This command is intentionally non-mutating and exits nonzero while it waits for confirmation: -`python3 /scripts/mlx-agent wire apply --target --path --json` +`python3 /scripts/mlx-agent wire apply --target --path --json` Show that returned diff and preview hash. Do not write configuration files directly. Only after the user explicitly confirms that exact preview, run: -`python3 /scripts/mlx-agent wire apply --target --path --confirm --preview-hash --json` +`python3 /scripts/mlx-agent wire apply --target --path --confirm --preview-hash --json` Never download model weights without an explicit confirmation. Report the transaction receipt returned by the CLI. diff --git a/packages/codex/mlx-agent/skills/mlx-wire/scripts/mlx-agent b/packages/codex/mlx-agent/skills/mlx-wire/scripts/mlx-agent deleted file mode 100755 index 54a5a80..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/scripts/mlx-agent +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -"""Run the bundled MLX agent core relative to this bootstrap, never the CWD.""" - -import sys -from pathlib import Path - - -sys.dont_write_bytecode = True -SCRIPT_DIR = Path(__file__).resolve().parent -sys.path.insert(0, str(SCRIPT_DIR.parent / "src")) - -from mlx_agent.cli import main # noqa: E402 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/__init__.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/__init__.py deleted file mode 100644 index 67dcb84..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Public API for the dependency-free MLX agent core.""" - -from .contracts import ResultEnvelope - -__all__ = ["ResultEnvelope"] -__version__ = "0.4.0" diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/__main__.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/__main__.py deleted file mode 100644 index 94dda3d..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/__main__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Run the MLX agent core with ``python -m mlx_agent``.""" - -import sys - -from .cli import main - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/adoption.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/adoption.py deleted file mode 100644 index e7c8c48..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/adoption.py +++ /dev/null @@ -1,870 +0,0 @@ -"""Provider-neutral, resumable model adoption state machine.""" - -from __future__ import annotations - -import hashlib -import json -import uuid -from concurrent.futures import ThreadPoolExecutor -from copy import deepcopy -from dataclasses import dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Dict, List, Tuple - -from .discovery import DiscoveryRequest, DiscoveryService -from .transactions import ( - ConcurrentTransactionError, - _atomic_in_directory, - _read_target, - _target_locks, -) -from .verification import ( - EvidenceStrength, - ROLE_PROBE_IDS, - TOOL_USE_PROBE_ID, - VerificationEvidence, - VerificationStatus, - Verifier, -) - - -ADOPTION_SCHEMA_VERSION = "1.3" -LEGACY_ADOPTION_SCHEMA_VERSIONS = ("1.1", "1.2") -PHASES = ("inspect", "discover", "shortlist", "verify", "measure", "compare", "recommend", "complete") -MEASURE_LIMIT = 6 -UTILITY_ROLES = {"general", "embedding"} -ALLOWED_ROLES = { - "general", "coding", "reasoning", "vision", "embedding", "tool-use" -} -EVIDENCE_SCORES = { - EvidenceStrength.RUNTIME_MEASURED.value: 500, - EvidenceStrength.RUNTIME_TESTED.value: 400, - EvidenceStrength.RUNTIME_INVENTORY.value: 300, - EvidenceStrength.METADATA_ONLY.value: 200, - EvidenceStrength.HEURISTIC_ONLY.value: 100, -} -ROLE_PROBE_BONUS = 50 - - -class AdoptionStateConflictError(ValueError): - """A persisted handoff changed or is locked by another workflow.""" - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -@dataclass(frozen=True) -class AdoptionRequest: - """Bounded request fields that are safe to persist in a handoff.""" - - roles: Tuple[str, ...] = ("general",) - state_path: object = None - shortlist_limit: int = 4 - allow_network: bool = True - offline: bool = False - refresh: bool = False - memory_gb: object = None - quantization: object = None - licenses: object = None - include_gated: bool = True - publishers: object = None - runtime: object = None - fast: bool = False - measure: bool = False - - def __post_init__(self): - if isinstance(self.roles, str): - supplied_roles = (self.roles,) - elif isinstance(self.roles, (list, tuple)): - supplied_roles = tuple(self.roles) - else: - raise TypeError("roles must be a string or an array of strings") - if any(not isinstance(role, str) or not role for role in supplied_roles): - raise TypeError("roles must contain non-empty strings") - roles = tuple(supplied_roles) - if not roles or any(role not in ALLOWED_ROLES for role in roles) or len(roles) > 6: - raise ValueError("roles must contain one to six supported roles") - if len(set(roles)) != len(roles): - raise ValueError("roles must not contain duplicates") - if not isinstance(self.shortlist_limit, int) or isinstance(self.shortlist_limit, bool): - raise TypeError("shortlist_limit must be an integer") - if self.shortlist_limit < 1 or self.shortlist_limit > 20: - raise ValueError("shortlist_limit must be between 1 and 20") - if not isinstance(self.allow_network, bool): - raise TypeError("allow_network must be a boolean") - for name in ("offline", "refresh", "include_gated", "fast", "measure"): - if not isinstance(getattr(self, name), bool): - raise TypeError("{0} must be a boolean".format(name)) - if self.memory_gb is not None and ( - not isinstance(self.memory_gb, (int, float)) or isinstance(self.memory_gb, bool) - ): - raise TypeError("memory_gb must be a number or null") - for name in ("quantization", "runtime"): - value = getattr(self, name) - if value is not None and not isinstance(value, str): - raise TypeError("{0} must be a string or null".format(name)) - object.__setattr__(self, "roles", roles) - - def to_dict(self): - return { - "roles": list(self.roles), - "shortlist_limit": self.shortlist_limit, - "allow_network": self.allow_network, - "offline": bool(self.offline), - "refresh": bool(self.refresh), - "memory_gb": self.memory_gb, - "quantization": self.quantization, - "licenses": _string_list(self.licenses), - "include_gated": bool(self.include_gated), - "publishers": _string_list(self.publishers), - "runtime": self.runtime, - "fast": bool(self.fast), - "measure": bool(self.measure), - } - - @classmethod - def from_dict(cls, value, state_path=None): - if not isinstance(value, dict): - raise TypeError("adoption request must be an object") - allowed = { - "roles", "state_path", "state", "shortlist_limit", "limit", - "allow_network", "offline", "refresh", "memory_gb", "quantization", - "licenses", "include_gated", "publishers", "runtime", "fast", - "measure", - } - unknown = sorted(set(value) - allowed) - if unknown: - raise ValueError("adoption request has unexpected keys: {0}".format(unknown)) - return cls( - roles=value.get("roles", ("general",)), - state_path=state_path or value.get("state_path") or value.get("state"), - shortlist_limit=value.get("shortlist_limit", value.get("limit", 4)), - allow_network=value.get("allow_network", True), - offline=value.get("offline", False), - refresh=value.get("refresh", False), - memory_gb=value.get("memory_gb"), - quantization=value.get("quantization"), - licenses=value.get("licenses"), - include_gated=value.get("include_gated", True), - publishers=value.get("publishers"), - runtime=value.get("runtime"), - fast=value.get("fast", False), - measure=value.get("measure", False), - ) - - -@dataclass -class AdoptionState: - """Schema-versioned, serializable adoption handoff.""" - - workflow_id: str - phase: str - status: str - request: Dict[str, Any] - state_path: Path = field(repr=False) - revision: int = 0 - completed_phases: List[str] = field(default_factory=list) - host: Dict[str, Any] = field(default_factory=dict) - discovery: Dict[str, Any] = field(default_factory=dict) - shortlist: List[Dict[str, Any]] = field(default_factory=list) - evidence: List[Dict[str, Any]] = field(default_factory=list) - comparisons: List[Dict[str, Any]] = field(default_factory=list) - recommendations: List[Dict[str, Any]] = field(default_factory=list) - warnings: List[Dict[str, str]] = field(default_factory=list) - errors: List[Dict[str, str]] = field(default_factory=list) - created_at: str = field(default_factory=_utc_now) - updated_at: str = field(default_factory=_utc_now) - schema_version: str = ADOPTION_SCHEMA_VERSION - _source_hash: object = field(default=None, repr=False, compare=False) - - def to_dict(self): - return { - "schema_version": self.schema_version, - "workflow_id": self.workflow_id, - "revision": self.revision, - "phase": self.phase, - "status": self.status, - "request": self.request, - "completed_phases": list(self.completed_phases), - "host": self.host, - "discovery": self.discovery, - "shortlist": self.shortlist, - "evidence": self.evidence, - "comparisons": self.comparisons, - "recommendations": self.recommendations, - "warnings": self.warnings, - "errors": self.errors, - "created_at": self.created_at, - "updated_at": self.updated_at, - } - - @classmethod - def from_dict(cls, value, state_path): - value = _migrate_state(value) - _validate_state(value) - return cls( - workflow_id=value["workflow_id"], - revision=value["revision"], - phase=value["phase"], - status=value["status"], - request=dict(value["request"]), - state_path=Path(state_path), - completed_phases=list(value["completed_phases"]), - host=dict(value["host"]), - discovery=dict(value["discovery"]), - shortlist=list(value["shortlist"]), - evidence=list(value["evidence"]), - comparisons=list(value["comparisons"]), - recommendations=list(value["recommendations"]), - warnings=list(value["warnings"]), - errors=list(value["errors"]), - created_at=value["created_at"], - updated_at=value["updated_at"], - schema_version=value["schema_version"], - ) - - -class AdoptionWorkflow: - """Advance adoption one durable phase at a time.""" - - def __init__(self, discovery_service=None, verifier=None, state_path=None): - self.discovery_service = discovery_service or DiscoveryService() - metadata_client = getattr(self.discovery_service, "_huggingface", None) - self.verifier = verifier or Verifier(metadata_client=metadata_client) - self.state_path = Path(state_path) if state_path is not None else None - - def start(self, request): - if isinstance(request, dict): - request = AdoptionRequest.from_dict(request) - if not isinstance(request, AdoptionRequest): - raise TypeError("request must be an AdoptionRequest or object") - path_value = request.state_path or self.state_path - if path_value is None: - raise ValueError("a state path is required") - state = AdoptionState( - workflow_id=str(uuid.uuid4()), - phase="inspect", - status="running", - request=request.to_dict(), - state_path=Path(path_value), - ) - self._persist(state) - return state - - def advance(self, state): - if not isinstance(state, AdoptionState): - raise TypeError("state must be an AdoptionState") - state.phase = _first_incomplete(state.completed_phases) - if state.phase == "complete": - state.status = "complete" - return state - - phase = state.phase - getattr(self, "_phase_{0}".format(phase))(state) - if phase not in state.completed_phases: - state.completed_phases.append(phase) - state.phase = _first_incomplete(state.completed_phases) - state.status = "complete" if state.phase == "complete" else "running" - self._persist(state) - return state - - def resume(self, path): - state_path = Path(path) - content, exists, _mode = _read_target(state_path) - if not exists: - raise ValueError("adoption state does not exist") - value = json.loads(content.decode("utf-8")) - state = AdoptionState.from_dict(value, state_path) - state._source_hash = hashlib.sha256(content).hexdigest() - state.phase = _first_incomplete(state.completed_phases) - state.status = "complete" if state.phase == "complete" else "running" - return state - - def status(self, path): - return self.resume(path) - - def _phase_inspect(self, state): - host = self.discovery_service.host - if hasattr(host, "to_dict"): - host = host.to_dict() - if not isinstance(host, dict): - raise TypeError("host inventory must be an object") - state.host = dict(host) - - def _phase_discover(self, state): - request = state.request - roles = request["roles"] - result = self.discovery_service.discover(DiscoveryRequest( - role=roles[0] if len(roles) == 1 else None, - memory_gb=request.get("memory_gb"), - quantization=request.get("quantization"), - licenses=request.get("licenses"), - include_gated=request.get("include_gated", True), - publishers=request.get("publishers"), - runtime=request.get("runtime"), - refresh=request.get("refresh", False), - offline=request.get("offline", False), - limit=request["shortlist_limit"], - fast=request.get("fast", False), - )) - if result.status != "ok": - error = result.to_dict()["error"] - state.errors.append({key: str(error[key]) for key in ("code", "message", "remediation")}) - self._persist(state) - raise RuntimeError("discovery failed: {0}".format(error["message"])) - state.discovery = dict(result.data) - state.warnings.extend(result.warnings) - - def _phase_shortlist(self, state): - requested = set(state.request["roles"]) - limit = state.request["shortlist_limit"] - shortlisted = [] - roles = state.discovery.get("roles", {}) - if not isinstance(roles, dict): - raise TypeError("discovery roles must be an object") - for role in state.request["roles"]: - if role not in requested: - continue - for candidate in (roles.get(role) or [])[:limit]: - record = dict(candidate) - record["role"] = role - shortlisted.append(record) - state.shortlist = shortlisted - - def _phase_verify(self, state): - clear_inventory_cache = getattr(self.verifier, "clear_inventory_cache", None) - if callable(clear_inventory_cache): - clear_inventory_cache() - workers = verification_concurrency(state.host.get("ram_gb")) - - def verify_one(candidate): - try: - evidence = self.verifier.verify( - candidate, - state.host, - allow_network=state.request.get("allow_network", True), - ) - if not isinstance(evidence, VerificationEvidence): - raise TypeError("verifier returned an invalid evidence record") - return evidence.to_dict() - except Exception as error: - repo = str( - candidate.get("repo") - or candidate.get("repository") - or "unknown" - ) - role = str(candidate.get("role") or "general") - if role == "tool-use": - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.HEURISTIC_ONLY, - status=VerificationStatus.FAILED, - available_locally=False, - loads=None, - reasoning_confirmed=None, - runtime=None, - note=_bounded("Candidate tool-use verification failed."), - details={ - "probe_id": TOOL_USE_PROBE_ID, - "reason": "verification_exception", - }, - ).to_dict() - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.HEURISTIC_ONLY, - status=VerificationStatus.FAILED, - available_locally=False, - loads=None, - reasoning_confirmed=None, - runtime=None, - note=_bounded("Candidate verification failed: {0}".format(error)), - details={"error": _bounded(str(error), 160)}, - ).to_dict() - - with ThreadPoolExecutor(max_workers=workers) as executor: - state.evidence = list(executor.map(verify_one, state.shortlist)) - - def _phase_measure(self, state): - if not state.request.get("measure", False): - return - from .bench import BenchError, measure_runtime - - runtime_clients = getattr(self.verifier, "runtime_clients", None) - clients = {} - if callable(runtime_clients): - for client in runtime_clients(): - name = getattr(client, "name", None) - if isinstance(name, str): - clients[name] = client - measured = 0 - for index, candidate in enumerate(state.shortlist): - if measured >= MEASURE_LIMIT: - break - if index >= len(state.evidence): - break - evidence = state.evidence[index] - if ( - evidence.get("strength") != EvidenceStrength.RUNTIME_TESTED.value - or evidence.get("status") != VerificationStatus.VERIFIED.value - ): - continue - client = clients.get(evidence.get("runtime")) - if client is None or not callable(getattr(client, "stream_generate", None)): - continue - repo = candidate.get("repo") or candidate.get("repository") - try: - measurement = measure_runtime( - repo, client, chip=state.host.get("chip") - ) - except BenchError as error: - state.warnings.append({ - "code": "measure_skipped", - "message": _bounded( - "{0}: {1}".format(repo, error) - ), - }) - continue - except Exception as error: - state.warnings.append({ - "code": "measure_skipped", - "message": _bounded("{0}: {1}".format(repo, error)), - }) - continue - details = dict(evidence.get("details") or {}) - details["bench"] = measurement.to_dict() - evidence["details"] = details - evidence["strength"] = EvidenceStrength.RUNTIME_MEASURED.value - measured += 1 - - def _phase_compare(self, state): - evidence_by_candidate = { - (item["repo"], item["role"]): item for item in state.evidence - } - comparisons = [] - for candidate in state.shortlist: - repo = candidate.get("repo") or candidate.get("repository") - role = candidate.get("role") - evidence = evidence_by_candidate.get((repo, role), {}) - reasons = [] - if candidate.get("fits") is False: - reasons.append("memory_budget") - if evidence.get("loads") is False: - reasons.append("runtime_generation_failed") - utility_role = role in UTILITY_ROLES or state.request.get("fast", False) - if utility_role and evidence.get("reasoning_confirmed") is True: - reasons.append("confirmed_reasoner_for_utility_role") - evidence_status = evidence.get( - "status", VerificationStatus.FAILED.value - ) - if ( - role == "tool-use" - and evidence_status != VerificationStatus.VERIFIED.value - ): - reasons.append("tool_use_not_verified") - score = EVIDENCE_SCORES.get(evidence.get("strength"), 0) - score += int(candidate.get("rank_score") or 0) - score += 20 if candidate.get("trusted") else 0 - probe_outcome = _role_probe_outcome_for(role, evidence) - if probe_outcome is not None: - if probe_outcome.get("valid") is True: - score += ROLE_PROBE_BONUS - elif probe_outcome.get("reason") != "unsupported_runtime": - reasons.append("role_probe_failed") - comparisons.append({ - "repo": repo, - "role": role, - "eligible": not reasons, - "score": score, - "evidence_strength": evidence.get("strength", EvidenceStrength.HEURISTIC_ONLY.value), - "evidence_status": evidence_status, - "verification_status": evidence_status, - "rejection_reasons": reasons, - }) - state.comparisons = comparisons - - def _phase_recommend(self, state): - candidates_by_candidate = { - ( - item.get("repo") or item.get("repository"), - item.get("role"), - ): item - for item in state.shortlist - } - recommendations = [] - for role in state.request["roles"]: - eligible = [ - item for item in state.comparisons - if item["role"] == role and item["eligible"] - and ( - role != "tool-use" - or item.get("evidence_status") - == VerificationStatus.VERIFIED.value - ) - ] - eligible.sort(key=lambda item: (-item["score"], item["repo"])) - if not eligible: - continue - chosen = eligible[0] - candidate = candidates_by_candidate[(chosen["repo"], chosen["role"])] - recommendations.append({ - "role": role, - "repo": chosen["repo"], - "evidence_strength": chosen["evidence_strength"], - "evidence_status": chosen.get( - "evidence_status", - chosen.get("verification_status", VerificationStatus.FAILED.value), - ), - "estimated_ram_gb": candidate.get("est_ram_gb"), - "wiring": candidate.get("wiring"), - "alternatives": [item["repo"] for item in eligible[1:4]], - }) - state.recommendations = recommendations - - def _persist(self, state): - destination = state.state_path - try: - with _target_locks([destination], create_parents=True): - current, exists, _mode = _read_target(destination) - if state._source_hash is None: - if state.revision != 0 or exists: - raise AdoptionStateConflictError( - "adoption state already exists or lacks a CAS baseline" - ) - else: - if not exists or hashlib.sha256(current).hexdigest() != state._source_hash: - raise AdoptionStateConflictError( - "adoption state changed after it was resumed" - ) - try: - current_value = json.loads(current.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as error: - raise AdoptionStateConflictError( - "adoption state changed after it was resumed" - ) from error - if current_value.get("revision") != state.revision: - raise AdoptionStateConflictError( - "adoption state revision changed after it was resumed" - ) - - next_revision = state.revision + 1 - next_updated_at = _utc_now() - value = state.to_dict() - value["revision"] = next_revision - value["updated_at"] = next_updated_at - _validate_state(value) - content = ( - json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n" - ).encode("utf-8") - - def validate_staged(text): - _validate_state(json.loads(text)) - - _atomic_in_directory( - destination.parent, - destination.name, - content, - 0o600, - validator=validate_staged, - ) - state.revision = next_revision - state.updated_at = next_updated_at - state._source_hash = hashlib.sha256(content).hexdigest() - except ConcurrentTransactionError as error: - raise AdoptionStateConflictError( - "another adoption workflow is updating this handoff" - ) from error - - -def _role_probe_outcome_for(role, evidence): - """Return the role-fit probe outcome recorded in evidence, if any.""" - probe_id = ROLE_PROBE_IDS.get(role) - if probe_id is None or not isinstance(evidence, dict): - return None - details = evidence.get("details") - if not isinstance(details, dict): - return None - probes = details.get("probes") - if not isinstance(probes, list): - return None - for probe in probes: - if not isinstance(probe, dict) or probe.get("probe_id") != probe_id: - continue - outcome = probe.get("outcome") - return outcome if isinstance(outcome, dict) else None - return None - - -def verification_concurrency(ram_gb): - try: - memory = int(float(ram_gb)) - except (TypeError, ValueError, OverflowError): - memory = 0 - return int(min(4, max(1, memory // 16))) - - -def _first_incomplete(completed): - completed_set = set(completed) - for phase in PHASES[:-1]: - if phase not in completed_set: - return phase - return "complete" - - -def _migrate_state(value): - if not isinstance(value, dict): - raise TypeError("adoption state must be an object") - version = value.get("schema_version") - if version == ADOPTION_SCHEMA_VERSION: - return value - if version not in LEGACY_ADOPTION_SCHEMA_VERSIONS: - raise ValueError("unsupported adoption state schema version") - - migrated = deepcopy(value) - if version == "1.1": - request = migrated.get("request") - legacy_roles = ( - list(request.get("roles", [])) - if isinstance(request, dict) and isinstance(request.get("roles", []), list) - else [] - ) - for collection in ("shortlist", "evidence", "comparisons", "recommendations"): - records = migrated.get(collection) - if isinstance(records, list): - legacy_roles.extend( - item.get("role") for item in records if isinstance(item, dict) - ) - if "tool-use" in legacy_roles: - raise ValueError("legacy adoption state cannot contain tool-use") - - status_by_strength = { - EvidenceStrength.RUNTIME_TESTED.value: VerificationStatus.VERIFIED.value, - EvidenceStrength.RUNTIME_INVENTORY.value: VerificationStatus.FAILED.value, - EvidenceStrength.METADATA_ONLY.value: VerificationStatus.METADATA_ONLY.value, - EvidenceStrength.HEURISTIC_ONLY.value: VerificationStatus.METADATA_ONLY.value, - } - for item in migrated.get("evidence", []): - if not isinstance(item, dict): - continue - strength = item.get("strength") - if strength in status_by_strength: - item["status"] = status_by_strength[strength] - - request = migrated.get("request") - if isinstance(request, dict) and "measure" not in request: - request["measure"] = False - completed = migrated.get("completed_phases") - if ( - isinstance(completed, list) - and "verify" in completed - and "measure" not in completed - ): - completed.insert(completed.index("verify") + 1, "measure") - migrated["schema_version"] = ADOPTION_SCHEMA_VERSION - return migrated - - -def _validate_state(value): - required = { - "schema_version", "workflow_id", "revision", "phase", "status", "request", - "completed_phases", "host", "discovery", "shortlist", "evidence", - "comparisons", "recommendations", "warnings", "errors", "created_at", "updated_at", - } - if not isinstance(value, dict): - raise TypeError("adoption state must be an object") - missing = sorted(required - set(value)) - unexpected = sorted(set(value) - required) - if missing: - raise ValueError("adoption state is missing keys: {0}".format(missing)) - if unexpected: - raise ValueError("adoption state has unexpected keys: {0}".format(unexpected)) - if value["schema_version"] != ADOPTION_SCHEMA_VERSION: - raise ValueError("unsupported adoption state schema version") - if not isinstance(value["workflow_id"], str) or not value["workflow_id"]: - raise ValueError("workflow_id must be a non-empty string") - if ( - not isinstance(value["revision"], int) - or isinstance(value["revision"], bool) - or value["revision"] < 1 - ): - raise ValueError("revision must be a positive integer") - if value["phase"] not in PHASES: - raise ValueError("invalid adoption phase") - if value["status"] not in ("running", "complete"): - raise ValueError("invalid adoption status") - _validate_completed_phases(value) - if not isinstance(value["request"], dict): - raise TypeError("adoption request must be an object") - _validate_request_shape(value["request"]) - AdoptionRequest.from_dict(value["request"]) - for key in ("host", "discovery"): - if not isinstance(value[key], dict): - raise TypeError("{0} must be an object".format(key)) - bounds = { - "shortlist": 100, - "evidence": 100, - "comparisons": 100, - "recommendations": 6, - "warnings": 50, - "errors": 50, - } - for key, maximum in bounds.items(): - if not isinstance(value[key], list): - raise TypeError("{0} must be an array".format(key)) - if len(value[key]) > maximum: - raise ValueError("{0} exceeds its maximum size".format(key)) - _validate_timestamp(value["created_at"], "created_at") - _validate_timestamp(value["updated_at"], "updated_at") - _validate_evidence(value["evidence"]) - _validate_phase_artifacts(value) - - -def _validate_completed_phases(value): - completed = value["completed_phases"] - if not isinstance(completed, list): - raise TypeError("completed_phases must be an array") - if len(completed) > len(PHASES) - 1: - raise ValueError("completed_phases contains an invalid phase") - expected = list(PHASES[:len(completed)]) - if completed != expected: - raise ValueError("completed_phases must be a unique contiguous phase prefix") - expected_phase = PHASES[len(completed)] - if value["phase"] != expected_phase: - raise ValueError("phase does not match completed_phases") - expected_status = "complete" if expected_phase == "complete" else "running" - if value["status"] != expected_status: - raise ValueError("status does not match phase") - - -def _validate_request_shape(request): - required = { - "roles", "shortlist_limit", "allow_network", "offline", "refresh", - "memory_gb", "quantization", "licenses", "include_gated", "publishers", - "runtime", "fast", "measure", - } - if set(request) != required: - raise ValueError("adoption request does not match the persisted schema") - if not isinstance(request["roles"], list) or not request["roles"]: - raise ValueError("request.roles must be a non-empty array") - if len(request["roles"]) > 6 or len(set(request["roles"])) != len(request["roles"]): - raise ValueError("request.roles must be bounded and unique") - if any(not isinstance(role, str) or not role for role in request["roles"]): - raise TypeError("request.roles must contain non-empty strings") - if not isinstance(request["shortlist_limit"], int) or isinstance(request["shortlist_limit"], bool): - raise TypeError("request.shortlist_limit must be an integer") - if not 1 <= request["shortlist_limit"] <= 20: - raise ValueError("request.shortlist_limit is out of range") - for key in ("allow_network", "offline", "refresh", "include_gated", "fast"): - if not isinstance(request[key], bool): - raise TypeError("request.{0} must be a boolean".format(key)) - if request["memory_gb"] is not None and ( - not isinstance(request["memory_gb"], (int, float)) or isinstance(request["memory_gb"], bool) - ): - raise TypeError("request.memory_gb must be a number or null") - for key in ("quantization", "runtime"): - if request[key] is not None and not isinstance(request[key], str): - raise TypeError("request.{0} must be a string or null".format(key)) - for key in ("licenses", "publishers"): - if not isinstance(request[key], list) or len(request[key]) > 20: - raise ValueError("request.{0} must be a bounded array".format(key)) - if any(not isinstance(item, str) for item in request[key]): - raise TypeError("request.{0} must contain strings".format(key)) - - -def _validate_timestamp(timestamp, field_name): - if not isinstance(timestamp, str): - raise TypeError("{0} must be an ISO-8601 timestamp".format(field_name)) - normalized = timestamp[:-1] + "+00:00" if timestamp.endswith("Z") else timestamp - try: - parsed = datetime.fromisoformat(normalized) - except ValueError: - raise ValueError("{0} must be an ISO-8601 timestamp".format(field_name)) - if parsed.tzinfo is None: - raise ValueError("{0} must include a timezone".format(field_name)) - - -def _validate_evidence(evidence): - required = { - "repo", "role", "strength", "status", "available_locally", "loads", - "reasoning_confirmed", "runtime", "note", "details", - } - strengths = set(EVIDENCE_SCORES) - statuses = {status.value for status in VerificationStatus} - for index, item in enumerate(evidence): - prefix = "evidence[{0}]".format(index) - if not isinstance(item, dict): - raise TypeError("{0} must be an object".format(prefix)) - if set(item) != required: - raise ValueError("{0} does not match the evidence schema".format(prefix)) - for key in ("repo", "role", "note"): - if not isinstance(item[key], str) or not item[key]: - raise ValueError("{0}.{1} must be a non-empty string".format(prefix, key)) - if len(item["note"]) > 300: - raise ValueError("{0}.note exceeds its maximum length".format(prefix)) - if item["strength"] not in strengths: - raise ValueError("{0}.strength is invalid".format(prefix)) - if item["status"] not in statuses: - raise ValueError("{0}.status is invalid".format(prefix)) - if not isinstance(item["available_locally"], bool): - raise TypeError("{0}.available_locally must be a boolean".format(prefix)) - for key in ("loads", "reasoning_confirmed"): - if item[key] is not None and not isinstance(item[key], bool): - raise TypeError("{0}.{1} must be a boolean or null".format(prefix, key)) - if item["runtime"] is not None and not isinstance(item["runtime"], str): - raise TypeError("{0}.runtime must be a string or null".format(prefix)) - if not isinstance(item["details"], dict): - raise TypeError("{0}.details must be an object".format(prefix)) - - -def _validate_phase_artifacts(value): - completed = value["completed_phases"] - verify_complete = "verify" in completed - compare_complete = "compare" in completed - recommend_complete = "recommend" in completed - shortlist = value["shortlist"] - evidence = value["evidence"] - comparisons = value["comparisons"] - recommendations = value["recommendations"] - if not verify_complete and evidence: - raise ValueError("evidence is not valid before verification completes") - if verify_complete: - _validate_records_match(shortlist, evidence, "evidence") - if not compare_complete and comparisons: - raise ValueError("comparisons are not valid before comparison completes") - if compare_complete: - _validate_records_match(shortlist, comparisons, "comparisons") - if not recommend_complete and recommendations: - raise ValueError("recommendations are not valid before recommendation completes") - - -def _validate_records_match(shortlist, records, field_name): - if len(shortlist) != len(records): - raise ValueError("{0} does not cover the current shortlist".format(field_name)) - for index, (candidate, record) in enumerate(zip(shortlist, records)): - if not isinstance(candidate, dict) or not isinstance(record, dict): - raise TypeError("{0}[{1}] must be an object".format(field_name, index)) - candidate_repo = candidate.get("repo") or candidate.get("repository") - if not isinstance(candidate_repo, str) or not candidate_repo: - raise ValueError("shortlist[{0}] has no repository".format(index)) - if not isinstance(candidate.get("role"), str) or not candidate["role"]: - raise ValueError("shortlist[{0}] has no role".format(index)) - if record.get("repo") != candidate_repo or record.get("role") != candidate["role"]: - raise ValueError("{0}[{1}] does not match the current shortlist".format(field_name, index)) - - -def _string_list(value): - if value is None: - return [] - if isinstance(value, str): - return [value] - return [str(item) for item in value] - - -def _bounded(value, limit=300): - text = " ".join(str(value).split()) - if len(text) <= limit: - return text - return text[: max(0, limit - 3)] + "..." diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/bench.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/bench.py deleted file mode 100644 index ba6d8cd..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/bench.py +++ /dev/null @@ -1,431 +0,0 @@ -"""Bounded, read-only performance measurement of a locally served model. - -Bench never installs, downloads, or starts anything. It measures a model that -is already served by an already-running loopback runtime, using the same -safety envelope as verification: validated loopback URLs, absolute monotonic -deadlines, bounded responses, and redacted errors. -""" - -from __future__ import annotations - -import json -import os -import statistics -import time -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Dict, List, Optional - -from .verification import ( - EvidenceStrength, - VerificationEvidence, - VerificationStatus, - _bounded, - _is_installed, - _runtime_name, - _safe_error, - installed_model_ids, -) - - -BENCH_PROBE_ID = "bench-v1" -BENCH_PROMPT = ( - "You are a careful technical writer. Write a detailed, factual explanation " - "of how unified memory architecture on Apple Silicon affects local " - "inference of large language models. Cover memory bandwidth, quantization " - "tradeoffs, KV-cache growth with context length, and the difference " - "between dense and mixture-of-experts models. Organize the answer into " - "clear sections with concrete numbers where possible." -) -BENCH_PROMPT_TOKEN_ESTIMATE = 96 -RUNS_MIN, RUNS_MAX, RUNS_DEFAULT = 1, 10, 3 -GEN_TOKENS_MIN, GEN_TOKENS_MAX, GEN_TOKENS_DEFAULT = 16, 2048, 128 -TIMEOUT_DEFAULT = 120.0 - - -class BenchError(RuntimeError): - """Classified bench failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -@dataclass(frozen=True) -class BenchMeasurement: - """Aggregate measured performance for one model on one runtime.""" - - repo: str - runtime: str - runs: int - prompt_tokens: int - gen_tokens: int - ttft_ms: Optional[float] - decode_toks: float - prefill_toks: Optional[float] - spread_pct: float - chip: Optional[str] - measured_at: str - samples: List[Dict[str, Any]] = field(default_factory=list) - - def to_dict(self) -> Dict[str, Any]: - return { - "probe_id": BENCH_PROBE_ID, - "outcome": "measured", - "repo": self.repo, - "runtime": self.runtime, - "runs": self.runs, - "prompt_tokens": self.prompt_tokens, - "gen_tokens": self.gen_tokens, - "ttft_ms": self.ttft_ms, - "decode_toks": self.decode_toks, - "prefill_toks": self.prefill_toks, - "peak_mem_gb": None, - "spread_pct": self.spread_pct, - "chip": self.chip, - "measured_at": self.measured_at, - "samples": list(self.samples), - } - - def to_evidence(self, role="general") -> VerificationEvidence: - return VerificationEvidence( - repo=self.repo, - role=role, - strength=EvidenceStrength.RUNTIME_MEASURED, - status=VerificationStatus.VERIFIED, - available_locally=True, - loads=True, - reasoning_confirmed=None, - runtime=self.runtime, - note=_bounded( - "Measured {0} run(s) on this host; median decode " - "{1} tok/s.".format(self.runs, self.decode_toks) - ), - details=self.to_dict(), - ) - - -def validate_bench_bounds(runs, gen_tokens, timeout): - if not isinstance(runs, int) or isinstance(runs, bool): - raise TypeError("runs must be an integer") - if not RUNS_MIN <= runs <= RUNS_MAX: - raise ValueError("runs must be between {0} and {1}".format(RUNS_MIN, RUNS_MAX)) - if not isinstance(gen_tokens, int) or isinstance(gen_tokens, bool): - raise TypeError("gen_tokens must be an integer") - if not GEN_TOKENS_MIN <= gen_tokens <= GEN_TOKENS_MAX: - raise ValueError( - "gen_tokens must be between {0} and {1}".format(GEN_TOKENS_MIN, GEN_TOKENS_MAX) - ) - if not isinstance(timeout, (int, float)) or isinstance(timeout, bool): - raise TypeError("timeout must be a number") - if not 1.0 <= float(timeout) <= 600.0: - raise ValueError("timeout must be between 1 and 600 seconds") - - -def measure_runtime(repo, runtime, runs=RUNS_DEFAULT, gen_tokens=GEN_TOKENS_DEFAULT, - timeout=TIMEOUT_DEFAULT, chip=None, clock=time.monotonic, - now=None): - """Measure one already-served model; never starts or downloads anything.""" - validate_bench_bounds(runs, gen_tokens, timeout) - if not isinstance(repo, str) or not repo.strip(): - raise BenchError( - "invalid_repo", - "bench requires a non-empty model repository identifier.", - "Pass --repo as publisher/model exactly as the runtime serves it.", - ) - stream_generate = getattr(runtime, "stream_generate", None) - if not callable(stream_generate): - raise BenchError( - "unsupported_runtime", - "This runtime does not support the bounded streaming measurement.", - "Serve the model with a runtime that supports streaming generation.", - ) - runtime_name = _runtime_name(runtime) - try: - installed = installed_model_ids(runtime.list_models()) - except Exception as error: - raise BenchError( - "runtime_unreachable", - "The runtime inventory could not be read: {0}".format(_safe_error(error)), - "Confirm the local runtime is running and reachable on its loopback port.", - ) - if not _is_installed(repo, installed): - raise BenchError( - "model_not_serving", - "The model is not served by the {0} runtime.".format(runtime_name), - "Load the model in the runtime first; bench never downloads models.", - ) - - per_run_timeout = float(timeout) / runs - try: - _timed_run(stream_generate, repo, gen_tokens, per_run_timeout, clock) - samples = [ - _timed_run(stream_generate, repo, gen_tokens, per_run_timeout, clock) - for _ in range(runs) - ] - except BenchError: - raise - except Exception as error: - raise BenchError( - "measurement_failed", - "The measurement run failed: {0}".format(_safe_error(error)), - "Confirm the model responds to a normal generation request, then retry.", - ) - - decode_values = [sample["decode_toks"] for sample in samples] - median_decode = statistics.median(decode_values) - spread = 0.0 - if median_decode > 0 and len(decode_values) > 1: - spread = (max(decode_values) - min(decode_values)) / median_decode * 100.0 - ttft_values = [sample["ttft_ms"] for sample in samples if sample["ttft_ms"] is not None] - prefill_values = [ - sample["prefill_toks"] for sample in samples if sample["prefill_toks"] is not None - ] - measured_at = now() if callable(now) else _utc_now() - return BenchMeasurement( - repo=repo, - runtime=runtime_name, - runs=runs, - prompt_tokens=BENCH_PROMPT_TOKEN_ESTIMATE, - gen_tokens=gen_tokens, - ttft_ms=round(statistics.median(ttft_values), 1) if ttft_values else None, - decode_toks=round(median_decode, 1), - prefill_toks=round(statistics.median(prefill_values), 1) if prefill_values else None, - spread_pct=round(spread, 1), - chip=chip, - measured_at=measured_at, - samples=samples, - ) - - -def _timed_run(stream_generate, repo, gen_tokens, timeout, clock): - started = clock() - first_content_at = None - last_event_at = None - content_events = 0 - usage = None - for event in stream_generate(repo, BENCH_PROMPT, gen_tokens, timeout=timeout): - if not isinstance(event, dict): - continue - last_event_at = clock() - text = _stream_event_text(event) - if text: - content_events += 1 - if first_content_at is None: - first_content_at = last_event_at - event_usage = _stream_event_usage(event) - if event_usage: - usage = event_usage - if content_events == 0 and usage is None: - raise BenchError( - "empty_stream", - "The runtime streamed no tokens for the measurement prompt.", - "Confirm the model generates normally, then retry.", - ) - finished = clock() - sample = { - "content_events": content_events, - "elapsed_ms": round((finished - started) * 1000.0, 1), - } - if first_content_at is not None: - sample["ttft_ms"] = round((first_content_at - started) * 1000.0, 1) - else: - sample["ttft_ms"] = None - - if usage and usage.get("eval_count") and usage.get("eval_duration"): - sample["decode_toks"] = round( - usage["eval_count"] / (usage["eval_duration"] / 1e9), 1 - ) - if usage.get("prompt_eval_count") and usage.get("prompt_eval_duration"): - sample["prefill_toks"] = round( - usage["prompt_eval_count"] / (usage["prompt_eval_duration"] / 1e9), 1 - ) - sample["prompt_tokens"] = usage["prompt_eval_count"] - elif first_content_at is not None and content_events > 1 and last_event_at > first_content_at: - sample["decode_toks"] = round( - (content_events - 1) / (last_event_at - first_content_at), 1 - ) - ttft_seconds = first_content_at - started - if ttft_seconds > 0: - sample["prefill_toks"] = round(BENCH_PROMPT_TOKEN_ESTIMATE / ttft_seconds, 1) - elif usage and usage.get("completion_tokens") and finished > started: - sample["decode_toks"] = round(usage["completion_tokens"] / (finished - started), 1) - else: - sample["decode_toks"] = round(content_events / max(finished - started, 1e-9), 1) - sample.setdefault("prefill_toks", None) - return sample - - -def _stream_event_text(event): - message = event.get("message") - if isinstance(message, dict) and isinstance(message.get("content"), str): - return message["content"] - choices = event.get("choices") - if isinstance(choices, list) and choices and isinstance(choices[0], dict): - delta = choices[0].get("delta") - if isinstance(delta, dict) and isinstance(delta.get("content"), str): - return delta["content"] - return "" - - -def _stream_event_usage(event): - usage = event.get("usage") - normalized = {} - if isinstance(usage, dict): - if isinstance(usage.get("completion_tokens"), int): - normalized["completion_tokens"] = usage["completion_tokens"] - if isinstance(usage.get("prompt_tokens"), int): - normalized["prompt_tokens"] = usage["prompt_tokens"] - for key in ("eval_count", "eval_duration", "prompt_eval_count", "prompt_eval_duration"): - value = event.get(key) - if isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0: - normalized[key] = value - return normalized or None - - -def _utc_now(): - from datetime import datetime, timezone - - return datetime.now(timezone.utc).isoformat() - - -BENCH_EXPORT_SCHEMA_VERSION = "1.0" -BENCH_AGGREGATE_SCHEMA_VERSION = "1.0" -MAX_EXPORT_LINE_BYTES = 4096 -MAX_EXPORT_LINES_PER_FILE = 5000 -MAX_AGGREGATE_ENTRIES = 5000 -_COMMUNITY_BENCH_PATH = Path(__file__).resolve().parent / "resources" / "bench-community.json" -_community_bench_cache = None - - -def export_record(measurement, tool_version): - """Build one bounded, anonymized export line from a measurement.""" - record = { - "schema_version": BENCH_EXPORT_SCHEMA_VERSION, - "repo": measurement.repo, - "chip": measurement.chip, - "runtime": measurement.runtime, - "ttft_ms": measurement.ttft_ms, - "decode_toks": measurement.decode_toks, - "prefill_toks": measurement.prefill_toks, - "measured_at": measurement.measured_at, - "tool_version": tool_version, - } - validate_export_record(record) - return record - - -def validate_export_record(record): - if not isinstance(record, dict): - raise ValueError("bench export record must be an object") - if record.get("schema_version") != BENCH_EXPORT_SCHEMA_VERSION: - raise ValueError("bench export record has an unsupported schema version") - for key in ("repo", "chip", "runtime", "measured_at", "tool_version"): - if not isinstance(record.get(key), str) or not record[key]: - raise ValueError("bench export record {0} must be a non-empty string".format(key)) - if len(record["repo"]) > 300 or len(record["chip"]) > 120: - raise ValueError("bench export record identifiers exceed their bounds") - for key in ("ttft_ms", "decode_toks", "prefill_toks"): - value = record.get(key) - if value is None: - continue - if not isinstance(value, (int, float)) or isinstance(value, bool): - raise ValueError("bench export record {0} must be numeric or null".format(key)) - if not 0 <= value <= 1e9: - raise ValueError("bench export record {0} is outside bench bounds".format(key)) - if record.get("decode_toks") is None: - raise ValueError("bench export record requires decode_toks") - return record - - -def append_export(path, record): - """Append one validated export line; never reads or rewrites the file.""" - line = json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" - if len(line.encode("utf-8")) > MAX_EXPORT_LINE_BYTES: - raise BenchError( - "invalid_arguments", - "The export record exceeds its size bound.", - "Export only records produced by bench run.", - ) - destination = Path(path) - flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, "O_NOFOLLOW", 0) - descriptor = os.open(str(destination), flags, 0o600) - try: - os.write(descriptor, line.encode("utf-8")) - finally: - os.close(descriptor) - return destination - - -def aggregate_exports(paths): - """Deduplicate and median-aggregate validated export records.""" - newest = {} - for path in paths: - location = Path(path) - try: - handle = location.open("r", encoding="utf-8") - except OSError: - continue - with handle: - for index, line in enumerate(handle): - if index >= MAX_EXPORT_LINES_PER_FILE: - break - stripped = line.strip() - if not stripped or len(stripped.encode("utf-8")) > MAX_EXPORT_LINE_BYTES: - continue - try: - record = validate_export_record(json.loads(stripped)) - except (ValueError, json.JSONDecodeError): - continue - key = (record["repo"], record["chip"], record["runtime"]) - if key not in newest or record["measured_at"] > newest[key]["measured_at"]: - newest[key] = record - groups = {} - for record in newest.values(): - groups.setdefault((record["repo"], record["chip"]), []).append(record) - entries = [] - for (repo, chip), records in sorted(groups.items()): - if len(entries) >= MAX_AGGREGATE_ENTRIES: - break - def _median(key): - values = [item[key] for item in records if item.get(key) is not None] - return round(statistics.median(values), 1) if values else None - - entries.append({ - "repo": repo, - "chip": chip, - "decode_toks": _median("decode_toks"), - "prefill_toks": _median("prefill_toks"), - "ttft_ms": _median("ttft_ms"), - "samples": len(records), - "updated_at": max(item["measured_at"] for item in records), - }) - return {"schema_version": BENCH_AGGREGATE_SCHEMA_VERSION, "entries": entries} - - -def load_community_bench(path=None): - """Load the bundled community aggregate once; the empty stub yields {}.""" - global _community_bench_cache - if _community_bench_cache is not None and path is None: - return _community_bench_cache - location = Path(path) if path is not None else _COMMUNITY_BENCH_PATH - entries = {} - try: - value = json.loads(location.read_text(encoding="utf-8")) - if isinstance(value, dict) and isinstance(value.get("entries"), list): - for entry in value["entries"]: - if not isinstance(entry, dict): - continue - repo = entry.get("repo") - chip = entry.get("chip") - if isinstance(repo, str) and isinstance(chip, str): - entries[(repo, chip)] = { - key: entry.get(key) - for key in ("decode_toks", "prefill_toks", "ttft_ms", "samples") - } - except (OSError, ValueError, json.JSONDecodeError): - entries = {} - if path is None: - _community_bench_cache = entries - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/blueprint.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/blueprint.py deleted file mode 100644 index 483d7cf..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/blueprint.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Deterministic dataset-creation blueprint from a DomainIntent. - -Guidance only: no download, no training, no execution. Templates are fixed so -the same intent always yields the same blueprint text and structure. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Dict, Tuple - -from .interview import DomainIntent - - -@dataclass(frozen=True) -class DatasetBlueprint: - goal: str - schema_fields: Tuple[str, ...] - labeling_notes: str - split_guidance: str - license_privacy: str - mlx_next_steps: str - - def to_dict(self) -> Dict[str, object]: - return { - "goal": self.goal, - "schema_fields": list(self.schema_fields), - "labeling_notes": self.labeling_notes, - "split_guidance": self.split_guidance, - "license_privacy": self.license_privacy, - "mlx_next_steps": self.mlx_next_steps, - } - - def to_markdown(self) -> str: - lines = [ - "### Goal", - "", - self.goal, - "", - "### Suggested schema", - "", - ] - for field_name in self.schema_fields: - lines.append("- `{0}`".format(field_name)) - lines.extend([ - "", - "### Labeling notes", - "", - self.labeling_notes, - "", - "### Train / val split", - "", - self.split_guidance, - "", - "### License and privacy", - "", - self.license_privacy, - "", - "### MLX fine-tune next steps", - "", - self.mlx_next_steps, - "", - ]) - return "\n".join(lines) - - -_DEFAULT_FIELDS = ( - "id", - "input_text", - "label", - "source", - "license", - "notes", -) - - -def build_dataset_blueprint(intent: DomainIntent) -> DatasetBlueprint: - """Build a deterministic dataset blueprint for an empty catalog result.""" - roles = ", ".join(intent.roles) or "general" - keywords = ", ".join(intent.keywords) or "none" - licenses = ", ".join(intent.license_allow) or "document and respect upstream licenses" - notes = intent.notes.strip() or "none recorded" - - schema = list(_DEFAULT_FIELDS) - if "vision" in intent.roles: - schema.insert(1, "image_path") - if "embedding" in intent.roles: - schema.insert(-2, "embedding_text") - - goal = ( - "Create a small, domain-specific dataset for **{0}** covering roles " - "({1}) and keywords ({2}). Prefer quality over volume; keep examples " - "representative of real production inputs." - ).format(intent.domain, roles, keywords) - - labeling = ( - "Define clear positive/negative or multi-class labels tied to the domain. " - "Double-check borderline cases. Keywords to cover: {0}. Extra notes: {1}." - ).format(keywords, notes) - - split = ( - "Hold out 10–20% for validation with no leakage from near-duplicates. " - "Stratify by label when classes are imbalanced. Keep a tiny smoke-test " - "slice for local MLX dry-runs." - ) - - license_privacy = ( - "Allowed licenses for this intent: {0}. Do not include secrets, credentials, " - "or personal data without consent and redaction. Record provenance per row." - ).format(licenses) - - next_steps = ( - "1. Draft the schema above as JSONL or Parquet locally (no Hub upload required).\n" - "2. Re-run `mlx-agent research` after datasets appear on the Hub, or keep this " - "blueprint as the creation guide.\n" - "3. When a base MLX model is chosen, use `mlx-agent adopt start` to verify and " - "wire it — research/blueprint never downloads or trains.\n" - "4. Fine-tune offline with your preferred MLX tooling against the local dataset." - ) - - return DatasetBlueprint( - goal=goal, - schema_fields=tuple(schema), - labeling_notes=labeling, - split_guidance=split, - license_privacy=license_privacy, - mlx_next_steps=next_steps, - ) diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/cli.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/cli.py deleted file mode 100644 index bbe2888..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/cli.py +++ /dev/null @@ -1,1924 +0,0 @@ -"""Command-line entry points for the dependency-free MLX agent core.""" - -import argparse -import json -import os -import sys -import urllib.parse -from datetime import datetime, timezone -from pathlib import Path - -from .adoption import ADOPTION_SCHEMA_VERSION, AdoptionRequest, AdoptionWorkflow -from .bench import ( - BENCH_PROBE_ID, - GEN_TOKENS_DEFAULT, - RUNS_DEFAULT, - TIMEOUT_DEFAULT, - BenchError, - measure_runtime, -) -from .contracts import ErrorDetail, ResultEnvelope -from .convert import ( - Q_BITS_CHOICES, - ConvertError, - plan_convert, - start_convert, - status_convert, -) -from .discovery import DiscoveryRequest, DiscoveryService -from .fleet import ( - FleetConfigAdapter, - FleetError, - assignments_from_adoption, - parse_assignments, - parse_runtime_map, -) -from .fuse import ( - FuseError, - plan_fuse, - start_fuse, - status_fuse, -) -from .host import HostInventory -from .huggingface import HuggingFaceClient -from .installer import Installer, InstallerConflictError -from .interview import build_intent, run_interview -from .lora import ( - BATCH_DEFAULT, - ITERS_DEFAULT, - LAYERS_DEFAULT, - LR_DEFAULT, - LoraError, - plan_lora, - start_lora, - status_lora, -) -from .modality import ALL_FACET_IDS, FOUNDATION_IDS, resolve_facets, resolve_modalities -from .models import DISCOVERY_ROLES, render_md, wire -from .project_blueprint import ( - build_brief, - generate_design_pack, - render_design_pack, - write_design_pack, -) -from .providers import ProviderRegistry -from .research import generate_pack, render_pack, write_pack -from .serve import ( - MAX_TOKENS_DEFAULT, - ServeError, - default_model_present, - load_recipes, - plan_start, - receipts_root, - start_serve, - status_serve, - stop_serve, - wired_port_claim, -) -from .transactions import ( - COOPERATIVE_CONCURRENCY_NOTE, - ConcurrentTransactionError, - Receipt, - Transaction, - _assert_safe_target, - _read_regular, - preview_rollback, - rollback, -) -from .verification import ( - LMStudioRuntimeClient, - MLXVLMRuntimeClient, - OllamaRuntimeClient, - OpenAICompatibleRuntimeClient, - Verifier, -) -from .watch import ( - WatchError, - build_snapshot, - collect_owned, - diff_snapshots, - read_baseline, - snapshot_candidates, - write_snapshot, -) -from .wiring import ConfigAdapter - - -FIXTURE_WARNING = { - "code": "synthetic_fixture", - "message": "Fixture-backed discovery; this is not live Hugging Face evidence.", -} - - -def _fixture_http_get(payload): - - def get(url, timeout=10.0): - del timeout - if "/api/datasets" in url: - return payload.get("datasets", []) - if "/tree/main" in url: - repo = urllib.parse.unquote(url.split("/api/models/", 1)[1].split("/tree/main", 1)[0]) - return payload["trees"].get(repo, []) - if "/api/models/" in url: - repo = urllib.parse.unquote(url.split("/api/models/", 1)[1]) - return payload["details"].get(repo, {}) - if url.startswith("http://127.0.0.1") or url.startswith("http://localhost"): - raise OSError("fixture does not emulate local runtime endpoints") - query = urllib.parse.urlsplit(url).query - params = urllib.parse.parse_qs(query) - filters = params.get("filter", []) - if "peft" in filters: - return payload.get("adapters", []) - return payload["models"] - - return get - - -def _discovery_service_from_environment(state_dir=None): - fixture = os.environ.get("MLX_AGENT_FIXTURE") - if not fixture: - return DiscoveryService(state_dir=state_dir), None, None - try: - payload = json.loads(Path(fixture).read_text()) - _validate_fixture(payload) - except (OSError, ValueError, TypeError, json.JSONDecodeError) as error: - return None, None, ResultEnvelope.fail( - "discover", "invalid_fixture", "MLX_AGENT_FIXTURE is invalid: {0}".format(error), - "Use a valid test fixture or unset MLX_AGENT_FIXTURE to run live discovery.", - ) - service = DiscoveryService( - host=HostInventory(**payload["host"]), - huggingface=HuggingFaceClient( - http_get=_fixture_http_get(payload), - card_get=lambda url, timeout=8: None, - ), - state_dir=state_dir, - cache_enabled=False, - ) - return service, FIXTURE_WARNING, None - - -def _validate_fixture(payload): - if not isinstance(payload, dict): - raise ValueError("fixture root must be an object") - if not isinstance(payload.get("models"), list): - raise ValueError("fixture.models must be a list") - for index, model in enumerate(payload["models"]): - if not isinstance(model, dict): - raise ValueError("fixture.models[{0}] must be an object".format(index)) - identifiers = [model[key] for key in ("id", "modelId") if key in model] - if not identifiers: - raise ValueError("fixture.models[{0}] must contain id or modelId".format(index)) - if any(not isinstance(identifier, str) or not identifier for identifier in identifiers): - raise ValueError("fixture.models[{0}].id and modelId must be non-empty strings".format(index)) - for counter in ("downloads", "likes"): - if counter in model and (not isinstance(model[counter], int) or isinstance(model[counter], bool)): - raise ValueError("fixture.models[{0}].{1} must be an integer".format(index, counter)) - if not isinstance(payload.get("details"), dict): - raise ValueError("fixture.details must be an object") - if not isinstance(payload.get("trees"), dict): - raise ValueError("fixture.trees must be an object") - host = payload.get("host") - if not isinstance(host, dict): - raise ValueError("fixture.host must be an object") - if set(host) != {"ram_gb", "chip", "ollama", "lmstudio"}: - raise ValueError("fixture.host must contain only ram_gb, chip, ollama, and lmstudio") - if host["ram_gb"] is not None and (not isinstance(host["ram_gb"], int) or isinstance(host["ram_gb"], bool)): - raise ValueError("fixture.host.ram_gb must be an integer or null") - if host["chip"] is not None and not isinstance(host["chip"], str): - raise ValueError("fixture.host.chip must be a string or null") - if not isinstance(host["ollama"], bool) or not isinstance(host["lmstudio"], bool): - raise ValueError("fixture.host runtime flags must be booleans") - - -def _add_discovery_arguments(parser): - parser.add_argument("--role", choices=DISCOVERY_ROLES) - parser.add_argument("--limit", type=int, default=6) - parser.add_argument("--memory-gb", type=float, help="maximum host memory budget in GB (keeps 20%% runtime headroom)") - parser.add_argument("--quantization", help="normalized quantization such as 4bit or q8") - parser.add_argument("--license", dest="licenses", action="append", help="allow only this license (repeatable)") - parser.add_argument("--publisher", dest="publishers", action="append", help="allow only this publisher (repeatable)") - parser.add_argument("--runtime", choices=["ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"], help="require a runtime compatible with the model role") - parser.add_argument("--context", dest="context_tokens", type=int, default=None, help="tighten fit checks to weights plus KV cache at this context length") - parser.add_argument("--exclude-gated", dest="include_gated", action="store_false", default=True, help="exclude gated repositories") - parser.add_argument("--include-gated", dest="include_gated", action="store_true", help="include gated repositories (the legacy default)") - cache_group = parser.add_mutually_exclusive_group() - cache_group.add_argument("--refresh", action="store_true", help="bypass a fresh cache and fetch live evidence") - cache_group.add_argument("--offline", action="store_true", help="use only a matching local cache entry") - parser.add_argument("--state-dir", help="directory for versioned discovery cache entries") - parser.add_argument("--new", action="store_true", help="sort by most-recently-updated") - parser.add_argument("--fast", action="store_true", help="skip per-model enrichment (name heuristics only)") - parser.add_argument("--json", action="store_true") - parser.add_argument("--wire", metavar="REPO", help="emit setup + config for a model, instead of discovering") - parser.add_argument("--target", choices=["ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"], default="mlx_lm") - parser.add_argument("--port", type=int, default=8080) - - -def _run_discovery(arguments, legacy): - if arguments.wire: - print(wire(arguments.wire, arguments.target, arguments.port)) - return 0 - if arguments.context_tokens is not None and not 1024 <= arguments.context_tokens <= 1048576: - result = ResultEnvelope.fail( - "discover", - "invalid_arguments", - "--context must be between 1024 and 1048576 tokens.", - "Pass a bounded context length, or omit --context for weights-only fit.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - error = result.to_dict()["error"] - print("discover failed [{0}]: {1}\nremediation: {2}".format(error["code"], error["message"], error["remediation"])) - return 2 - service, fixture_warning, fixture_error = _discovery_service_from_environment(arguments.state_dir) - result = fixture_error or service.discover(DiscoveryRequest( - role=arguments.role, - memory_gb=arguments.memory_gb, - quantization=arguments.quantization, - licenses=arguments.licenses, - include_gated=arguments.include_gated, - publishers=arguments.publishers, - runtime=arguments.runtime, - refresh=arguments.refresh, - offline=arguments.offline, - limit=arguments.limit, - new=arguments.new, - fast=arguments.fast, - context_tokens=arguments.context_tokens, - )) - if fixture_warning: - result = ResultEnvelope.ok("discover", result.data, warnings=[fixture_warning]) - print("warning: synthetic fixture-backed discovery; not live Hugging Face evidence.", file=sys.stderr) - value = result.to_dict() - if legacy: - report = value["data"] if result.status == "ok" else {"host": service.host.to_dict() if service else HostInventory().to_dict(), "error": value["error"]["message"], "roles": {}} - print(json.dumps(report, indent=2) if arguments.json else render_md(report)) - return 0 if result.status == "ok" else 2 - if arguments.json: - print(json.dumps(value, indent=2)) - elif result.status == "ok": - print(render_md(value["data"])) - else: - error = value["error"] - print("discover failed [{0}]: {1}\nremediation: {2}".format(error["code"], error["message"], error["remediation"])) - return 0 if result.status == "ok" else 2 - - -def _run_inspect_host(arguments): - host, warnings = HostInventory.inspect(HuggingFaceClient()._http_get) - result = ResultEnvelope.ok("inspect-host", host.to_dict(), warnings=warnings) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Host inventory: {0}".format(json.dumps(host.to_dict(), sort_keys=True))) - for warning in warnings: - print("warning [{0}/{1}]: {2}".format(warning["code"], warning["probe"], warning["message"])) - return 0 - - -def _adoption_state_path(arguments): - return arguments.state or os.environ.get("MLX_AGENT_ADOPTION_STATE") - - -def _emit_adoption_result(result, as_json): - value = result.to_dict() - if as_json: - print(json.dumps(value, indent=2)) - elif result.status == "ok": - state = result.data["state"] - print("Adoption {0}: {1}".format(state["status"], state["phase"])) - if state["recommendations"]: - for item in state["recommendations"]: - print("{0}: {1} [{2}]".format(item["role"], item["repo"], item["evidence_strength"])) - requested_roles = state.get("request", {}).get("roles", []) - recommended_roles = { - item.get("role") for item in state.get("recommendations", []) - } - if ( - state.get("status") == "complete" - and "tool-use" in requested_roles - and "tool-use" not in recommended_roles - ): - print("No verified tool-use model was found.") - print( - "No model was downloaded. Install a shortlisted candidate in a " - "supported local runtime and start adoption again." - ) - else: - error = value["error"] - print("{0} failed [{1}]: {2}\nremediation: {3}".format( - result.operation, error["code"], error["message"], error["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _run_adoption(arguments): - operation = "adopt-{0}".format(arguments.adopt_command) - state_path = _adoption_state_path(arguments) - if not state_path: - return _emit_adoption_result(ResultEnvelope.fail( - operation, - "state_path_required", - "No adoption state path was supplied.", - "Pass --state PATH or set MLX_AGENT_ADOPTION_STATE.", - ), arguments.json) - - if arguments.adopt_command == "status": - workflow = AdoptionWorkflow( - discovery_service=DiscoveryService(), verifier=Verifier(), state_path=state_path - ) - try: - state = workflow.status(state_path) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - return _emit_adoption_result(ResultEnvelope.fail( - operation, - "adoption_state_invalid", - "Adoption state could not be read: {0}".format(error), - "Check --state PATH and restore a schema-version {0} adoption handoff.".format( - ADOPTION_SCHEMA_VERSION - ), - ), arguments.json) - return _emit_adoption_result( - ResultEnvelope.ok(operation, {"state": state.to_dict()}), arguments.json - ) - - service, fixture_warning, fixture_error = _discovery_service_from_environment() - if fixture_error: - error = fixture_error.to_dict()["error"] - return _emit_adoption_result(ResultEnvelope.fail( - operation, - error["code"], - error["message"], - error["remediation"], - retryable=error["retryable"], - ), arguments.json) - workflow = AdoptionWorkflow( - discovery_service=service, - verifier=Verifier(metadata_client=getattr(service, "_huggingface", None)), - state_path=state_path, - ) - try: - if arguments.adopt_command == "start": - state = workflow.start(AdoptionRequest( - roles=tuple(arguments.roles or ("general",)), - state_path=state_path, - shortlist_limit=arguments.shortlist_limit, - allow_network=arguments.allow_network and not arguments.offline, - offline=arguments.offline, - refresh=arguments.refresh, - fast=arguments.fast, - )) - else: - state = workflow.resume(state_path) - while state.phase != "complete": - state = workflow.advance(state) - except (OSError, TypeError, ValueError, RuntimeError, json.JSONDecodeError) as error: - return _emit_adoption_result(ResultEnvelope.fail( - operation, - "adoption_failed", - "Adoption workflow could not continue: {0}".format(error), - "Inspect the saved state with 'adopt status', resolve the reported issue, and resume.", - ), arguments.json) - warnings = [fixture_warning] if fixture_warning else [] - return _emit_adoption_result( - ResultEnvelope.ok(operation, {"state": state.to_dict()}, warnings=warnings), - arguments.json, - ) - - -def _add_adoption_arguments(parser): - actions = parser.add_subparsers(dest="adopt_command", required=True) - start = actions.add_parser("start", help="start and durably run a model adoption workflow") - start.add_argument("--state", help="adoption handoff path") - start.add_argument("--role", dest="roles", action="append", choices=DISCOVERY_ROLES) - start.add_argument("--shortlist-limit", type=int, default=4) - start.add_argument("--offline", action="store_true", help="use cached discovery and no metadata network requests") - start.add_argument("--refresh", action="store_true", help="refresh model discovery") - start.add_argument("--fast", action="store_true", help="use heuristic-only discovery enrichment") - start.add_argument("--no-network", dest="allow_network", action="store_false", default=True, help="do not inspect missing-model metadata") - start.add_argument("--measure", action="store_true", help="measure verified shortlist candidates (bench) before ranking") - start.add_argument("--json", action="store_true") - for name in ("resume", "status"): - action = actions.add_parser(name, help="{0} an adoption handoff".format(name)) - action.add_argument("--state", help="adoption handoff path") - action.add_argument("--json", action="store_true") - - -def _add_research_arguments(parser): - parser.add_argument("--domain", help="one-line domain description (required unless --interview)") - parser.add_argument("--role", dest="roles", action="append", choices=DISCOVERY_ROLES, help="model role to research (repeatable)") - parser.add_argument("--keyword", dest="keywords", action="append", help="domain keyword to prioritize (repeatable)") - parser.add_argument( - "--modality", - dest="modalities", - action="append", - choices=list(FOUNDATION_IDS), - help="foundational modality to seed (repeatable): audio, video, document-vision", - ) - parser.add_argument( - "--facet", - dest="facets", - action="append", - choices=list(ALL_FACET_IDS), - help="modality facet to prioritize (repeatable)", - ) - parser.add_argument("--license", dest="licenses", action="append", help="allow only this license (repeatable)") - parser.add_argument("--memory-gb", type=float, help="host memory budget in GB") - parser.add_argument("--notes", default="", help="free-form constraints") - parser.add_argument("--project", default=str(Path.cwd()), help="project root; the pack is written under /mlx-research") - parser.add_argument("--limit", type=int, default=6, help="candidates fetched per role during discovery; the pack is also capped at this many total") - parser.add_argument("--interview", action="store_true", help="ask questions interactively on stdin") - parser.add_argument("--no-write", dest="write", action="store_false", default=True, help="render the pack without writing a file") - parser.add_argument("--json", action="store_true") - - -def _stdin_reader(question): - prompt = question["prompt"] - if question.get("kind") == "multi": - prompt += " [{0}]".format(", ".join(question.get("choices", ()))) - return input("{0}\n> ".format(prompt)) - - -def _emit_research(result, arguments, markdown=None): - value = result.to_dict() - if arguments.json: - print(json.dumps(value, indent=2)) - elif result.status == "ok": - if "path" in result.data: - print("Research pack written to {0}".format(result.data["path"])) - else: - print(markdown if markdown is not None else "Research pack generated.") - else: - error = value["error"] - print("research failed [{0}]: {1}\nremediation: {2}".format( - error["code"], error["message"], error["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _run_research(arguments): - operation = "research" - try: - cli_modalities = tuple(arguments.modalities or ()) - cli_facets = tuple(arguments.facets or ()) - if arguments.interview: - intent = run_interview( - _stdin_reader, - preset_modalities=cli_modalities, - preset_facets=cli_facets, - ) - elif not arguments.domain: - return _emit_research(ResultEnvelope.fail( - operation, "domain_required", - "No domain description was supplied.", - "Pass --domain \"...\" or use --interview.", - ), arguments) - else: - detect_text = " ".join( - [arguments.domain] + list(arguments.keywords or []) - ) - modalities = resolve_modalities(cli=cli_modalities, text=detect_text) - if not modalities: - return _emit_research(ResultEnvelope.fail( - operation, "modality_required", - "No foundational modality was supplied or detected from the domain text.", - "Pass --modality audio|video|document-vision (repeatable), " - "or use --interview to choose interactively.", - ), arguments) - facets = resolve_facets(modalities, cli=cli_facets, text=detect_text) - intent = build_intent({ - "domain": arguments.domain, - "roles": arguments.roles or [], - "keywords": ",".join(arguments.keywords or []), - "license": ",".join(arguments.licenses or []), - "memory_gb": arguments.memory_gb, - "notes": arguments.notes, - "modalities": list(modalities), - "facets": list(facets), - }) - except (ValueError, EOFError) as error: - return _emit_research(ResultEnvelope.fail( - operation, "invalid_intent", str(error), - "Correct the interview answers or flags and retry.", - ), arguments) - - service, fixture_warning, fixture_error = _discovery_service_from_environment() - if fixture_error: - error = fixture_error.to_dict()["error"] - return _emit_research(ResultEnvelope.fail( - operation, error["code"], error["message"], error["remediation"], - ), arguments) - hf_client = getattr(service, "_huggingface", None) or HuggingFaceClient() - moment = datetime.now(timezone.utc) - try: - pack = generate_pack(intent, service, hf_client, limit=arguments.limit, now=moment) - except (OSError, TypeError, ValueError) as error: - return _emit_research(ResultEnvelope.fail( - operation, "research_failed", - "Research could not complete: {0}".format(error), - "Check network access to huggingface.co, or unset MLX_AGENT_FIXTURE.", - ), arguments) - markdown = render_pack(pack) - data = {"pack": pack.to_dict()} - if arguments.write: - try: - data["path"] = str(write_pack( - markdown, intent, root=arguments.project, now=moment, pack=pack, - )) - except (OSError, ValueError) as error: - return _emit_research(ResultEnvelope.fail( - operation, "write_failed", - "Research pack could not be written: {0}".format(error), - "Choose a writable --project directory without a symlinked mlx-research folder.", - ), arguments) - warnings = [fixture_warning] if fixture_warning else [] - return _emit_research( - ResultEnvelope.ok(operation, data, warnings=warnings), arguments, markdown - ) - - -def _add_blueprint_arguments(parser): - parser.add_argument("--goal", help="one-line project goal (required)") - parser.add_argument( - "--modality", - dest="modalities", - action="append", - choices=list(FOUNDATION_IDS), - help="foundational modality to include (repeatable)", - ) - parser.add_argument("--memory-gb", type=float, help="host memory budget in GB") - parser.add_argument("--notes", default="", help="free-form constraints") - parser.add_argument( - "--project", - default=str(Path.cwd()), - help="project root; the pack is written under /mlx-blueprints", - ) - parser.add_argument( - "--no-write", - dest="write", - action="store_false", - default=True, - help="render the pack without writing a file", - ) - parser.add_argument("--json", action="store_true") - - -def _emit_blueprint(result, arguments, markdown=None): - value = result.to_dict() - if arguments.json: - print(json.dumps(value, indent=2)) - elif result.status == "ok": - if "path" in result.data: - print("Project design pack written to {0}".format(result.data["path"])) - else: - print(markdown if markdown is not None else "Project design pack generated.") - else: - error = value["error"] - print("blueprint failed [{0}]: {1}\nremediation: {2}".format( - error["code"], error["message"], error["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _run_blueprint(arguments): - operation = "blueprint" - try: - if not arguments.goal: - return _emit_blueprint(ResultEnvelope.fail( - operation, "goal_required", - "No project goal was supplied.", - "Pass --goal \"...\".", - ), arguments) - brief = build_brief({ - "goal": arguments.goal, - "modalities": arguments.modalities or [], - "memory_gb": arguments.memory_gb, - "notes": arguments.notes, - }) - except (ValueError, TypeError) as error: - return _emit_blueprint(ResultEnvelope.fail( - operation, "invalid_brief", str(error), - "Correct the flags and retry.", - ), arguments) - - moment = datetime.now(timezone.utc) - pack = generate_design_pack(brief, now=moment) - markdown = render_design_pack(pack) - data = {"pack": pack.to_dict()} - if arguments.write: - try: - data["path"] = str(write_design_pack( - markdown, brief, root=arguments.project, now=moment, pack=pack, - )) - except (OSError, ValueError) as error: - return _emit_blueprint(ResultEnvelope.fail( - operation, "write_failed", - "Design pack could not be written: {0}".format(error), - "Choose a writable --project directory without a symlinked mlx-blueprints folder.", - ), arguments) - return _emit_blueprint(ResultEnvelope.ok(operation, data), arguments, markdown) - - -def _emit_wire_result(result, as_json, human=None): - if as_json: - print(json.dumps(result.to_dict(), indent=2)) - elif human is not None: - print(human) - elif result.status == "ok": - print(json.dumps(result.data, indent=2)) - else: - error = result.to_dict()["error"] - print("{0} failed [{1}]: {2}\nremediation: {3}".format( - result.operation, error["code"], error["message"], error["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _receipt_data(receipt): - value = receipt.to_dict() - value["receipt_path"] = receipt.receipt_path - return value - - -def _wire_failure(operation, code, message, remediation, data=None): - return ResultEnvelope( - operation=operation, status="error", data=data or {}, - warnings=[{"code": "cooperative_concurrency", "message": COOPERATIVE_CONCURRENCY_NOTE}], - error=ErrorDetail(code, message, remediation), - ) - - -def _wire_ok(operation, data): - return ResultEnvelope.ok(operation, data, warnings=[{ - "code": "cooperative_concurrency", "message": COOPERATIVE_CONCURRENCY_NOTE, - }]) - - -def _wire_render(arguments): - path = _assert_safe_target(arguments.path) - existing = _read_regular(path).decode("utf-8") - adapter = ConfigAdapter.detect(path, runtime=arguments.target) - content = adapter.render(arguments.model, arguments.target, existing) - adapter.validate(content) - return path, adapter, content - - -def _run_wire(arguments): - operation = "wire-{0}".format(arguments.wire_command) - try: - if arguments.wire_command == "status": - location = _assert_safe_target(arguments.receipt) - receipt = Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) - return _emit_wire_result(_wire_ok(operation, {"receipt": _receipt_data(receipt)}), arguments.json) - if arguments.wire_command == "rollback": - if not arguments.confirm: - preview = preview_rollback(arguments.receipt) - result = _wire_ok( - operation, {"preview": preview, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print(json.dumps(preview, indent=2)) - print( - "Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH." - ) - return 2 - if not arguments.preview_hash: - return _emit_wire_result(_wire_failure( - operation, - "preview_hash_required", - "--confirm requires the hash from a reviewed rollback preview.", - "Run wire rollback without --confirm, inspect the current-state preview, then pass --preview-hash.", - ), arguments.json) - receipt = rollback(arguments.receipt, preview_hash=arguments.preview_hash) - result = _wire_ok(operation, {"receipt": _receipt_data(receipt)}) if receipt.status == "rolled_back" else _wire_failure( - operation, receipt.status, "Rollback did not complete; receipt status is {0}.".format(receipt.status), - "Inspect the receipt validations and restore the verified backup manually.", {"receipt": _receipt_data(receipt)}, - ) - return _emit_wire_result(result, arguments.json) - - path, adapter, content = _wire_render(arguments) - if arguments.wire_command == "render": - return _emit_wire_result(_wire_ok(operation, { - "path": str(path), "runtime": arguments.target, "config": content, - "validation": {"parse": True}, - }), arguments.json, human=content) - transaction = Transaction(receipts_dir=arguments.receipts_dir) - preview = transaction.preview([{ - "path": str(path), "content": content, "runtime": arguments.target, - "adapter": adapter, "endpoint": arguments.endpoint, - }]) - if not arguments.confirm: - result = _wire_ok(operation, {"preview": preview, "requires_confirmation": True}) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print(preview["diff"]) - print("Confirmation required: rerun with --confirm to apply this transaction.") - return 2 - if not arguments.preview_hash: - return _emit_wire_result(_wire_failure( - operation, "preview_hash_required", "--confirm requires the preview hash from a prior preview.", - "Run wire apply without --confirm, inspect the preview, then pass its --preview-hash value.", - {"preview": preview}, - ), arguments.json) - if arguments.preview_hash != preview["preview_hash"]: - return _emit_wire_result(_wire_failure( - operation, "preview_stale", "The supplied preview hash does not match the current preview.", - "Generate and inspect a new preview before confirming this mutation.", - {"preview": preview}, - ), arguments.json) - if not arguments.json: - print(preview["diff"]) - receipt = transaction.apply(arguments.preview_hash) - data = {"preview": preview, "receipt": _receipt_data(receipt)} - result = _wire_ok(operation, data) if receipt.status == "applied" else _wire_failure( - operation, receipt.status, "Wire did not apply; receipt status is {0}.".format(receipt.status), - "Inspect the receipt validation results and use the recorded recovery path before retrying.", data, - ) - return _emit_wire_result(result, arguments.json) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - code = "cooperative_lock_busy" if isinstance(error, ConcurrentTransactionError) else ("preview_stale" if str(error).startswith("preview is stale") else "wire_failed") - return _emit_wire_result(_wire_failure( - operation, code, "Wire could not complete: {0}".format(error), - "Correct the target configuration or receipt, then render a new preview.", - ), arguments.json) - - -_BENCH_RUNTIMES = ("ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm") - - -def _bench_runtime_client(name): - if name == "ollama": - return OllamaRuntimeClient() - if name == "lmstudio": - return LMStudioRuntimeClient() - if name == "mlx-vlm": - return MLXVLMRuntimeClient() - if name == "mlx_lm": - return OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080") - if name == "litellm": - return OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000") - raise ValueError("unsupported bench runtime: {0}".format(name)) - - -def _add_bench_arguments(parser): - actions = parser.add_subparsers(dest="bench_command", required=True) - run = actions.add_parser( - "run", - help="measure a model already served by a local runtime (never downloads)", - ) - run.add_argument("--repo", required=True, help="model identifier exactly as the runtime serves it") - run.add_argument("--runtime", required=True, choices=_BENCH_RUNTIMES) - run.add_argument("--role", default="general", choices=DISCOVERY_ROLES) - run.add_argument("--runs", type=int, default=RUNS_DEFAULT, help="timed runs after one warm-up (1-10)") - run.add_argument("--gen-tokens", type=int, default=GEN_TOKENS_DEFAULT, help="tokens generated per run (16-2048)") - run.add_argument("--timeout", type=float, default=TIMEOUT_DEFAULT, help="total measurement deadline in seconds") - run.add_argument("--export", default=None, help="append an anonymized result line to this JSONL file") - run.add_argument("--json", action="store_true") - aggregate = actions.add_parser( - "aggregate", - help="deduplicate and median-aggregate bench export files", - ) - aggregate.add_argument("--exports", required=True, help="directory of .jsonl export files") - aggregate.add_argument("--out", default=None, help="aggregate output path (default: print only)") - aggregate.add_argument("--json", action="store_true") - - -def _run_bench(arguments): - operation = "bench-{0}".format(arguments.bench_command) - try: - if arguments.bench_command == "aggregate": - from .bench import aggregate_exports - - exports_dir = Path(arguments.exports) - if not exports_dir.is_dir(): - raise BenchError( - "invalid_arguments", - "The exports directory does not exist: {0}".format(arguments.exports), - "Point --exports at a directory of bench export .jsonl files.", - ) - files = sorted(exports_dir.glob("*.jsonl")) - aggregate = aggregate_exports(files) - if arguments.out: - destination = Path(arguments.out) - content = (json.dumps(aggregate, indent=2, sort_keys=True) + "\n").encode("utf-8") - destination.write_bytes(content) - data = { - "entries": aggregate["entries"], - "sources": len(files), - "written": str(arguments.out) if arguments.out else None, - } - result = ResultEnvelope.ok(operation, data) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Aggregate: {0} entries from {1} export file(s){2}".format( - len(aggregate["entries"]), len(files), - " -> {0}".format(arguments.out) if arguments.out else "", - )) - return 0 - chip = None - try: - chip = HostInventory.inspect().chip - except Exception: - chip = None - measurement = measure_runtime( - arguments.repo, - _bench_runtime_client(arguments.runtime), - runs=arguments.runs, - gen_tokens=arguments.gen_tokens, - timeout=arguments.timeout, - chip=chip, - ) - evidence = measurement.to_evidence(role=arguments.role).to_dict() - data = {"measurement": measurement.to_dict(), "evidence": evidence} - if arguments.export: - from . import __version__ - from .bench import append_export, export_record - - destination = append_export( - arguments.export, export_record(measurement, __version__) - ) - data["exported"] = str(destination) - result = ResultEnvelope.ok(operation, data) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Bench: {0} on {1} ({2} runs)".format( - measurement.repo, measurement.runtime, measurement.runs - )) - ttft = "{0} ms".format(measurement.ttft_ms) if measurement.ttft_ms is not None else "n/a" - prefill = ( - "{0} tok/s".format(measurement.prefill_toks) - if measurement.prefill_toks is not None - else "n/a" - ) - print(" decode: {0} tok/s (spread {1}%)".format(measurement.decode_toks, measurement.spread_pct)) - print(" ttft: {0}".format(ttft)) - print(" prefill: {0}".format(prefill)) - return 0 - except BenchError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (TypeError, ValueError) as error: - result = ResultEnvelope.fail( - operation, - "invalid_arguments", - str(error), - "Correct the bench arguments and retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("bench failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _add_serve_arguments(parser): - actions = parser.add_subparsers(dest="serve_command", required=True) - start = actions.add_parser( - "start", - help="preview, then confirmation-gated launch of a local MLX server", - ) - start.add_argument("--repo", required=True, help="publisher/model present in the local Hugging Face cache") - start.add_argument("--runtime", required=True, choices=["mlx_lm", "mlx-vlm"]) - start.add_argument("--port", type=int, default=None, help="loopback port (defaults per runtime recipe)") - start.add_argument("--max-tokens", type=int, default=MAX_TOKENS_DEFAULT) - start.add_argument("--adapter-path", default=None, help="LoRA adapter directory (mlx_lm only)") - start.add_argument("--launchd", action="store_true", help="install a launchd agent plist instead of spawning now") - start.add_argument("--launchd-dir", default=None, help="launchd target directory (defaults to ~/Library/LaunchAgents)") - start.add_argument("--confirm", action="store_true", help="authorize this reviewed server launch") - start.add_argument("--preview-hash", help="hash returned by the separately reviewed serve start preview") - start.add_argument("--receipts-dir", default=None, help="receipt root (defaults to the current directory)") - start.add_argument("--hf-cache", default=None, help="Hugging Face cache root for the local-model gate") - start.add_argument("--json", action="store_true") - stop = actions.add_parser("stop", help="stop exactly the server a serve receipt owns") - stop.add_argument("--port", type=int, required=True) - stop.add_argument("--receipts-dir", default=None) - stop.add_argument("--json", action="store_true") - status = actions.add_parser("status", help="cross-check serve receipts against live processes") - status.add_argument("--receipts-dir", default=None) - status.add_argument("--json", action="store_true") - - -def _run_serve(arguments): - operation = "serve-{0}".format(arguments.serve_command) - try: - if arguments.serve_command == "status": - entries = status_serve(arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, {"servers": entries}), arguments.json, - human=_serve_status_human(entries), - ) - if arguments.serve_command == "stop": - outcome = stop_serve(arguments.port, arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="serve {0}: port {1} (pid {2})".format( - outcome["status"], outcome["port"], outcome["pid"] - ), - ) - recipes = load_recipes() - plan = plan_start( - arguments.repo, - arguments.runtime, - recipes, - port=arguments.port, - max_tokens=arguments.max_tokens, - adapter_path=arguments.adapter_path, - ) - if arguments.launchd: - return _run_serve_launchd(arguments, plan, operation) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Serve plan: {0} on 127.0.0.1:{1}".format(plan["repo"], plan["port"])) - print(" argv: {0}".format(" ".join(plan["argv"]))) - print(" readiness: {0}".format(plan["readiness"])) - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - wired_claims = wired_port_claim([ - arguments.receipts_dir or str(Path.cwd()) - ]) - outcome = start_serve( - plan, - receipts_dir=arguments.receipts_dir, - confirm=True, - preview_hash=arguments.preview_hash, - model_present=default_model_present(arguments.hf_cache), - wired_claims=wired_claims, - ) - receipt = outcome["receipt"] - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="serve started: {0} on 127.0.0.1:{1} (pid {2})\n log: {3}".format( - receipt["repo"], receipt["port"], receipt["pid"], receipt["log_path"] - ), - ) - except ServeError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "serve_failed", - str(error), - "Correct the serve arguments or receipt state, then retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("serve failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _run_serve_launchd(arguments, plan, operation): - """Preview-confirm installation of a launchd agent for a serve plan.""" - from .serve import LaunchdPlistAdapter, launchd_label, render_launchd_plist - - launchd_dir = Path(arguments.launchd_dir) if arguments.launchd_dir else ( - Path.home() / "Library" / "LaunchAgents" - ) - target = launchd_dir / "{0}.plist".format(launchd_label(plan["port"])) - log_path = str(receipts_root(arguments.receipts_dir) / "{0}.log".format(plan["port"])) - content = render_launchd_plist(plan, log_path=log_path) - adapter = LaunchdPlistAdapter(target) - adapter.validate(content) - if target.exists(): - result = ResultEnvelope.fail( - operation, - "output_exists", - "A launchd plist already exists at {0}.".format(target), - "Unload and remove it yourself; serve never overwrites launchd agents.", - ) - return _emit_serve_result(result, arguments.json) - if not launchd_dir.is_dir(): - result = ResultEnvelope.fail( - operation, - "invalid_arguments", - "The launchd directory does not exist: {0}".format(launchd_dir), - "Create it yourself, or pass --launchd-dir.", - ) - return _emit_serve_result(result, arguments.json) - if not default_model_present(arguments.hf_cache)(plan["repo"]): - result = ResultEnvelope.fail( - operation, - "model_not_local", - "The model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; serve never downloads.", - ) - return _emit_serve_result(result, arguments.json) - transaction = Transaction(receipts_dir=arguments.receipts_dir) - preview = transaction.preview([{ - "path": str(target), "content": content, "runtime": "launchd", - "adapter": adapter, "endpoint": None, - }]) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"preview": preview, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print(preview["diff"]) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - if not arguments.preview_hash: - result = ResultEnvelope.fail( - operation, - "preview_hash_required", - "--confirm requires the preview hash from a prior preview.", - "Run serve start --launchd without --confirm, inspect the preview, then pass --preview-hash.", - ) - return _emit_serve_result(result, arguments.json) - if arguments.preview_hash != preview["preview_hash"]: - result = ResultEnvelope.fail( - operation, - "preview_stale", - "The supplied preview hash does not match the current preview.", - "Generate and inspect a new preview before confirming this mutation.", - ) - return _emit_serve_result(result, arguments.json) - receipt = transaction.apply(arguments.preview_hash) - data = {"receipt": _receipt_data(receipt), "plist": str(target)} - result = ResultEnvelope.ok(operation, data) - return _emit_serve_result( - result, arguments.json, - human="launchd agent installed: {0}\n load it with: launchctl bootstrap gui/$(id -u) {1}".format( - target, target - ), - ) - - -def _emit_serve_result(result, as_json, human=None): - if as_json: - print(json.dumps(result.to_dict(), indent=2)) - elif human: - print(human) - elif result.status == "ok": - print(json.dumps(result.to_dict()["data"], indent=2)) - else: - payload = result.to_dict()["error"] - print("{0} failed [{1}]: {2}\nremediation: {3}".format( - result.operation, payload["code"], payload["message"], payload["remediation"] - )) - return 0 if result.status == "ok" else 2 - - -def _serve_status_human(entries): - if not entries: - return "No serve receipts." - lines = [] - for entry in entries: - state = "alive" if entry["alive"] else "dead" - if entry["alive"] and not entry["argv_match"]: - state = "alive (argv mismatch)" - lines.append(" 127.0.0.1:{0} {1} ({2}) pid {3} - {4}".format( - entry["port"], entry["repo"], entry["runtime"], entry["pid"], state - )) - return "Serve receipts:\n" + "\n".join(lines) - - -def _add_fleet_arguments(parser): - actions = parser.add_subparsers(dest="fleet_command", required=True) - for name in ("render", "apply"): - action = actions.add_parser( - name, - help="{0} a one-shot per-role router configuration".format(name), - ) - action.add_argument("--path", required=True, help="target router configuration file") - action.add_argument("--assign", action="append", default=None, metavar="ROLE=REPO", - help="per-role model assignment (repeatable)") - action.add_argument("--from-adoption", default=None, metavar="STATE", - help="read role assignments from a completed adopt handoff") - action.add_argument("--runtime-map", action="append", default=None, metavar="ROLE=RUNTIME", - help="per-role runtime override: mlx_lm or mlx-vlm (repeatable)") - action.add_argument("--allow-missing", action="store_true", - help="warn instead of failing when a model is not in a local inventory") - action.add_argument("--json", action="store_true") - if name == "apply": - action.add_argument("--confirm", action="store_true", help="explicitly authorize this reviewed mutation") - action.add_argument("--preview-hash", help="hash returned by the separately reviewed fleet apply preview") - action.add_argument("--receipts-dir", help="directory for non-secret transaction receipts") - action.add_argument("--endpoint", help="optional local runtime health endpoint") - - -def _fleet_inputs(arguments): - if arguments.assign and arguments.from_adoption: - raise FleetError( - "invalid_arguments", - "--assign and --from-adoption are mutually exclusive.", - "Pick explicit assignments or an adoption handoff, not both.", - ) - if arguments.from_adoption: - assignments = assignments_from_adoption(arguments.from_adoption) - else: - assignments = parse_assignments(arguments.assign) - runtime_map = parse_runtime_map(arguments.runtime_map) - return assignments, runtime_map - - -def _fleet_missing_models(assignments): - from .model_doctor import ( - collect_runtime_inventory, - default_hf_cache, - inspect_hf_cache, - ) - - known = {item["id"].casefold() for item in inspect_hf_cache(default_hf_cache())} - runtime_clients = [ - OllamaRuntimeClient(), - LMStudioRuntimeClient(), - OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080"), - MLXVLMRuntimeClient(), - OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000"), - ] - runtime_inventory, _errors = collect_runtime_inventory(runtime_clients) - known.update(item["id"].casefold() for item in runtime_inventory) - return sorted( - repo for repo in set(assignments.values()) if repo.casefold() not in known - ) - - -def _run_fleet(arguments): - operation = "fleet-{0}".format(arguments.fleet_command) - try: - assignments, runtime_map = _fleet_inputs(arguments) - path = _assert_safe_target(arguments.path) - existing = _read_regular(path).decode("utf-8") - adapter = FleetConfigAdapter(path) - content = adapter.render(assignments, runtime_map, existing=existing) - missing = _fleet_missing_models(assignments) - warnings = [] - if missing and not arguments.allow_missing: - return _emit_wire_result(_wire_failure( - operation, - "model_not_local", - "Models are not present in a local inventory: {0}".format(", ".join(missing)), - "Download them first, or pass --allow-missing to record a reviewed warning.", - ), arguments.json) - if missing: - warnings.append({ - "code": "model_not_local", - "message": "Not in a local inventory: {0}".format(", ".join(missing)), - }) - if arguments.fleet_command == "render": - result = _wire_ok(operation, { - "path": str(path), "assignments": assignments, "config": content, - "validation": {"parse": True}, - }) - result.warnings.extend(warnings) - return _emit_wire_result(result, arguments.json, human=content) - transaction = Transaction(receipts_dir=arguments.receipts_dir) - preview = transaction.preview([{ - "path": str(path), "content": content, "runtime": "litellm", - "adapter": adapter, "endpoint": arguments.endpoint, - }]) - if not arguments.confirm: - result = _wire_ok(operation, {"preview": preview, "requires_confirmation": True}) - result.warnings.extend(warnings) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print(preview["diff"]) - print("Confirmation required: rerun with --confirm to apply this transaction.") - return 2 - if not arguments.preview_hash: - return _emit_wire_result(_wire_failure( - operation, "preview_hash_required", "--confirm requires the preview hash from a prior preview.", - "Run fleet apply without --confirm, inspect the preview, then pass its --preview-hash value.", - {"preview": preview}, - ), arguments.json) - if arguments.preview_hash != preview["preview_hash"]: - return _emit_wire_result(_wire_failure( - operation, "preview_stale", "The supplied preview hash does not match the current preview.", - "Generate and inspect a new preview before confirming this mutation.", - {"preview": preview}, - ), arguments.json) - if not arguments.json: - print(preview["diff"]) - receipt = transaction.apply(arguments.preview_hash) - data = {"preview": preview, "receipt": _receipt_data(receipt)} - result = _wire_ok(operation, data) if receipt.status == "applied" else _wire_failure( - operation, receipt.status, "Fleet did not apply; receipt status is {0}.".format(receipt.status), - "Inspect the receipt validation results and use the recorded recovery path before retrying.", data, - ) - result.warnings.extend(warnings) - return _emit_wire_result(result, arguments.json) - except FleetError as error: - return _emit_wire_result(_wire_failure( - operation, error.code, str(error), error.remediation, - ), arguments.json) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - code = "cooperative_lock_busy" if isinstance(error, ConcurrentTransactionError) else ("preview_stale" if str(error).startswith("preview is stale") else "fleet_failed") - return _emit_wire_result(_wire_failure( - operation, code, "Fleet could not complete: {0}".format(error), - "Correct the assignments or target configuration, then render a new preview.", - ), arguments.json) - - -def _watch_runtime_clients(): - return [ - OllamaRuntimeClient(), - LMStudioRuntimeClient(), - OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080"), - MLXVLMRuntimeClient(), - OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000"), - ] - - -def _add_watch_arguments(parser): - actions = parser.add_subparsers(dest="watch_command", required=True) - for name in ("snapshot", "diff"): - action = actions.add_parser( - name, - help="{0} the owned-model Hugging Face watch state".format(name), - ) - action.add_argument("--state-dir", default="./.mlx-agent-watch", - help="watch state root (default ./.mlx-agent-watch)") - action.add_argument("--limit", type=int, default=6, - help="discovery results per role (default 6)") - action.add_argument("--json", action="store_true") - - -def _run_watch(arguments): - operation = "watch-{0}".format(arguments.watch_command) - try: - service, fixture_warning, fixture_error = _discovery_service_from_environment() - if fixture_error is not None: - return _emit_wire_result(fixture_error, arguments.json) - result = service.discover(DiscoveryRequest(limit=arguments.limit)) - if result.status != "ok": - error = result.to_dict()["error"] - return _emit_wire_result(ResultEnvelope.fail( - operation, error["code"], error["message"], error["remediation"] - ), arguments.json) - owned = collect_owned(_watch_runtime_clients()) - candidates = snapshot_candidates(result.data) - if arguments.watch_command == "snapshot": - snapshot = build_snapshot(owned, candidates) - destination = write_snapshot(arguments.state_dir, snapshot) - data = { - "state": str(destination), - "owned": len(snapshot["owned"]), - "candidates": len(snapshot["candidates"]), - "rotated_previous": snapshot["previous"] is not None, - } - result = ResultEnvelope.ok(operation, data) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Watch snapshot: {0} owned, {1} tracked repos -> {2}".format( - data["owned"], data["candidates"], data["state"] - )) - return 0 - baseline = read_baseline(arguments.state_dir) - current = build_snapshot(owned, candidates) - findings = diff_snapshots(baseline, current) - data = { - "baseline_created_at": baseline["created_at"], - "findings": findings, - } - warnings = [ - {"code": finding["code"], "message": "{0}: {1}".format(finding["repo"], finding["detail"])} - for finding in findings - ] - result = ResultEnvelope.ok(operation, data, warnings=warnings) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Watch diff against {0}: {1} finding(s)".format( - baseline["created_at"], len(findings) - )) - for finding in findings: - print(" [{0}] {1} - {2}".format(finding["code"], finding["repo"], finding["detail"])) - return 0 - except WatchError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "watch_failed", - str(error), - "Correct the state directory and retry; watch only writes its own state file.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("watch failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _add_convert_arguments(parser): - actions = parser.add_subparsers(dest="convert_command", required=True) - start = actions.add_parser( - "start", - help="preview, then confirmation-gated quantization of a cached model", - ) - start.add_argument("--repo", required=True, help="publisher/model present in the local Hugging Face cache") - start.add_argument("--q-bits", type=int, default=4, choices=Q_BITS_CHOICES) - start.add_argument("--out", default=None, help="output directory (default -MLX-bit)") - start.add_argument("--confirm", action="store_true", help="authorize this reviewed conversion") - start.add_argument("--preview-hash", help="hash returned by the separately reviewed convert preview") - start.add_argument("--receipts-dir", default=None) - start.add_argument("--hf-cache", default=None) - start.add_argument("--json", action="store_true") - status = actions.add_parser("status", help="cross-check convert receipts against live processes") - status.add_argument("--receipts-dir", default=None) - status.add_argument("--json", action="store_true") - - -def _run_convert(arguments): - operation = "convert-{0}".format(arguments.convert_command) - try: - if arguments.convert_command == "status": - entries = status_convert(arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, {"jobs": entries}), arguments.json, - human=_convert_status_human(entries), - ) - plan = plan_convert(arguments.repo, q_bits=arguments.q_bits, out=arguments.out) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Convert plan: {0} -> {1} ({2}bit)".format( - plan["repo"], plan["out"], plan["q_bits"] - )) - print(" argv: {0}".format(" ".join(plan["argv"]))) - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - outcome = start_convert( - plan, - receipts_dir=arguments.receipts_dir, - confirm=True, - preview_hash=arguments.preview_hash, - model_present=default_model_present(arguments.hf_cache), - ) - receipt = outcome["receipt"] - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="convert started: {0} -> {1} (pid {2})\n log: {3}".format( - receipt["repo"], receipt["out"], receipt["pid"], receipt["log_path"] - ), - ) - except ConvertError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "convert_failed", - str(error), - "Correct the convert arguments or receipt state, then retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("convert failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _convert_status_human(entries): - if not entries: - return "No convert receipts." - lines = [] - for entry in entries: - lines.append(" {0} ({1}bit) -> {2}: {3}".format( - entry["repo"], entry["q_bits"], entry["out"], entry["state"] - )) - return "Convert jobs:\n" + "\n".join(lines) - - -def _add_lora_arguments(parser): - actions = parser.add_subparsers(dest="lora_command", required=True) - start = actions.add_parser( - "start", - help="preview, then confirmation-gated LoRA training on a cached model", - ) - start.add_argument("--repo", required=True, help="publisher/model base present in the local Hugging Face cache") - start.add_argument("--data", required=True, help="dataset directory with train.jsonl") - start.add_argument("--iters", type=int, default=ITERS_DEFAULT) - start.add_argument("--batch-size", type=int, default=BATCH_DEFAULT) - start.add_argument("--learning-rate", type=float, default=LR_DEFAULT) - start.add_argument("--num-layers", type=int, default=LAYERS_DEFAULT, help="-1 trains all layers") - start.add_argument("--out", default=None, help="adapter output directory (default -lora)") - start.add_argument("--confirm", action="store_true", help="authorize this reviewed training run") - start.add_argument("--preview-hash", help="hash returned by the separately reviewed lora preview") - start.add_argument("--receipts-dir", default=None) - start.add_argument("--hf-cache", default=None) - start.add_argument("--json", action="store_true") - status = actions.add_parser("status", help="cross-check lora receipts against live processes") - status.add_argument("--receipts-dir", default=None) - status.add_argument("--json", action="store_true") - - -def _run_lora(arguments): - operation = "lora-{0}".format(arguments.lora_command) - try: - if arguments.lora_command == "status": - entries = status_lora(arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, {"jobs": entries}), arguments.json, - human=_lora_status_human(entries), - ) - plan = plan_lora( - arguments.repo, - arguments.data, - iters=arguments.iters, - batch_size=arguments.batch_size, - learning_rate=arguments.learning_rate, - num_layers=arguments.num_layers, - out=arguments.out, - ) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("LoRA plan: {0} + {1} train lines -> {2}".format( - plan["repo"], plan["dataset"]["train_lines"], plan["out"] - )) - print(" argv: {0}".format(" ".join(plan["argv"]))) - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - outcome = start_lora( - plan, - receipts_dir=arguments.receipts_dir, - confirm=True, - preview_hash=arguments.preview_hash, - model_present=default_model_present(arguments.hf_cache), - ) - receipt = outcome["receipt"] - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="lora started: {0} -> {1} (pid {2})\n log: {3}".format( - receipt["repo"], receipt["out"], receipt["pid"], receipt["log_path"] - ), - ) - except LoraError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "lora_failed", - str(error), - "Correct the lora arguments or receipt state, then retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("lora failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _lora_status_human(entries): - if not entries: - return "No lora receipts." - lines = [] - for entry in entries: - lines.append(" {0} -> {1}: {2}".format( - entry["repo"], entry["out"], entry["state"] - )) - return "LoRA jobs:\n" + "\n".join(lines) - - -def _add_fuse_arguments(parser): - actions = parser.add_subparsers(dest="fuse_command", required=True) - start = actions.add_parser( - "start", - help="preview, then confirmation-gated LoRA fuse into the base model", - ) - start.add_argument("--repo", required=True, help="publisher/model base present in the local Hugging Face cache") - start.add_argument("--adapter", required=True, help="completed lora adapter directory") - start.add_argument("--out", default=None, help="fused output directory (default -fused)") - start.add_argument("--confirm", action="store_true", help="authorize this reviewed fuse") - start.add_argument("--preview-hash", help="hash returned by the separately reviewed fuse preview") - start.add_argument("--receipts-dir", default=None) - start.add_argument("--hf-cache", default=None) - start.add_argument("--json", action="store_true") - status = actions.add_parser("status", help="cross-check fuse receipts against live processes") - status.add_argument("--receipts-dir", default=None) - status.add_argument("--json", action="store_true") - - -def _run_fuse(arguments): - operation = "fuse-{0}".format(arguments.fuse_command) - try: - if arguments.fuse_command == "status": - entries = status_fuse(arguments.receipts_dir) - return _emit_serve_result( - ResultEnvelope.ok(operation, {"jobs": entries}), arguments.json, - human=_fuse_status_human(entries), - ) - plan = plan_fuse(arguments.repo, arguments.adapter, out=arguments.out) - if not arguments.confirm: - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - print("Fuse plan: {0} + {1} -> {2}".format( - plan["repo"], plan["adapter"], plan["out"] - )) - print(" argv: {0}".format(" ".join(plan["argv"]))) - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - return 2 - outcome = start_fuse( - plan, - receipts_dir=arguments.receipts_dir, - confirm=True, - preview_hash=arguments.preview_hash, - model_present=default_model_present(arguments.hf_cache), - ) - receipt = outcome["receipt"] - return _emit_serve_result( - ResultEnvelope.ok(operation, outcome), arguments.json, - human="fuse started: {0} -> {1} (pid {2})\n log: {3}".format( - receipt["repo"], receipt["out"], receipt["pid"], receipt["log_path"] - ), - ) - except FuseError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - result = ResultEnvelope.fail( - operation, - "fuse_failed", - str(error), - "Correct the fuse arguments or receipt state, then retry.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("fuse failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - - -def _fuse_status_human(entries): - if not entries: - return "No fuse receipts." - lines = [] - for entry in entries: - lines.append(" {0} -> {1}: {2}".format( - entry["repo"], entry["out"], entry["state"] - )) - return "Fuse jobs:\n" + "\n".join(lines) - - -def _add_wire_arguments(parser): - actions = parser.add_subparsers(dest="wire_command", required=True) - for name in ("render", "apply"): - action = actions.add_parser(name, help="{0} a deterministic runtime configuration ({1})".format(name, "advisory lock protects cooperative writers" if name == "apply" else "safe render")) - action.add_argument("model", help="Hugging Face model repository") - action.add_argument("--target", choices=["ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"], default="mlx_lm") - action.add_argument("--path", required=True, help="target configuration file") - action.add_argument("--json", action="store_true") - if name == "apply": - action.add_argument("--confirm", action="store_true", help="explicitly authorize this reviewed mutation") - action.add_argument("--preview-hash", help="hash returned by the separately reviewed wire apply preview") - action.add_argument("--receipts-dir", help="directory for non-secret transaction receipts") - action.add_argument("--endpoint", help="optional local runtime health endpoint") - status = actions.add_parser("status", help="inspect a Wire receipt") - status.add_argument("receipt") - status.add_argument("--json", action="store_true") - restore = actions.add_parser("rollback", help="restore a Wire receipt's exact backup") - restore.add_argument("receipt") - restore.add_argument("--confirm", action="store_true", help="explicitly authorize this rollback") - restore.add_argument("--preview-hash", help="hash returned by the separately reviewed rollback preview") - restore.add_argument("--json", action="store_true") - - -def _installer_registry(): - root = Path(__file__).resolve().parents[2] - home = os.environ.get("MLX_AGENT_HOME") - config_root = os.environ.get("MLX_AGENT_CONFIG_ROOT") or os.environ.get("XDG_STATE_HOME") - xdg_config_home = os.environ.get("XDG_CONFIG_HOME") - return ProviderRegistry( - root / "plugin.json", home=home, config_root=config_root, - xdg_config_home=xdg_config_home, - ) - - -def _add_installer_arguments(parser, include_providers=True): - if include_providers: - parser.add_argument("providers", nargs="*", help="explicit provider IDs; omit to inspect detected choices") - parser.add_argument("--scope", choices=["user", "project"], default="user") - parser.add_argument("--project", default=str(Path.cwd()), help="project root for project-scoped installation") - parser.add_argument("--dry-run", action="store_true", help="show the reviewed plan without mutating files") - parser.add_argument("--confirm", action="store_true", help="authorize a separately reviewed installer preview") - parser.add_argument("--preview-hash", help="preview hash returned by the installer dry run") - parser.add_argument("--json", action="store_true") - - -def _add_doctor_arguments(parser): - _add_installer_arguments(parser) - parser.add_argument( - "--wired-root", - action="append", - default=None, - help="root scanned for Wire-managed configs (repeatable; defaults to the project root)", - ) - parser.add_argument( - "--hf-cache", - default=None, - help="Hugging Face cache root (defaults to ~/.cache/huggingface/hub)", - ) - parser.add_argument( - "--prune", - action="store_true", - help="preview deletion of incomplete HF cache snapshots (requires --confirm --preview-hash to execute)", - ) - - -def _run_model_doctor(arguments): - from .model_doctor import ( - PruneError, - default_hf_cache, - execute_prune, - plan_prune, - run_model_doctor, - ) - - operation = "doctor-models" - wired_roots = arguments.wired_root or [arguments.project] - hf_cache = arguments.hf_cache or default_hf_cache() - if arguments.prune: - try: - plan = plan_prune(hf_cache) - outcome = execute_prune( - plan, - confirm=arguments.confirm, - preview_hash=arguments.preview_hash, - ) - except PruneError as error: - result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("doctor failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - if outcome["status"] == "preview": - result = ResultEnvelope.ok( - operation, {"plan": plan, "requires_confirmation": True} - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - if not plan["candidates"]: - print("Nothing to prune: no incomplete cache snapshots.") - for candidate in plan["candidates"]: - print(" prune: {0} ({1:.1f} GB) - {2}".format( - candidate["repo"], candidate["bytes"] / 1e9, candidate["path"] - )) - if plan["candidates"]: - print("IRREVERSIBLE: these cache directories will be deleted permanently.") - print(" preview_hash: {0}".format(plan["preview_hash"])) - print("Confirmation required: rerun with --prune --confirm --preview-hash PREVIEW_HASH.") - return 0 if not plan["candidates"] else 2 - result = ResultEnvelope.ok(operation, outcome) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - removed = [item for item in outcome["removed"] if item.get("removed")] - print("Pruned {0} incomplete snapshot(s).".format(len(removed))) - for item in removed: - print(" removed: {0}".format(item["repo"])) - return 0 - runtime_clients = [ - OllamaRuntimeClient(), - LMStudioRuntimeClient(), - OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080"), - MLXVLMRuntimeClient(), - OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000"), - ] - try: - report = run_model_doctor(wired_roots, hf_cache, runtime_clients) - except (OSError, TypeError, ValueError) as error: - result = ResultEnvelope.fail( - operation, - "doctor_failed", - str(error), - "Correct the scanned roots and retry; doctor never mutates anything.", - ) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - payload = result.to_dict()["error"] - print("doctor failed [{0}]: {1}\nremediation: {2}".format( - payload["code"], payload["message"], payload["remediation"] - )) - return 2 - warnings = [ - {"code": finding["code"], "message": "{0}: {1}".format( - finding.get("path") or finding.get("model") or finding["code"], - finding["remediation"], - )} - for finding in report["findings"] - ] - result = ResultEnvelope.ok(operation, report, warnings=warnings) - if arguments.json: - print(json.dumps(result.to_dict(), indent=2)) - else: - summary = report["summary"] - print("Doctor: {0} model(s), {1} wired config(s), {2} finding(s)".format( - summary["models"], summary["wired_configs"], summary["findings"] - )) - print(" hf-cache disk: {0:.1f} GB".format(summary["hf_cache_bytes"] / 1e9)) - for finding in report["findings"]: - location = finding.get("path") or finding.get("model") or "" - print(" [{0}] {1} - {2}".format(finding["code"], location, finding["remediation"])) - for endpoint in report["endpoints"]: - print(" endpoint {0}: {1}".format(endpoint["endpoint"], endpoint["status"])) - for runtime_error in report["runtime_errors"]: - print(" runtime unreachable: {0}".format(runtime_error)) - return 0 - - -def _installer_result(operation, data, as_json, status="ok", error=None): - result = ResultEnvelope.ok(operation, data) if status == "ok" else ResultEnvelope.fail( - operation, error[0], error[1], error[2] - ) - if as_json: - print(json.dumps(result.to_dict(), indent=2)) - elif status == "ok": - if data.get("selection_required"): - available = [item["id"] for item in data["providers"] if item["available"]] - print("No provider was selected. Detected providers: {0}".format(", ".join(available) if available else "none")) - elif "preview" in data: - compatibility = data.get("plan", {}).get("compatibility", []) - if compatibility: - print("Provider compatibility:") - for item in compatibility: - details = "{0}: {1}".format(item["id"], item["state"]) - if item.get("version"): - details += " (version {0})".format(item["version"]) - if item.get("error"): - details += " - {0}".format(item["error"]) - print(details) - print(data["preview"]["diff"]) - print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") - else: - print(json.dumps(data, indent=2)) - else: - print("{0} failed [{1}]: {2}\nremediation: {3}".format(operation, error[0], error[1], error[2])) - return 0 if status == "ok" else 2 - - -def _run_installer(arguments): - operation = arguments.command - if operation == "doctor" and "models" in (getattr(arguments, "providers", None) or []): - return _run_model_doctor(arguments) - try: - installer = Installer(_installer_registry(), project_root=arguments.project) - detections = [item.to_dict() for item in installer.detected()] - selected = tuple(getattr(arguments, "providers", ()) or ()) - if operation == "providers" or not selected: - return _installer_result(operation, { - "providers": detections, - "selection_required": operation != "providers", - "mutated": False, - }, arguments.json) - plan = installer.plan(operation, selected, arguments.scope, arguments.project) - if arguments.dry_run or (operation != "doctor" and not arguments.confirm): - return _installer_result(operation, {"plan": plan.to_dict(), "preview": plan.preview, "mutated": False}, arguments.json) - if operation != "doctor" and not arguments.preview_hash: - return _installer_result(operation, {"plan": plan.to_dict(), "preview": plan.preview}, arguments.json, "error", ( - "preview_hash_required", "--confirm requires the hash from an inspected preview.", - "Run the command with --dry-run, inspect its preview, then pass --preview-hash.", - )) - if operation != "doctor" and arguments.preview_hash != plan.preview["preview_hash"]: - return _installer_result(operation, {"plan": plan.to_dict(), "preview": plan.preview}, arguments.json, "error", ( - "preview_stale", "The supplied preview hash does not match the current plan.", - "Generate and inspect a new preview before confirming this mutation.", - )) - result = installer.execute(plan, arguments.preview_hash if operation != "doctor" else False) - data = result if isinstance(result, dict) else result.to_dict() - return _installer_result(operation, {"result": data, "mutated": operation != "doctor"}, arguments.json) - except InstallerConflictError as error: - lock_code = "legacy_lock_busy" if str(error).startswith("legacy_lock_busy") else ( - "legacy_lock_recreated" if str(error).startswith("legacy_lock_recreated") else "artifact_conflict" - ) - return _installer_result(operation, {}, arguments.json, "error", ( - lock_code, str(error), - "Stop older mlx-agent processes and remove only recreated legacy locks." if lock_code != "artifact_conflict" else "Preserve the modified file or restore it to the receipt hash before retrying.", - )) - except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: - return _installer_result(operation, {}, arguments.json, "error", ( - "installer_failed", str(error), "Inspect the provider selection and preview, then correct the reported path or manifest issue.", - )) - - -def legacy_scout_main(argv=None): - parser = argparse.ArgumentParser(description="Discover MLX models on HuggingFace for this host.") - _add_discovery_arguments(parser) - return _run_discovery(parser.parse_args(argv), legacy=True) - - -def build_parser(): - parser = argparse.ArgumentParser(description="MLX agent command-line core.") - subcommands = parser.add_subparsers(dest="command", required=True) - discover = subcommands.add_parser("discover", help="discover MLX models for this host") - _add_discovery_arguments(discover) - inspect_host = subcommands.add_parser("inspect-host", help="inspect local Apple Silicon and runtime inventory without model discovery") - inspect_host.add_argument("--json", action="store_true") - adopt = subcommands.add_parser("adopt", help="run or inspect resumable model adoption") - _add_adoption_arguments(adopt) - research = subcommands.add_parser("research", help="build a read-only domain research pack (markdown)") - _add_research_arguments(research) - blueprint = subcommands.add_parser( - "blueprint", - help="emit a read-only MLX project design pack (quant/train guidance; no scaffolding)", - ) - _add_blueprint_arguments(blueprint) - wire_command = subcommands.add_parser("wire", help="render, apply, inspect, or roll back runtime wiring") - _add_wire_arguments(wire_command) - bench_command = subcommands.add_parser("bench", help="measure a locally served model without downloading it") - _add_bench_arguments(bench_command) - serve_command = subcommands.add_parser("serve", help="preview and launch a local MLX server (confirmation-gated)") - _add_serve_arguments(serve_command) - fleet_command = subcommands.add_parser("fleet", help="render or apply a one-shot per-role router configuration") - _add_fleet_arguments(fleet_command) - watch_command = subcommands.add_parser("watch", help="snapshot and diff owned models against Hugging Face") - _add_watch_arguments(watch_command) - convert_command = subcommands.add_parser("convert", help="preview and quantize a cached model (confirmation-gated)") - _add_convert_arguments(convert_command) - lora_command = subcommands.add_parser("lora", help="preview and run LoRA training on a cached model (confirmation-gated)") - _add_lora_arguments(lora_command) - fuse_command = subcommands.add_parser("fuse", help="preview and fuse a LoRA adapter into its base (confirmation-gated)") - _add_fuse_arguments(fuse_command) - providers_command = subcommands.add_parser("providers", help="list detected supported provider CLIs") - _add_installer_arguments(providers_command, include_providers=False) - for name in ("install", "update", "uninstall", "doctor"): - installer_command = subcommands.add_parser(name, help="{0} declared provider artifacts safely".format(name)) - if name == "doctor": - _add_doctor_arguments(installer_command) - else: - _add_installer_arguments(installer_command) - return parser - - -def main(argv=None): - parser = build_parser() - arguments = parser.parse_args(argv) - if arguments.command == "adopt": - return _run_adoption(arguments) - if arguments.command == "wire": - return _run_wire(arguments) - if arguments.command == "bench": - return _run_bench(arguments) - if arguments.command == "serve": - return _run_serve(arguments) - if arguments.command == "fleet": - return _run_fleet(arguments) - if arguments.command == "watch": - return _run_watch(arguments) - if arguments.command == "convert": - return _run_convert(arguments) - if arguments.command == "lora": - return _run_lora(arguments) - if arguments.command == "fuse": - return _run_fuse(arguments) - if arguments.command in {"providers", "install", "update", "uninstall", "doctor"}: - return _run_installer(arguments) - if arguments.command == "inspect-host": - return _run_inspect_host(arguments) - if arguments.command == "research": - return _run_research(arguments) - if arguments.command == "blueprint": - return _run_blueprint(arguments) - return _run_discovery(arguments, legacy=False) diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/command_executor.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/command_executor.py deleted file mode 100644 index 737ecaf..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/command_executor.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Bounded stdin executor for native provider custom tools.""" - -from __future__ import annotations - -import argparse -import io -import json -import sys - -from .gemini_args import GeminiArgumentError, parse_gemini_arguments - - -MAX_ARGUMENT_BYTES = 4096 -_PROVIDERS = ("opencode",) -_CAPABILITIES = ("scout", "adopt", "wire", "bench") - - -class CommandExecutorError(ValueError): - """Custom-tool input or execution failed without exposing raw arguments.""" - - -def read_command_arguments(stream): - """Read one small UTF-8 argument payload from a byte stream.""" - if not hasattr(stream, "read"): - raise CommandExecutorError("custom tool input was rejected") - try: - value = stream.read(MAX_ARGUMENT_BYTES + 1) - except (OSError, ValueError, TypeError) as error: - raise CommandExecutorError("custom tool input was rejected") from error - if not isinstance(value, bytes) or len(value) > MAX_ARGUMENT_BYTES: - raise CommandExecutorError("custom tool input was rejected") - try: - return value.decode("utf-8") - except UnicodeDecodeError as error: - raise CommandExecutorError("custom tool input was rejected") from error - - -def execute_command(provider, capability, stream, core=None): - """Validate stdin and execute only the capability's allowlisted core argv.""" - if provider not in _PROVIDERS or capability not in _CAPABILITIES: - raise CommandExecutorError("custom tool request was rejected") - try: - arguments = parse_gemini_arguments(capability, read_command_arguments(stream)) - except (GeminiArgumentError, CommandExecutorError) as error: - raise CommandExecutorError("custom tool request was rejected") from error - if core is None: - from .cli import main as core - try: - result = core(arguments) - except Exception as error: - raise CommandExecutorError("custom tool execution failed") from error - try: - exit_code = 0 if result is None else int(result) - except (TypeError, ValueError) as error: - raise CommandExecutorError("custom tool execution failed") from error - return {"status": "ok", "provider": provider, "capability": capability, "exit_code": exit_code} - - -def main(argv=None, stdin=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--provider", required=True, choices=_PROVIDERS) - parser.add_argument("--capability", required=True, choices=_CAPABILITIES) - arguments = parser.parse_args(argv) - try: - result = execute_command(arguments.provider, arguments.capability, stdin or sys.stdin.buffer) - except CommandExecutorError: - print(json.dumps({"status": "error", "provider": arguments.provider, "capability": arguments.capability, "error": "command rejected"}, sort_keys=True)) - return 2 - print(json.dumps(result, sort_keys=True)) - return result["exit_code"] - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/contextfit.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/contextfit.py deleted file mode 100644 index b355316..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/contextfit.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Bounded KV-cache arithmetic for context-aware memory fit.""" - -from __future__ import annotations - - -_MAX_INT = 2 ** 63 - 1 -MAX_ARCH_VALUE = 1000000 -DEFAULT_DTYPE_BYTES = 2 -CONTEXT_MIN, CONTEXT_MAX = 1024, 1048576 - - -def extract_architecture(config): - """Return bounded architecture facts from a Hugging Face config object.""" - if not isinstance(config, dict): - return None - - def _positive_int(key): - value = config.get(key) - if ( - not isinstance(value, int) - or isinstance(value, bool) - or not 0 < value <= MAX_ARCH_VALUE - ): - return None - return value - - layers = _positive_int("num_hidden_layers") - attention_heads = _positive_int("num_attention_heads") - kv_heads = _positive_int("num_key_value_heads") or attention_heads - head_dim = _positive_int("head_dim") - if head_dim is None: - hidden_size = _positive_int("hidden_size") - if hidden_size is not None and attention_heads: - if hidden_size % attention_heads != 0: - return None - head_dim = hidden_size // attention_heads - if layers is None or kv_heads is None or head_dim is None: - return None - return { - "layers": layers, - "kv_heads": kv_heads, - "head_dim": head_dim, - "max_position_embeddings": _positive_int("max_position_embeddings"), - } - - -def kv_bytes_per_token(architecture, dtype_bytes=DEFAULT_DTYPE_BYTES): - if not isinstance(architecture, dict): - return None - layers = architecture.get("layers") - kv_heads = architecture.get("kv_heads") - head_dim = architecture.get("head_dim") - if not all(isinstance(value, int) and value > 0 for value in (layers, kv_heads, head_dim)): - return None - return min(2 * layers * kv_heads * head_dim * dtype_bytes, _MAX_INT) - - -def kv_cache_bytes(architecture, context_tokens, dtype_bytes=DEFAULT_DTYPE_BYTES): - per_token = kv_bytes_per_token(architecture, dtype_bytes) - if per_token is None: - return None - if not isinstance(context_tokens, int) or isinstance(context_tokens, bool): - raise TypeError("context_tokens must be an integer") - if not CONTEXT_MIN <= context_tokens <= CONTEXT_MAX: - raise ValueError( - "context_tokens must be between {0} and {1}".format(CONTEXT_MIN, CONTEXT_MAX) - ) - return min(per_token * context_tokens, _MAX_INT) - - -def max_context_tokens(architecture, weight_bytes, budget_bytes, dtype_bytes=DEFAULT_DTYPE_BYTES): - """Largest context whose weights + KV fit the budget; None when unknown.""" - per_token = kv_bytes_per_token(architecture, dtype_bytes) - if per_token is None: - return None - if not isinstance(weight_bytes, (int, float)) or not isinstance(budget_bytes, (int, float)): - return None - if weight_bytes < 0 or budget_bytes <= 0: - return None - headroom = budget_bytes - weight_bytes - if headroom <= 0: - return 0 - maximum = int(headroom // per_token) - cap = architecture.get("max_position_embeddings") - if isinstance(cap, int) and cap > 0: - maximum = min(maximum, cap) - return min(maximum, CONTEXT_MAX) diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/contracts.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/contracts.py deleted file mode 100644 index 7f1accf..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/contracts.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Versioned result contracts shared by every provider adapter.""" - -from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional - - -SCHEMA_VERSION = "1.0" - - -def _require_non_empty_string(value: Any, name: str) -> None: - if not isinstance(value, str): - raise TypeError("{0} must be a string".format(name)) - if not value: - raise ValueError("{0} must not be empty".format(name)) - - -def _validate_warnings(warnings: Any) -> List[Dict[str, str]]: - if warnings is None: - return [] - if not isinstance(warnings, list): - raise TypeError("warnings must be a list of objects of strings") - - validated: List[Dict[str, str]] = [] - for index, warning in enumerate(warnings): - if not isinstance(warning, dict) or not all( - isinstance(item, str) for item in warning.values() - ): - raise TypeError("warnings[{0}] must be an object of strings".format(index)) - validated.append(dict(warning)) - return validated - - -@dataclass(frozen=True) -class ErrorDetail: - """A stable, actionable description of an unsuccessful operation.""" - - code: str - message: str - remediation: str - retryable: bool = False - - -@dataclass(frozen=True) -class ResultEnvelope: - """The provider-neutral result returned by MLX agent operations.""" - - operation: str - status: str - data: Dict[str, Any] = field(default_factory=dict) - warnings: List[Dict[str, str]] = field(default_factory=list) - error: Optional[ErrorDetail] = None - schema_version: str = SCHEMA_VERSION - generated_at: str = field( - default_factory=lambda: datetime.now(timezone.utc).isoformat() - ) - - @classmethod - def ok(cls, operation: str, data: Dict[str, Any], warnings=None): - """Create a successful versioned result.""" - _require_non_empty_string(operation, "operation") - if not isinstance(data, dict): - raise TypeError("data must be an object") - return cls( - operation=operation, - status="ok", - data=dict(data), - warnings=_validate_warnings(warnings), - ) - - @classmethod - def fail( - cls, - operation: str, - code: str, - message: str, - remediation: str, - retryable=False, - ): - """Create an error result with a machine-readable recovery path.""" - _require_non_empty_string(operation, "operation") - _require_non_empty_string(code, "code") - _require_non_empty_string(message, "message") - _require_non_empty_string(remediation, "remediation") - if not isinstance(retryable, bool): - raise TypeError("retryable must be a boolean") - return cls( - operation=operation, - status="error", - error=ErrorDetail(code, message, remediation, retryable), - ) - - def to_dict(self) -> Dict[str, Any]: - """Serialize the envelope without an empty error object on success.""" - value = asdict(self) - if self.error is None: - value.pop("error") - return value diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/convert.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/convert.py deleted file mode 100644 index 908d885..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/convert.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Confirmation-gated local MLX quantization jobs.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from pathlib import Path - -from .serve import ( - _argv_matches, - _pid_alive, - _pid_command, - _terminate_pid, -) -from .transactions import _atomic_in_directory -from .wiring import _MODEL - - -CONVERT_RECEIPT_SCHEMA_VERSION = "1.0" -CONVERT_RECEIPT_KIND = "convert" -Q_BITS_CHOICES = (4, 8) -EXECUTABLE = "mlx_lm.convert" -MAX_LOG_TAIL_BYTES = 64 * 1024 - - -class ConvertError(RuntimeError): - """Classified convert failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def receipts_root(root=None, kind="convert"): - base = Path(root) if root is not None else Path.cwd() - return base / ".mlx-agent-receipts" / kind - - -def plan_convert(repo, q_bits=4, out=None): - """Render the exact conversion plan; pure and side-effect free.""" - if not isinstance(repo, str) or not _MODEL.fullmatch(repo): - raise ConvertError( - "invalid_repo", - "convert requires a safe publisher/model identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - if not isinstance(q_bits, int) or isinstance(q_bits, bool): - raise ConvertError( - "invalid_arguments", "q_bits must be an integer.", - "Pass --q-bits 4 or --q-bits 8.", - ) - if q_bits not in Q_BITS_CHOICES: - raise ConvertError( - "invalid_arguments", - "q_bits must be one of {0}.".format(list(Q_BITS_CHOICES)), - "Only bounded 4bit and 8bit recipes are supported in v1.", - ) - if out is None: - name = repo.split("/", 1)[1] - out = "{0}-MLX-{1}bit".format(name, q_bits) - plan = { - "repo": repo, - "q_bits": q_bits, - "out": str(out), - "argv": [ - EXECUTABLE, - "--hf-path", repo, - "--mlx-path", str(out), - "--q-bits", str(q_bits), - ], - } - canonical = json.dumps(plan, sort_keys=True, separators=(",", ":")) - plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() - return plan - - -def _receipt_path(root, plan): - return root / "{0}-{1}bit.json".format( - plan["repo"].split("/", 1)[1], plan["q_bits"] - ) - - -def _read_receipt(path, kind=CONVERT_RECEIPT_KIND): - try: - value = json.loads(Path(path).read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - return None - if not isinstance(value, dict) or value.get("kind") != kind: - return None - if not isinstance(value.get("pid"), int) or isinstance(value.get("pid"), bool): - return None - if not isinstance(value.get("argv"), list) or not isinstance(value.get("repo"), str): - return None - return value - - -def _write_receipt(root, receipt, filename): - content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") - _atomic_in_directory(root, filename, content, 0o600) - - -def start_convert(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, spawn=None, now=_utc_now, - pid_alive=None): - """Execute a reviewed conversion plan; the only mutating entry point.""" - from .serve import _default_spawn, _default_which - - which = which or _default_which - spawn = spawn or _default_spawn - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise ConvertError( - "preview_hash_required", - "--confirm requires the hash from a reviewed convert preview.", - "Run convert start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise ConvertError( - "preview_stale", - "The supplied preview hash does not match this convert plan.", - "Re-run convert start without --confirm and review the fresh plan.", - ) - if which(EXECUTABLE) is None: - raise ConvertError( - "runtime_not_installed", - "The {0} executable is not installed.".format(EXECUTABLE), - "Install mlx-lm yourself (pip install mlx-lm); convert never installs runtimes.", - ) - if model_present is not None and not model_present(plan["repo"]): - raise ConvertError( - "model_not_local", - "The source model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; convert never downloads.", - ) - if Path(plan["out"]).exists(): - raise ConvertError( - "output_exists", - "The output path already exists: {0}".format(plan["out"]), - "Pick a fresh --out path; convert never overwrites.", - ) - for receipt_path in sorted(root.glob("*.json")) if root.is_dir() else []: - existing = _read_receipt(receipt_path) - if existing is None or existing.get("completed_at"): - continue - if pid_alive(existing["pid"]): - raise ConvertError( - "job_in_progress", - "A convert job is still running (pid {0}).".format(existing["pid"]), - "Wait for it to finish (convert status) before starting another.", - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}-{1}bit.log".format(plan["repo"].split("/", 1)[1], plan["q_bits"]) - pid = spawn(plan["argv"], str(log_path)) - receipt = { - "schema_version": CONVERT_RECEIPT_SCHEMA_VERSION, - "kind": CONVERT_RECEIPT_KIND, - "repo": plan["repo"], - "q_bits": plan["q_bits"], - "out": plan["out"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "started_at": now(), - "preview_hash": plan["preview_hash"], - "completed_at": None, - "exit_status": None, - } - _write_receipt( - root, receipt, - "{0}-{1}bit.json".format(receipt["repo"].split("/", 1)[1], receipt["q_bits"]), - ) - return {"status": "started", "receipt": receipt} - - -def status_convert(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - now=_utc_now): - """Cross-check convert receipts against live processes; marks exits once.""" - root = receipts_root(receipts_dir) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - receipt = _read_receipt(path) - if receipt is None: - continue - entry = { - "receipt": str(path), - "repo": receipt["repo"], - "q_bits": receipt["q_bits"], - "out": receipt["out"], - "pid": receipt["pid"], - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - "completed_at": receipt.get("completed_at"), - } - if receipt.get("completed_at"): - entry["state"] = receipt.get("exit_status") or "done" - entries.append(entry) - continue - if pid_alive(receipt["pid"]): - command = pid_command(receipt["pid"]) - entry["state"] = "running" if _argv_matches( - {"argv": receipt["argv"], "port": None, "repo": receipt["repo"]}, - command, - require_port=False, - ) else "unknown" - entries.append(entry) - continue - exit_status = "done" if Path(receipt["out"]).exists() else "failed" - receipt["completed_at"] = now() - receipt["exit_status"] = exit_status - _write_receipt( - root, receipt, - "{0}-{1}bit.json".format(receipt["repo"].split("/", 1)[1], receipt["q_bits"]), - ) - entry["state"] = exit_status - entry["completed_at"] = receipt["completed_at"] - entries.append(entry) - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/discovery.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/discovery.py deleted file mode 100644 index 889e34d..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/discovery.py +++ /dev/null @@ -1,528 +0,0 @@ -"""Model discovery, filtering, ranking, and cached structured Scout results.""" - -from __future__ import annotations - -import hashlib -import json -import os -import tempfile -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path - -from .contracts import ResultEnvelope -from .host import HostInventory -from .huggingface import HuggingFaceClient -from .models import ( - DISCOVERY_ROLES, - REPUTABLE, - base_name, - classify, - classify_roles, - infer_quantization, - quant_rank, - resolve_ram, - resolve_reasoning, - wiring, -) - - -CACHE_SCHEMA_VERSION = "2.0" -CACHE_TTL_SECONDS = 24 * 60 * 60 - - -@dataclass(frozen=True) -class DiscoveryRequest: - """Bounded, explicit discovery policy. Legacy ``new`` and ``fast`` remain supported.""" - - role: object = None - memory_gb: object = None - quantization: object = None - licenses: object = None - include_gated: bool = True - publishers: object = None - runtime: object = None - refresh: bool = False - offline: bool = False - limit: int = 6 - new: bool = False - fast: bool = False - context_tokens: object = None - - def cache_request(self): - """The reproducible request shape, excluding transport-only cache switches.""" - return { - "role": self.role, - "memory_gb": self.memory_gb, - "quantization": _canonical_quantization(self.quantization), - "licenses": _normalise_values(self.licenses), - "include_gated": self.include_gated, - "publishers": _normalise_values(self.publishers), - "runtime": self.runtime, - "limit": self.limit, - "new": self.new, - "fast": self.fast, - "context_tokens": self.context_tokens, - } - - -def _normalise_values(value): - if value is None: - return [] - if isinstance(value, str): - return [value.lower()] - return sorted(str(item).lower() for item in value) - - -def _canonical_quantization(value): - if value is None: - return None - normalized = infer_quantization(str(value)) - return normalized or str(value).strip().lower() - - -def _utc_now(): - return datetime.now(timezone.utc) - - -def _parse_timestamp(value): - try: - return datetime.fromisoformat(value.replace("Z", "+00:00")) - except (AttributeError, TypeError, ValueError): - return None - - -def _community_bench_for(repo, chip): - """Return the community bench entry matching repo and host chip, if any.""" - if not isinstance(repo, str) or not isinstance(chip, str) or not chip: - return None - try: - from .bench import load_community_bench - - entry = load_community_bench().get((repo, chip)) - except Exception: - return None - if not isinstance(entry, dict) or entry.get("decode_toks") is None: - return None - result = dict(entry) - result["chip"] = chip - return result - - -def _kv_estimate(architecture, ram_gb, budget_gb, context_tokens): - """Attach KV-cache math when architecture, weights, and budget are known.""" - if not isinstance(architecture, dict) or ram_gb is None or budget_gb is None: - return None - from .contextfit import ( - DEFAULT_DTYPE_BYTES, - kv_bytes_per_token, - kv_cache_bytes, - max_context_tokens, - ) - - if kv_bytes_per_token(architecture) is None: - return None - weight_bytes = float(ram_gb) * 1e9 - budget_bytes = float(budget_gb) * 0.8 * 1e9 - block = { - "max_context_tokens": max_context_tokens(architecture, weight_bytes, budget_bytes), - "dtype_bytes": DEFAULT_DTYPE_BYTES, - "src": "huggingface_model_metadata", - } - if context_tokens is not None: - kv_bytes = kv_cache_bytes(architecture, context_tokens) - block["context_tokens"] = context_tokens - block["kv_gb"] = round(kv_bytes / 1e9, 1) if kv_bytes is not None else None - return block - - -def _estimates(ram, budget, kv_block): - estimates = {"ram_gb": ram, "memory_budget_gb": budget, "headroom_fraction": 0.2} - if kv_block is not None: - estimates["kv"] = kv_block - return estimates - - -class DiscoveryService: - def __init__(self, host=None, huggingface=None, state_dir=None, cache_enabled=None, cache_ttl_seconds=CACHE_TTL_SECONDS): - self._host = host - self._huggingface = huggingface or HuggingFaceClient() - injected_state_dir = state_dir or os.environ.get("MLX_AGENT_STATE_DIR") - self._state_dir = Path(injected_state_dir) if injected_state_dir else None - self._cache_enabled = self._state_dir is not None if cache_enabled is None else cache_enabled - self._cache_ttl_seconds = cache_ttl_seconds - - @property - def host(self): - if self._host is None: - self._host = HostInventory.detect(self._huggingface.http_get) - return self._host - - def discover(self, request): - cache = self._read_cache(request) - if request.offline: - if cache is None: - return ResultEnvelope.fail( - "discover", - "offline_cache_missing", - "No cached discovery response matches this request.", - "Run discovery while online first, or remove --offline.", - ) - data, stale = cache - warnings = [_warning("stale_cache")] if stale else [] - return ResultEnvelope.ok("discover", self._response_data(request, data, "stale" if stale else "fresh"), warnings=warnings) - if cache is not None and not cache[1] and not request.refresh: - return ResultEnvelope.ok("discover", self._response_data(request, cache[0], "fresh")) - - host = self.host - list_sort = "lastModified" if request.new else "trendingScore" - list_url_builder = getattr(self._huggingface, "list_models_url", None) - list_url = ( - list_url_builder(sort=list_sort, limit_fetch=300) - if callable(list_url_builder) else None - ) - try: - raw = self._huggingface.list_models(sort=list_sort) - except Exception as error: - return ResultEnvelope.fail( - "discover", - "network_unavailable", - "HuggingFace query failed: {0}".format(error), - "Check network access to huggingface.co and retry the discovery command.", - retryable=True, - ) - - data = self._build_response(request, raw, host.to_dict(), list_url=list_url) - self._write_cache(request, data) - return ResultEnvelope.ok("discover", self._response_data(request, data, "refreshed" if request.refresh else "miss")) - - def _build_response(self, request, raw, host_data, list_url=None): - buckets = {role: [] for role in DISCOVERY_ROLES} - rejected = {} - seen_repo, seen_base = set(), {} - rejection_limit = max(1, int(request.limit or 1)) * 3 - for model in raw: - repo = model.get("id") or model.get("modelId") - if not repo or repo in seen_repo: - continue - seen_repo.add(repo) - if ( - request.role - and request.role != "tool-use" - and classify(repo) != request.role - ): - continue - enrichment = {} if request.fast else self._huggingface.inspect_model(repo) - roles, tool_use = classify_roles(repo, enrichment) - selected_roles = ( - (request.role,) if request.role in roles else () - ) if request.role else roles - if not selected_roles: - continue - accepted = False - rejected_candidate = None - for role in selected_roles: - candidate = self._candidate( - repo, - model, - role, - roles, - tool_use, - host_data, - enrichment, - request, - list_url, - ) - reasons = self._rejection_reasons(candidate, request) - candidate["rejection_reasons"] = reasons - if reasons: - rejected_candidate = rejected_candidate or candidate - continue - accepted = True - key = (role, candidate["base"]) - current = seen_base.get(key) - if current is not None: - if self._selection_key(candidate) > self._selection_key(current): - buckets[role].remove(current) - buckets[role].append(candidate) - seen_base[key] = candidate - continue - seen_base[key] = candidate - buckets[role].append(candidate) - if not accepted and rejected_candidate is not None: - if len(rejected) < rejection_limit: - rejected[repo] = rejected_candidate - - for role in buckets: - buckets[role].sort(key=self._selection_key, reverse=True) - buckets[role] = buckets[role][: max(0, int(request.limit))] - - return { - "host": host_data, - "fast": request.fast, - "roles": {role: items for role, items in buckets.items() if items}, - "rejected": rejected, - "request": request.cache_request(), - } - - def _candidate( - self, - repo, - model, - role, - roles, - tool_use, - host_data, - enrichment, - request, - list_url=None, - ): - ram, ram_source = resolve_ram(repo, enrichment) - reasoning, reason_source = resolve_reasoning(repo, enrichment) - publisher = repo.split("/", 1)[0].lower() - quantization = infer_quantization(repo) - downloads = model.get("downloads", 0) - likes = model.get("likes", 0) - trusted = publisher in REPUTABLE - budget = request.memory_gb if request.memory_gb is not None else host_data.get("ram_gb") - architecture = enrichment.get("architecture") - kv_block = _kv_estimate(architecture, ram, budget, request.context_tokens) - fits = ram is None or budget is None or ram < float(budget) * 0.8 - if request.context_tokens is not None and kv_block is not None: - fits = (ram + kv_block["kv_gb"]) < float(budget) * 0.8 - community_bench = _community_bench_for(repo, host_data.get("chip")) - rank_score = (1000000 if trusted else 0) + (quant_rank(repo) * 10000) + min(downloads, 9999) + min(likes, 999) - metadata_available = enrichment.get("metadata_available") is True - gated_status = "unknown" - if metadata_available: - gated_status = "gated" if enrichment.get("gated") else "public" - facts = { - "repository": repo, - "downloads": downloads, - "likes": likes, - "gated": gated_status, - "license": enrichment.get("license") if metadata_available else None, - } - if enrichment.get("tree_available") is True and enrichment.get("weight_bytes"): - facts["weight_bytes"] = enrichment["weight_bytes"] - candidate = { - # Legacy report keys. - "repo": repo, - "role": role, - "roles": list(roles), - "downloads": downloads, - "likes": likes, - "trusted": trusted, - "base": base_name(repo), - "qrank": quant_rank(repo), - "est_ram_gb": ram, - "ram_src": ram_source, - "fits": fits, - "reasoning": reasoning, - "reason_src": reason_source, - "gated": enrichment.get("gated") if metadata_available else None, - "license": facts["license"], - "wiring": wiring(repo, role, host_data), - # Explainable structured fields. Memory remains an estimate, even when - # the weights came from a measured file-size listing. - "facts": facts, - "estimates": _estimates(ram, budget, kv_block), - "tool_use": tool_use, - "heuristics": { - "role": role, - "primary_role": roles[0], - "roles": list(roles), - "quantization": quantization, - "trusted_publisher": trusted, - }, - "provenance": self._provenance( - repo, - role, - tool_use, - enrichment, - request.fast, - ram_source, - reason_source, - list_url, - ), - "rank_score": rank_score, - "selection_reasons": self._selection_reasons(role, quantization, trusted, fits), - "rejection_reasons": [], - } - if community_bench is not None: - candidate["community_bench"] = community_bench - return candidate - - @staticmethod - def _provenance( - repo, - role, - tool_use, - enrichment, - fast, - ram_source, - reason_source, - list_url=None, - ): - encoded = repo.replace("/", "%2F") - records = [{ - "source": "huggingface_model_list", - "url": list_url or "https://huggingface.co/api/models", - "fields": ["repository", "downloads", "likes"], - }] - if not fast and enrichment.get("metadata_available") is True: - metadata_fields = ["gated", "license"] - if reason_source in ("chat_template", "tags", "checked"): - metadata_fields.append("reasoning") - if tool_use["source"] in ("chat_template", "tags", "checked"): - metadata_fields.append("tool_use") - records.append({ - "source": "huggingface_model_metadata", - "url": enrichment.get("metadata_url") or "https://huggingface.co/api/models/{0}".format(encoded), - "fields": metadata_fields, - }) - if not fast and enrichment.get("tree_available") is True and enrichment.get("weight_bytes"): - records.append({ - "source": "huggingface_repository_tree", - "url": enrichment.get("tree_url") or "https://huggingface.co/api/models/{0}/tree/main?recursive=true".format(encoded), - "fields": ["weight_bytes"], - }) - if tool_use["source"] in ("chat_template", "tags", "checked"): - role_basis = ( - "primary role from repository name; tool-use signal from " - "Hugging Face metadata ({0})" - ).format(tool_use["source"]) - elif tool_use["source"] == "name": - role_basis = "primary role and tool-use signal from repository name" - else: - role_basis = "primary role from repository name; no tool-use signal" - records.append({ - "source": "local_role_derivation", - "fields": ["role", "roles", "primary_role"], - "basis": role_basis, - }) - name_fields = ["quantization", "trusted_publisher"] - if tool_use["source"] == "name": - name_fields.append("tool_use") - if reason_source == "name": - name_fields.append("reasoning") - records.append({ - "source": "local_name_derivation", - "fields": name_fields, - }) - records.append({ - "source": "local_memory_estimate", - "fields": ["ram_gb", "fits"], - "basis": ram_source or "unavailable", - }) - records.append({ - "source": "local_ranking_derivation", - "fields": ["rank_score", "selection_reasons", "rejection_reasons"], - }) - return records - - @staticmethod - def _selection_reasons(role, quantization, trusted, fits): - reasons = ["role membership: {0}".format(role)] - if quantization: - reasons.append("quantization inferred from repository name: {0}".format(quantization)) - if trusted: - reasons.append("publisher is in the configured reputable set") - if fits: - reasons.append("estimated weights fit within the memory budget with headroom") - return reasons - - @staticmethod - def _selection_key(candidate): - # The final repository ID makes ranking stable across equal evidence. - return (candidate["trusted"], candidate["qrank"], candidate["downloads"], candidate["likes"], candidate["repo"]) - - @staticmethod - def _rejection_reasons(candidate, request): - reasons = [] - licenses = set(_normalise_values(request.licenses)) - publishers = set(_normalise_values(request.publishers)) - wanted_quantization = _canonical_quantization(request.quantization) - if not request.include_gated and candidate["facts"]["gated"] == "gated": - reasons.append("gated") - if not request.include_gated and candidate["facts"]["gated"] == "unknown": - reasons.append("gated_status_unknown") - if licenses and (candidate["license"] or "").lower() not in licenses: - reasons.append("license") - if publishers and candidate["repo"].split("/", 1)[0].lower() not in publishers: - reasons.append("publisher") - if wanted_quantization and candidate["heuristics"]["quantization"] != wanted_quantization: - reasons.append("quantization") - if request.memory_gb is not None and not candidate["fits"]: - reasons.append("memory_budget") - if request.runtime and not HostInventory.runtime_supports(request.runtime, candidate["heuristics"]["role"]): - reasons.append("runtime") - return reasons - - def _cache_path(self, request): - encoded = json.dumps(request.cache_request(), sort_keys=True, separators=(",", ":")).encode("utf-8") - return self._state_dir / "discovery-{0}.json".format(hashlib.sha256(encoded).hexdigest()[:20]) - - def _read_cache(self, request): - if not self._cache_enabled: - return None - try: - payload = json.loads(self._cache_path(request).read_text()) - if payload.get("schema_version") != CACHE_SCHEMA_VERSION or payload.get("request") != request.cache_request(): - return None - fetched_at = _parse_timestamp(payload.get("fetched_at")) - if fetched_at is None or not isinstance(payload.get("response"), dict): - return None - stale = (_utc_now() - fetched_at).total_seconds() > self._cache_ttl_seconds - return payload["response"], stale - except (OSError, TypeError, ValueError, json.JSONDecodeError): - return None - - def _write_cache(self, request, response): - if not self._cache_enabled: - return - try: - self._state_dir.mkdir(parents=True, exist_ok=True) - payload = { - "schema_version": CACHE_SCHEMA_VERSION, - "fetched_at": _utc_now().isoformat(), - "request": request.cache_request(), - "response": response, - } - destination = self._cache_path(request) - with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=str(destination.parent), delete=False) as temporary: - json.dump(payload, temporary, sort_keys=True) - temporary.flush() - os.fsync(temporary.fileno()) - temporary_name = temporary.name - os.replace(temporary_name, destination) - except OSError: - # Discovery evidence is still useful if the local cache cannot be written. - return - - def _cached_data(self, response, status): - data = dict(response) - data["cache"] = {"status": status, "ttl_seconds": self._cache_ttl_seconds} - return data - - def _response_data(self, request, response, status): - if not self._cache_enabled and self._legacy_compatible(request): - return {key: response[key] for key in ("host", "fast", "roles")} - return self._cached_data(response, status) - - @staticmethod - def _legacy_compatible(request): - return ( - request.memory_gb is None - and request.quantization is None - and request.licenses is None - and request.include_gated - and request.publishers is None - and request.runtime is None - and not request.refresh - and not request.offline - ) - - -def _warning(code): - return {"code": code, "message": "Cached discovery response is older than the 24 hour TTL."} diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/fleet.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/fleet.py deleted file mode 100644 index 71de98f..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/fleet.py +++ /dev/null @@ -1,218 +0,0 @@ -"""One-shot per-role routing configuration backed by Wire transactions.""" - -from __future__ import annotations - -import json -import re -from pathlib import Path - -from .wiring import _MODEL, require_secret_free_config - - -FLEET_RUNTIMES = ("mlx_lm", "mlx-vlm") -ROLE_DEFAULT_RUNTIME = { - "general": "mlx_lm", - "coding": "mlx_lm", - "reasoning": "mlx_lm", - "embedding": "mlx_lm", - "tool-use": "mlx_lm", - "vision": "mlx-vlm", -} -RUNTIME_PORTS = {"mlx_lm": 8080, "mlx-vlm": 8083} -_ROLE = re.compile(r"^[a-z][a-z0-9-]{0,31}$") -MAX_ASSIGNMENTS = 6 - -_HEADER = "# MLX_AGENT_WIRE v1" - - -class FleetError(ValueError): - """Classified fleet failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def parse_assignments(values): - """Parse repeatable role=repo assignments into an ordered mapping.""" - assignments = {} - for value in values or []: - if not isinstance(value, str) or "=" not in value: - raise FleetError( - "invalid_assignment", - "Assignments must use role=repo form: {0}".format(value), - "Pass --assign coding=publisher/model.", - ) - role, repo = value.split("=", 1) - role = role.strip() - repo = repo.strip() - if role not in ROLE_DEFAULT_RUNTIME: - raise FleetError( - "invalid_role", - "Unknown fleet role: {0}".format(role), - "Use one of: {0}.".format(", ".join(sorted(ROLE_DEFAULT_RUNTIME))), - ) - if role in assignments: - raise FleetError( - "duplicate_role", - "Role {0} is assigned twice.".format(role), - "Assign each role at most once.", - ) - if not _MODEL.fullmatch(repo): - raise FleetError( - "invalid_repo", - "Model must be a safe publisher/model identifier: {0}".format(repo), - "Pass the Hugging Face publisher/model form.", - ) - assignments[role] = repo - if not assignments: - raise FleetError( - "missing_assignments", - "Fleet requires at least one role assignment.", - "Pass --assign role=repo or --from-adoption .", - ) - if len(assignments) > MAX_ASSIGNMENTS: - raise FleetError( - "invalid_assignment", - "Fleet supports at most {0} role assignments.".format(MAX_ASSIGNMENTS), - "Reduce the number of assigned roles.", - ) - return assignments - - -def assignments_from_adoption(state_path): - """Read role -> repo recommendations from a completed adopt handoff.""" - try: - value = json.loads(Path(state_path).read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError) as error: - raise FleetError( - "invalid_state", - "The adoption handoff could not be read: {0}".format(error), - "Pass a readable adopt state file produced by adopt start/resume.", - ) - recommendations = value.get("recommendations") - if not isinstance(recommendations, list) or not recommendations: - raise FleetError( - "empty_recommendations", - "The adoption handoff contains no recommendations.", - "Complete the adoption workflow first, then rerun fleet.", - ) - assignments = {} - for item in recommendations: - if not isinstance(item, dict): - continue - role = item.get("role") - repo = item.get("repo") - if role in ROLE_DEFAULT_RUNTIME and isinstance(repo, str) and role not in assignments: - assignments[role] = repo - return parse_assignments(["{0}={1}".format(role, repo) for role, repo in assignments.items()]) - - -def parse_runtime_map(values): - """Parse repeatable role=runtime overrides.""" - runtime_map = {} - for value in values or []: - if not isinstance(value, str) or "=" not in value: - raise FleetError( - "invalid_runtime_map", - "Runtime overrides must use role=runtime form: {0}".format(value), - "Pass --runtime-map vision=mlx-vlm.", - ) - role, runtime = value.split("=", 1) - role = role.strip() - runtime = runtime.strip() - if role not in ROLE_DEFAULT_RUNTIME: - raise FleetError( - "invalid_role", - "Unknown fleet role: {0}".format(role), - "Use one of: {0}.".format(", ".join(sorted(ROLE_DEFAULT_RUNTIME))), - ) - if runtime not in FLEET_RUNTIMES: - raise FleetError( - "unsupported_runtime", - "Fleet routes only through: {0}.".format(", ".join(FLEET_RUNTIMES)), - "Ollama and LM Studio manage their own routing; use mlx_lm or mlx-vlm.", - ) - runtime_map[role] = runtime - return runtime_map - - -class FleetConfigAdapter: - """Validate the exact bounded multi-model router subset fleet renders.""" - - version = "1.0" - runtime = "litellm-fleet" - - def __init__(self, path=None): - self.path = Path(path) if path is not None else None - - def render(self, assignments, runtime_map=None, existing=""): - runtime_map = runtime_map or {} - if not assignments: - raise FleetError( - "missing_assignments", - "Fleet requires at least one role assignment.", - "Pass --assign role=repo or --from-adoption .", - ) - if not isinstance(existing, str): - raise TypeError("existing config content must be text") - require_secret_free_config(existing) - if existing.strip() and not existing.startswith(_HEADER): - raise FleetError( - "existing_config_unmanaged", - "The target file exists and is not fleet-managed.", - "Pick a new --path or remove the unmanaged file yourself.", - ) - lines = [_HEADER, "model_list:"] - for role in sorted(assignments): - runtime = runtime_map.get(role, ROLE_DEFAULT_RUNTIME[role]) - if runtime not in FLEET_RUNTIMES: - raise FleetError( - "unsupported_runtime", - "Fleet routes only through: {0}.".format(", ".join(FLEET_RUNTIMES)), - "Use mlx_lm or mlx-vlm for every role.", - ) - port = RUNTIME_PORTS[runtime] - lines.extend([ - " - model_name: {0}".format(role), - " litellm_params:", - " model: openai/{0}".format(assignments[role]), - " api_base: http://127.0.0.1:{0}/v1".format(port), - " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY", - ]) - content = "\n".join(lines) + "\n" - self.validate(content) - return content - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("configuration content must be text") - require_secret_free_config(content) - lines = content.splitlines() - if len(lines) < 7 or (len(lines) - 2) % 5 != 0: - raise ValueError("fleet configuration must contain whole five-line entries") - if lines[0] != _HEADER or lines[1] != "model_list:": - raise ValueError("fleet configuration must start with the managed header") - roles = [] - for offset in range(2, len(lines), 5): - header, params, model, api_base, api_key = lines[offset:offset + 5] - role = re.fullmatch(r" - model_name: ([a-z][a-z0-9-]{0,31})", header) - if role is None or not _ROLE.fullmatch(role.group(1)): - raise ValueError("fleet entry has an invalid model_name") - if params != " litellm_params:": - raise ValueError("fleet entry is missing litellm_params") - model_match = re.fullmatch(r" model: openai/(" + _MODEL.pattern[1:-1] + r")", model) - if model_match is None: - raise ValueError("fleet entry has an invalid model identifier") - if api_base not in ( - " api_base: http://127.0.0.1:8080/v1", - " api_base: http://127.0.0.1:8083/v1", - ): - raise ValueError("fleet entry api_base must be a bounded loopback port") - if api_key != " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY": - raise ValueError("fleet entry must reference the managed API key env var") - roles.append(role.group(1)) - if len(roles) != len(set(roles)): - raise ValueError("fleet configuration contains duplicate roles") - return True diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/fuse.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/fuse.py deleted file mode 100644 index b35f13a..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/fuse.py +++ /dev/null @@ -1,213 +0,0 @@ -"""Confirmation-gated LoRA fuse jobs on the convert machinery.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from pathlib import Path - -from .convert import ( - _read_receipt, - _write_receipt, - receipts_root, -) -from .serve import _argv_matches, _pid_alive, _pid_command -from .wiring import _MODEL - - -FUSE_RECEIPT_SCHEMA_VERSION = "1.0" -FUSE_RECEIPT_KIND = "fuse" -EXECUTABLE = "mlx_lm.fuse" - - -class FuseError(RuntimeError): - """Classified fuse failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def validate_adapter(adapter_path): - """Confirm the adapter directory looks like an mlx_lm LoRA output.""" - root = Path(adapter_path) - if not root.is_dir(): - raise FuseError( - "adapter_invalid", - "The adapter path is not a directory: {0}".format(adapter_path), - "Point --adapter at a completed lora output directory.", - ) - config = root / "adapter_config.json" - if not config.is_file(): - raise FuseError( - "adapter_invalid", - "The adapter directory lacks adapter_config.json.", - "Point --adapter at a completed lora output directory.", - ) - try: - value = json.loads(config.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError) as error: - raise FuseError( - "adapter_invalid", - "adapter_config.json could not be parsed: {0}".format(error), - "Repair or regenerate the adapter before fusing.", - ) - if not isinstance(value, dict): - raise FuseError( - "adapter_invalid", - "adapter_config.json must contain a JSON object.", - "Repair or regenerate the adapter before fusing.", - ) - return {"adapter_config": True} - - -def plan_fuse(repo, adapter, out=None): - """Validate inputs and render the exact fuse plan; side-effect free.""" - if not isinstance(repo, str) or not _MODEL.fullmatch(repo): - raise FuseError( - "invalid_repo", - "fuse requires a safe publisher/model base identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - validate_adapter(adapter) - if out is None: - out = "{0}-fused".format(repo.split("/", 1)[1]) - plan = { - "repo": repo, - "adapter": str(adapter), - "out": str(out), - "argv": [ - EXECUTABLE, - "--model", repo, - "--adapter-path", str(adapter), - "--save-path", str(out), - ], - } - canonical = json.dumps(plan, sort_keys=True, separators=(",", ":")) - plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() - return plan - - -def start_fuse(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, spawn=None, now=_utc_now, - pid_alive=None): - """Execute a reviewed fuse plan; the only mutating entry point.""" - from .serve import _default_spawn, _default_which - - which = which or _default_which - spawn = spawn or _default_spawn - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir, FUSE_RECEIPT_KIND) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise FuseError( - "preview_hash_required", - "--confirm requires the hash from a reviewed fuse preview.", - "Run fuse start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise FuseError( - "preview_stale", - "The supplied preview hash does not match this fuse plan.", - "Re-run fuse start without --confirm and review the fresh plan.", - ) - if which(EXECUTABLE) is None: - raise FuseError( - "runtime_not_installed", - "The {0} executable is not installed.".format(EXECUTABLE), - "Install mlx-lm yourself (pip install mlx-lm); fuse never installs runtimes.", - ) - if model_present is not None and not model_present(plan["repo"]): - raise FuseError( - "model_not_local", - "The base model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; fuse never downloads.", - ) - if Path(plan["out"]).exists(): - raise FuseError( - "output_exists", - "The fused output path already exists: {0}".format(plan["out"]), - "Pick a fresh --out path; fuse never overwrites.", - ) - for receipt_path in sorted(root.glob("*.json")) if root.is_dir() else []: - existing = _read_receipt(receipt_path, FUSE_RECEIPT_KIND) - if existing is None or existing.get("completed_at"): - continue - if pid_alive(existing["pid"]): - raise FuseError( - "job_in_progress", - "A fuse job is still running (pid {0}).".format(existing["pid"]), - "Wait for it to finish (fuse status) before starting another.", - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}-fuse.log".format(plan["repo"].split("/", 1)[1]) - pid = spawn(plan["argv"], str(log_path)) - receipt = { - "schema_version": FUSE_RECEIPT_SCHEMA_VERSION, - "kind": FUSE_RECEIPT_KIND, - "repo": plan["repo"], - "adapter": plan["adapter"], - "out": plan["out"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "started_at": now(), - "preview_hash": plan["preview_hash"], - "completed_at": None, - "exit_status": None, - } - _write_receipt(root, receipt, "{0}-fuse.json".format(plan["repo"].split("/", 1)[1])) - return {"status": "started", "receipt": receipt} - - -def status_fuse(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - now=_utc_now): - """Cross-check fuse receipts against live processes; marks exits once.""" - root = receipts_root(receipts_dir, FUSE_RECEIPT_KIND) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - receipt = _read_receipt(path, FUSE_RECEIPT_KIND) - if receipt is None: - continue - entry = { - "receipt": str(path), - "repo": receipt["repo"], - "adapter": receipt.get("adapter"), - "out": receipt["out"], - "pid": receipt["pid"], - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - "completed_at": receipt.get("completed_at"), - } - if receipt.get("completed_at"): - entry["state"] = receipt.get("exit_status") or "done" - entries.append(entry) - continue - if pid_alive(receipt["pid"]): - command = pid_command(receipt["pid"]) - entry["state"] = "running" if _argv_matches( - {"argv": receipt["argv"], "port": None, "repo": receipt["repo"]}, - command, - require_port=False, - ) else "unknown" - entries.append(entry) - continue - exit_status = "done" if Path(receipt["out"]).exists() else "failed" - receipt["completed_at"] = now() - receipt["exit_status"] = exit_status - _write_receipt(root, receipt, path.name) - entry["state"] = exit_status - entry["completed_at"] = receipt["completed_at"] - entries.append(entry) - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/gemini_args.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/gemini_args.py deleted file mode 100644 index 886acaf..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/gemini_args.py +++ /dev/null @@ -1,340 +0,0 @@ -"""Strict argument grammar for Gemini custom-command input. - -Gemini TOML commands pass user text to the model, not directly to a process. -This module converts that opaque text into an allowlisted MLX-agent argv list -before a bundled skill may execute the structured CLI. -""" - -from __future__ import annotations - -import argparse -import json -import re -import shlex -import urllib.parse -from pathlib import Path - -from .models import ROLES - - -class GeminiArgumentError(ValueError): - """Gemini command input is outside the documented capability grammar.""" - - -_RUNTIMES = {"ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"} -_ROLES = {role for role, _keywords, _label in ROLES} -_MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,191}$") -_SAFE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+:-]{0,127}$") -_SAFE_QUANTIZATION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") -_METACHARACTERS = set(";|&`$<>") - - -def _tokens(raw): - if not isinstance(raw, str): - raise GeminiArgumentError("arguments must be text") - if len(raw) > 4096 or any(ord(character) < 32 or ord(character) == 127 for character in raw): - raise GeminiArgumentError("arguments must not contain control characters") - if any(character in _METACHARACTERS for character in raw): - raise GeminiArgumentError("shell metacharacters are not allowed") - try: - tokens = shlex.split(raw, posix=True) - except ValueError as error: - raise GeminiArgumentError("arguments must use balanced shell-style quotes: {0}".format(error)) - if len(tokens) > 32 or any(not token or len(token) > 512 for token in tokens): - raise GeminiArgumentError("arguments exceed the supported command grammar") - return tokens - - -def _value(tokens, index, flag): - if index + 1 >= len(tokens) or tokens[index + 1].startswith("--"): - raise GeminiArgumentError("{0} requires a value".format(flag)) - return tokens[index + 1] - - -def _path(value, label): - if not isinstance(value, str) or not value or value.startswith("~") or "\\" in value: - raise GeminiArgumentError("{0} must be a portable non-empty path".format(label)) - path = Path(value) - if any(part in {"", ".", ".."} for part in path.parts): - raise GeminiArgumentError("{0} must not traverse parent directories".format(label)) - return value - - -def _model(value): - if not _MODEL.fullmatch(value): - raise GeminiArgumentError("model must be a publisher/model identifier") - return value - - -def _integer(value, label, minimum, maximum): - if not re.fullmatch(r"[0-9]+", value): - raise GeminiArgumentError("{0} must be an integer".format(label)) - number = int(value) - if number < minimum or number > maximum: - raise GeminiArgumentError("{0} must be between {1} and {2}".format(label, minimum, maximum)) - return str(number) - - -def _decimal(value, label, minimum, maximum): - if not re.fullmatch(r"(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+)", value): - raise GeminiArgumentError("{0} must be a decimal number".format(label)) - number = float(value) - if number < minimum or number > maximum: - raise GeminiArgumentError("{0} must be between {1} and {2}".format(label, minimum, maximum)) - return value - - -def _endpoint(value): - try: - parsed = urllib.parse.urlsplit(value) - hostname = parsed.hostname - username = parsed.username - password = parsed.password - port = parsed.port - except (UnicodeError, ValueError) as error: - raise GeminiArgumentError("endpoint is not a valid local HTTP URL") from error - if ( - parsed.scheme not in {"http", "https"} - or hostname not in {"127.0.0.1", "localhost", "::1"} - or username is not None - or password is not None - or parsed.query - or parsed.fragment - ): - raise GeminiArgumentError("endpoint must be a credential-free local HTTP URL") - if port is not None and not 1 <= port <= 65535: - raise GeminiArgumentError("endpoint port must be between 1 and 65535") - return value - - -def _parse_scout(tokens): - argv, index = ["discover"], 0 - seen, cache = set(), None - values = { - "--role": (lambda value: value if value in _ROLES else None, "role"), - "--limit": (lambda value: _integer(value, "limit", 1, 50), "limit"), - "--memory-gb": (lambda value: _decimal(value, "memory-gb", 0.25, 1024), "memory-gb"), - "--quantization": (lambda value: value if _SAFE_QUANTIZATION.fullmatch(value) else None, "quantization"), - "--runtime": (lambda value: value if value in _RUNTIMES else None, "runtime"), - "--context": (lambda value: _integer(value, "context", 1024, 1048576), "context"), - "--state-dir": (lambda value: _path(value, "state-dir"), "state-dir"), - "--wire": (lambda value: _model(value), "wire"), - "--target": (lambda value: value if value in _RUNTIMES else None, "target"), - "--port": (lambda value: _integer(value, "port", 1, 65535), "port"), - } - repeatable = {"--license", "--publisher"} - booleans = {"--exclude-gated", "--include-gated", "--new", "--fast", "--json"} - while index < len(tokens): - flag = tokens[index] - if flag in booleans: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - if flag in {"--exclude-gated", "--include-gated"} and {"--exclude-gated", "--include-gated"} & seen: - raise GeminiArgumentError("gated-repository flags are mutually exclusive") - seen.add(flag) - argv.append(flag) - index += 1 - elif flag in {"--refresh", "--offline"}: - if cache is not None: - raise GeminiArgumentError("--refresh and --offline are mutually exclusive") - cache = flag - argv.append(flag) - index += 1 - elif flag in repeatable: - value = _value(tokens, index, flag) - if not _SAFE_NAME.fullmatch(value): - raise GeminiArgumentError("{0} must be a safe identifier".format(flag)) - argv.extend([flag, value]) - index += 2 - elif flag in values: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - value = values[flag][0](_value(tokens, index, flag)) - if value is None: - raise GeminiArgumentError("invalid {0} value".format(values[flag][1])) - seen.add(flag) - argv.extend([flag, value]) - index += 2 - else: - raise GeminiArgumentError("unsupported Scout argument: {0}".format(flag)) - return argv - - -def _parse_adopt(tokens): - if not tokens or tokens[0] not in {"start", "resume", "status"}: - raise GeminiArgumentError("Adopt requires start, resume, or status") - action, argv, index = tokens[0], ["adopt", tokens[0]], 1 - state = None - allowed_booleans = {"--json"} if action != "start" else {"--offline", "--refresh", "--fast", "--no-network", "--measure", "--json"} - while index < len(tokens): - flag = tokens[index] - if flag == "--state": - if state is not None: - raise GeminiArgumentError("duplicate flag: --state") - state = _path(_value(tokens, index, flag), "state") - argv.extend([flag, state]) - index += 2 - elif action == "start" and flag == "--role": - role = _value(tokens, index, flag) - if role not in _ROLES: - raise GeminiArgumentError("unknown adoption role") - argv.extend([flag, role]) - index += 2 - elif action == "start" and flag == "--shortlist-limit": - argv.extend([flag, _integer(_value(tokens, index, flag), "shortlist-limit", 1, 20)]) - index += 2 - elif flag in allowed_booleans and flag not in argv: - argv.append(flag) - index += 1 - else: - raise GeminiArgumentError("unsupported Adopt argument: {0}".format(flag)) - if state is None: - raise GeminiArgumentError("Adopt requires an explicit --state path") - return argv - - -def _parse_wire(tokens): - if not tokens or tokens[0] not in {"render", "apply", "status", "rollback"}: - raise GeminiArgumentError("Wire requires render, apply, status, or rollback") - action, argv, index = tokens[0], ["wire", tokens[0]], 1 - if action in {"status", "rollback"}: - if index >= len(tokens): - raise GeminiArgumentError("Wire {0} requires a receipt path".format(action)) - argv.append(_path(tokens[index], "receipt")) - index += 1 - allowed = ( - {"--json"} - if action == "status" - else {"--confirm", "--preview-hash", "--json"} - ) - else: - if index >= len(tokens): - raise GeminiArgumentError("Wire {0} requires a model".format(action)) - argv.append(_model(tokens[index])) - index += 1 - allowed = {"--target", "--path", "--json"} - if action == "apply": - allowed.update({"--confirm", "--preview-hash", "--receipts-dir", "--endpoint"}) - path_seen = False - while index < len(tokens): - flag = tokens[index] - if flag not in allowed: - raise GeminiArgumentError("unsupported Wire argument: {0}".format(flag)) - if flag in argv: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - if flag in {"--json", "--confirm"}: - argv.append(flag) - index += 1 - else: - value = _value(tokens, index, flag) - if flag == "--target": - if value not in _RUNTIMES: - raise GeminiArgumentError("unsupported Wire target") - elif flag in {"--path", "--receipts-dir"}: - value = _path(value, flag[2:]) - path_seen = path_seen or flag == "--path" - elif flag == "--preview-hash": - if not re.fullmatch(r"[0-9a-f]{64}", value): - raise GeminiArgumentError("preview-hash must be a SHA-256 digest") - elif flag == "--endpoint": - value = _endpoint(value) - argv.extend([flag, value]) - index += 2 - if action in {"render", "apply"} and not path_seen: - raise GeminiArgumentError("Wire {0} requires --path".format(action)) - return argv - - -def _parse_bench(tokens): - if not tokens or tokens[0] not in {"run", "aggregate"}: - raise GeminiArgumentError("Bench requires the run or aggregate action") - if tokens[0] == "aggregate": - return _parse_bench_aggregate(tokens) - argv, index = ["bench", "run"], 1 - seen = set() - values = { - "--repo": (lambda value: _model(value), "repo"), - "--runtime": (lambda value: value if value in _RUNTIMES else None, "runtime"), - "--role": (lambda value: value if value in _ROLES else None, "role"), - "--runs": (lambda value: _integer(value, "runs", 1, 10), "runs"), - "--gen-tokens": (lambda value: _integer(value, "gen-tokens", 16, 2048), "gen-tokens"), - "--timeout": (lambda value: _decimal(value, "timeout", 1, 600), "timeout"), - "--export": (lambda value: _path(value, "export"), "export"), - } - while index < len(tokens): - flag = tokens[index] - if flag == "--json": - if flag in seen: - raise GeminiArgumentError("duplicate flag: --json") - seen.add(flag) - argv.append(flag) - index += 1 - elif flag in values: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - value = values[flag][0](_value(tokens, index, flag)) - if value is None: - raise GeminiArgumentError("invalid {0} value".format(values[flag][1])) - seen.add(flag) - argv.extend([flag, value]) - index += 2 - else: - raise GeminiArgumentError("unsupported Bench argument: {0}".format(flag)) - for required in ("--repo", "--runtime"): - if required not in seen: - raise GeminiArgumentError("Bench run requires {0}".format(required)) - return argv - - -def _parse_bench_aggregate(tokens): - argv, index = ["bench", "aggregate"], 1 - seen = set() - while index < len(tokens): - flag = tokens[index] - if flag == "--json": - if flag in seen: - raise GeminiArgumentError("duplicate flag: --json") - seen.add(flag) - argv.append(flag) - index += 1 - elif flag in {"--exports", "--out"}: - if flag in seen: - raise GeminiArgumentError("duplicate flag: {0}".format(flag)) - seen.add(flag) - argv.extend([flag, _path(_value(tokens, index, flag), flag[2:])]) - index += 2 - else: - raise GeminiArgumentError("unsupported Bench argument: {0}".format(flag)) - if "--exports" not in seen: - raise GeminiArgumentError("Bench aggregate requires --exports") - return argv - - -def parse_gemini_arguments(capability, raw): - """Return a validated argv list; this function never runs a process.""" - tokens = _tokens(raw) - if capability == "scout": - return _parse_scout(tokens) - if capability == "adopt": - return _parse_adopt(tokens) - if capability == "wire": - return _parse_wire(tokens) - if capability == "bench": - return _parse_bench(tokens) - raise GeminiArgumentError("unknown Gemini capability") - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("capability", choices=("scout", "adopt", "wire", "bench")) - parser.add_argument("raw", help="one opaque custom-command argument string") - arguments = parser.parse_args(argv) - try: - print(json.dumps({"argv": parse_gemini_arguments(arguments.capability, arguments.raw)})) - except GeminiArgumentError as error: - parser.error(str(error)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/gemini_executor.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/gemini_executor.py deleted file mode 100644 index 3a0fc12..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/gemini_executor.py +++ /dev/null @@ -1,140 +0,0 @@ -"""No-shell executor for Gemini custom-command argument files.""" - -from __future__ import annotations - -import argparse -import json -import os -import stat -import tempfile -from pathlib import Path - -from .gemini_args import GeminiArgumentError, parse_gemini_arguments - - -MAX_ARGS_BYTES = 4096 -_ARGS_DIRECTORY = "mlx-agent-gemini-command-args" - - -class GeminiCommandError(ValueError): - """A custom-command argument file is unsafe or invalid.""" - - -def command_args_root(): - """Return the private OS-temporary root reserved for Gemini argument files.""" - root = Path(tempfile.gettempdir()) / _ARGS_DIRECTORY - try: - root.mkdir(mode=0o700, parents=True, exist_ok=True) - details = os.lstat(str(root)) - except OSError as error: - raise GeminiCommandError("private argument storage is unavailable") from error - if ( - stat.S_ISLNK(details.st_mode) - or not stat.S_ISDIR(details.st_mode) - or details.st_uid != os.getuid() - or stat.S_IMODE(details.st_mode) != 0o700 - ): - raise GeminiCommandError("private argument storage is unsafe") - return root - - -def _safe_args_path(value): - root = command_args_root() - path = Path(value) - if not path.is_absolute() or path.parent != root or ".." in path.parts or not path.name: - raise GeminiCommandError("argument file path is outside private storage") - return path - - -def _discard_if_safe(path, opened=None): - """Remove only the named regular file we opened, or a rejected symlink.""" - try: - current = os.lstat(str(path)) - if stat.S_ISLNK(current.st_mode): - os.unlink(str(path)) - elif stat.S_ISREG(current.st_mode) and (opened is None or (current.st_dev, current.st_ino) == (opened.st_dev, opened.st_ino)): - os.unlink(str(path)) - except OSError: - pass - - -def _read_args_file(value): - path = _safe_args_path(value) - opened = None - descriptor = None - try: - try: - before = os.lstat(str(path)) - except OSError as error: - raise GeminiCommandError("argument file is unavailable") from error - if ( - stat.S_ISLNK(before.st_mode) - or not stat.S_ISREG(before.st_mode) - or before.st_uid != os.getuid() - or stat.S_IMODE(before.st_mode) != 0o600 - ): - raise GeminiCommandError("argument file must be a regular file") - if before.st_size > MAX_ARGS_BYTES: - raise GeminiCommandError("argument file exceeds the maximum size") - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - try: - descriptor = os.open(str(path), flags) - opened = os.fstat(descriptor) - except OSError as error: - raise GeminiCommandError("argument file cannot be opened safely") from error - if ( - not stat.S_ISREG(opened.st_mode) - or opened.st_uid != os.getuid() - or stat.S_IMODE(opened.st_mode) != 0o600 - or (opened.st_dev, opened.st_ino) != (before.st_dev, before.st_ino) - ): - raise GeminiCommandError("argument file changed before it could be read") - data = os.read(descriptor, MAX_ARGS_BYTES + 1) - if len(data) > MAX_ARGS_BYTES: - raise GeminiCommandError("argument file exceeds the maximum size") - try: - return data.decode("utf-8") - except UnicodeDecodeError as error: - raise GeminiCommandError("argument file must be UTF-8") from error - finally: - if descriptor is not None: - os.close(descriptor) - _discard_if_safe(path, opened) - - -def execute_gemini_command(capability, args_file, core=None): - """Read, validate, execute, and discard one trusted Gemini argument file.""" - try: - raw = _read_args_file(args_file) - argv = parse_gemini_arguments(capability, raw) - except (GeminiArgumentError, GeminiCommandError) as error: - raise GeminiCommandError("Gemini command arguments were rejected") from error - if core is None: - from .cli import main as core - try: - result = core(argv) - except Exception as error: - raise GeminiCommandError("Gemini command execution failed") from error - exit_code = 0 if result is None else int(result) - return { - "status": "ok" if exit_code == 0 else "error", - "capability": capability, - "exit_code": exit_code, - } - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--capability", required=True, choices=("scout", "adopt", "wire")) - parser.add_argument("--args-file", required=True) - arguments = parser.parse_args(argv) - try: - result = execute_gemini_command(arguments.capability, arguments.args_file) - print(json.dumps(result, sort_keys=True)) - except GeminiCommandError: - parser.error("Gemini command arguments were rejected") - return result["exit_code"] - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/gemini_transport.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/gemini_transport.py deleted file mode 100644 index bf7a648..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/gemini_transport.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Deterministic contract simulation for Gemini custom-command TOML transport. - -This module simulates documented ``{{args}}`` prompt substitution for tests and -local adapters. It is not a claim about Gemini model or tool behavior. -""" - -from __future__ import annotations - -import os -import subprocess -import tempfile -from pathlib import Path - -from .gemini_executor import MAX_ARGS_BYTES, GeminiCommandError, command_args_root, execute_gemini_command - - -_OPEN = "\n" -_CLOSE = "\n" - - -class GeminiTransportError(ValueError): - """The simulated TOML transport could not safely carry command data.""" - - -def substitute_toml_args(prompt, raw_args): - """Simulate one documented TOML placeholder substitution without parsing it.""" - if not isinstance(prompt, str) or not isinstance(raw_args, str) or prompt.count("{{args}}") != 1: - raise GeminiTransportError("TOML transport template is invalid") - return prompt.replace("{{args}}", raw_args) - - -def extract_untrusted_payload(rendered_prompt): - """Extract the delimited payload exactly, preserving embedded delimiter text.""" - if not isinstance(rendered_prompt, str): - raise GeminiTransportError("TOML transport payload is invalid") - start = rendered_prompt.find(_OPEN) - end = rendered_prompt.rfind(_CLOSE) - if start < 0 or end < start + len(_OPEN): - raise GeminiTransportError("TOML transport delimiters are invalid") - return rendered_prompt[start + len(_OPEN):end] - - -def write_private_args_file(payload): - """Write opaque transport data to a 0600 private file without a shell.""" - if not isinstance(payload, str): - raise GeminiTransportError("TOML transport payload is invalid") - encoded = payload.encode("utf-8") - if len(encoded) > MAX_ARGS_BYTES: - raise GeminiTransportError("TOML transport payload exceeds the maximum size") - root = command_args_root() - descriptor = None - name = None - complete = False - try: - descriptor, name = tempfile.mkstemp(prefix="transport-", suffix=".args", dir=str(root)) - os.fchmod(descriptor, 0o600) - os.write(descriptor, encoded) - complete = True - return name - except OSError as error: - raise GeminiTransportError("private transport storage is unavailable") from error - finally: - if descriptor is not None: - os.close(descriptor) - if name is not None and not complete: - try: - os.unlink(name) - except OSError: - pass - - -def build_structured_executor_call(skill_dir, capability, args_file): - """Build the only permitted non-shell Gemini executor process shape.""" - skill_path = Path(skill_dir) - argument_path = Path(args_file) - root = command_args_root() - if ( - capability not in {"scout", "adopt", "wire"} - or not skill_path.is_absolute() - or not argument_path.is_absolute() - or argument_path.parent != root - or not argument_path.name - ): - raise GeminiTransportError("structured executor fields are invalid") - return { - "program": "python3", - "argv": ["-m", "mlx_agent.gemini_executor", "--capability", capability, "--args-file", str(argument_path)], - "env": {"PYTHONPATH": str(skill_path / "src")}, - "shell": False, - } - - -def run_structured_executor_call(call, extra_env=None): - """Run a validated structured executor fixture through subprocess without a shell.""" - if not isinstance(call, dict) or set(call) != {"program", "argv", "env", "shell"}: - raise GeminiTransportError("structured executor fields are invalid") - program, argv, env, shell = call["program"], call["argv"], call["env"], call["shell"] - if ( - program != "python3" - or not isinstance(argv, list) - or len(argv) != 6 - or argv[:3] != ["-m", "mlx_agent.gemini_executor", "--capability"] - or argv[3] not in {"scout", "adopt", "wire"} - or argv[4] != "--args-file" - or not isinstance(argv[5], str) - or not isinstance(env, dict) - or set(env) != {"PYTHONPATH"} - or not isinstance(env["PYTHONPATH"], str) - ): - raise GeminiTransportError("structured executor fields are invalid") - if shell is not False or not env["PYTHONPATH"]: - raise GeminiTransportError("structured executor fields are invalid") - environment = dict(os.environ) - if extra_env is not None: - environment.update(extra_env) - environment.update(env) - return subprocess.run([program] + argv, env=environment, shell=False, text=True, capture_output=True, check=False) - - -def simulate_toml_transport(prompt, raw_args, capability, core=None): - """Simulate TOML prompt transport and dispatch only through the executor.""" - rendered = substitute_toml_args(prompt, raw_args) - payload = extract_untrusted_payload(rendered) - args_file = write_private_args_file(payload) - try: - return execute_gemini_command(capability, args_file, core=core) - except GeminiCommandError as error: - raise GeminiTransportError("TOML transport payload was rejected") from error diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/host.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/host.py deleted file mode 100644 index a7ee4c7..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/host.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Local host inventory probes used by Scout.""" - -import subprocess -from dataclasses import asdict, dataclass - - -@dataclass(frozen=True) -class HostInventory: - ram_gb: object = None - chip: object = None - ollama: bool = False - lmstudio: bool = False - - @staticmethod - def runtime_supports(runtime, role): - """Whether a runtime can serve the requested model class, independent of install state.""" - if runtime == "mlx-vlm": - return role == "vision" - if role == "vision": - return False - return runtime in ("mlx_lm", "ollama", "lmstudio", "litellm") - - @classmethod - def detect(cls, http_get, check_output=None): - return cls.inspect(http_get, check_output=check_output)[0] - - @classmethod - def inspect(cls, http_get, check_output=None): - """Return bounded local inventory plus classified unavailable-probe warnings.""" - check_output = check_output or subprocess.check_output - values = {"ram_gb": None, "chip": None, "ollama": False, "lmstudio": False} - warnings = [] - try: - values["ram_gb"] = round(int(check_output(["sysctl", "-n", "hw.memsize"]).strip()) / 1073741824) - except Exception: - warnings.append({"code": "host_probe_unavailable", "probe": "memory", "message": "Host memory probe unavailable."}) - try: - values["chip"] = check_output(["sysctl", "-n", "machdep.cpu.brand_string"], text=True).strip() - except Exception: - warnings.append({"code": "host_probe_unavailable", "probe": "chip", "message": "Host chip probe unavailable."}) - try: - http_get("http://127.0.0.1:11434/api/tags", timeout=3) - values["ollama"] = True - except Exception: - warnings.append({"code": "runtime_probe_unavailable", "probe": "ollama", "message": "Local runtime probe unavailable."}) - try: - http_get("http://localhost:1234/v1/models", timeout=3) - values["lmstudio"] = True - except Exception: - warnings.append({"code": "runtime_probe_unavailable", "probe": "lmstudio", "message": "Local runtime probe unavailable."}) - return cls(**values), warnings - - def to_dict(self): - return asdict(self) diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/huggingface.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/huggingface.py deleted file mode 100644 index 3a84ed5..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/huggingface.py +++ /dev/null @@ -1,457 +0,0 @@ -"""Dependency-free Hugging Face Hub access for Scout.""" - -import http.client -import json -import queue -import re -import threading -import time -import urllib.parse - -from .contextfit import extract_architecture -from .models import ( - REASONER_HINTS, - TEMPLATE_REASON, - TEMPLATE_TOOL_USE, - TOOL_USE_HINTS, - TOOL_USE_TAGS, -) - - -HF_API = "https://huggingface.co/api/models" -HF_DATASETS_API = "https://huggingface.co/api/datasets" -HF_API_HOST = "huggingface.co" -UA = {"User-Agent": "mlx-scout/0.2 (+https://github.com/cavi-ai/mlx-agent)"} -HF_RESPONSE_MAX_BYTES = 8 * 1024 * 1024 -_HTTP_READ_CHUNK_BYTES = 64 * 1024 -HF_CARD_HOST = "huggingface.co" -MODEL_CARD_MAX_BYTES = 512 * 1024 -_CARD_PATH_SUFFIX = "/raw/main/README.md" - - -def _is_allowed_api_path(path): - if path == "/api/models" or path.startswith("/api/models/"): - return True - if path == "/api/datasets" or path.startswith("/api/datasets/"): - return True - return False - - -def _is_valid_card_path(path): - """Accept model or dataset README raw paths on the fixed card host.""" - if not path.endswith(_CARD_PATH_SUFFIX): - return False - slash_count = path.count("/") - if slash_count == 5 and not path.startswith("/datasets/"): - return True - if slash_count == 6 and path.startswith("/datasets/"): - return True - return False - - -def http_json( - url, - timeout=10.0, - connection_factory=None, - clock=time.monotonic, - completion_wait=None, -): - """Read bounded JSON from the fixed Hugging Face API under one deadline.""" - parsed = urllib.parse.urlsplit(url) - if parsed.scheme != "https" or parsed.hostname != HF_API_HOST: - raise ValueError("Hugging Face URL must use the fixed HTTPS API host") - if parsed.port not in (None, 443): - raise ValueError("Hugging Face URL must use the default HTTPS port") - if parsed.username is not None or parsed.password is not None: - raise ValueError("Hugging Face URL must not contain credentials") - if parsed.fragment: - raise ValueError("Hugging Face URL must not contain a fragment") - if not _is_allowed_api_path(parsed.path): - raise ValueError("Hugging Face URL must target the models or datasets API") - - deadline = clock() + timeout - remaining = _deadline_remaining(deadline, clock) - if connection_factory is None: - connection = http.client.HTTPSConnection( - HF_API_HOST, - 443, - timeout=remaining, - ) - else: - connection = connection_factory(HF_API_HOST, 443, remaining) - target = parsed.path or "/" - if parsed.query: - target = "{0}?{1}".format(target, parsed.query) - return _run_http_worker( - connection, - lambda: _http_json_operation(connection, target, deadline, clock), - deadline, - clock, - completion_wait, - ) - - -def _run_http_worker(connection, operation, deadline, clock, completion_wait): - results = queue.Queue(maxsize=1) - completion = threading.Event() - - def run(): - try: - result = (True, operation()) - except BaseException as error: # noqa: BLE001 - propagated to caller - result = (False, error) - results.put_nowait(result) - completion.set() - - worker = threading.Thread( - target=run, - name="mlx-agent-huggingface-request", - daemon=True, - ) - try: - worker.start() - except BaseException: - connection.close() - raise - - try: - remaining = _deadline_remaining(deadline, clock) - except TimeoutError: - connection.close() - raise - try: - if completion_wait is None: - completed = completion.wait(remaining) - else: - completed = completion_wait(completion, remaining) - except BaseException: - connection.close() - raise - if not completed: - connection.close() - raise TimeoutError("Hugging Face request exceeded its overall deadline") - - try: - remaining = _deadline_remaining(deadline, clock) - except TimeoutError: - connection.close() - raise - worker.join(timeout=remaining) - if worker.is_alive(): - connection.close() - raise TimeoutError("Hugging Face request exceeded its overall deadline") - - succeeded, value = results.get_nowait() - if succeeded: - return value - raise value - - -def _http_json_operation(connection, target, deadline, clock): - try: - connection.request("GET", target, headers=UA) - _set_connection_timeout( - connection, - _deadline_remaining(deadline, clock), - ) - response = connection.getresponse() - _deadline_remaining(deadline, clock) - if 300 <= response.status < 400: - raise http.client.HTTPException( - "redirect responses are not allowed for Hugging Face requests" - ) - if not 200 <= response.status < 300: - raise http.client.HTTPException( - "Hugging Face returned HTTP status {0}".format(response.status) - ) - content_length = response.headers.get("Content-Length") - if content_length is not None: - try: - declared_length = int(content_length) - except (TypeError, ValueError) as error: - raise ValueError("invalid Hugging Face Content-Length") from error - if declared_length < 0: - raise ValueError("invalid Hugging Face Content-Length") - if declared_length > HF_RESPONSE_MAX_BYTES: - raise ValueError("Hugging Face response exceeds size limit") - body = _read_bounded_body( - response, - connection, - deadline, - clock, - ) - return json.loads(body.decode("utf-8")) - finally: - connection.close() - - -def http_card_text( - url, - timeout=8.0, - connection_factory=None, - clock=time.monotonic, - completion_wait=None, -): - """Read bounded README/model-card text from the fixed card host.""" - parsed = urllib.parse.urlsplit(url) - if parsed.scheme != "https" or parsed.hostname != HF_CARD_HOST: - raise ValueError("card URL must use the fixed HTTPS card host") - if parsed.port not in (None, 443): - raise ValueError("card URL must use the default HTTPS port") - if parsed.username is not None or parsed.password is not None: - raise ValueError("card URL must not contain credentials") - if parsed.fragment: - raise ValueError("card URL must not contain a fragment") - if not _is_valid_card_path(parsed.path): - raise ValueError( - "card URL must target //raw/main/README.md " - "or datasets///raw/main/README.md" - ) - - deadline = clock() + timeout - remaining = _deadline_remaining(deadline, clock) - if connection_factory is None: - connection = http.client.HTTPSConnection(HF_CARD_HOST, 443, timeout=remaining) - else: - connection = connection_factory(HF_CARD_HOST, 443, remaining) - target = parsed.path - if parsed.query: - target = "{0}?{1}".format(target, parsed.query) - return _run_http_worker( - connection, - lambda: _http_text_operation(connection, target, deadline, clock), - deadline, - clock, - completion_wait, - ) - - -def _http_text_operation(connection, target, deadline, clock): - try: - connection.request("GET", target, headers=UA) - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - response = connection.getresponse() - _deadline_remaining(deadline, clock) - if 300 <= response.status < 400: - raise http.client.HTTPException( - "redirect responses are not allowed for card requests" - ) - if not 200 <= response.status < 300: - raise http.client.HTTPException( - "card host returned HTTP status {0}".format(response.status) - ) - content_length = response.headers.get("Content-Length") - if content_length is not None: - try: - declared_length = int(content_length) - except (TypeError, ValueError) as error: - raise ValueError("invalid card Content-Length") from error - if declared_length < 0 or declared_length > MODEL_CARD_MAX_BYTES: - raise ValueError("card response exceeds size limit") - body = _read_bounded_body( - response, - connection, - deadline, - clock, - max_bytes=MODEL_CARD_MAX_BYTES, - ) - return body.decode("utf-8", errors="replace") - finally: - connection.close() - - -def _read_bounded_body(response, connection, deadline, clock, max_bytes=HF_RESPONSE_MAX_BYTES): - chunks = [] - total = 0 - while True: - _set_connection_timeout( - connection, - _deadline_remaining(deadline, clock), - ) - chunk = response.read( - min(_HTTP_READ_CHUNK_BYTES, max_bytes - total + 1) - ) - _deadline_remaining(deadline, clock) - if not chunk: - break - total += len(chunk) - if total > max_bytes: - raise ValueError("Hugging Face response exceeds size limit") - chunks.append(chunk) - return b"".join(chunks) - - -def _deadline_remaining(deadline, clock): - remaining = deadline - clock() - if remaining <= 0: - raise TimeoutError("Hugging Face request exceeded its overall deadline") - return remaining - - -def _set_connection_timeout(connection, timeout): - sock = getattr(connection, "sock", None) - if sock is not None: - sock.settimeout(timeout) - - -class HuggingFaceClient: - def __init__(self, http_get=http_json, card_get=http_card_text): - self._http_get = http_get - self._card_get = card_get - - @property - def http_get(self): - return self._http_get - - def fetch_model_card(self, repo, timeout=8): - """Return bounded README/model-card text, or None on any failure.""" - quoted = "/".join(urllib.parse.quote(part) for part in repo.split("/")) - url = "https://{0}/{1}/raw/main/README.md".format(HF_CARD_HOST, quoted) - try: - return self._card_get(url, timeout=timeout) - except Exception: - return None - - def fetch_dataset_card(self, repo, timeout=8): - """Return bounded dataset README text, or None on any failure.""" - quoted = "/".join(urllib.parse.quote(part) for part in repo.split("/")) - url = "https://{0}/datasets/{1}/raw/main/README.md".format(HF_CARD_HOST, quoted) - try: - return self._card_get(url, timeout=timeout) - except Exception: - return None - - @staticmethod - def list_models_url(sort="trendingScore", limit_fetch=300): - query = urllib.parse.urlencode({"filter": "mlx", "sort": sort, "direction": "-1", "limit": limit_fetch}) - return "{0}?{1}".format(HF_API, query) - - @staticmethod - def list_adapters_url(search="", limit_fetch=20): - params = { - "filter": "peft", - "sort": "downloads", - "direction": "-1", - "limit": limit_fetch, - } - if search: - params["search"] = search - return "{0}?{1}".format(HF_API, urllib.parse.urlencode(params)) - - @staticmethod - def list_datasets_url(search="", limit_fetch=20): - params = { - "sort": "downloads", - "direction": "-1", - "limit": limit_fetch, - } - if search: - params["search"] = search - return "{0}?{1}".format(HF_DATASETS_API, urllib.parse.urlencode(params)) - - def list_models(self, sort="trendingScore", limit_fetch=300): - return self._http_get(self.list_models_url(sort=sort, limit_fetch=limit_fetch)) - - def list_adapters(self, search="", limit_fetch=20, timeout=10): - """List PEFT/LoRA adapter model rows from the Hub (read-only).""" - rows = self._http_get( - self.list_adapters_url(search=search, limit_fetch=limit_fetch), - timeout=timeout, - ) - return rows if isinstance(rows, list) else [] - - def list_datasets(self, search="", limit_fetch=20, timeout=10): - """List dataset rows from the Hub (read-only).""" - rows = self._http_get( - self.list_datasets_url(search=search, limit_fetch=limit_fetch), - timeout=timeout, - ) - return rows if isinstance(rows, list) else [] - - def inspect_model_metadata(self, repo, timeout=8): - """Inspect model metadata without fetching the recursive repository tree.""" - quoted = urllib.parse.quote(repo) - model_url = "{0}/{1}".format(HF_API, quoted) - tree_url = "{0}/{1}/tree/main?recursive=true".format(HF_API, quoted) - output = { - "weight_bytes": None, - "tags": [], - "gated": None, - "license": None, - "reasoning": None, - "reason_src": None, - "tool_use": None, - "tool_use_src": None, - "tool_use_confidence": "none", - "params_total": None, - "architecture": None, - "metadata_available": False, - "tree_available": False, - "metadata_url": model_url, - "tree_url": tree_url, - "repository_url": "https://huggingface.co/{0}".format(repo), - } - try: - metadata = self._http_get(model_url, timeout=timeout) - output["metadata_available"] = True - tags = metadata.get("tags", []) or [] - output["tags"] = tags - output["gated"] = bool(metadata.get("gated")) - config = metadata.get("config") or {} - card_data = metadata.get("cardData") or {} - output["license"] = card_data.get("license") or next((tag.split("license:", 1)[1] for tag in tags if tag.startswith("license:")), None) - output["params_total"] = (metadata.get("safetensors") or {}).get("total") - output["architecture"] = extract_architecture(config) - template = ((config.get("tokenizer_config") or {}).get("chat_template") or "") - lower_tags = [tag.lower() for tag in tags] - normalized_tags = { - re.sub(r"\s+", "-", str(tag).strip().lower()) - for tag in tags - } - if TEMPLATE_REASON.search(template): - output["reasoning"], output["reason_src"] = True, "chat_template" - elif any(tag in ("reasoning", "thinking", "chain-of-thought") for tag in lower_tags): - output["reasoning"], output["reason_src"] = True, "tags" - elif REASONER_HINTS.search(repo): - output["reasoning"], output["reason_src"] = True, "name" - else: - output["reasoning"], output["reason_src"] = False, "checked" - if TEMPLATE_TOOL_USE.search(template): - output.update({ - "tool_use": True, - "tool_use_src": "chat_template", - "tool_use_confidence": "explicit", - }) - elif normalized_tags & TOOL_USE_TAGS: - output.update({ - "tool_use": True, - "tool_use_src": "tags", - "tool_use_confidence": "explicit", - }) - elif TOOL_USE_HINTS.search(repo): - output.update({ - "tool_use": True, - "tool_use_src": "name", - "tool_use_confidence": "weak", - }) - else: - output.update({ - "tool_use": False, - "tool_use_src": "checked", - "tool_use_confidence": "explicit", - }) - except Exception: - pass - return output - - def inspect_model(self, repo): - output = self.inspect_model_metadata(repo, timeout=8) - tree_url = output["tree_url"] - try: - tree = self._http_get(tree_url, timeout=8) - output["tree_available"] = True - weights = sum(item.get("size", 0) for item in tree if item.get("path", "").endswith((".safetensors", ".gguf", ".bin"))) - output["weight_bytes"] = weights or None - except Exception: - pass - return output diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/installer.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/installer.py deleted file mode 100644 index 06819e5..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/installer.py +++ /dev/null @@ -1,696 +0,0 @@ -"""Scoped, receipt-owned installer built on the Transaction safety boundary.""" - -from __future__ import annotations - -import hashlib -import json -import os -import uuid -from dataclasses import dataclass, field -from pathlib import Path - -from .providers import detect_providers -from .transactions import ( - LegacyLockError, Receipt, Transaction, _assert_safe_directory, _assert_safe_target, - _atomic_in_directory, _read_regular, _read_target, _walk_directory, - _rollback_receipt_hash, legacy_lock_problem, rollback, -) -from .wiring import redact_secrets - - -class InstallerConflictError(ValueError): - """A requested operation would touch an unowned or modified artifact.""" - - -class _ArtifactAdapter: - version = "installer-1.0" - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("installer artifacts must be UTF-8 text") - redacted = redact_secrets(content) - try: - contains_persisted_secret = json.loads(redacted) != json.loads(content) - except (TypeError, ValueError, json.JSONDecodeError): - contains_persisted_secret = redacted != content - if contains_persisted_secret: - raise ValueError("installer artifacts must not contain persisted secrets") - return True - - -class _SourceCodeArtifactAdapter: - """Source modules may contain secret-pattern examples but never persisted values.""" - - version = "installer-source-1.0" - secret_scan_exempt = True - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("installer artifacts must be UTF-8 text") - return True - - -@dataclass(frozen=True) -class _PlannedTransaction: - provider_id: str - changes: tuple - preview_hash: str - destinations: tuple - transaction_id: str = None - expected_receipt_path: str = None - child_index: int = None - - -@dataclass -class InstallerReceipt: - status: str - receipts: list = field(default_factory=list) - receipt_path: str = "" - targets: list = field(default_factory=list) - batch_path: str = "" - - def to_dict(self): - return { - "status": self.status, - "receipt_path": self.receipt_path or None, - "batch_path": self.batch_path or None, - "targets": list(self.targets), - "receipts": [_receipt_data(item) for item in self.receipts], - } - - -@dataclass -class InstallPlan: - action: str - provider_ids: tuple - scope: str - project_root: Path - preview: dict - transactions: list = field(default_factory=list) - rollback_receipts: list = field(default_factory=list) - uninstall_hashes: dict = field(default_factory=dict) - restore_changes: tuple = field(default_factory=tuple) - binding: dict = field(default_factory=dict) - compatibility: tuple = field(default_factory=tuple) - noop: bool = False - - def to_dict(self): - return { - "action": self.action, - "providers": list(self.provider_ids), - "scope": self.scope, - "project_root": str(self.project_root), - "preview": self.preview, - "compatibility": list(self.compatibility), - "noop": self.noop, - } - - -def _sha256(content): - return hashlib.sha256(content).hexdigest() - - -def _receipt_data(receipt): - value = receipt.to_dict() - value["receipt_path"] = receipt.receipt_path - return value - - -def _digest(value): - return _sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")) - - -def _read_optional_target(path): - """Read an artifact without following links; missing parent paths mean absent.""" - try: - content, exists, _mode = _read_target(path) - return content if exists else None - except ValueError as error: - if str(error).startswith("path component does not exist:"): - return None - raise - - -def _adapter_for_artifact(artifact): - is_runtime_source = "src" in artifact.source.parts and "mlx_agent" in artifact.source.parts - return _SourceCodeArtifactAdapter() if is_runtime_source else _ArtifactAdapter() - - -class Installer: - """Plan first; execute only the exact reviewed transaction previews.""" - - def __init__(self, registry, project_root=None, executable_lookup=None, env=None, - transaction_factory=None, rollback_func=None, probe_runner=None): - self.registry = registry - self.project_root = Path(project_root).resolve() if project_root else Path.cwd().resolve() - self.executable_lookup = executable_lookup - self.env = env - self.transaction_factory = transaction_factory or Transaction - self.rollback_func = rollback_func or rollback - self.probe_runner = probe_runner - - def detected(self): - return detect_providers( - self.registry.definitions().values(), self.env, - executable_lookup=self.executable_lookup, - probe_runner=self.probe_runner, - ) - - def _compatibility(self, provider_ids): - definitions = self.registry.definitions() - return tuple( - item.to_dict() for item in detect_providers( - [definitions[provider_id] for provider_id in provider_ids], - self.env, - executable_lookup=self.executable_lookup, - probe_runner=self.probe_runner, - ) - ) - - def plan(self, action, providers, scope="user", project_root=None): - if action not in {"install", "update", "uninstall", "doctor"}: - raise ValueError("unsupported installer action: {0}".format(action)) - if scope not in {"user", "project"}: - raise ValueError("scope must be 'user' or 'project'") - project = Path(project_root or self.project_root).resolve() - if scope == "project" and not project.is_dir(): - raise ValueError("project root must be an existing directory") - definitions = self.registry.definitions() - selected = tuple(providers or ()) - if not selected: - raise ValueError("provider selection is required; inspect detected providers first") - if len(selected) != len(set(selected)) or any(item not in definitions for item in selected): - raise ValueError("providers must be unique supported provider IDs") - if action == "doctor": - return self._doctor_plan(selected, definitions, scope, project) - if action == "uninstall": - return self._uninstall_plan(selected, definitions, scope, project) - return self._install_plan(action, selected, definitions, scope, project) - - def execute(self, plan, confirmed=False): - if plan.action == "doctor": - return self._doctor(plan) - if plan.noop: - return InstallerReceipt("noop") - if confirmed is not True and confirmed != plan.preview["preview_hash"]: - raise PermissionError("explicit confirmation for this installer preview is required") - self._verify_plan_binding(plan) - batch_path = self._create_batch(plan) - if plan.action == "uninstall": - return self._execute_uninstall(plan, batch_path) - return self._execute_install(plan, batch_path, self._batch_operations(plan, batch_path)) - - def _execute_install(self, plan, batch_path, operations): - successful, all_receipts = [], [] - try: - prepared = [(operation, *self._repreview(operation, plan.scope, plan.project_root)) for operation in operations] - except PermissionError as error: - self._batch_update(batch_path, "rolled_back", successful, error=str(error)) - raise - try: - for operation, transaction, preview in prepared: - self._batch_update(batch_path, "pending", all_receipts, active=self._active_operation(operation)) - receipt = transaction.apply(operation.preview_hash) - all_receipts.append(receipt) - self._batch_update(batch_path, "pending", all_receipts) - if receipt.status != "applied": - raise InstallerConflictError("provider transaction did not apply: {0}".format(receipt.status)) - successful.append(receipt) - self._batch_update(batch_path, "complete", successful) - return InstallerReceipt("applied", all_receipts, successful[-1].receipt_path, [target for item in successful for target in item.targets], str(batch_path)) - except LegacyLockError as error: - self._batch_update(batch_path, "rolled_back", all_receipts, error=str(error)) - raise InstallerConflictError(str(error)) - except Exception as error: - recovered, unresolved = self._compensate_install(all_receipts) - child_requires_recovery = any(item.status in {"recovery_required", "rollback_failed"} for item in all_receipts) - status = "rolled_back" if recovered and not child_requires_recovery else "recovery_required" - remediation = unresolved or ([{"paths": list(all_receipts[-1].targets), "message": "Child receipt requires manual recovery: {0}".format(all_receipts[-1].receipt_path)}] if child_requires_recovery and all_receipts else []) - self._batch_update(batch_path, status, all_receipts, error=str(error), remediation=remediation) - return InstallerReceipt(status, all_receipts, all_receipts[-1].receipt_path if all_receipts else "", [target for item in all_receipts for target in item.targets], str(batch_path)) - - def _execute_uninstall(self, plan, batch_path): - completed, attempted = [], [] - try: - for receipt in plan.rollback_receipts: - self._batch_update(batch_path, "pending", attempted, active={"provider": "uninstall", "transaction_id": receipt.transaction_id, "expected_receipt_path": receipt.receipt_path}) - result = self.rollback_func( - receipt.receipt_path, - expected_after_hashes=receipt.after_hashes, - expected_receipt_hash=_rollback_receipt_hash(receipt), - ) - recorded = self._rollback_result(result, receipt, plan.scope, plan.project_root) - attempted.append(recorded) - self._batch_update(batch_path, "pending", attempted) - if recorded.status != "rolled_back": - raise InstallerConflictError("uninstall rollback did not complete: {0}".format(recorded.status)) - completed.append(recorded) - self._cleanup_empty_artifact_dirs(plan) - self._batch_update(batch_path, "complete", attempted) - return InstallerReceipt("rolled_back", attempted, attempted[-1].receipt_path if attempted else "", [target for item in attempted for target in item.targets], str(batch_path)) - except Exception as error: - unresolved_children = [item for item in attempted if item.status != "rolled_back"] - if not completed: - status = "recovery_required" if unresolved_children else "rolled_back" - remediation = self._unresolved_child_remediation(unresolved_children) - self._batch_update(batch_path, status, attempted, error=str(error), remediation=remediation) - message = "uninstall recovery is required" if unresolved_children else "uninstall aborted before mutation" - raise InstallerConflictError("{0}: {1}".format(message, error)) - recovered, unresolved = self._compensate_uninstall(plan.restore_changes, completed, plan.scope, plan.project_root) - status = "rolled_back" if recovered and not unresolved_children else "recovery_required" - remediation = self._unresolved_child_remediation(unresolved_children) + unresolved - self._batch_update(batch_path, status, attempted, error=str(error), remediation=remediation) - if status == "recovery_required": - raise InstallerConflictError("uninstall recovery is required: {0}".format(error)) - return InstallerReceipt(status, attempted, attempted[-1].receipt_path if attempted else "", [target for item in attempted for target in item.targets], str(batch_path)) - - def _install_plan(self, action, selected, definitions, scope, project): - transactions, summary = [], [] - for provider_id in selected: - definition = definitions[provider_id] - changes = [] - for artifact in definition.artifacts: - if not definition.applies_to(scope, artifact): - continue - target = definition.artifact_destination(scope, project, artifact) - desired_bytes, source_exists, _source_mode = _read_target(artifact.source) - if not source_exists: - raise ValueError("provider artifact source disappeared: {0}".format(artifact.source)) - try: - desired = desired_bytes.decode("utf-8") - except UnicodeDecodeError as error: - raise ValueError("provider artifact source must be UTF-8") from error - current = _read_optional_target(target) - if current is not None: - history = self._artifact_history(definition, target, scope, project) - if not history or _sha256(current) != history[-1].after_hashes[str(target)]: - raise InstallerConflictError("refusing to overwrite unowned or modified artifact: {0}".format(target)) - if current == desired.encode("utf-8"): - continue - adapter = _adapter_for_artifact(artifact) - changes.append({"path": str(target), "content": desired, "adapter": adapter}) - if changes: - transaction = self._new_transaction(scope, project) - preview = transaction.preview(changes) - transactions.append(_PlannedTransaction(provider_id, tuple(changes), preview["preview_hash"], tuple(str(item["path"]) for item in changes))) - summary.append({"provider": provider_id, "changes": preview["changes"], "diff": preview["diff"]}) - return self._make_plan(action, selected, scope, project, transactions=transactions, summary=summary, definitions=definitions) - - def _uninstall_plan(self, selected, definitions, scope, project): - steps, summary, uninstall_hashes, restore_changes = [], [], {}, {} - seen = set() - for provider_id in selected: - definition = definitions[provider_id] - provider_steps = [] - for artifact in definition.artifacts: - if not definition.applies_to(scope, artifact): - continue - target = definition.artifact_destination(scope, project, artifact) - history = self._artifact_history(definition, target, scope, project) - if not history: - continue - oldest, latest = history[0], history[-1] - target_name = str(target) - if oldest.backup_paths.get(target_name) is not None: - raise InstallerConflictError("artifact was not receipt-owned at installation: {0}".format(target)) - location = _assert_safe_target(target) - content = _read_regular(location) - if _sha256(content) != latest.after_hashes[target_name]: - raise InstallerConflictError("refusing to remove user-modified artifact: {0}".format(target)) - uninstall_hashes[target_name] = latest.after_hashes[target_name] - restore_changes.setdefault(provider_id, []).append({ - "path": target_name, - "content": content.decode("utf-8"), - "adapter": _adapter_for_artifact(artifact), - }) - provider_steps.extend(history) - for receipt in sorted({item.receipt_path: item for item in provider_steps}.values(), key=lambda item: item.timestamp, reverse=True): - if receipt.receipt_path not in seen: - seen.add(receipt.receipt_path) - steps.append(receipt) - summary.append({"provider": provider_id, "changes": len(provider_steps), "diff": "remove receipt-owned artifacts"}) - return self._make_plan("uninstall", selected, scope, project, rollback_receipts=steps, - uninstall_hashes=uninstall_hashes, restore_changes=tuple((provider_id, tuple(changes)) for provider_id, changes in restore_changes.items()), - summary=summary, definitions=definitions) - - def _doctor_plan(self, selected, definitions, scope, project): - return self._make_plan("doctor", selected, scope, project, definitions=definitions) - - def _cleanup_empty_artifact_dirs(self, plan): - """Remove only emptied receipt-owned descendants; retain host-owned roots.""" - definitions = self.registry.definitions() - for provider_id in plan.provider_ids: - definition = definitions[provider_id] - provider_root = definition.destination(plan.scope, plan.project_root) - for artifact in definition.artifacts: - if not definition.applies_to(plan.scope, artifact): - continue - destination = definition.artifact_destination(plan.scope, plan.project_root, artifact) - try: - relative = destination.relative_to(provider_root) - except ValueError: - # Some providers own companion artifacts (for example, - # Gemini command files) beneath a host-wide directory - # rather than their package directory. Removing empty - # parents there could remove host-owned structure. - continue - stop = provider_root / relative.parts[0] - current = destination.parent - while current != stop: - try: - _parent, descriptor = _walk_directory(current.parent) - try: - os.rmdir(current.name, dir_fd=descriptor) - os.fsync(descriptor) - finally: - os.close(descriptor) - except (FileNotFoundError, OSError, ValueError): - break - current = current.parent - - def _doctor(self, plan): - problems, checked, provider_states = [], [], [] - definitions = self.registry.definitions() - detections = { - item.id: item for item in detect_providers( - [definitions[provider_id] for provider_id in plan.provider_ids], - self.env, - executable_lookup=self.executable_lookup, - probe_runner=self.probe_runner, - ) - } - for provider_id in plan.provider_ids: - definition = definitions[provider_id] - artifact_problems = [] - for artifact in definition.artifacts: - if not definition.applies_to(plan.scope, artifact): - continue - target = definition.artifact_destination(plan.scope, plan.project_root, artifact) - checked.append(str(target)) - try: - safe_root = _assert_safe_directory(plan.project_root if plan.scope == "project" else definition.user_root) - location = _assert_safe_target(target) - location.relative_to(safe_root) - history = self._artifact_history(definition, target, plan.scope, plan.project_root) - if not history or _sha256(_read_regular(location)) != history[-1].after_hashes[str(target)]: - raise ValueError("artifact hash is not receipt-owned") - except (OSError, ValueError): - problem = {"provider": provider_id, "code": "artifact_invalid", "path": str(target)} - problems.append(problem) - artifact_problems.append(problem) - detection = detections[provider_id] - if definition.install_mode == "portable": - integration_state = "portable" - elif definition.install_mode == "staged": - integration_state = "staged" - problems.append({ - "provider": provider_id, - "code": "provider_staged", - "message": "artifacts are staged but native host visibility is not proven", - }) - elif detection.state == "native-visible": - integration_state = "native-visible" - else: - integration_state = "staged" - code = "provider_unsupported" if detection.state == "unsupported" else "provider_absent" - problems.append({ - "provider": provider_id, - "code": code, - "message": detection.error or "provider compatibility could not be proven", - }) - provider_states.append({ - "provider": provider_id, - "artifact_state": "invalid" if artifact_problems else "installed", - "integration_state": integration_state, - "compatibility": detection.to_dict(), - }) - problems.extend(self._batch_problems(plan.scope, plan.project_root)) - lock_root = self._receipts_dir(plan.scope, plan.project_root) / "locks" - for provider_id in plan.provider_ids: - definition = definitions[provider_id] - targets = [ - str(definition.artifact_destination(plan.scope, plan.project_root, artifact)) - for artifact in definition.artifacts - if definition.applies_to(plan.scope, artifact) - ] - try: - legacy_problem = legacy_lock_problem(targets, lock_root) - except ValueError as error: - problems.append({"provider": provider_id, "code": "legacy_lock_invalid", "message": str(error)}) - continue - problems.extend({"provider": provider_id, **item} for item in legacy_problem) - return {"healthy": not problems, "problems": problems, "checked": checked, - "providers": provider_states, - "cooperative_concurrency": "Transaction advisory locks protect cooperative writers."} - - def _batch_problems(self, scope, project): - batches = self._receipts_dir(scope, project) / "batches" - try: - safe_batches = _assert_safe_directory(batches) - except ValueError: - return [] - problems = [] - for path in safe_batches.glob("*/batch.json"): - try: - batch = self._read_batch(path) - except (OSError, ValueError, json.JSONDecodeError): - problems.append({"code": "batch_recovery_required", "path": str(path)}) - continue - if batch["status"] in {"pending", "recovery_required"}: - active = batch.get("active") - if isinstance(active, dict) and active.get("expected_receipt_path"): - expected = active["expected_receipt_path"] - try: - child = _assert_safe_target(expected) - child_value = json.loads(_read_regular(child).decode("utf-8")) - if child_value.get("transaction_id") != active.get("transaction_id"): - raise ValueError("active child receipt does not match batch transaction") - except (OSError, ValueError, json.JSONDecodeError): - expected = "missing or untrusted active receipt: {0}".format(expected) - problems.append({"code": "batch_recovery_required", "path": str(path), "active_receipt": expected}) - else: - problems.append({"code": "batch_recovery_required", "path": str(path)}) - return problems - - def _make_plan(self, action, selected, scope, project, transactions=(), rollback_receipts=(), - uninstall_hashes=None, restore_changes=(), summary=(), definitions=None): - definitions = definitions or self.registry.definitions() - compatibility = self._compatibility(selected) - destinations = [{"provider": provider_id, "targets": [str(definitions[provider_id].artifact_destination(scope, project, artifact)) for artifact in definitions[provider_id].artifacts if definitions[provider_id].applies_to(scope, artifact)]} for provider_id in selected] - binding = { - "action": action, "providers": list(selected), "scope": scope, "project": str(project), - "destinations": destinations, - "inner_previews": [{"provider": item.provider_id, "hash": item.preview_hash, "targets": list(item.destinations)} for item in transactions], - "rollback_chain": [self._receipt_identity(item) for item in rollback_receipts], - "compatibility": list(compatibility), - } - preview = {"changes": sum(item["changes"] for item in summary), "providers": list(selected), - "diff": "\n".join(item["diff"] for item in summary), "preview_hash": _digest(binding), - "requires_confirmation": action != "doctor"} - return InstallPlan(action, selected, scope, project, preview, list(transactions), list(rollback_receipts), - dict(uninstall_hashes or {}), tuple(restore_changes), binding, compatibility, - noop=not transactions and not rollback_receipts) - - def _verify_plan_binding(self, plan): - definitions = self.registry.definitions() - binding = { - "action": plan.action, "providers": list(plan.provider_ids), "scope": plan.scope, - "project": str(plan.project_root), - "destinations": [{"provider": provider_id, "targets": [str(definitions[provider_id].artifact_destination(plan.scope, plan.project_root, artifact)) for artifact in definitions[provider_id].artifacts if definitions[provider_id].applies_to(plan.scope, artifact)]} for provider_id in plan.provider_ids], - "inner_previews": [{"provider": item.provider_id, "hash": item.preview_hash, "targets": list(item.destinations)} for item in plan.transactions], - "rollback_chain": [self._receipt_identity(self._load_receipt(item.receipt_path, plan.scope, plan.project_root)) for item in plan.rollback_receipts], - "compatibility": list(self._compatibility(plan.provider_ids)), - } - if binding != plan.binding or _digest(binding) != plan.preview["preview_hash"]: - raise PermissionError("installer preview is stale; regenerate and review it before confirmation") - - def _repreview(self, operation, scope, project): - transaction = self._new_transaction(scope, project, operation.transaction_id) - preview = transaction.preview(list(operation.changes)) - if preview["preview_hash"] != operation.preview_hash: - raise PermissionError("installer preview is stale; an inner preview changed") - return transaction, preview - - def _new_transaction(self, scope, project, transaction_id=None): - receipts_dir = self._receipts_dir(scope, project) - return self.transaction_factory( - receipts_dir=receipts_dir, - lock_root=receipts_dir / "locks", - create_target_parents=True, - transaction_id=transaction_id, - ) - - @staticmethod - def _operation_transaction_id(batch_id, child_index, provider_id): - return str(uuid.uuid5(uuid.UUID(batch_id), "{0}:{1}".format(child_index, provider_id))) - - def _artifact_history(self, definition, target, scope, project): - target_name = str(target) - allowed = {str(definition.artifact_destination(scope, project, artifact)) for artifact in definition.artifacts if definition.applies_to(scope, artifact)} - return [item for item in self._all_receipts(scope, project) if target_name in item.targets and set(item.targets).issubset(allowed) and item.status == "applied"] - - def _all_receipts(self, scope, project): - receipts_dir = self._receipts_dir(scope, project) - try: - safe_dir = _assert_safe_directory(receipts_dir) - except ValueError: - return [] - result = [] - for candidate in safe_dir.glob("*/receipt.json"): - try: - result.append(self._load_receipt(candidate, scope, project)) - except (OSError, ValueError, json.JSONDecodeError): - continue - return sorted(result, key=lambda item: item.timestamp) - - def _load_receipt(self, path, scope, project): - receipts_dir = _assert_safe_directory(self._receipts_dir(scope, project)) - location = _assert_safe_target(path) - if location.parent.parent != receipts_dir: - raise ValueError("installer receipt escapes its scoped receipt directory") - return Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) - - @staticmethod - def _receipt_identity(receipt): - return {"transaction_id": receipt.transaction_id, "receipt_path": receipt.receipt_path, - "targets": list(receipt.targets), "before_hashes": receipt.before_hashes, - "after_hashes": receipt.after_hashes, "backup_paths": receipt.backup_paths, - "rollback_receipt_hash": _rollback_receipt_hash(receipt)} - - def _rollback_result(self, result, planned, scope, project): - if all(hasattr(result, field) for field in ("transaction_id", "receipt_path", "targets", "before_hashes", "after_hashes", "backup_paths")): - return result - recorded = self._load_receipt(planned.receipt_path, scope, project) - recorded.status = result.status - return recorded - - @staticmethod - def _unresolved_child_remediation(receipts): - return [{"paths": list(receipt.targets), "message": "Rollback child requires manual recovery: {0}".format(receipt.receipt_path)} for receipt in receipts] - - def _compensate_install(self, receipts): - unresolved = [] - for receipt in reversed(receipts): - if receipt.status == "rolled_back": - continue - try: - expected = { - "expected_after_hashes": receipt.after_hashes, - "expected_receipt_hash": _rollback_receipt_hash(receipt), - } if receipt.status == "applied" else {} - result = self.rollback_func(receipt.receipt_path, **expected) - if result.status != "rolled_back": - unresolved.append({"paths": list(receipt.targets), "message": "Could not recover child receipt: {0}".format(receipt.receipt_path)}) - except Exception as error: - unresolved.append({"paths": list(receipt.targets), "message": redact_secrets(str(error))}) - return not unresolved, unresolved - - def _compensate_uninstall(self, changes, completed, scope, project): - if not changes: - return True, [] - expected = {} - for receipt in completed: - for target in receipt.targets: - expected[target] = {"hash": receipt.before_hashes[target], - "exists": receipt.backup_paths[target] is not None, - "mode": receipt.target_modes[target]} - unresolved = [] - try: - affected = {target for receipt in completed for target in receipt.targets} - for _provider_id, provider_changes in changes: - provider_changes = [change for change in provider_changes if change["path"] in affected] - if not provider_changes: - continue - expected_current = {change["path"]: expected[change["path"]] for change in provider_changes} - transaction = self._new_transaction(scope, project) - preview = transaction.preview(provider_changes, expected_current=expected_current) - if transaction.apply(preview["preview_hash"], expected_current=expected_current).status != "applied": - unresolved.append({"paths": [change["path"] for change in provider_changes], "message": "Compensation transaction did not apply."}) - return not unresolved, unresolved - except Exception as error: - return False, [{"paths": sorted(expected), "message": redact_secrets(str(error))}] - - def _receipts_dir(self, scope, project): - return (self.registry.config_root / "mlx-agent" / "installer-receipts") if scope == "user" else (project / ".mlx-agent" / "installer-receipts") - - def _create_batch(self, plan): - batches = self._receipts_dir(plan.scope, plan.project_root) / "batches" - directory, descriptor = _walk_directory(batches, create=True) - batch_id = str(uuid.uuid4()) - try: - os.mkdir(batch_id, 0o700, dir_fd=descriptor) - os.fsync(descriptor) - finally: - os.close(descriptor) - path = directory / batch_id / "batch.json" - operations = [] - for index, item in enumerate(plan.transactions): - operation = self._active_operation(item, batch_id, index) - operation["expected_receipt_path"] = str(self._receipts_dir(plan.scope, plan.project_root) / operation["transaction_id"] / "receipt.json") - operations.append(operation) - immutable = { - "schema_version": "1.0", "batch_id": batch_id, "action": plan.action, "scope": plan.scope, - "project_root": str(plan.project_root), "preview_hash": plan.preview["preview_hash"], - "binding": plan.binding, - "operations": operations, - } - self._write_batch(path, dict(immutable, status="pending", children=[], active=None, remediation=[])) - return path - - def _batch_update(self, path, status, receipts, active=None, error=None, remediation=None): - value = self._read_batch(path) - children = {item["receipt_path"]: item for item in value.get("children", [])} - for receipt in receipts: - children[receipt.receipt_path] = dict(self._receipt_identity(receipt), status=receipt.status) - value["children"] = list(children.values()) - value["status"] = status - value["active"] = active - if error: - value.setdefault("remediation", []).append({"paths": [], "message": redact_secrets(str(error))}) - if remediation: - value.setdefault("remediation", []).extend(remediation) - self._write_batch(path, value) - - def _read_batch(self, path): - location = _assert_safe_target(path) - value = json.loads(_read_regular(location).decode("utf-8")) - required = {"schema_version", "batch_id", "action", "scope", "project_root", "preview_hash", "binding", "operations", "status", "children", "active", "remediation"} - if not isinstance(value, dict) or set(value) != required: - raise ValueError("installer batch journal is malformed") - return value - - def _write_batch(self, path, value): - root = _assert_safe_directory(Path(path).parent) - _atomic_in_directory(root, "batch.json", (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600) - - def _batch_operations(self, plan, path): - batch = self._read_batch(path) - operations = batch["operations"] - if len(operations) != len(plan.transactions): - raise ValueError("installer batch operations do not match the confirmed plan") - result = [] - for index, (planned, operation) in enumerate(zip(plan.transactions, operations)): - expected = self._active_operation(planned, batch["batch_id"], index) - expected["expected_receipt_path"] = str(self._receipts_dir(plan.scope, plan.project_root) / expected["transaction_id"] / "receipt.json") - if operation != expected: - raise ValueError("installer batch operation binding is stale") - result.append(_PlannedTransaction(planned.provider_id, planned.changes, planned.preview_hash, - planned.destinations, operation["transaction_id"], - operation["expected_receipt_path"], operation["child_index"])) - return result - - @staticmethod - def _active_operation(operation, batch_id=None, child_index=None): - if operation.transaction_id is None: - if batch_id is None or child_index is None: - raise ValueError("batch operation identity is required") - transaction_id = Installer._operation_transaction_id(batch_id, child_index, operation.provider_id) - expected_receipt_path = None - else: - transaction_id = operation.transaction_id - expected_receipt_path = operation.expected_receipt_path - return {"provider": operation.provider_id, - "child_index": operation.child_index if operation.child_index is not None else child_index, - "transaction_id": transaction_id, - "expected_receipt_path": expected_receipt_path, - "preview_hash": operation.preview_hash, "targets": list(operation.destinations)} diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/interview.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/interview.py deleted file mode 100644 index d4711cb..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/interview.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Domain-agnostic interview engine producing a validated DomainIntent. - -The question set is a deterministic, config-driven backbone. An optional -``assist`` callable may refine answers, but every assist result is re-validated -through ``build_intent`` and can never bypass validation or inject unknown -roles. Modality foundations seed roles/keywords via ``modality`` profiles. -No network access, no LLM dependency for the deterministic path. -""" - -from __future__ import annotations - -import math - -from dataclasses import dataclass -from typing import Callable, Dict, List, Mapping, Optional, Sequence, Tuple - -from .modality import ( - FACET_CHOICES, - MODALITY_CHOICES, - apply_modality_profile, - detect_facets, - detect_modalities, - resolve_facets, - validate_facets, - validate_modalities, -) -from .models import DISCOVERY_ROLES - - -ROLE_CHOICES: Dict[str, str] = { - "Vision / OCR": "vision", - "Embeddings": "embedding", - "Coding": "coding", - "Reasoning": "reasoning", - "General chat": "general", - "Tool use / function calling": "tool-use", -} - - -QUESTIONS: Tuple[Dict[str, object], ...] = ( - { - "id": "domain", - "prompt": "What are you building? Describe the domain in a sentence.", - "kind": "text", - }, - { - "id": "roles", - "prompt": "Which model roles do you need? (choose one or more)", - "kind": "multi", - "choices": tuple(ROLE_CHOICES.keys()), - }, - { - "id": "keywords", - "prompt": "Any domain keywords to prioritize? (comma-separated, optional)", - "kind": "text", - }, - { - "id": "license", - "prompt": "Restrict to specific licenses? (comma-separated, optional)", - "kind": "text", - }, - { - "id": "memory_gb", - "prompt": "Memory budget in GB? (optional, e.g. 32)", - "kind": "text", - }, - { - "id": "notes", - "prompt": "Any other constraints or notes? (optional)", - "kind": "text", - }, -) - -_FOLLOW_ON_QUESTIONS = tuple(question for question in QUESTIONS if question["id"] != "domain") - - -@dataclass(frozen=True) -class DomainIntent: - """A validated, deterministic description of what the user wants.""" - - domain: str - roles: Tuple[str, ...] - keywords: Tuple[str, ...] = () - license_allow: Tuple[str, ...] = () - memory_gb: Optional[float] = None - notes: str = "" - modalities: Tuple[str, ...] = () - facets: Tuple[str, ...] = () - - def to_dict(self) -> Dict[str, object]: - return { - "domain": self.domain, - "roles": list(self.roles), - "keywords": list(self.keywords), - "license_allow": list(self.license_allow), - "memory_gb": self.memory_gb, - "notes": self.notes, - "modalities": list(self.modalities), - "facets": list(self.facets), - } - - -def _dedupe(values: Sequence[str]) -> Tuple[str, ...]: - seen = [] - for value in values: - if value and value not in seen: - seen.append(value) - return tuple(seen) - - -def _split_csv(raw: object) -> Tuple[str, ...]: - if raw is None: - return () - if isinstance(raw, (list, tuple)): - parts = [str(item) for item in raw] - else: - parts = str(raw).split(",") - return _dedupe([part.strip().lower() for part in parts if part.strip()]) - - -def _split_labels(raw: object) -> Tuple[str, ...]: - if raw is None or raw == "": - return () - if isinstance(raw, (list, tuple)): - return _dedupe([str(item).strip() for item in raw if str(item).strip()]) - return _dedupe([part.strip() for part in str(raw).split(",") if part.strip()]) - - -def _resolve_roles(raw: object) -> Tuple[str, ...]: - if raw is None or raw == "": - labels: Sequence[str] = () - elif isinstance(raw, (list, tuple)): - labels = [str(item) for item in raw] - else: - labels = [part.strip() for part in str(raw).split(",")] - roles = [] - for label in labels: - cleaned = label.strip() - if not cleaned: - continue - if cleaned in ROLE_CHOICES: - roles.append(ROLE_CHOICES[cleaned]) - elif cleaned in DISCOVERY_ROLES: - roles.append(cleaned) - else: - raise ValueError("unknown role: {0}".format(cleaned)) - return _dedupe(roles) - - -def _resolve_memory(raw: object) -> Optional[float]: - if raw is None: - return None - text = str(raw).strip() - if not text: - return None - try: - value = float(text) - except ValueError as error: - raise ValueError("memory_gb must be a number") from error - if not math.isfinite(value) or value <= 0: - raise ValueError("memory_gb must be a positive number") - return value - - -def _resolve_modality_ids(raw: object) -> Tuple[str, ...]: - labels = _split_labels(raw) - ids = [] - for label in labels: - if label in MODALITY_CHOICES: - ids.append(MODALITY_CHOICES[label]) - else: - ids.append(label) - return validate_modalities(ids) - - -def _resolve_facet_ids(raw: object, modalities: Sequence[str]) -> Tuple[str, ...]: - labels = _split_labels(raw) - if not labels: - return () - label_to_id = {} - for modality in modalities: - label_to_id.update(FACET_CHOICES.get(modality, {})) - ids = [] - for label in labels: - if label in label_to_id: - ids.append(label_to_id[label]) - else: - ids.append(label) - return validate_facets(ids, modalities=modalities) - - -def build_intent( - answers: Mapping[str, object], - assist: Optional[Callable[["DomainIntent"], Mapping[str, object]]] = None, -) -> DomainIntent: - """Validate raw answers into a DomainIntent. Deterministic given answers. - - If ``assist`` is provided it receives the validated intent and may return a - mapping of overrides; those overrides are merged and the intent is rebuilt - through this same validator (assist can never bypass validation). - """ - domain = str(answers.get("domain", "")).strip() - if not domain: - raise ValueError("domain is required") - - modalities_raw = answers.get("modalities") - if modalities_raw is None or modalities_raw == "": - modalities = detect_modalities(domain) - keyword_text = " ".join(_split_csv(answers.get("keywords"))) - if keyword_text: - modalities = _dedupe(list(modalities) + list(detect_modalities(keyword_text))) - else: - modalities = _resolve_modality_ids(modalities_raw) - - facets_raw = answers.get("facets") - detect_text = "{0} {1}".format(domain, " ".join(_split_csv(answers.get("keywords")))) - if facets_raw is None or facets_raw == "": - facets = resolve_facets(modalities, cli=(), text=detect_text) if modalities else () - else: - facets = _resolve_facet_ids(facets_raw, modalities) - - roles = _resolve_roles(answers.get("roles")) - keywords = _split_csv(answers.get("keywords")) - if modalities: - roles, keywords = apply_modality_profile(roles, keywords, modalities, facets) - if not roles: - roles = ("general",) - - intent = DomainIntent( - domain=domain, - roles=roles, - keywords=keywords, - license_allow=_split_csv( - answers["license"] if "license" in answers else answers.get("license_allow") - ), - memory_gb=_resolve_memory(answers.get("memory_gb")), - notes=str(answers.get("notes", "")).strip(), - modalities=modalities, - facets=facets, - ) - if assist is None: - return intent - overrides = assist(intent) - if not overrides: - return intent - merged = intent.to_dict() - merged.update(dict(overrides)) - return build_intent(merged, assist=None) - - -def run_interview( - reader: Callable[[Mapping[str, object]], object], - assist: Optional[Callable[["DomainIntent"], Mapping[str, object]]] = None, - preset_modalities: Sequence[str] = (), - preset_facets: Sequence[str] = (), -) -> DomainIntent: - """Ask questions via ``reader`` with detect-then-ask modality resolution.""" - answers: Dict[str, object] = {} - domain_question = QUESTIONS[0] - answers["domain"] = reader(domain_question) - domain = str(answers["domain"]).strip() - - modalities = validate_modalities(preset_modalities) if preset_modalities else detect_modalities(domain) - if not modalities: - modality_question = { - "id": "modalities", - "prompt": ( - "Which foundational modalities apply? " - "(audio, video, document extraction / vision)" - ), - "kind": "multi", - "choices": tuple(MODALITY_CHOICES.keys()), - } - modalities = _resolve_modality_ids(reader(modality_question)) - if not modalities: - raise ValueError("at least one foundational modality is required") - answers["modalities"] = list(modalities) - - facets: List[str] = list(validate_facets(preset_facets, modalities=modalities)) if preset_facets else [] - if not facets: - for modality in modalities: - detected = detect_facets(modality, domain) - if detected: - facets.extend(detected) - else: - facet_question = { - "id": "facets:{0}".format(modality), - "prompt": "Which {0} facets matter? (optional multi-select)".format(modality), - "kind": "multi", - "choices": tuple(FACET_CHOICES[modality].keys()), - } - response = reader(facet_question) - if response is None or response == "": - continue - facets.extend(_resolve_facet_ids(response, (modality,))) - if not facets and modalities: - facets = list(resolve_facets(modalities, cli=(), text=domain)) - answers["facets"] = list(_dedupe(facets)) - - for question in _FOLLOW_ON_QUESTIONS: - answers[str(question["id"])] = reader(question) - return build_intent(answers, assist=assist) diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/lora.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/lora.py deleted file mode 100644 index 336b859..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/lora.py +++ /dev/null @@ -1,326 +0,0 @@ -"""Confirmation-gated LoRA training jobs on the convert machinery.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from pathlib import Path - -from .convert import ( - _read_receipt, - _write_receipt, - receipts_root, -) -from .serve import _argv_matches, _pid_alive, _pid_command -from .wiring import _MODEL - - -LORA_RECEIPT_SCHEMA_VERSION = "1.0" -LORA_RECEIPT_KIND = "lora" -EXECUTABLE = "mlx_lm.lora" -ITERS_MIN, ITERS_MAX, ITERS_DEFAULT = 1, 100000, 1000 -BATCH_MIN, BATCH_MAX, BATCH_DEFAULT = 1, 64, 4 -LR_MIN, LR_MAX, LR_DEFAULT = 1e-6, 1e-2, 1e-5 -LAYERS_MIN, LAYERS_MAX, LAYERS_DEFAULT = -1, 128, 16 -MAX_DATASET_LINES = 10000 -DATASET_SAMPLE_LINES = 50 - - -class LoraError(RuntimeError): - """Classified lora failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def validate_dataset(data_dir): - """Validate an mlx_lm training dataset directory; bounded and read-only.""" - root = Path(data_dir) - train = root / "train.jsonl" - if not root.is_dir() or not train.is_file(): - raise LoraError( - "dataset_invalid", - "The dataset directory must contain train.jsonl: {0}".format(data_dir), - "Provide a directory with train.jsonl (text or messages per line).", - ) - summary = {"train_lines": _validate_jsonl(train), "valid_lines": 0} - valid = root / "valid.jsonl" - if valid.is_file(): - summary["valid_lines"] = _validate_jsonl(valid) - return summary - - -def _validate_jsonl(path): - count = 0 - try: - with open(path, "r", encoding="utf-8") as handle: - for line in handle: - count += 1 - if count > MAX_DATASET_LINES: - raise LoraError( - "dataset_invalid", - "{0} exceeds {1} lines.".format(path.name, MAX_DATASET_LINES), - "Split the dataset; lora validates bounded datasets only.", - ) - if count > DATASET_SAMPLE_LINES: - continue - stripped = line.strip() - if not stripped: - continue - try: - record = json.loads(stripped) - except json.JSONDecodeError: - raise LoraError( - "dataset_invalid", - "{0} line {1} is not valid JSON.".format(path.name, count), - "Fix the dataset line; every line must be one JSON object.", - ) - _validate_record(record, path.name, count) - except OSError as error: - raise LoraError( - "dataset_invalid", - "The dataset could not be read: {0}".format(error), - "Pass a readable dataset directory.", - ) - if count == 0: - raise LoraError( - "dataset_invalid", - "{0} is empty.".format(path.name), - "Provide at least one training example.", - ) - return count - - -def _validate_record(record, name, line): - if not isinstance(record, dict): - raise LoraError( - "dataset_invalid", - "{0} line {1} must be a JSON object.".format(name, line), - "Use {\"text\": ...} or {\"messages\": [...]} per line.", - ) - text = record.get("text") - if isinstance(text, str) and text.strip(): - return - messages = record.get("messages") - if isinstance(messages, list) and messages: - for message in messages: - if ( - not isinstance(message, dict) - or not isinstance(message.get("role"), str) - or not isinstance(message.get("content"), str) - ): - raise LoraError( - "dataset_invalid", - "{0} line {1} has a malformed message.".format(name, line), - "Each message needs string role and content fields.", - ) - return - raise LoraError( - "dataset_invalid", - "{0} line {1} needs a non-empty text field or a messages array.".format(name, line), - "Use {\"text\": ...} or {\"messages\": [{\"role\": ..., \"content\": ...}]}.", - ) - - -def _validate_hyperparameters(iters, batch_size, learning_rate, num_layers): - if not isinstance(iters, int) or isinstance(iters, bool) or not ITERS_MIN <= iters <= ITERS_MAX: - raise LoraError( - "invalid_arguments", - "iters must be between {0} and {1}.".format(ITERS_MIN, ITERS_MAX), - "Pass a bounded --iters value.", - ) - if not isinstance(batch_size, int) or isinstance(batch_size, bool) or not BATCH_MIN <= batch_size <= BATCH_MAX: - raise LoraError( - "invalid_arguments", - "batch-size must be between {0} and {1}.".format(BATCH_MIN, BATCH_MAX), - "Pass a bounded --batch-size value.", - ) - if ( - not isinstance(learning_rate, (int, float)) - or isinstance(learning_rate, bool) - or not LR_MIN <= float(learning_rate) <= LR_MAX - ): - raise LoraError( - "invalid_arguments", - "learning-rate must be between {0} and {1}.".format(LR_MIN, LR_MAX), - "Pass a bounded --learning-rate value.", - ) - if not isinstance(num_layers, int) or isinstance(num_layers, bool) or not LAYERS_MIN <= num_layers <= LAYERS_MAX: - raise LoraError( - "invalid_arguments", - "num-layers must be between {0} and {1}.".format(LAYERS_MIN, LAYERS_MAX), - "Pass a bounded --num-layers value (-1 trains all layers).", - ) - - -def plan_lora(repo, data, iters=ITERS_DEFAULT, batch_size=BATCH_DEFAULT, - learning_rate=LR_DEFAULT, num_layers=LAYERS_DEFAULT, out=None): - """Validate inputs and render the exact training plan; side-effect free.""" - if not isinstance(repo, str) or not _MODEL.fullmatch(repo): - raise LoraError( - "invalid_repo", - "lora requires a safe publisher/model base identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - _validate_hyperparameters(iters, batch_size, learning_rate, num_layers) - dataset = validate_dataset(data) - if out is None: - out = "{0}-lora".format(repo.split("/", 1)[1]) - plan = { - "repo": repo, - "data": str(data), - "iters": iters, - "batch_size": batch_size, - "learning_rate": float(learning_rate), - "num_layers": num_layers, - "out": str(out), - "dataset": dataset, - "argv": [ - EXECUTABLE, - "--model", repo, - "--train", - "--data", str(data), - "--iters", str(iters), - "--batch-size", str(batch_size), - "--learning-rate", repr(float(learning_rate)), - "--num-layers", str(num_layers), - "--adapter-path", str(out), - ], - } - canonical = json.dumps( - {key: plan[key] for key in sorted(plan) if key != "preview_hash"}, - sort_keys=True, separators=(",", ":"), - ) - plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() - return plan - - -def _receipt_filename(plan): - return "{0}-lora.json".format(plan["repo"].split("/", 1)[1]) - - -def start_lora(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, spawn=None, now=_utc_now, - pid_alive=None): - """Execute a reviewed training plan; the only mutating entry point.""" - from .serve import _default_spawn, _default_which - - which = which or _default_which - spawn = spawn or _default_spawn - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir, LORA_RECEIPT_KIND) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise LoraError( - "preview_hash_required", - "--confirm requires the hash from a reviewed lora preview.", - "Run lora start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise LoraError( - "preview_stale", - "The supplied preview hash does not match this lora plan.", - "Re-run lora start without --confirm and review the fresh plan.", - ) - if which(EXECUTABLE) is None: - raise LoraError( - "runtime_not_installed", - "The {0} executable is not installed.".format(EXECUTABLE), - "Install mlx-lm yourself (pip install mlx-lm); lora never installs runtimes.", - ) - if model_present is not None and not model_present(plan["repo"]): - raise LoraError( - "model_not_local", - "The base model is not present in the local Hugging Face cache.", - "Download it with the runtime's own pull command first; lora never downloads.", - ) - if Path(plan["out"]).exists(): - raise LoraError( - "output_exists", - "The adapter output path already exists: {0}".format(plan["out"]), - "Pick a fresh --out path; lora never overwrites.", - ) - for receipt_path in sorted(root.glob("*.json")) if root.is_dir() else []: - existing = _read_receipt(receipt_path, LORA_RECEIPT_KIND) - if existing is None or existing.get("completed_at"): - continue - if pid_alive(existing["pid"]): - raise LoraError( - "job_in_progress", - "A lora job is still running (pid {0}).".format(existing["pid"]), - "Wait for it to finish (lora status) before starting another.", - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}-lora.log".format(plan["repo"].split("/", 1)[1]) - pid = spawn(plan["argv"], str(log_path)) - receipt = { - "schema_version": LORA_RECEIPT_SCHEMA_VERSION, - "kind": LORA_RECEIPT_KIND, - "repo": plan["repo"], - "data": plan["data"], - "iters": plan["iters"], - "out": plan["out"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "started_at": now(), - "preview_hash": plan["preview_hash"], - "completed_at": None, - "exit_status": None, - } - _write_receipt(root, receipt, _receipt_filename(plan)) - return {"status": "started", "receipt": receipt} - - -def status_lora(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - now=_utc_now): - """Cross-check lora receipts against live processes; marks exits once.""" - root = receipts_root(receipts_dir, LORA_RECEIPT_KIND) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - receipt = _read_receipt(path, LORA_RECEIPT_KIND) - if receipt is None: - continue - entry = { - "receipt": str(path), - "repo": receipt["repo"], - "data": receipt.get("data"), - "out": receipt["out"], - "pid": receipt["pid"], - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - "completed_at": receipt.get("completed_at"), - } - if receipt.get("completed_at"): - entry["state"] = receipt.get("exit_status") or "done" - entries.append(entry) - continue - if pid_alive(receipt["pid"]): - command = pid_command(receipt["pid"]) - entry["state"] = "running" if _argv_matches( - {"argv": receipt["argv"], "port": None, "repo": receipt["repo"]}, - command, - require_port=False, - ) else "unknown" - entries.append(entry) - continue - exit_status = "done" if Path(receipt["out"]).exists() else "failed" - receipt["completed_at"] = now() - receipt["exit_status"] = exit_status - _write_receipt(root, receipt, path.name) - entry["state"] = exit_status - entry["completed_at"] = receipt["completed_at"] - entries.append(entry) - return entries diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/modality.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/modality.py deleted file mode 100644 index 03205fb..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/modality.py +++ /dev/null @@ -1,283 +0,0 @@ -"""Foundational modality layers for research intent seeding. - -Three foundations (audio, video, document-vision) seed existing discovery roles -and keywords. No new discovery roles or runtimes. Detection is deterministic -lexicon matching only — no LLM classification. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Dict, Iterable, Mapping, Optional, Sequence, Tuple - - -@dataclass(frozen=True) -class FacetProfile: - id: str - label: str - lexicon: Tuple[str, ...] - seed_keywords: Tuple[str, ...] - - -@dataclass(frozen=True) -class ModalityProfile: - id: str - label: str - lexicon: Tuple[str, ...] - default_roles: Tuple[str, ...] - seed_keywords: Tuple[str, ...] - facets: Tuple[FacetProfile, ...] - - -MODALITY_PROFILES: Tuple[ModalityProfile, ...] = ( - ModalityProfile( - id="audio", - label="Audio (ASR / TTS / music)", - lexicon=( - "audio", "speech", "whisper", "asr", "tts", "voice", "music", - "sound", "transcri", - ), - default_roles=("general",), - seed_keywords=("audio", "speech", "whisper"), - facets=( - FacetProfile( - id="asr", - label="ASR / speech-to-text", - lexicon=("asr", "speech-to-text", "speech to text", "transcri", "whisper", "stt"), - seed_keywords=("asr", "whisper", "transcription", "speech-to-text"), - ), - FacetProfile( - id="tts", - label="TTS / text-to-speech", - lexicon=("tts", "text-to-speech", "text to speech", "narrat", "voice synthes"), - seed_keywords=("tts", "text-to-speech", "speech synthesis"), - ), - FacetProfile( - id="music", - label="Music", - lexicon=("music", "melody", "song", "soundtrack", "audio generation"), - seed_keywords=("music", "melody", "song"), - ), - ), - ), - ModalityProfile( - id="video", - label="Video", - lexicon=("video", "footage", "clip", "mp4", "frames"), - default_roles=("vision", "general"), - seed_keywords=("video",), - facets=( - FacetProfile( - id="understanding", - label="Understanding / captioning", - lexicon=("understanding", "caption", "describe video", "video-llm", "vlm video"), - seed_keywords=("video", "caption", "understanding"), - ), - FacetProfile( - id="generation", - label="Generation", - lexicon=("generation", "generate video", "diffusion", "text-to-video", "text to video"), - seed_keywords=("video", "generation", "diffusion", "text-to-video"), - ), - FacetProfile( - id="action", - label="Action / tracking", - lexicon=("action recognition", "action", "tracking", "pose", "motion"), - seed_keywords=("video", "action", "tracking"), - ), - ), - ), - ModalityProfile( - id="document-vision", - label="Document extraction / vision", - lexicon=( - "ocr", "document", "pdf", "scanned", "invoice", "layout", - "handwriting", "vision", "vlm", "image understanding", - ), - default_roles=("vision",), - seed_keywords=("ocr", "document", "vision"), - facets=( - FacetProfile( - id="ocr", - label="OCR / text extraction", - lexicon=("ocr", "optical character", "scanned", "handwriting", "text extraction"), - seed_keywords=("ocr", "handwriting", "scanned"), - ), - FacetProfile( - id="layout", - label="Layout / structure", - lexicon=("layout", "form parsing", "table extraction", "document structure"), - seed_keywords=("layout", "document", "pdf"), - ), - FacetProfile( - id="general-vision", - label="General vision / VLM", - lexicon=("general vision", "vlm", "image caption", "visual qa", "vision-language"), - seed_keywords=("vision", "vlm", "image"), - ), - ), - ), -) - -FOUNDATION_IDS: Tuple[str, ...] = tuple(profile.id for profile in MODALITY_PROFILES) -ALL_FACET_IDS: Tuple[str, ...] = tuple( - facet.id for profile in MODALITY_PROFILES for facet in profile.facets -) -MODALITY_CHOICES: Dict[str, str] = {profile.label: profile.id for profile in MODALITY_PROFILES} -FACET_CHOICES: Dict[str, Dict[str, str]] = { - profile.id: {facet.label: facet.id for facet in profile.facets} - for profile in MODALITY_PROFILES -} - -_PROFILES_BY_ID: Mapping[str, ModalityProfile] = {profile.id: profile for profile in MODALITY_PROFILES} -_FACETS_BY_ID: Mapping[str, FacetProfile] = { - facet.id: facet for profile in MODALITY_PROFILES for facet in profile.facets -} -_FACET_TO_MODALITY: Mapping[str, str] = { - facet.id: profile.id for profile in MODALITY_PROFILES for facet in profile.facets -} - - -def _dedupe(values: Iterable[str]) -> Tuple[str, ...]: - seen = [] - for value in values: - if value and value not in seen: - seen.append(value) - return tuple(seen) - - -def _haystack(text: str) -> str: - return " ".join(str(text or "").lower().split()) - - -def _lexicon_hit(haystack: str, lexicon: Sequence[str]) -> bool: - return any(token in haystack for token in lexicon) - - -def detect_modalities(text: str) -> Tuple[str, ...]: - """Return foundation ids matched by deterministic lexicon hits.""" - haystack = _haystack(text) - if not haystack: - return () - hits = [] - for profile in MODALITY_PROFILES: - if _lexicon_hit(haystack, profile.lexicon): - hits.append(profile.id) - return tuple(hits) - - -def detect_facets(modality: str, text: str) -> Tuple[str, ...]: - """Return facet ids for one foundation matched in text.""" - profile = _PROFILES_BY_ID.get(modality) - if profile is None: - raise ValueError("unknown modality: {0}".format(modality)) - haystack = _haystack(text) - if not haystack: - return () - hits = [] - for facet in profile.facets: - if _lexicon_hit(haystack, facet.lexicon): - hits.append(facet.id) - return tuple(hits) - - -def validate_modalities(modalities: Sequence[str]) -> Tuple[str, ...]: - resolved = _dedupe([str(item).strip() for item in modalities if str(item).strip()]) - for modality in resolved: - if modality not in _PROFILES_BY_ID: - raise ValueError("unknown modality: {0}".format(modality)) - return resolved - - -def validate_facets(facets: Sequence[str], modalities: Optional[Sequence[str]] = None) -> Tuple[str, ...]: - resolved = _dedupe([str(item).strip() for item in facets if str(item).strip()]) - allowed = None - if modalities is not None: - allowed = set() - for modality in validate_modalities(modalities): - allowed.update(facet.id for facet in _PROFILES_BY_ID[modality].facets) - for facet in resolved: - if facet not in _FACETS_BY_ID: - raise ValueError("unknown facet: {0}".format(facet)) - if allowed is not None and facet not in allowed: - raise ValueError( - "facet {0} is not valid for modalities {1}".format( - facet, ",".join(modalities) or "none" - ) - ) - return resolved - - -def resolve_modalities(cli: Sequence[str] = (), text: str = "") -> Tuple[str, ...]: - """CLI modalities win; otherwise detect from text.""" - if cli: - return validate_modalities(cli) - return detect_modalities(text) - - -def resolve_facets( - modalities: Sequence[str], - cli: Sequence[str] = (), - text: str = "", -) -> Tuple[str, ...]: - """Resolve facets: CLI wins; else detect; else seed all facets for each modality.""" - modalities = validate_modalities(modalities) - if not modalities: - return () - if cli: - return validate_facets(cli, modalities=modalities) - detected = [] - for modality in modalities: - detected.extend(detect_facets(modality, text)) - detected = _dedupe(detected) - if detected: - return validate_facets(detected, modalities=modalities) - # Lock: seed all facet ids for selected foundations when none detected. - all_facets = [] - for modality in modalities: - all_facets.extend(facet.id for facet in _PROFILES_BY_ID[modality].facets) - return tuple(all_facets) - - -def apply_modality_profile( - roles: Sequence[str], - keywords: Sequence[str], - modalities: Sequence[str], - facets: Sequence[str] = (), -) -> Tuple[Tuple[str, ...], Tuple[str, ...]]: - """Union user roles/keywords with profile seeds. User values are preserved.""" - modalities = validate_modalities(modalities) - facets = validate_facets(facets, modalities=modalities) if facets else () - seeded_roles = list(roles) - seeded_keywords = list(keywords) - for modality in modalities: - profile = _PROFILES_BY_ID[modality] - for role in profile.default_roles: - if role not in seeded_roles: - seeded_roles.append(role) - for keyword in profile.seed_keywords: - if keyword not in seeded_keywords: - seeded_keywords.append(keyword) - facet_ids = facets - if modalities and not facet_ids: - facet_ids = resolve_facets(modalities, cli=(), text="") - for facet_id in facet_ids: - facet = _FACETS_BY_ID[facet_id] - for keyword in facet.seed_keywords: - if keyword not in seeded_keywords: - seeded_keywords.append(keyword) - return tuple(seeded_roles), tuple(seeded_keywords) - - -def modality_label(modality_id: str) -> str: - profile = _PROFILES_BY_ID.get(modality_id) - if profile is None: - raise ValueError("unknown modality: {0}".format(modality_id)) - return profile.label - - -def facet_label(facet_id: str) -> str: - facet = _FACETS_BY_ID.get(facet_id) - if facet is None: - raise ValueError("unknown facet: {0}".format(facet_id)) - return facet.label diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/model_doctor.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/model_doctor.py deleted file mode 100644 index b10511a..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/model_doctor.py +++ /dev/null @@ -1,495 +0,0 @@ -"""Read-only model inventory, wiring drift, and endpoint health diagnostics. - -Doctor never deletes, moves, repairs, downloads, or starts anything. It reads -the Hugging Face cache, queries already-running loopback runtimes, parses -Wire-managed configuration, and reports classified findings with remediation -text. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path - -from .verification import _safe_error, installed_model_ids -from .wiring import ( - _OLLAMA, - _STATE_CHANGING_PATH_PARTS, - ConfigAdapter, - validate_health_endpoint, -) - - -MAX_INVENTORY_ENTRIES = 500 -MAX_FINDINGS = 100 -MAX_ENDPOINTS = 50 -MAX_WALK_FILES = 2000 -MAX_CONFIG_BYTES = 1024 * 1024 -MAX_RECEIPTS = 50 -_WIRE_MARKER = "MLX_AGENT_WIRE" -_SKIP_DIRECTORIES = { - ".git", ".hg", ".svn", "node_modules", "__pycache__", ".venv", "venv", - ".tox", ".mypy_cache", ".pytest_cache", "Library", -} -_CONFIG_SUFFIXES = {".json", ".yaml", ".yml", ".modelfile"} -_HEALTH_TIMEOUT = 2.0 - - -def default_hf_cache(): - return Path.home() / ".cache" / "huggingface" / "hub" - - -def inspect_hf_cache(cache_dir): - """Summarize model repos in a Hugging Face cache without reading weights.""" - root = Path(cache_dir) - inventory = [] - if not root.is_dir(): - return inventory - for entry in sorted(root.iterdir()): - if len(inventory) >= MAX_INVENTORY_ENTRIES: - break - if not entry.is_dir() or not entry.name.startswith("models--"): - continue - repo = entry.name[len("models--"):].replace("--", "/") - refs_main = _read_text_quiet(entry / "refs" / "main") - snapshots = entry / "snapshots" - revisions = [] - total_bytes = 0 - missing_blobs = 0 - seen_blobs = set() - if snapshots.is_dir(): - for revision in sorted(snapshots.iterdir()): - if not revision.is_dir(): - continue - revisions.append(revision.name) - for dirpath, dirnames, filenames in os.walk(revision): - dirnames[:] = [name for name in dirnames if not name.startswith(".")] - for filename in filenames: - location = Path(dirpath) / filename - try: - resolved = location.resolve(strict=True) - except OSError: - missing_blobs += 1 - continue - if resolved in seen_blobs: - continue - seen_blobs.add(resolved) - try: - total_bytes += resolved.stat().st_size - except OSError: - missing_blobs += 1 - complete = bool(refs_main and revisions and missing_blobs == 0) - inventory.append({ - "id": repo, - "source": "hf-cache", - "bytes": total_bytes, - "revisions": revisions[:8], - "complete": complete, - }) - return inventory - - -def _read_text_quiet(path): - try: - return path.read_text(encoding="utf-8").strip() or None - except OSError: - return None - - -def collect_runtime_inventory(runtime_clients): - """Collect served model IDs from already-running loopback runtimes.""" - inventory = [] - errors = [] - for runtime in runtime_clients: - name = str(getattr(runtime, "name", runtime.__class__.__name__)) - try: - response = runtime.list_models() - for model_id in sorted(installed_model_ids(response)): - if len(inventory) >= MAX_INVENTORY_ENTRIES: - break - inventory.append({ - "id": model_id, - "source": name, - "bytes": _runtime_model_size(response, model_id), - "complete": True, - }) - except Exception as error: - errors.append("{0}: {1}".format(name, _safe_error(error))) - return inventory, errors - - -def _runtime_model_size(response, model_id): - if not isinstance(response, dict): - return None - for record in response.get("models", []) or []: - if not isinstance(record, dict): - continue - if str(record.get("name") or record.get("id") or "") != model_id: - continue - size = record.get("size") - if isinstance(size, int) and not isinstance(size, bool) and size >= 0: - return size - return None - - -def scan_wired_configs(roots): - """Find Wire-managed configs under roots and extract their references.""" - wired = [] - for root in roots: - for location in _walk_config_files(root): - if len(wired) >= MAX_ENDPOINTS: - return wired - content = _read_bounded(location) - if content is None or _WIRE_MARKER not in content: - continue - try: - adapter = ConfigAdapter.detect(location) - reference = adapter_references(adapter, content) - except (TypeError, ValueError): - continue - if reference is None: - continue - wired.append({ - "path": str(location), - "runtime": reference.get("runtime") or adapter.runtime, - "model": reference.get("model"), - "endpoint": reference.get("endpoint"), - }) - return wired - - -def _walk_config_files(root): - root = Path(root) - if root.is_file(): - yield root - return - if not root.is_dir(): - return - yielded = 0 - for dirpath, dirnames, filenames in os.walk(root): - dirnames[:] = [ - name for name in dirnames - if name not in _SKIP_DIRECTORIES and not name.startswith(".mlx-agent-receipts") - ] - for filename in sorted(filenames): - if yielded >= MAX_WALK_FILES: - return - location = Path(dirpath) / filename - if location.suffix.lower() not in _CONFIG_SUFFIXES and filename.lower() != "modelfile": - continue - yielded += 1 - yield location - - -def _read_bounded(location): - try: - if location.stat().st_size > MAX_CONFIG_BYTES: - return None - return location.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - return None - - -def adapter_references(adapter, content): - """Extract the model and endpoint a Wire-managed config references.""" - if adapter.runtime == "ollama": - match = _OLLAMA.fullmatch(content) - if match is None: - return None - return {"model": match.group(1), "endpoint": None} - if adapter.runtime == "litellm": - model = re.search(r"(?m)^ model: openai/([^\s]+)$", content) - base = re.search(r"(?m)^ api_base: (http[^\s]+)$", content) - if model is None: - return None - return { - "model": model.group(1), - "endpoint": base.group(1) if base else None, - } - try: - document = json.loads(content) - except (TypeError, ValueError, json.JSONDecodeError): - return None - if not isinstance(document, dict): - return None - marker = document.get("mlx_agent_wire") - if not isinstance(marker, dict): - return None - provider = marker.get("provider") if isinstance(marker.get("provider"), dict) else {} - runtime = marker.get("runtime") - return { - "model": marker.get("model"), - "endpoint": provider.get("base_url"), - "runtime": runtime if isinstance(runtime, str) else None, - } - - -def scan_receipt_after_hashes(roots): - """Read after_hashes from Wire receipts without enforcing receipt validity.""" - records = [] - for root in roots: - root = Path(root) - candidates = [] - if (root / ".mlx-agent-receipts").is_dir(): - candidates.append(root / ".mlx-agent-receipts") - else: - for dirpath, dirnames, _filenames in os.walk(root): - if ".mlx-agent-receipts" in dirnames: - candidates.append(Path(dirpath) / ".mlx-agent-receipts") - dirnames.remove(".mlx-agent-receipts") - for receipts_dir in candidates: - for receipt in sorted(receipts_dir.glob("*/receipt.json")): - if len(records) >= MAX_RECEIPTS: - return records - try: - value = json.loads(receipt.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - continue - after_hashes = value.get("after_hashes") - if not isinstance(after_hashes, dict): - continue - records.append({ - "receipt": str(receipt), - "transaction_id": value.get("transaction_id"), - "status": value.get("status"), - "after_hashes": { - str(key): str(item) for key, item in after_hashes.items() - }, - }) - return records - - -def check_endpoint(endpoint, timeout=_HEALTH_TIMEOUT): - """Classify a wired loopback endpoint as ok, reachable, or down.""" - if not endpoint: - return None - try: - url = validate_health_endpoint(endpoint) - parsed_path = urllib.parse.urlparse(url).path.rstrip("/") - if parsed_path.endswith("/v1"): - url = url.rstrip("/") + "/models" - request = urllib.request.Request(url, method="GET") - opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - with opener.open(request, timeout=timeout) as response: - status = response.status - except urllib.error.HTTPError as error: - status = error.code - except (OSError, ValueError, urllib.error.URLError): - return "down" - return "ok" if 200 <= status < 400 else "reachable" - - -def run_model_doctor(wired_roots, hf_cache, runtime_clients): - """Run every read-only check and return the doctor report.""" - wired_roots = [Path(root) for root in wired_roots] - inventory = inspect_hf_cache(hf_cache) - runtime_inventory, runtime_errors = collect_runtime_inventory(runtime_clients) - inventory.extend(runtime_inventory) - known_ids = {item["id"].casefold() for item in inventory} - - wired = scan_wired_configs(wired_roots) - receipts = scan_receipt_after_hashes(wired_roots) - - findings = [] - for config in wired: - model = config.get("model") - if model and model.casefold() not in known_ids: - findings.append({ - "code": "drift_missing_model", - "path": config["path"], - "model": model, - "remediation": ( - "Re-download the model into a local inventory, or re-run " - "wire render for a model that is present." - ), - }) - - for receipt in receipts: - for target, expected in receipt["after_hashes"].items(): - location = Path(target) - if not location.is_file(): - findings.append({ - "code": "drift_missing_file", - "path": target, - "receipt_id": receipt["transaction_id"], - "remediation": ( - "The wired file is gone; restore it from the receipt " - "backup or roll the receipt back." - ), - }) - continue - try: - digest = hashlib.sha256(location.read_bytes()).hexdigest() - except OSError: - continue - if digest != expected: - findings.append({ - "code": "drift_hash_mismatch", - "path": target, - "receipt_id": receipt["transaction_id"], - "remediation": ( - "The wired file changed after wiring; inspect it, then " - "re-apply or roll back the receipt." - ), - }) - - endpoints = [] - claims = {} - for config in wired: - endpoint = config.get("endpoint") - if not endpoint: - continue - origin = _endpoint_origin_key(endpoint) - if origin is None: - continue - previous = claims.get(origin) - if previous is not None and previous["runtime"] != config["runtime"]: - findings.append({ - "code": "drift_endpoint_conflict", - "path": config["path"], - "model": config.get("model"), - "remediation": ( - "Two wired configs for different runtimes claim {0}; " - "move one runtime to its own port.".format(origin) - ), - }) - else: - claims.setdefault(origin, config) - if len(endpoints) < MAX_ENDPOINTS: - endpoints.append({ - "endpoint": origin, - "path": config["path"], - "status": check_endpoint(endpoint), - }) - - for item in inventory: - if item["source"] == "hf-cache" and not item.get("complete", True): - findings.append({ - "code": "hf_cache_incomplete", - "path": None, - "model": item["id"], - "remediation": ( - "The cached snapshot is incomplete; re-download the model " - "with the runtime's own pull command." - ), - }) - - findings = findings[:MAX_FINDINGS] - total_bytes = sum( - item["bytes"] or 0 for item in inventory if item["source"] == "hf-cache" - ) - return { - "inventory": inventory, - "summary": { - "models": len(inventory), - "hf_cache_bytes": total_bytes, - "wired_configs": len(wired), - "findings": len(findings), - }, - "wired": wired, - "endpoints": endpoints, - "findings": findings, - "runtime_errors": runtime_errors, - } - - -def _endpoint_origin_key(endpoint): - try: - url = validate_health_endpoint(endpoint) - except ValueError: - return None - parsed = urllib.parse.urlparse(url) - port = parsed.port or (443 if parsed.scheme == "https" else 80) - decoded = parsed.path or "/" - parts = {part for part in decoded.lower().split("/") if part} - if parts & _STATE_CHANGING_PATH_PARTS: - return None - return "{0}://{1}:{2}".format(parsed.scheme.lower(), parsed.hostname.lower(), port) - - -PRUNE_PREVIEW_VERSION = "1.0" -MAX_PRUNE_CANDIDATES = 100 - - -class PruneError(RuntimeError): - """Classified prune failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def plan_prune(cache_dir): - """List incomplete HF cache snapshots eligible for deletion; read-only.""" - root = Path(cache_dir) - candidates = [] - for item in inspect_hf_cache(root): - if item.get("complete", True): - continue - location = root / "models--{0}".format(item["id"].replace("/", "--")) - candidates.append({ - "repo": item["id"], - "path": str(location), - "bytes": item.get("bytes") or 0, - }) - if len(candidates) >= MAX_PRUNE_CANDIDATES: - break - canonical = json.dumps(candidates, sort_keys=True, separators=(",", ":")) - return { - "cache": str(root), - "candidates": candidates, - "total_bytes": sum(item["bytes"] for item in candidates), - "preview_hash": hashlib.sha256(canonical.encode("utf-8")).hexdigest(), - } - - -def execute_prune(plan, confirm=False, preview_hash=None, remover=None): - """Delete exactly the reviewed candidates; irreversible by design.""" - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise PruneError( - "preview_hash_required", - "--confirm requires the hash from a reviewed prune preview.", - "Run doctor models --prune without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise PruneError( - "preview_stale", - "The supplied preview hash does not match the current prune plan.", - "Re-run doctor models --prune and review the fresh candidate list.", - ) - if remover is None: - import shutil - - def remover(path): - shutil.rmtree(path) - - cache = Path(plan["cache"]).resolve() - removed = [] - for candidate in plan["candidates"]: - location = Path(candidate["path"]) - resolved = location.resolve() - if resolved == cache or cache not in resolved.parents: - raise PruneError( - "unsafe_target", - "Refusing to remove a path outside the cache: {0}".format(candidate["path"]), - "Only cache-owned model directories are ever pruned.", - ) - if not location.is_dir(): - removed.append({"repo": candidate["repo"], "removed": False, "reason": "already gone"}) - continue - remover(str(location)) - removed.append({ - "repo": candidate["repo"], - "removed": not location.exists(), - "bytes": candidate["bytes"], - }) - return {"status": "pruned", "removed": removed} diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/models.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/models.py deleted file mode 100644 index ef170de..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/models.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Pure model classification, sizing, wiring, and rendering helpers.""" - -from __future__ import annotations - -import json -import re - -from .runtime_preference import prefer_runtime_for_role, wiring_for_preference - - -ROLES = [ - ("vision", ["-vl", "vision", "vlm", "ocr", "-omni"], "Vision / OCR (needs mlx-vlm)"), - ("embedding", ["embed", "embedding", "bge", "nomic", "gte"], "Embeddings"), - ("coding", ["coder", "-code", "devstral", "codestral", "starcoder"], "Coding"), - ("reasoning", ["gpt-oss", "-a3b", "thinking", "reason", "qwq", "-r1", "deepseek-r"], "Reasoning"), - ("general", ["instruct", "-it", "chat", "mistral", "gemma", "qwen", "llama", "phi"], "General"), -] -REASONER_HINTS = re.compile(r"(thinking|reason|-a3b\b|qwq|-r1\b|deepseek-r|gpt-oss)", re.I) -TEMPLATE_REASON = re.compile(r"(reasoning_effort||<\|think|channel\|>analysis)", re.I) -TOOL_USE_HINTS = re.compile( - r"\b(?:tool[-_]?use|(?:function|tool)(?:[-_]call|[ -]calling))\b", - re.I, -) -TEMPLATE_TOOL_USE = re.compile( - r"(?:|\btool_calls\b|\bfunction_call\b|\bavailable_tools\b|" - r"(?:\{\{|\{%)[^}]*\btools\b[^}]*(?:\}\}|%\}))", - re.I, -) -TOOL_USE_TAGS = { - "tool-use", - "tool_use", - "function-calling", - "function_calling", - "tools", -} -DISCOVERY_ROLES = tuple(role for role, _keywords, _label in ROLES) + ("tool-use",) -REPUTABLE = {"mlx-community", "lmstudio-community", "unsloth", "mlx-omni", "mlxvlm", "qwen", "google", "mistralai", "meta-llama", "nvidia"} -QUANT_TOK = re.compile(r"[-_.]?(mlx|mxfp4|nvfp4|dwq|optiq|turboquant|rotorquant|oq4e?|mtp|q\d(_k(_[ms])?)?|int[48]|fp16|fp8|bf16|\d+\.?\d*bpw|\d+bit|e[24]b|4bit|8bit)$", re.I) -PARAM_RE = re.compile(r"(\d+(?:\.\d+)?)\s*b(?![a-z])", re.I) -QUANT_GB_PER_B = [ - (re.compile(r"(fp16|bf16|-16bit|f16)", re.I), 2.0), - (re.compile(r"(8bit|-q8|int8|fp8|mxfp8)", re.I), 1.0), - (re.compile(r"(6bit|-q6)", re.I), 0.75), - (re.compile(r"(4bit|-q4|mxfp4|nvfp4|int4)", re.I), 0.55), - (re.compile(r"(3bit|-q3)", re.I), 0.45), - (re.compile(r"(2bit|-q2)", re.I), 0.35), -] -QUANT_RANK = [("bf16", 6), ("fp16", 6), ("8bit", 5), ("q8", 5), ("6bit", 4), ("q6", 4), ("4bit", 3), ("q4", 3), ("mxfp4", 3), ("3bit", 2), ("2bit", 1)] - - -def infer_quantization(value): - """Return a normalized name-derived quantization, never a measured property.""" - low = value.lower() - for name, patterns in [ - ("fp16", ("fp16", "bf16", "f16")), - ("8bit", ("8bit", "q8", "int8", "fp8")), - ("6bit", ("6bit", "q6")), - ("4bit", ("4bit", "q4", "int4", "mxfp4", "nvfp4")), - ("3bit", ("3bit", "q3")), - ("2bit", ("2bit", "q2")), - ]: - if any(pattern in low for pattern in patterns): - return name - return None - - -def name_ram_gb(repo): - match = PARAM_RE.search(repo) - if not match: - return None - for pattern, value in QUANT_GB_PER_B: - if pattern.search(repo): - return round(float(match.group(1)) * value, 1) - return round(float(match.group(1)) * 0.55, 1) - - -def classify(repo): - low = repo.lower() - for role, keywords, _label in ROLES: - if any(keyword in low for keyword in keywords): - return role - return "general" - - -def classify_roles(repo, enrichment): - """Return ordered role memberships and the canonical tool-use signal.""" - primary = classify(repo) - explicit = enrichment.get("tool_use") - if explicit is not None: - source = enrichment.get("tool_use_src") or "checked" - signal = { - "supported": bool(explicit), - "source": source, - "confidence": "weak" if source == "name" else "explicit", - } - elif TOOL_USE_HINTS.search(repo): - signal = { - "supported": True, - "source": "name", - "confidence": "weak", - } - else: - signal = { - "supported": False, - "source": "none", - "confidence": "none", - } - roles = (primary, "tool-use") if signal["supported"] else (primary,) - return roles, signal - - -def base_name(repo): - name = repo.split("/")[-1] - previous = None - while name != previous: - previous = name - name = QUANT_TOK.sub("", name).rstrip("-_.") - return name.lower() - - -def quant_rank(repo): - low = repo.lower() - for token, rank in QUANT_RANK: - if token in low: - return rank - return 0 - - -def resolve_ram(repo, enrichment): - if enrichment.get("weight_bytes"): - return round(enrichment["weight_bytes"] / 1e9, 1), "estimated_from_weight_bytes" - if enrichment.get("params_total"): - for pattern, value in QUANT_GB_PER_B: - if pattern.search(repo): - return round(enrichment["params_total"] * value / 1e9, 1), "estimated_from_parameter_count" - return round(enrichment["params_total"] * 0.55 / 1e9, 1), "estimated_from_parameter_count" - estimate = name_ram_gb(repo) - return (estimate, "estimated_from_name") if estimate is not None else (None, None) - - -def resolve_reasoning(repo, enrichment): - if enrichment.get("reasoning") is not None: - return enrichment["reasoning"], enrichment.get("reason_src") - return bool(REASONER_HINTS.search(repo)), "name" - - -def wiring(repo, role, host): - preference = prefer_runtime_for_role(role, host or {}) - return wiring_for_preference(repo, role, preference) - - -def render_md(report): - host = report["host"] - out = ["# mlx-scout report", ""] - out.append("**Host:** {0} · {1}GB · Ollama {2} · LM Studio {3}".format( - host.get("chip") or "?", host.get("ram_gb") or "?", "✓" if host["ollama"] else "✗", "✓" if host["lmstudio"] else "✗") + (" _(fast mode: name heuristics)_" if report.get("fast") else "")) - if report.get("error"): - return "\n".join(out + ["\n> ⚠ {0}".format(report["error"])]) - labels = {role: label for role, _keywords, label in ROLES} - for role, items in report["roles"].items(): - out.extend(["\n## {0}".format(labels.get(role, role)), "| model | RAM | reasoning | fits | license | wiring |", "|---|---|---|---|---|---|"]) - for item in items: - ram = "{0}GB".format(item["est_ram_gb"]) if item.get("est_ram_gb") is not None else "?" - reason = "⚠ {0}".format(item.get("reason_src") or "") if item.get("reasoning") else "no" - license_name = (item.get("license") or "?") + (" 🔒" if item.get("gated") else "") - out.append("| `{0}`{1} | {2} | {3} | {4} | {5} | {6} |".format(item["repo"], " ⭐" if item.get("trusted") else "", ram, reason, "✓" if item["fits"] else "✗", license_name, item["wiring"])) - out.append("\n_RAM is always an estimate; measured model-file bytes, when available, are retained separately as evidence. Add KV-cache headroom (~1–4GB) beyond weights._") - out.append("_`reasoning ⚠` sources: chat_template > tags > name (weakest). Verify with a test-generate._") - return "\n".join(out) - - -def wire(repo, target, port): - short = repo.split("/")[-1] - if target == "ollama": - return "\n".join(["# Ollama runs curated tags / HF GGUF repos (not arbitrary mlx-community weights).", "ollama pull hf.co/{0} # GGUF repos; else use the matching library tag".format(repo), "# agent ref: ollama/"]) - if target == "lmstudio": - return "\n".join(["lms get {0} # download into LM Studio".format(repo), "lms server start # OpenAI-compatible API on http://localhost:1234/v1", "# agent ref: lmstudio/{0}".format(short.lower())]) - if target == "mlx_lm": - return "\n".join(["pip install mlx-lm", "mlx_lm.server --model {0} --port {1} --max-tokens 8192".format(repo, port), "# openai-compatible provider:", json.dumps({"id": "mlxlm", "type": "openai", "baseURL": "http://127.0.0.1:{0}/v1".format(port), "apiKey": "local"}, indent=2), "# agent ref: mlxlm/{0}".format(short)]) - if target == "mlx-vlm": - return "\n".join(["pip install mlx-vlm", "mlx_vlm.server --model {0} --port {1}".format(repo, port), json.dumps({"id": "mlxvlm", "type": "openai", "baseURL": "http://127.0.0.1:{0}/v1".format(port), "apiKey": "local"}, indent=2), "# agent ref: mlxvlm/{0}".format(short)]) - if target == "litellm": - return "\n".join(["# litellm config.yaml (run a native server first, e.g. mlx_lm.server):", "model_list:", " - model_name: {0}".format(short.lower()), " litellm_params:", " model: openai/{0}".format(repo), " api_base: http://127.0.0.1:{0}/v1".format(port), " api_key: local"]) - return "unknown target: {0}".format(target) diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/probe_assets.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/probe_assets.py deleted file mode 100644 index 52e0c98..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/probe_assets.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Deterministic synthetic assets for role-fit verification probes.""" - -from __future__ import annotations - -import base64 -import struct -import zlib - - -VISION_PROBE_TEXT = "MLX42" - -_FONT_5X7 = { - "M": ( - "X...X", - "XX.XX", - "X.X.X", - "X.X.X", - "X...X", - "X...X", - "X...X", - ), - "L": ( - "X....", - "X....", - "X....", - "X....", - "X....", - "X....", - "XXXXX", - ), - "X": ( - "X...X", - "X...X", - ".X.X.", - "..X..", - ".X.X.", - "X...X", - "X...X", - ), - "4": ( - "...X.", - "..XX.", - ".X.X.", - "X..X.", - "XXXXX", - "...X.", - "...X.", - ), - "2": ( - ".XXX.", - "X...X", - "....X", - "...X.", - "..X..", - ".X...", - "XXXXX", - ), -} - -_cached_image_b64 = None - - -def _png_chunk(tag, data): - return ( - struct.pack(">I", len(data)) - + tag - + data - + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) - ) - - -def _png_gray(width, height, rows): - ihdr = struct.pack(">IIBBBBB", width, height, 8, 0, 0, 0, 0) - raw = b"".join(b"\x00" + bytes(row) for row in rows) - return ( - b"\x89PNG\r\n\x1a\n" - + _png_chunk(b"IHDR", ihdr) - + _png_chunk(b"IDAT", zlib.compress(raw)) - + _png_chunk(b"IEND", b"") - ) - - -def _render_text_bitmap(text, scale=6, padding=8): - glyphs = [_FONT_5X7[character] for character in text] - glyph_width = 5 - spacing = 1 - bitmap_width = len(glyphs) * (glyph_width + spacing) - spacing - bitmap_height = 7 - width = bitmap_width * scale + 2 * padding - height = bitmap_height * scale + 2 * padding - rows = [bytearray(width) for _ in range(height)] - for index, glyph in enumerate(glyphs): - origin_x = padding + index * (glyph_width + spacing) * scale - for gy, line in enumerate(glyph): - for gx, mark in enumerate(line): - if mark != "X": - continue - for dy in range(scale): - row = rows[padding + gy * scale + dy] - start = origin_x + gx * scale - for dx in range(scale): - row[start + dx] = 255 - return width, height, rows - - -def vision_probe_image_png(): - """Render the fixed probe word as a small deterministic grayscale PNG.""" - width, height, rows = _render_text_bitmap(VISION_PROBE_TEXT) - return _png_gray(width, height, rows) - - -def vision_probe_image_base64(): - global _cached_image_b64 - if _cached_image_b64 is None: - _cached_image_b64 = base64.b64encode(vision_probe_image_png()).decode("ascii") - return _cached_image_b64 diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/project_blueprint.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/project_blueprint.py deleted file mode 100644 index c0872fb..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/project_blueprint.py +++ /dev/null @@ -1,257 +0,0 @@ -"""MLX project design packs: guidance-only blueprints for downstream agents. - -No scaffolding, no downloads, no training. Deterministic templates from a short -project brief. Distinct from dataset blueprints in ``blueprint.py``. -""" - -from __future__ import annotations - -import json -import math -import os -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from typing import Dict, List, Mapping, Optional, Tuple - -from .modality import validate_modalities -from .research import slugify - - -@dataclass(frozen=True) -class ProjectBrief: - goal: str - modalities: Tuple[str, ...] = () - notes: str = "" - memory_gb: Optional[float] = None - - def to_dict(self) -> Dict[str, object]: - return { - "goal": self.goal, - "modalities": list(self.modalities), - "notes": self.notes, - "memory_gb": self.memory_gb, - } - - -@dataclass(frozen=True) -class ProjectDesignPack: - brief: ProjectBrief - generated_at: str - quantization_ideas: Tuple[str, ...] - training_loop: Tuple[str, ...] - lora_notes: Tuple[str, ...] - mtx_notes: Tuple[str, ...] - study_materials: Tuple[str, ...] - stack_path: Tuple[str, ...] - next_steps: Tuple[str, ...] - - def to_dict(self) -> Dict[str, object]: - return { - "brief": self.brief.to_dict(), - "generated_at": self.generated_at, - "quantization_ideas": list(self.quantization_ideas), - "training_loop": list(self.training_loop), - "lora_notes": list(self.lora_notes), - "mtx_notes": list(self.mtx_notes), - "study_materials": list(self.study_materials), - "stack_path": list(self.stack_path), - "next_steps": list(self.next_steps), - } - - -def _resolve_memory(raw: object) -> Optional[float]: - if raw is None: - return None - text = str(raw).strip() - if not text: - return None - try: - value = float(text) - except ValueError as error: - raise ValueError("memory_gb must be a number") from error - if not math.isfinite(value) or value <= 0: - raise ValueError("memory_gb must be a positive number") - return value - - -def build_brief(answers: Mapping[str, object]) -> ProjectBrief: - goal = str(answers.get("goal") or answers.get("domain") or "").strip() - if not goal: - raise ValueError("goal is required") - modalities_raw = answers.get("modalities") or () - if isinstance(modalities_raw, str): - modalities_raw = [part.strip() for part in modalities_raw.split(",") if part.strip()] - modalities = validate_modalities(list(modalities_raw)) if modalities_raw else () - return ProjectBrief( - goal=goal, - modalities=modalities, - notes=str(answers.get("notes", "")).strip(), - memory_gb=_resolve_memory(answers.get("memory_gb")), - ) - - -def generate_design_pack( - brief: ProjectBrief, - now: Optional[datetime] = None, -) -> ProjectDesignPack: - """Build a deterministic project design pack. Guidance only.""" - moment = now or datetime.now(timezone.utc) - visionish = any(item in brief.modalities for item in ("document-vision", "video")) - audio = "audio" in brief.modalities - memory = brief.memory_gb - - quant: List[str] = [ - "Start from published MLX community 4-bit / 8-bit weights when available.", - "Prefer 4-bit for chat/tooling under tight unified memory; keep 8-bit for quality-sensitive OCR or reasoning.", - "Record the exact Hub repo + quant tag in the research pack before fine-tuning.", - ] - if memory is not None: - quant.append( - "Respect the {0:g} GB budget with ~20% runtime headroom beyond weight size.".format(memory) - ) - if visionish: - quant.append( - "Vision stacks need mlx-vlm-compatible VLM quants; do not plan Ollama-only vision paths." - ) - if audio: - quant.append( - "Audio ASR/TTS often ships as specialty checkpoints — verify native MLX serve path before training plans." - ) - - training = ( - "Define a tiny offline JSONL/Parquet dataset before any train loop.", - "Dry-run one batch on Apple Silicon with mlx-lm / project tooling; abort on OOM.", - "Use LoRA/QLoRA-style adapters before full fine-tunes; keep base weights frozen.", - "Hold out 10–20% validation with no near-duplicate leakage.", - "This blueprint does not train or download — execution stays outside mlx-agent.", - ) - - lora = ( - "Search adapters in `mlx-agent research` (PEFT/LoRA section) before training new ones.", - "Serve adapters with mlx_lm `--adapter-path` after verify; keep base + adapter versions pinned.", - "If no Hub adapter fits, use the research dataset blueprint to draft labels locally.", - ) - - mtx = ( - "Treat experimental MTX / mixed-precision experiments as optional research notes, not production defaults.", - "Document any experimental kernel or quant recipe separately from the shipping wire config.", - "Prefer proven mlx-community quants for the first vertical slice; revisit MTX only after a working baseline.", - ) - - study = ( - "Apple MLX documentation: model convert / quantize / generate flows.", - "mlx-lm and mlx-vlm server READMEs for OpenAI-compatible local serving.", - "mlx-scout runtimes reference: LM Studio vs mlx_lm vs Ollama trade-offs.", - "PEFT/LoRA primers focused on low-rank adapters for domain dial-in.", - ) - - stack = ( - "1. Run `mlx-agent research` with the same goal/modalities to rank models, adapters, and datasets.", - "2. Follow the pack's Runtime preference (mlx-vlm / LM Studio / mlx_lm; Ollama only as curated alternate).", - "3. `mlx-agent adopt start` to verify a candidate on a local runtime.", - "4. `mlx-agent wire` to apply a confirmation-gated config — never skip preview hashes.", - ) - - next_steps = ( - "Freeze the goal and modalities above; do not expand scope mid-slice.", - "Generate a research pack and pick one base model + optional adapter.", - "Create or obtain a small labeled dataset offline.", - "Implement training outside mlx-agent; keep this design pack as the agent-ingestible plan.", - ) - if brief.notes: - next_steps = next_steps + ("Honor project notes: {0}".format(brief.notes),) - - return ProjectDesignPack( - brief=brief, - generated_at=moment.isoformat(), - quantization_ideas=tuple(quant), - training_loop=training, - lora_notes=lora, - mtx_notes=mtx, - study_materials=study, - stack_path=stack, - next_steps=next_steps, - ) - - -def render_design_pack(pack: ProjectDesignPack) -> str: - brief = pack.brief - lines = [ - "# MLX Project Design Pack: {0}".format(brief.goal), - "", - "- Generated: {0}".format(pack.generated_at), - "- Modalities: {0}".format(", ".join(brief.modalities) or "none"), - "- Memory budget (GB): {0}".format( - brief.memory_gb if brief.memory_gb is not None else "none" - ), - ] - if brief.notes: - lines.append("- Notes: {0}".format(brief.notes)) - lines.extend([ - "", - "## Goal", - "", - brief.goal, - "", - "## Recommended stack path", - "", - ]) - for item in pack.stack_path: - lines.append("- {0}".format(item)) - lines.extend(["", "## Quantization ideas", ""]) - for item in pack.quantization_ideas: - lines.append("- {0}".format(item)) - lines.extend(["", "## Training loop sketch", ""]) - for item in pack.training_loop: - lines.append("- {0}".format(item)) - lines.extend(["", "## LoRA / adapter notes", ""]) - for item in pack.lora_notes: - lines.append("- {0}".format(item)) - lines.extend(["", "## Experimental MTX notes", ""]) - for item in pack.mtx_notes: - lines.append("- {0}".format(item)) - lines.extend(["", "## Study materials", ""]) - for item in pack.study_materials: - lines.append("- {0}".format(item)) - lines.extend(["", "## Next steps", ""]) - for index, item in enumerate(pack.next_steps, start=1): - lines.append("{0}. {1}".format(index, item)) - lines.extend([ - "", - "## Notes", - "", - "This pack is guidance only for downstream agents. " - "mlx-agent blueprint does not scaffold repositories, download weights, or run training.", - "", - ]) - return "\n".join(lines) - - -def write_design_pack( - markdown: str, - brief: ProjectBrief, - root: Optional[object] = None, - now: Optional[datetime] = None, - pack: Optional[ProjectDesignPack] = None, -) -> Path: - """Write under ``/mlx-blueprints`` and never outside it.""" - moment = now or datetime.now(timezone.utc) - base = Path(root) if root is not None else Path.cwd() - out_dir = (base / "mlx-blueprints").resolve() - unresolved_dir = base / "mlx-blueprints" - if unresolved_dir.is_symlink(): - raise ValueError("refusing to write design pack through a symlinked folder") - timestamp = moment.strftime("%Y%m%dT%H%M%SZ") - filename = "{0}-{1}.md".format(slugify(brief.goal), timestamp) - path = (out_dir / filename).resolve() - if os.path.commonpath([str(out_dir), str(path)]) != str(out_dir): - raise ValueError("refusing to write design pack outside the project folder") - out_dir.mkdir(parents=True, exist_ok=True) - path.write_text(markdown, encoding="utf-8") - if pack is not None: - sidecar = path.with_suffix(".json") - if os.path.commonpath([str(out_dir), str(sidecar.resolve())]) != str(out_dir): - raise ValueError("refusing to write design pack outside the project folder") - sidecar.write_text(json.dumps(pack.to_dict(), indent=2) + "\n", encoding="utf-8") - return path diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/providers.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/providers.py deleted file mode 100644 index 8e5fad1..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/providers.py +++ /dev/null @@ -1,436 +0,0 @@ -"""Manifest-backed provider definitions and non-mutating provider detection.""" - -from __future__ import annotations - -import json -import os -import re -import selectors -import shutil -import subprocess -import time -from dataclasses import dataclass -from pathlib import Path - - -_GEMINI_EXTENSION_SUFFIX = Path(".gemini") / "extensions" / "mlx-agent" - - -@dataclass(frozen=True) -class ProviderArtifact: - source: Path - destination: Path - project_destination: Path = None - scopes: tuple = ("user", "project") - - -@dataclass(frozen=True) -class ProviderDefinition: - id: str - detect_commands: tuple - user_root: Path - project_root: Path - invocation_kind: str - invocation_prefix: str - minimum_version: str - last_tested_version: str - version_probe_args: tuple - version_probe_timeout: float - install_mode: str - artifacts: tuple - config_paths: tuple - - def destination(self, scope, project_root=None): - if scope == "user": - return self.user_root - if scope != "project": - raise ValueError("scope must be 'user' or 'project'") - if project_root is None: - raise ValueError("project scope requires a project root") - return Path(project_root).resolve() / self.project_root - - def artifact_destination(self, scope, project_root, artifact): - """Return the receipt-owned target for one declared artifact.""" - if scope == "project" and artifact.project_destination is not None: - return Path(project_root).resolve() / artifact.project_destination - return self.destination(scope, project_root) / artifact.destination - - @staticmethod - def applies_to(scope, artifact): - return scope in artifact.scopes - - -@dataclass(frozen=True) -class ProviderDetection: - id: str - available: bool - command: str = "" - command_path: str = None - state: str = "absent" - version: str = None - minimum_version: str = None - last_tested_version: str = None - error: str = None - install_mode: str = "direct" - - def to_dict(self): - return { - "id": self.id, - "available": self.available, - "command": self.command or None, - "command_path": self.command_path, - "state": self.state, - "version": self.version, - "minimum_version": self.minimum_version, - "last_tested_version": self.last_tested_version, - "error": self.error, - "install_mode": self.install_mode, - } - - -def _safe_relative(value, field, allow_current=False): - path = Path(value) - if allow_current and str(value) == ".": - return path - if path.is_absolute() or ".." in path.parts or not path.parts: - raise ValueError("{0} must be a safe relative path".format(field)) - return path - - -class ProviderRegistry: - """Load immutable installation data from the canonical plugin manifest.""" - - def __init__(self, manifest_path, home=None, config_root=None, xdg_config_home=None): - self.manifest_path = Path(manifest_path).resolve() - self.home = Path(home).expanduser().resolve() if home else Path.home().resolve() - self.config_root = ( - Path(config_root).expanduser().resolve() if config_root else self.home - ) - self.xdg_config_home = ( - Path(xdg_config_home).expanduser().resolve() - if xdg_config_home else (self.home / ".config").resolve() - ) - - def definitions(self): - try: - manifest = json.loads(self.manifest_path.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError) as error: - raise ValueError("provider manifest could not be read: {0}".format(error)) - providers = manifest.get("providers") - if not isinstance(providers, dict): - raise ValueError("provider manifest has no providers object") - definitions = {} - for provider_id, value in providers.items(): - definitions[provider_id] = self._definition(provider_id, value) - return definitions - - def _definition(self, provider_id, value): - if not isinstance(value, dict): - raise ValueError("provider {0} must be an object".format(provider_id)) - required = ( - "detect_commands", "user_root", "project_root", "artifacts", "config_paths", - "invocation", "minimum_version", "last_tested_version", "version_probe", - "install_mode", - ) - if any(key not in value for key in required): - raise ValueError("provider {0} has no installer definition".format(provider_id)) - commands = value["detect_commands"] - if not isinstance(commands, list) or not all(isinstance(item, str) and item for item in commands): - raise ValueError("provider {0}.detect_commands is invalid".format(provider_id)) - user_root = self._user_root(value["user_root"], provider_id) - project_root = self._project_root(value["project_root"], provider_id) - invocation = value["invocation"] - if not isinstance(invocation, dict) or set(invocation) != {"kind", "prefix"}: - raise ValueError("provider {0}.invocation is invalid".format(provider_id)) - invocation_kind, invocation_prefix = invocation["kind"], invocation["prefix"] - if invocation_kind not in {"command", "skill"} or not isinstance(invocation_prefix, str): - raise ValueError("provider {0}.invocation is invalid".format(provider_id)) - if invocation_kind == "command" and invocation_prefix != "/": - raise ValueError("provider {0}.invocation is invalid".format(provider_id)) - if invocation_kind == "skill" and invocation_prefix not in {"", "$"}: - raise ValueError("provider {0}.invocation is invalid".format(provider_id)) - minimum_version = value["minimum_version"] - last_tested_version = value["last_tested_version"] - for label, version in ( - ("minimum_version", minimum_version), - ("last_tested_version", last_tested_version), - ): - if version is not None and not _SEMVER.fullmatch(version): - raise ValueError("provider {0}.{1} is invalid".format(provider_id, label)) - probe = value["version_probe"] - if probe is None: - version_probe_args, version_probe_timeout = (), 0.0 - elif ( - not isinstance(probe, dict) - or set(probe) != {"args", "timeout_seconds"} - or not isinstance(probe["args"], list) - or not 1 <= len(probe["args"]) <= 4 - or not all(isinstance(item, str) and item and len(item) <= 64 for item in probe["args"]) - or not isinstance(probe["timeout_seconds"], (int, float)) - or isinstance(probe["timeout_seconds"], bool) - or not 0 < probe["timeout_seconds"] <= 5 - ): - raise ValueError("provider {0}.version_probe is invalid".format(provider_id)) - else: - version_probe_args = tuple(probe["args"]) - version_probe_timeout = float(probe["timeout_seconds"]) - install_mode = value["install_mode"] - if install_mode not in {"direct", "staged", "portable"}: - raise ValueError("provider {0}.install_mode is invalid".format(provider_id)) - if install_mode == "portable" and (commands or probe is not None or minimum_version is not None or last_tested_version is not None): - raise ValueError("portable providers must not declare CLI compatibility probes") - if install_mode != "portable" and (not commands or probe is None or minimum_version is None): - raise ValueError("native providers require a minimum version and bounded probe") - artifacts = [] - if not isinstance(value["artifacts"], list) or not value["artifacts"]: - raise ValueError("provider {0}.artifacts is invalid".format(provider_id)) - for index, artifact in enumerate(value["artifacts"]): - if not isinstance(artifact, dict) or set(artifact) not in ({"source", "destination"}, {"source", "destination", "scope"}, {"source", "destination", "project_destination"}, {"source", "destination", "project_destination", "scope"}): - raise ValueError("provider {0}.artifacts[{1}] is invalid".format(provider_id, index)) - source = _safe_relative(artifact["source"], "artifact source") - unresolved = self.manifest_path.parent / source - cursor = self.manifest_path.parent - for component in source.parts: - cursor = cursor / component - if cursor.is_symlink(): - raise ValueError( - "provider artifact source must not contain symlinks: {0}".format(source) - ) - location = unresolved.resolve() - if self.manifest_path.parent not in location.parents or not (location.is_file() or location.is_dir()): - raise ValueError("provider artifact source is outside the plugin or missing: {0}".format(source)) - destination = _safe_relative( - artifact["destination"], "artifact destination", allow_current=True - ) - project_destination = ( - _safe_relative(artifact["project_destination"], "artifact project_destination") - if "project_destination" in artifact else None - ) - scopes = (artifact.get("scope", "user"),) if "scope" in artifact else ("user", "project") - if scopes not in (("user",), ("project",)) and scopes != ("user", "project"): - raise ValueError("provider {0}.artifacts[{1}].scope is invalid".format(provider_id, index)) - if location.is_file(): - artifacts.append(ProviderArtifact(location, destination, project_destination, scopes)) - else: - for child in sorted(location.rglob("*")): - if child.is_symlink(): - raise ValueError( - "provider artifact source must not contain symlinks: {0}".format(child) - ) - resolved = child.resolve() - relative = child.relative_to(location) - if "__pycache__" in relative.parts or child.suffix == ".pyc": - continue - if child.is_file() and location in resolved.parents: - project_child = project_destination / child.relative_to(location) if project_destination else None - artifacts.append(ProviderArtifact(resolved, destination / child.relative_to(location), project_child, scopes)) - config_paths = value["config_paths"] - if not isinstance(config_paths, list) or not all(isinstance(item, str) for item in config_paths): - raise ValueError("provider {0}.config_paths is invalid".format(provider_id)) - if provider_id == "gemini": - self._validate_gemini_extension_layout(user_root, project_root, artifacts) - if provider_id == "opencode": - self._validate_opencode_layout(user_root, project_root, artifacts) - return ProviderDefinition( - id=provider_id, - detect_commands=tuple(commands), - user_root=user_root, - project_root=project_root, - invocation_kind=invocation_kind, - invocation_prefix=invocation_prefix, - minimum_version=minimum_version, - last_tested_version=last_tested_version, - version_probe_args=version_probe_args, - version_probe_timeout=version_probe_timeout, - install_mode=install_mode, - artifacts=tuple(artifacts), - config_paths=tuple(config_paths), - ) - - def _validate_gemini_extension_layout(self, user_root, project_root, artifacts): - """Keep Gemini's install roots aligned with its extension discovery layout.""" - if user_root != (self.home / _GEMINI_EXTENSION_SUFFIX).resolve(): - raise ValueError("provider gemini.user_root must target home/.gemini/extensions/mlx-agent") - if project_root != _GEMINI_EXTENSION_SUFFIX: - raise ValueError("provider gemini.project_root must target .gemini/extensions/mlx-agent") - if not any(item.destination == Path("gemini-extension.json") for item in artifacts): - raise ValueError("provider gemini must install gemini-extension.json") - for item in artifacts: - if item.project_destination is not None and item.project_destination.parts[:1] != (".gemini",): - raise ValueError("provider gemini project artifacts must stay under .gemini") - - def _validate_opencode_layout(self, user_root, project_root, artifacts): - """Keep OpenCode's documented global and project discovery locations exact.""" - if user_root != (self.xdg_config_home / "opencode").resolve(): - raise ValueError("provider opencode.user_root must target the OpenCode XDG config directory") - if project_root != Path(".opencode"): - raise ValueError("provider opencode.project_root must target .opencode") - if not any(item.destination == Path("commands/mlx-scout.md") for item in artifacts): - raise ValueError("provider opencode must install native command files") - if not any(item.destination == Path("agents/mlx-advisor.md") for item in artifacts): - raise ValueError("provider opencode must install its advisor agent") - if not any(item.destination == Path("skills/mlx-scout/SKILL.md") for item in artifacts): - raise ValueError("provider opencode must install native skills") - if any(item.destination == Path("opencode.json") for item in artifacts): - raise ValueError("provider opencode must not mutate global or project config") - if not any(item.destination == Path("plugins/mlx-agent-command.ts") for item in artifacts): - raise ValueError("provider opencode must install its native custom tool") - if not any(item.destination == Path("src/mlx_agent/command_executor.py") for item in artifacts): - raise ValueError("provider opencode must install its custom-tool runtime") - - def _user_root(self, template, provider_id): - if not isinstance(template, str) or "{project}" in template: - raise ValueError("provider {0}.user_root is invalid".format(provider_id)) - try: - path = Path(template.format( - home=str(self.home), config_root=str(self.config_root), - xdg_config_home=str(self.xdg_config_home), - )).resolve() - except (KeyError, ValueError) as error: - raise ValueError("provider {0}.user_root is invalid: {1}".format(provider_id, error)) - allowed_roots = (self.home, self.config_root, self.xdg_config_home) - if not any(root == path or root in path.parents for root in allowed_roots): - raise ValueError("provider {0}.user_root must be under an approved user configuration root".format(provider_id)) - return path - - def _project_root(self, template, provider_id): - if not isinstance(template, str) or "{project}" not in template: - raise ValueError("provider {0}.project_root is invalid".format(provider_id)) - marker = Path("/__mlx_agent_project_marker__") - try: - rendered = Path(template.format( - project=str(marker), home=str(self.home), config_root=str(self.config_root), - xdg_config_home=str(self.xdg_config_home), - )).resolve() - return rendered.relative_to(marker) - except (KeyError, ValueError): - raise ValueError("provider {0}.project_root is invalid".format(provider_id)) - - -_SEMVER = re.compile(r"(? _MAX_VERSION_OUTPUT: - raise ValueError("version probe output exceeded the bounded limit") - returncode = process.wait(timeout=max(0.01, deadline - time.monotonic())) - return subprocess.CompletedProcess(argv, returncode, bytes(output)) - finally: - selector.close() - if process.poll() is None: - process.kill() - process.wait() - if process.stdout is not None: - process.stdout.close() - - -def detect_providers(definitions, env=None, path=None, executable_lookup=None, probe_runner=None): - """Report installed provider CLIs only; this function never installs anything.""" - environment = dict(os.environ if env is None else env) - search_path = path if path is not None else environment.get("PATH", "") - lookup = executable_lookup or shutil.which - detections = [] - for definition in definitions: - if definition.install_mode == "portable": - detections.append(ProviderDetection( - definition.id, True, state="portable", install_mode="portable" - )) - continue - command_path = None - command = "" - for candidate in definition.detect_commands: - found = lookup(candidate, path=search_path) - if found: - command, command_path = candidate, str(found) - break - if not command_path: - detections.append(ProviderDetection( - definition.id, False, command, command_path, state="absent", - minimum_version=definition.minimum_version, - last_tested_version=definition.last_tested_version, - error="provider executable was not found", - install_mode=definition.install_mode, - )) - continue - try: - argv = [command_path] + list(definition.version_probe_args) - if probe_runner is None: - completed = _run_bounded_probe( - argv, - timeout=definition.version_probe_timeout, - env=environment, - ) - else: - completed = probe_runner( - argv, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=definition.version_probe_timeout, - check=False, - shell=False, - env=environment, - ) - raw = completed.stdout or b"" - if isinstance(raw, bytes): - output = raw[:_MAX_VERSION_OUTPUT].decode("utf-8", errors="replace") - else: - output = str(raw)[:_MAX_VERSION_OUTPUT] - match = _SEMVER.search(output) - if completed.returncode != 0 or match is None: - raise ValueError("version probe did not return a SemVer core") - version = match.group(1) - supported = definition.last_tested_version is not None and ( - _version_tuple(definition.minimum_version) - <= _version_tuple(version) - <= _version_tuple(definition.last_tested_version) - ) - state = "native-visible" if supported else "unsupported" - detections.append(ProviderDetection( - definition.id, supported, command, command_path, state=state, - version=version, minimum_version=definition.minimum_version, - last_tested_version=definition.last_tested_version, - error=None if supported else "provider version is outside the validated range", - install_mode=definition.install_mode, - )) - except (OSError, subprocess.SubprocessError, ValueError) as error: - detections.append(ProviderDetection( - definition.id, False, command, command_path, state="unsupported", - minimum_version=definition.minimum_version, - last_tested_version=definition.last_tested_version, - error=str(error), install_mode=definition.install_mode, - )) - return detections diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/research.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/research.py deleted file mode 100644 index c817b79..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/research.py +++ /dev/null @@ -1,502 +0,0 @@ -"""Read-only domain research packs: discovery -> scoring -> project-local markdown. - -No verification, no wiring, no downloads. Discovery and the Hugging Face client -are injected so this module is fully testable offline. Output is written only -inside the project-local ``mlx-research`` folder. -""" - -from __future__ import annotations - -import json -import os -import re -from dataclasses import dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -from .blueprint import DatasetBlueprint, build_dataset_blueprint -from .discovery import DiscoveryRequest -from .interview import DomainIntent -from .modality import FACET_CHOICES, facet_label, modality_label -from .runtime_preference import RuntimePreference, prefer_runtime, wiring_for_preference -from .scoring import ScoreResult, rank_scored, score_candidate - - -_SLUG_RE = re.compile(r"[^a-z0-9]+") -_CARD_EXCERPT_CHARS = 240 -_DEFAULT_CATALOG_LIMIT = 5 - - -@dataclass(frozen=True) -class ResearchCandidate: - repo: str - role: str - score: float - wiring: str - signals: Tuple[dict, ...] - provenance: Tuple[dict, ...] - card_present: bool - card_excerpt: str - - def to_dict(self): - return { - "repo": self.repo, - "role": self.role, - "score": self.score, - "wiring": self.wiring, - "signals": [dict(signal) for signal in self.signals], - "provenance": [dict(record) for record in self.provenance], - "card_present": self.card_present, - "card_excerpt": self.card_excerpt, - } - - -@dataclass(frozen=True) -class CatalogItem: - repo: str - kind: str # adapter | dataset - score: float - signals: Tuple[dict, ...] - provenance: Tuple[dict, ...] - card_excerpt: str - record: Dict[str, object] - - def to_dict(self): - return { - "repo": self.repo, - "kind": self.kind, - "score": self.score, - "signals": [dict(signal) for signal in self.signals], - "provenance": [dict(record) for record in self.provenance], - "card_excerpt": self.card_excerpt, - "record": dict(self.record), - } - - -@dataclass(frozen=True) -class ResearchPack: - intent: DomainIntent - candidates: Tuple[ResearchCandidate, ...] - generated_at: str - warnings: Tuple[dict, ...] = field(default_factory=tuple) - adapters: Tuple[CatalogItem, ...] = field(default_factory=tuple) - datasets: Tuple[CatalogItem, ...] = field(default_factory=tuple) - dataset_blueprint: Optional[DatasetBlueprint] = None - runtime_preference: Optional[RuntimePreference] = None - - def to_dict(self): - return { - "intent": self.intent.to_dict(), - "candidates": [candidate.to_dict() for candidate in self.candidates], - "generated_at": self.generated_at, - "warnings": [dict(warning) for warning in self.warnings], - "adapters": [item.to_dict() for item in self.adapters], - "datasets": [item.to_dict() for item in self.datasets], - "dataset_blueprint": ( - self.dataset_blueprint.to_dict() if self.dataset_blueprint else None - ), - "runtime_preference": ( - self.runtime_preference.to_dict() if self.runtime_preference else None - ), - } - - -def slugify(domain: str) -> str: - slug = _SLUG_RE.sub("-", domain.strip().lower()).strip("-") - return slug or "domain" - - -def candidate_metadata(candidate: Dict[str, object]) -> Dict[str, object]: - """Map a discovery candidate to the scoring core's metadata shape.""" - roles = candidate.get("roles") or ([candidate["role"]] if candidate.get("role") else []) - return { - "roles": list(roles), - "license": candidate.get("license"), - "downloads": candidate.get("downloads", 0), - "likes": candidate.get("likes", 0), - "est_ram_gb": candidate.get("est_ram_gb"), - "tags": candidate.get("tags", []), - } - - -def hub_row_metadata(row: Dict[str, object], intent: DomainIntent) -> Dict[str, object]: - """Map a Hub adapter/dataset row into the scoring metadata shape.""" - tags = list(row.get("tags") or []) - license_name = row.get("license") - if not license_name: - for tag in tags: - text = str(tag) - if text.startswith("license:"): - license_name = text.split(":", 1)[1] - break - return { - "roles": list(intent.roles), - "license": license_name, - "downloads": row.get("downloads", 0), - "likes": row.get("likes", 0), - "est_ram_gb": None, - "tags": tags, - } - - -def catalog_search_query(intent: DomainIntent) -> str: - parts = list(intent.keywords) - if intent.domain and intent.domain.lower() not in {p.lower() for p in parts}: - parts.append(intent.domain) - return " ".join(parts).strip() - - -def _excerpt(card_text: Optional[str]) -> str: - if not card_text: - return "" - collapsed = " ".join(card_text.split()) - return collapsed[:_CARD_EXCERPT_CHARS] - - -def _repo_id(row: Dict[str, object]) -> Optional[str]: - repo = row.get("id") or row.get("modelId") or row.get("repo") - if not repo: - return None - return str(repo) - - -def _score_catalog_rows( - intent: DomainIntent, - rows: List[Dict[str, object]], - kind: str, - fetch_card, - catalog_limit: int, -) -> Tuple[CatalogItem, ...]: - scored: List[Tuple[str, ScoreResult]] = [] - detail: Dict[str, Dict[str, object]] = {} - for row in rows: - repo = _repo_id(row) - if not repo or repo in detail: - continue - try: - card_text = fetch_card(repo) - except Exception: - card_text = None - result = score_candidate(intent, hub_row_metadata(row, intent), card_text) - scored.append((repo, result)) - detail[repo] = {"row": row, "card_text": card_text} - - ranked = rank_scored(scored)[:max(0, catalog_limit)] - items: List[CatalogItem] = [] - for repo, result in ranked: - row = detail[repo]["row"] - card_text = detail[repo]["card_text"] - items.append(CatalogItem( - repo=repo, - kind=kind, - score=result.score, - signals=tuple(signal.to_dict() for signal in result.signals), - provenance=result.provenance, - card_excerpt=_excerpt(card_text), - record=dict(row), - )) - return tuple(items) - - -def generate_pack( - intent: DomainIntent, - discovery_service, - hf_client, - limit: int = 6, - catalog_limit: int = _DEFAULT_CATALOG_LIMIT, - now: Optional[datetime] = None, -) -> ResearchPack: - """Build a research pack. Read-only: never verifies, wires, or downloads.""" - moment = now or datetime.now(timezone.utc) - seen: Dict[str, Dict[str, object]] = {} - warnings: List[dict] = [] - host: Dict[str, object] = {} - for role in intent.roles: - request = DiscoveryRequest( - role=role, - memory_gb=intent.memory_gb, - licenses=list(intent.license_allow) or None, - limit=limit, - ) - envelope = discovery_service.discover(request) - if envelope.status != "ok": - detail = envelope.error - warnings.append({ - "code": detail.code if detail else "discovery_failed", - "message": "{0}: {1}".format(role, detail.message if detail else "discovery failed"), - }) - continue - if not host and isinstance(envelope.data.get("host"), dict): - host = dict(envelope.data["host"]) - for bucket in envelope.data.get("roles", {}).values(): - for candidate in bucket: - repo = candidate.get("repo") - if repo and repo not in seen: - seen[repo] = candidate - - preference = prefer_runtime( - host, - roles=intent.roles, - modalities=intent.modalities, - ) - - scored: List[Tuple[str, ScoreResult]] = [] - detail_by_repo: Dict[str, Dict[str, object]] = {} - for repo, candidate in seen.items(): - try: - card_text = hf_client.fetch_model_card(repo) - except Exception: - card_text = None - result = score_candidate(intent, candidate_metadata(candidate), card_text) - scored.append((repo, result)) - detail_by_repo[repo] = {"candidate": candidate, "card_text": card_text} - - ranked = rank_scored(scored)[:max(0, limit)] - candidates = [] - for repo, result in ranked: - candidate = detail_by_repo[repo]["candidate"] - card_text = detail_by_repo[repo]["card_text"] - role = candidate.get("role", "") - candidates.append(ResearchCandidate( - repo=repo, - role=role, - score=result.score, - wiring=wiring_for_preference(repo, role, preference), - signals=tuple(signal.to_dict() for signal in result.signals), - provenance=result.provenance, - card_present=bool(card_text), - card_excerpt=_excerpt(card_text), - )) - - search = catalog_search_query(intent) - try: - adapter_rows = list(hf_client.list_adapters(search=search, limit_fetch=max(catalog_limit * 4, 20))) - except Exception as error: - warnings.append({ - "code": "adapter_search_failed", - "message": "adapter search failed: {0}".format(error), - }) - adapter_rows = [] - try: - dataset_rows = list(hf_client.list_datasets(search=search, limit_fetch=max(catalog_limit * 4, 20))) - except Exception as error: - warnings.append({ - "code": "dataset_search_failed", - "message": "dataset search failed: {0}".format(error), - }) - dataset_rows = [] - - adapters = _score_catalog_rows( - intent, - adapter_rows, - "adapter", - hf_client.fetch_model_card, - catalog_limit, - ) - datasets = _score_catalog_rows( - intent, - dataset_rows, - "dataset", - hf_client.fetch_dataset_card, - catalog_limit, - ) - blueprint = build_dataset_blueprint(intent) if not datasets else None - - return ResearchPack( - intent=intent, - candidates=tuple(candidates), - generated_at=moment.isoformat(), - warnings=tuple(warnings), - adapters=adapters, - datasets=datasets, - dataset_blueprint=blueprint, - runtime_preference=preference, - ) - - -def _render_catalog_section(title: str, items: Tuple[CatalogItem, ...], empty_message: str) -> List[str]: - lines = ["## {0}".format(title), ""] - if not items: - lines.append(empty_message) - lines.append("") - return lines - for index, item in enumerate(items, start=1): - lines.append("### {0}. `{1}` — score {2}/100".format(index, item.repo, item.score)) - lines.append("") - lines.append("- Kind: {0}".format(item.kind)) - matched = [s for s in item.signals if s["applicable"] and s["matched"]] - if matched: - lines.append("- Why it ranked:") - for signal in matched: - lines.append(" - {0}: {1}".format(signal["id"], signal["detail"])) - if item.card_excerpt: - lines.append("- Card excerpt: {0}".format(item.card_excerpt)) - lines.append("") - return lines - - -def render_pack(pack: ResearchPack) -> str: - intent = pack.intent - lines = [ - "# MLX Research Pack: {0}".format(intent.domain), - "", - "- Generated: {0}".format(pack.generated_at), - "- Roles: {0}".format(", ".join(intent.roles) or "none"), - "- Keywords: {0}".format(", ".join(intent.keywords) or "none"), - "- License filter: {0}".format(", ".join(intent.license_allow) or "none"), - "- Memory budget (GB): {0}".format(intent.memory_gb if intent.memory_gb is not None else "none"), - "- Modalities: {0}".format(", ".join(intent.modalities) or "none"), - "- Facets: {0}".format(", ".join(intent.facets) or "none"), - ] - if intent.notes: - lines.append("- Notes: {0}".format(intent.notes)) - lines.extend(["", "## Summary", ""]) - if pack.candidates: - lines.append( - "{0} candidate(s) ranked by transparent scoring. Scores are read-only " - "estimates from metadata and model-card text; no model was downloaded, " - "verified, or wired.".format(len(pack.candidates)) - ) - else: - lines.append("No candidates were found for this intent. See warnings below.") - summary_bits = [] - if pack.adapters: - summary_bits.append("{0} adapter(s)".format(len(pack.adapters))) - if pack.datasets: - summary_bits.append("{0} dataset(s)".format(len(pack.datasets))) - if pack.dataset_blueprint is not None: - summary_bits.append("dataset blueprint (no Hub datasets matched)") - if summary_bits: - lines.append("Catalog enrichment: {0}.".format("; ".join(summary_bits))) - if pack.warnings: - lines.extend(["", "## Warnings", ""]) - for warning in pack.warnings: - lines.append("- [{0}] {1}".format(warning.get("code", "warning"), warning.get("message", ""))) - lines.extend(["", "## Modality foundations", ""]) - if intent.modalities: - for modality in intent.modalities: - allowed = set(FACET_CHOICES.get(modality, {}).values()) - facet_ids = [facet for facet in intent.facets if facet in allowed] - lines.append("- `{0}` — {1}".format(modality, modality_label(modality))) - if facet_ids: - lines.append(" - Facets: {0}".format( - ", ".join( - "`{0}` ({1})".format(facet, facet_label(facet)) - for facet in facet_ids - ) - )) - lines.append("") - else: - lines.append("No foundational modalities were selected for this pack.") - lines.append("") - lines.extend(["", "## Runtime preference", ""]) - preference = pack.runtime_preference - if preference is not None: - lines.append("- Preferred: `{0}`".format(preference.preferred)) - lines.append( - "- Alternates: {0}".format( - ", ".join("`{0}`".format(item) for item in preference.alternates) - if preference.alternates - else "none" - ) - ) - lines.append("- Rationale: {0}".format(preference.rationale)) - snapshot = preference.host_snapshot - lines.append( - "- Host inventory: Ollama {0}; LM Studio {1}".format( - "up" if snapshot.get("ollama") else "down", - "up" if snapshot.get("lmstudio") else "down", - ) - ) - lines.append("") - else: - lines.append("No runtime preference was computed for this pack.") - lines.append("") - lines.extend(["", "## Candidates", ""]) - for index, candidate in enumerate(pack.candidates, start=1): - lines.append("### {0}. `{1}` — score {2}/100".format(index, candidate.repo, candidate.score)) - lines.append("") - lines.append("- Role: {0}".format(candidate.role)) - lines.append("- Suggested wiring: {0}".format(candidate.wiring)) - lines.append("- Model card: {0}".format("present" if candidate.card_present else "not found")) - matched = [s for s in candidate.signals if s["applicable"] and s["matched"]] - if matched: - lines.append("- Why it ranked:") - for signal in matched: - lines.append(" - {0}: {1}".format(signal["id"], signal["detail"])) - if candidate.card_excerpt: - lines.append("- Card excerpt: {0}".format(candidate.card_excerpt)) - lines.append("") - lines.extend(_render_catalog_section( - "Adapters / LoRAs", - pack.adapters, - "No PEFT/LoRA adapters matched this intent on the Hub (read-only search).", - )) - lines.extend(_render_catalog_section( - "Datasets", - pack.datasets, - "No datasets matched this intent on the Hub. See the dataset blueprint below.", - )) - if pack.dataset_blueprint is not None: - lines.extend([ - "## Dataset blueprint", - "", - "No Hub datasets ranked for this intent. Use this guidance-only blueprint " - "to create a local dataset — nothing is downloaded or trained by research.", - "", - pack.dataset_blueprint.to_markdown().rstrip(), - "", - ]) - lines.extend([ - "## Next steps", - "", - "1. Review candidates, adapters, and datasets above and pick a local stack.", - "2. Prefer the runtime in **Runtime preference** above" - + ( - " (`{0}`)".format(pack.runtime_preference.preferred) - if pack.runtime_preference is not None - else "" - ) - + "; Ollama remains a valid alternate for curated tags when listed.", - "3. Install a base model in that runtime (LM Studio, mlx_lm/mlx-vlm, or Ollama).", - "4. Run `mlx-agent adopt start` to verify and wire it — research does not verify or download.", - "5. If no datasets matched, follow the dataset blueprint to create labeled data locally.", - "", - "## Notes", - "", - "This pack is a read-only research artifact intended for ingestion by other agents. " - "All scores are estimates with per-signal provenance; verify capability before relying on any model. " - "Runtime preference is guidance from host inventory and modality/role rules — it does not change scores.", - "", - ]) - return "\n".join(lines) - - -def write_pack( - markdown: str, - intent: DomainIntent, - root: Optional[object] = None, - now: Optional[datetime] = None, - pack: Optional[ResearchPack] = None, -) -> Path: - """Write the pack under ``/mlx-research`` and never outside it.""" - moment = now or datetime.now(timezone.utc) - base = Path(root) if root is not None else Path.cwd() - out_dir = (base / "mlx-research").resolve() - unresolved_dir = base / "mlx-research" - if unresolved_dir.is_symlink(): - raise ValueError("refusing to write research pack through a symlinked folder") - timestamp = moment.strftime("%Y%m%dT%H%M%SZ") - filename = "{0}-{1}.md".format(slugify(intent.domain), timestamp) - path = (out_dir / filename).resolve() - if os.path.commonpath([str(out_dir), str(path)]) != str(out_dir): - raise ValueError("refusing to write research pack outside the project folder") - out_dir.mkdir(parents=True, exist_ok=True) - path.write_text(markdown, encoding="utf-8") - if pack is not None: - sidecar = path.with_suffix(".json") - if os.path.commonpath([str(out_dir), str(sidecar.resolve())]) != str(out_dir): - raise ValueError("refusing to write research pack outside the project folder") - sidecar.write_text(json.dumps(pack.to_dict(), indent=2) + "\n", encoding="utf-8") - return path diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/adapter-runtime.json b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/adapter-runtime.json deleted file mode 100644 index b0e6f55..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/adapter-runtime.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "schema_version": "1.0", - "purpose": "Bundled adapter runtime resource contract" -} diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/bench-community.json b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/bench-community.json deleted file mode 100644 index 4c3573d..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/bench-community.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "schema_version": "1.0", - "entries": [] -} diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/references/model-families.md b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/references/model-families.md deleted file mode 100644 index e331847..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/references/model-families.md +++ /dev/null @@ -1,35 +0,0 @@ -# Model family quirks (chat templates, tool calling, reasoning) - -Family-specific behaviors that change how you wire, prompt, and verify. Read this before debugging a "broken" model — it is usually a template mismatch. - -## Qwen (Qwen3 and later) - -- Ships **hybrid reasoning**: `enable_thinking` in the chat template toggles hidden `` blocks. `mlx-agent` detects this from the template and tags reasoning models so they stay out of fast/utility roles. -- Tool calling uses the Hermes-style `` JSON blocks; works well through `mlx_lm.server` and LM Studio. Verified tool-use is still per-model — template support is not behavior proof. -- MoE variants (`A3B`, `A35B`) are the speed pick on Apple Silicon; dense variants are the consistency pick. - -## Gemma - -- **No system role.** A system prompt must be folded into the first user turn; runtimes that auto-inject a system role produce degraded or refused outputs. -- Google ships **QAT (quantization-aware trained)** builds — their 4bit is unusually strong; prefer publisher QAT over generic 4bit re-quants. -- Tool calling is template-level and inconsistent across sizes; verify before assigning a tool-use role. - -## gpt-oss (OpenAI open weights) - -- **MXFP4 native** — use the MXFP4 builds; do not re-quantize to generic 4bit. -- Uses the **harmony** response format with `reasoning_effort` levels (low/medium/high). These are reasoning models by design: expect hidden analysis channels; never put them in a fast/cheap slot. -- Strong tool calling when served through runtimes that parse harmony (recent `mlx-lm`, LM Studio). - -## Llama (Meta, MLX ports) - -- Solid generic instruction following; tool calling depends heavily on the exact instruct template version — verify, don't assume. -- Community MLX ports vary in template fidelity; check the repo's `chat_template` presence before wiring (missing template → runtime default, often wrong). - -## Vision (Qwen-VL and friends) - -- Vision models need `mlx-vlm`; Ollama's engine does not run them. Wire them on their own port (`:8083` by convention) so they never collide with the text server. -- OCR quality varies more with input resolution handling than with quant; keep 4bit unless the documents are dense. - -## Embedding models - -- Served via `/v1/embeddings` (OpenAI-compatible) or `/api/embed` (Ollama). They cannot chat; a "ready" generation probe is not meaningful for them — rely on the embedding probe, not text generation. diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/references/quantization.md b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/references/quantization.md deleted file mode 100644 index 7600943..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/references/quantization.md +++ /dev/null @@ -1,25 +0,0 @@ -# MLX quantization guide - -How to pick a quant on Apple Silicon. Read this before recommending a specific quantized repo. - -## The tradeoff ladder - -| Quant | Size vs bf16 | Quality | When to pick | -| --- | --- | --- | --- | -| `bf16` / `fp16` | 1.0× | reference | Fine-tuning source, quality ceiling checks. Rarely for serving. | -| `8bit` | ~0.5× | near-reference | RAM is comfortable and the task is quality-sensitive (reasoning, code review). | -| `4bit` | ~0.25× | good | Default serving choice. Best size/quality point for most models. | -| `MXFP4` | ~0.25× | good | Native format for gpt-oss; use the publisher's own MXFP4 builds. | -| `3bit` and below | ≤0.2× | noticeably worse | Only when nothing else fits; expect degradation on reasoning and tool calling. | - -Rules of thumb: - -- **Spend RAM on a bigger model at 4bit, not a smaller model at 8bit.** A 32B at 4bit (~18 GB) usually beats a 14B at 8bit (~15 GB) on hard tasks. -- **Mixed MoE quants matter.** MoE models (e.g. Qwen3-A3B) activate few parameters per token, so at the same RAM they decode much faster than dense. On memory-bandwidth-bound Apple Silicon, active parameters set decode speed. -- **Weights are not the whole budget.** KV cache grows linearly with context: `2 × layers × kv_heads × head_dim × context × 2 bytes` (fp16). A model that "fits" can still die at long context. `mlx-agent discover --context N` computes this for you; the candidate's `estimates.kv.max_context_tokens` is the honest ceiling. -- **Publisher quants beat community re-quants** for quality-sensitive roles: mlx-community and the original publisher calibrate with different data and care. When both exist, prefer the publisher or `mlx-community` (trusted), then unsloth. -- **LoRA preserves quant.** `mlx_lm.lora` trains on the quantized base and `mlx_lm.fuse` keeps the quant, so you do not need bf16 weights to fine-tune. - -## Reasoning models and quant - -Reasoning models emit long hidden chains, so they are more sensitive to aggressive quant than chat models (errors compound over thousands of generated tokens). Prefer 8bit for a reasoning role when RAM allows; 4bit is acceptable for MXFP4-native families. diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/references/troubleshooting.md b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/references/troubleshooting.md deleted file mode 100644 index 4ae6b5c..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/references/troubleshooting.md +++ /dev/null @@ -1,33 +0,0 @@ -# Troubleshooting local MLX serving - -Symptom-first playbook. Each entry: cause, then the fix. Run `mlx-agent doctor models` first — half of these are visible there. - -## "It was working yesterday" - -1. `mlx-agent doctor models` — look for `drift_missing_model` (deleted weights), `drift_hash_mismatch` (hand-edited config), `endpoint_down`. -2. `mlx-agent serve status` — a serve receipt whose pid died, or an argv mismatch after a manual restart. - -## Metal / memory allocation failures at load or mid-generation - -- Cause: weights + KV cache exceed unified memory under pressure. macOS will swap-thrash before it errors cleanly. -- Fix: drop context length (KV is linear in context — check `estimates.kv.max_context_tokens` from discovery), close other GPU consumers (browsers, other servers), or drop a quant tier. Two loaded 30B+ models do not coexist on ≤64 GB. - -## Slow tokens/sec - -- Expected ceiling: decode tok/s ≈ memory bandwidth ÷ active bytes per token. Dense 32B 4bit on a high-end M-series is tens of tok/s, not hundreds. -- Checklist: MoE instead of dense (2–5× at the same RAM); native `mlx_lm` on native MLX weights instead of Ollama for MoE; shorter context (KV reads grow); don't bench while another server is loaded — measurements contend. -- Prompt processing (prefill) is compute-bound, not bandwidth-bound; long prompts are slow on every runtime. Measure with `mlx-agent bench run` before blaming the model. - -## Server answers but agent calls fail - -- Port collision: two configs on one port (`doctor models` → `drift_endpoint_conflict`). Convention: mlx_lm `:8080`, mlx-vlm `:8083`, LiteLLM `:4000`, Ollama `:11434`, LM Studio `:1234`. -- Auth header: wired configs reference `MLX_AGENT_LOCAL_API_KEY` by design; export any value, but the variable must exist for OpenAI-compatible clients that send a key. -- Chat template missing/wrong: outputs look like raw completion text, tool calls never parse. Check the repo ships a `chat_template`; see `references/model-families.md` for family quirks (Gemma has no system role). - -## Tool calling fails verification - -- Metadata ≠ behavior. Only a schema-valid synthetic call counts. If the probe fails with `malformed_arguments` or `missing_tool_call`, the model (or its template at that quant) does not do reliable tool use — pick a different model for the tool-use role; do not lower the bar. - -## Gated repositories - -- `gated: true` means HF auth is required before any runtime can pull. Accept the terms on the Hub page with the account that owns the local `HF_TOKEN`; nothing in this pack downloads, so nothing here can accept terms for you. diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/serve-recipes.json b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/serve-recipes.json deleted file mode 100644 index d297d5e..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/resources/serve-recipes.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "schema_version": "1.0", - "recipes": { - "mlx_lm": { - "executable": "mlx_lm.server", - "argv": ["{executable}", "--model", "{repo}", "--port", "{port}", "--max-tokens", "{max_tokens}"], - "adapter_argv": ["--adapter-path", "{adapter_path}"], - "default_port": 8080, - "readiness": "http://127.0.0.1:{port}/v1/models", - "install_hint": "pip install mlx-lm" - }, - "mlx-vlm": { - "executable": "mlx_vlm.server", - "argv": ["{executable}", "--model", "{repo}", "--port", "{port}"], - "adapter_argv": null, - "default_port": 8083, - "readiness": "http://127.0.0.1:{port}/v1/models", - "install_hint": "pip install mlx-vlm" - } - } -} diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/runtime_preference.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/runtime_preference.py deleted file mode 100644 index d458cc1..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/runtime_preference.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Justified MLX-native runtime preference for research packs and discovery wiring. - -Reporting only: does not change scoring. Uses existing HostInventory flags -(ollama, lmstudio) — no new runtime probes. Ollama remains a valid alternate -for text workloads and is never removed from wire/CLI surfaces. - -This module stays free of interview/models imports to avoid cycles. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Mapping, Sequence, Tuple - - -_VISION_MODALITIES = frozenset({"document-vision", "video"}) - - -@dataclass(frozen=True) -class RuntimePreference: - preferred: str - alternates: Tuple[str, ...] - rationale: str - host_snapshot: Mapping[str, object] - - def to_dict(self): - return { - "preferred": self.preferred, - "alternates": list(self.alternates), - "rationale": self.rationale, - "host_snapshot": { - "ollama": bool(self.host_snapshot.get("ollama")), - "lmstudio": bool(self.host_snapshot.get("lmstudio")), - "chip": self.host_snapshot.get("chip"), - "ram_gb": self.host_snapshot.get("ram_gb"), - }, - } - - -def _needs_vision_path(roles: Sequence[str], modalities: Sequence[str]) -> bool: - if "vision" in roles: - return True - return any(modality in _VISION_MODALITIES for modality in modalities) - - -def prefer_runtime( - host: Mapping[str, object], - roles: Sequence[str] = (), - modalities: Sequence[str] = (), -) -> RuntimePreference: - """Return a deterministic preferred runtime with alternates and rationale.""" - ollama = bool(host.get("ollama")) - lmstudio = bool(host.get("lmstudio")) - snapshot = { - "ollama": ollama, - "lmstudio": lmstudio, - "chip": host.get("chip"), - "ram_gb": host.get("ram_gb"), - } - - if _needs_vision_path(roles, modalities): - alternates = [] - if lmstudio: - alternates.append("lmstudio") - return RuntimePreference( - preferred="mlx-vlm", - alternates=tuple(alternates), - rationale=( - "Vision / document / video workloads need native mlx-vlm; " - "Ollama does not run VLMs. LM Studio can help download weights " - "when its server is up." - ), - host_snapshot=snapshot, - ) - - audio = "audio" in modalities - if lmstudio: - alternates = ["mlx_lm"] - if ollama: - alternates.append("ollama") - rationale = ( - "LM Studio is the lowest-friction native MLX path for arbitrary Hub " - "repos when its local server is up." - ) - if audio: - rationale += ( - " Audio packs still prefer native MLX weights; Ollama is limited " - "to curated tags." - ) - return RuntimePreference( - preferred="lmstudio", - alternates=tuple(alternates), - rationale=rationale, - host_snapshot=snapshot, - ) - - alternates = [] - if ollama: - alternates.append("ollama") - rationale = ( - "Prefer headless mlx_lm for native MLX quants and Hub repos when LM Studio " - "is not detected." - ) - if audio: - rationale += ( - " Audio packs benefit from native MLX serving; Ollama remains a valid " - "alternate for curated tags only." - ) - elif ollama: - rationale += ( - " Ollama remains a valid first choice for curated library tags." - ) - else: - rationale += " Start LM Studio or install mlx-lm to serve local models." - - return RuntimePreference( - preferred="mlx_lm", - alternates=tuple(alternates), - rationale=rationale, - host_snapshot=snapshot, - ) - - -def wiring_for_preference(repo: str, role: str, preference: RuntimePreference) -> str: - """Render a suggested wiring string for the preferred runtime.""" - del role - short = repo.split("/")[-1].lower() - name = repo.split("/")[-1] - preferred = preference.preferred - if preferred == "mlx-vlm": - return "mlx-vlm server → `mlxvlm/{0}`".format(name) - if preferred == "lmstudio": - return "LM Studio → `lmstudio/{0}`".format(short) - if preferred == "ollama": - return "Ollama → `ollama/` (curated / GGUF; not arbitrary mlx-community)" - return "`mlx_lm.server --model {0}` → custom provider".format(repo) - - -def prefer_runtime_for_role(role: str, host: Mapping[str, object]) -> RuntimePreference: - """Scout/discovery helper: preference from a single role + host inventory.""" - modalities = ("document-vision",) if role == "vision" else () - return prefer_runtime(host, roles=(role or "general",), modalities=modalities) diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/scoring.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/scoring.py deleted file mode 100644 index 2bad358..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/scoring.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Deterministic, transparent scoring of a model against a DomainIntent. - -Pure and side-effect free. Signals that do not apply to the intent are excluded -from both the numerator and denominator so a candidate is never penalized for a -constraint the user did not request. No network access. -""" - -from __future__ import annotations - -import math -from dataclasses import dataclass -from typing import List, Mapping, Optional, Sequence, Tuple - -from .interview import DomainIntent - - -SIGNAL_WEIGHTS = { - "role_match": 30.0, - "keyword_match": 25.0, - "popularity": 15.0, - "license_ok": 10.0, - "memory_fit": 10.0, - "card_quality": 10.0, -} - -_SIGNAL_SOURCES = { - "role_match": "local_role_derivation", - "keyword_match": "card_text_and_tags", - "popularity": "huggingface_model_list", - "license_ok": "huggingface_model_metadata", - "memory_fit": "local_memory_estimate", - "card_quality": "card_text", -} - -_POPULARITY_LOG_CEILING = 6.0 # ~1e6 downloads saturates the signal. -_MEMORY_HEADROOM_FRACTION = 0.8 - - -@dataclass(frozen=True) -class Signal: - id: str - weight: float - applicable: bool - matched: Optional[bool] - contribution: float - source: str - detail: str - - def to_dict(self): - return { - "id": self.id, - "weight": self.weight, - "applicable": self.applicable, - "matched": self.matched, - "contribution": round(self.contribution, 3), - "source": self.source, - "detail": self.detail, - } - - -@dataclass(frozen=True) -class ScoreResult: - score: float - signals: Tuple[Signal, ...] - provenance: Tuple[dict, ...] - - def to_dict(self): - return { - "score": self.score, - "signals": [signal.to_dict() for signal in self.signals], - "provenance": [dict(record) for record in self.provenance], - } - - -def _signal(signal_id, applicable, matched, fraction, detail): - weight = SIGNAL_WEIGHTS[signal_id] - contribution = weight * fraction if applicable else 0.0 - return Signal( - id=signal_id, - weight=weight, - applicable=applicable, - matched=matched if applicable else None, - contribution=contribution, - source=_SIGNAL_SOURCES[signal_id], - detail=detail, - ) - - -def _role_signal(intent: DomainIntent, metadata: Mapping[str, object]) -> Signal: - model_roles = set(metadata.get("roles") or []) - matched = [role for role in intent.roles if role in model_roles] - fraction = len(matched) / len(intent.roles) if intent.roles else 0.0 - return _signal( - "role_match", True, bool(matched), fraction, - "matched roles: {0}".format(", ".join(matched) or "none"), - ) - - -def _keyword_signal( - intent: DomainIntent, metadata: Mapping[str, object], card_text: Optional[str] -) -> Signal: - if not intent.keywords: - return _signal("keyword_match", False, None, 0.0, "no keywords requested") - tags = " ".join(str(tag).lower() for tag in (metadata.get("tags") or [])) - haystack = "{0} {1}".format((card_text or "").lower(), tags) - matched = [keyword for keyword in intent.keywords if keyword in haystack] - fraction = len(matched) / len(intent.keywords) - return _signal( - "keyword_match", True, bool(matched), fraction, - "matched keywords: {0}".format(", ".join(matched) or "none"), - ) - - -def _popularity_signal(metadata: Mapping[str, object]) -> Signal: - downloads = metadata.get("downloads") or 0 - try: - downloads = max(0, int(downloads)) - except (TypeError, ValueError): - downloads = 0 - fraction = min(1.0, math.log10(downloads + 1) / _POPULARITY_LOG_CEILING) - return _signal( - "popularity", True, downloads > 0, fraction, - "downloads: {0}".format(downloads), - ) - - -def _license_signal(intent: DomainIntent, metadata: Mapping[str, object]) -> Signal: - if not intent.license_allow: - return _signal("license_ok", False, None, 0.0, "no license restriction") - license_name = (metadata.get("license") or "").lower() - matched = license_name in set(intent.license_allow) - return _signal( - "license_ok", True, matched, 1.0 if matched else 0.0, - "license: {0}".format(license_name or "unknown"), - ) - - -def _memory_signal(intent: DomainIntent, metadata: Mapping[str, object]) -> Signal: - est_ram = metadata.get("est_ram_gb") - if intent.memory_gb is None or est_ram is None: - return _signal("memory_fit", False, None, 0.0, "no budget or estimate") - fits = float(est_ram) < intent.memory_gb * _MEMORY_HEADROOM_FRACTION - return _signal( - "memory_fit", True, fits, 1.0 if fits else 0.0, - "est_ram_gb {0} vs budget {1}".format(est_ram, intent.memory_gb), - ) - - -def _card_quality_signal(card_text: Optional[str]) -> Signal: - if not card_text: - return _signal("card_quality", True, False, 0.0, "no model card text") - quality = 0.5 - lowered = card_text.lower() - if "```" in card_text or "usage" in lowered or "example" in lowered or "how to" in lowered: - quality += 0.5 - return _signal( - "card_quality", True, True, quality, - "card present; usage/example: {0}".format(quality > 0.5), - ) - - -def score_candidate( - intent: DomainIntent, - metadata: Mapping[str, object], - card_text: Optional[str], -) -> ScoreResult: - signals = ( - _role_signal(intent, metadata), - _keyword_signal(intent, metadata, card_text), - _popularity_signal(metadata), - _license_signal(intent, metadata), - _memory_signal(intent, metadata), - _card_quality_signal(card_text), - ) - applicable = [signal for signal in signals if signal.applicable] - denominator = sum(signal.weight for signal in applicable) - numerator = sum(signal.contribution for signal in applicable) - score = round(100.0 * numerator / denominator, 1) if denominator else 0.0 - provenance = tuple( - { - "source": signal.source, - "fields": [signal.id], - "basis": signal.detail, - } - for signal in applicable - ) - return ScoreResult(score=score, signals=signals, provenance=provenance) - - -def rank_scored( - entries: Sequence[Tuple[str, ScoreResult]] -) -> List[Tuple[str, ScoreResult]]: - """Sort (repo, ScoreResult) entries by score desc, then repo asc.""" - return sorted(entries, key=lambda entry: (-entry[1].score, entry[0])) diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/serve.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/serve.py deleted file mode 100644 index 264cfca..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/serve.py +++ /dev/null @@ -1,528 +0,0 @@ -"""Confirmation-gated launcher for local MLX serving runtimes. - -Serve is the only component that spawns processes, so every mutation goes -through the wire-style preview -> confirm -> receipt flow. Serve never -installs runtimes, never downloads models, binds only to 127.0.0.1, and stops -only processes that it started itself, verified against their receipts. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import shutil -import signal -import subprocess -import time -import urllib.error -import urllib.request -from datetime import datetime, timezone -from pathlib import Path - -from .model_doctor import scan_wired_configs -from .transactions import _atomic_in_directory -from .wiring import require_secret_free_config, validate_health_endpoint - - -SERVE_RECEIPT_SCHEMA_VERSION = "1.0" -SERVE_RECEIPT_KIND = "serve" -MAX_TOKENS_MIN, MAX_TOKENS_MAX, MAX_TOKENS_DEFAULT = 256, 65536, 8192 -PORT_MIN, PORT_MAX = 1, 65535 -READINESS_DEADLINE_DEFAULT = 60.0 -STOP_DEADLINE_SECONDS = 10.0 -MAX_RECEIPTS = 50 - -_RECIPES_PATH = Path(__file__).resolve().parent / "resources" / "serve-recipes.json" - - -class ServeError(RuntimeError): - """Classified serve failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def load_recipes(path=None): - location = Path(path) if path is not None else _RECIPES_PATH - value = json.loads(location.read_text(encoding="utf-8")) - recipes = value.get("recipes") - if not isinstance(recipes, dict) or not recipes: - raise ValueError("serve recipe table is missing its recipes") - for name, recipe in recipes.items(): - for key in ("executable", "argv", "default_port", "readiness", "install_hint"): - if key not in recipe: - raise ValueError("serve recipe {0} is missing {1}".format(name, key)) - return recipes - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def _preview_hash(plan): - canonical = json.dumps( - {key: plan[key] for key in sorted(plan) if key != "preview_hash"}, - sort_keys=True, separators=(",", ":"), - ) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest() - - -def receipts_root(root=None): - base = Path(root) if root is not None else Path.cwd() - return base / ".mlx-agent-receipts" / "serve" - - -def plan_start(repo, runtime, recipes, port=None, max_tokens=MAX_TOKENS_DEFAULT, - adapter_path=None): - """Render the exact start plan; pure and side-effect free.""" - if not isinstance(repo, str) or not repo.strip() or "/" not in repo: - raise ServeError( - "invalid_repo", - "serve requires a publisher/model repository identifier.", - "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", - ) - recipe = recipes.get(runtime) - if recipe is None: - raise ServeError( - "unsupported_runtime", - "serve does not launch the {0} runtime.".format(runtime), - "Ollama and LM Studio manage their own servers; serve supports: {0}.".format( - ", ".join(sorted(recipes)) - ), - ) - if not isinstance(max_tokens, int) or isinstance(max_tokens, bool): - raise ServeError( - "invalid_arguments", "max_tokens must be an integer.", - "Pass --max-tokens between {0} and {1}.".format(MAX_TOKENS_MIN, MAX_TOKENS_MAX), - ) - if not MAX_TOKENS_MIN <= max_tokens <= MAX_TOKENS_MAX: - raise ServeError( - "invalid_arguments", - "max_tokens is outside {0}-{1}.".format(MAX_TOKENS_MIN, MAX_TOKENS_MAX), - "Pass a bounded --max-tokens value.", - ) - selected_port = recipe["default_port"] if port is None else port - if not isinstance(selected_port, int) or isinstance(selected_port, bool): - raise ServeError( - "invalid_arguments", "port must be an integer.", - "Pass --port between {0} and {1}.".format(PORT_MIN, PORT_MAX), - ) - if not PORT_MIN <= selected_port <= PORT_MAX: - raise ServeError( - "invalid_arguments", - "port is outside {0}-{1}.".format(PORT_MIN, PORT_MAX), - "Pass a valid loopback port.", - ) - if adapter_path is not None and not recipe.get("adapter_argv"): - raise ServeError( - "unsupported_runtime", - "The {0} recipe does not support adapter serving.".format(runtime), - "Serve adapters with a runtime whose recipe declares adapter support.", - ) - values = { - "executable": recipe["executable"], - "repo": repo, - "port": str(selected_port), - "max_tokens": str(max_tokens), - } - argv = [part.format(**values) for part in recipe["argv"]] - if adapter_path is not None: - argv.extend( - part.format(adapter_path=str(adapter_path)) for part in recipe["adapter_argv"] - ) - plan = { - "repo": repo, - "runtime": runtime, - "port": selected_port, - "max_tokens": max_tokens, - "adapter_path": str(adapter_path) if adapter_path is not None else None, - "argv": argv, - "readiness": recipe["readiness"].format(port=selected_port), - "bind": "127.0.0.1", - } - plan["preview_hash"] = _preview_hash(plan) - return plan - - -def _default_which(executable): - return shutil.which(executable) - - -def _default_port_free(port): - import socket - - probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - probe.settimeout(0.5) - return probe.connect_ex(("127.0.0.1", port)) != 0 - finally: - probe.close() - - -def _default_spawn(argv, log_path): - handle = open(log_path, "ab", buffering=0) - process = subprocess.Popen( - argv, - stdin=subprocess.DEVNULL, - stdout=handle, - stderr=subprocess.STDOUT, - start_new_session=True, - ) - return process.pid - - -def _default_readiness(url, deadline_seconds, clock=time.monotonic, sleep=time.sleep): - validate_health_endpoint(url) - deadline = clock() + deadline_seconds - while clock() < deadline: - try: - request = urllib.request.Request(url, method="GET") - opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - with opener.open(request, timeout=2.0) as response: - if 200 <= response.status < 400: - return True - except (OSError, ValueError, urllib.error.URLError): - pass - sleep(0.5) - return False - - -def _pid_alive(pid): - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True - except OSError: - return False - return True - - -def _pid_command(pid): - try: - result = subprocess.run( - ["ps", "-p", str(pid), "-o", "command="], - capture_output=True, text=True, timeout=5, - ) - except (OSError, subprocess.SubprocessError): - return None - if result.returncode != 0: - return None - return result.stdout.strip() or None - - -def start_serve(plan, receipts_dir=None, confirm=False, preview_hash=None, - which=None, model_present=None, port_free=None, - wired_claims=None, spawn=None, readiness=None, - readiness_deadline=READINESS_DEADLINE_DEFAULT, now=_utc_now, - pid_alive=None): - """Execute a reviewed start plan; the only mutating serve entry point.""" - which = which or _default_which - port_free = port_free or _default_port_free - spawn = spawn or _default_spawn - readiness = readiness or _default_readiness - pid_alive = pid_alive or _pid_alive - root = receipts_root(receipts_dir) - - if not confirm: - return {"status": "preview", "plan": plan, "requires_confirmation": True} - if not preview_hash: - raise ServeError( - "preview_hash_required", - "--confirm requires the hash from a reviewed serve preview.", - "Run serve start without --confirm, inspect the plan, then pass --preview-hash.", - ) - if preview_hash != plan["preview_hash"]: - raise ServeError( - "preview_stale", - "The supplied preview hash does not match this start plan.", - "Re-run serve start without --confirm and review the fresh plan.", - ) - - if which(plan["argv"][0]) is None: - recipe_hint = load_recipes()[plan["runtime"]]["install_hint"] - raise ServeError( - "runtime_not_installed", - "The {0} executable is not installed.".format(plan["argv"][0]), - "Install it yourself ({0}); serve never installs runtimes.".format(recipe_hint), - ) - if model_present is not None and not model_present(plan["repo"]): - raise ServeError( - "model_not_local", - "The model is not present in a local inventory.", - "Download it with the runtime's own pull command first; serve never downloads models.", - ) - if not port_free(plan["port"]): - raise ServeError( - "port_in_use", - "Port {0} is already bound on 127.0.0.1.".format(plan["port"]), - "Pick a free --port, or stop the process that owns this one.", - ) - if wired_claims is not None: - claimed = wired_claims(plan["port"], plan["runtime"]) - if claimed: - raise ServeError( - "port_in_use", - "A wired config already claims port {0}.".format(plan["port"]), - "Move this server to its own --port.", - ) - if plan["adapter_path"] is not None and not Path(plan["adapter_path"]).is_dir(): - raise ServeError( - "invalid_arguments", - "The adapter path does not exist: {0}".format(plan["adapter_path"]), - "Pass an existing --adapter-path directory produced by mlx_lm.lora.", - ) - - receipt_path = root / "{0}.json".format(plan["port"]) - if receipt_path.exists(): - existing = _read_receipt(receipt_path) - if existing is not None and pid_alive(existing.get("pid", -1)): - raise ServeError( - "port_in_use", - "A serve receipt for port {0} is still live.".format(plan["port"]), - "Run serve stop --port {0} first.".format(plan["port"]), - ) - - root.mkdir(parents=True, exist_ok=True) - log_path = root / "{0}.log".format(plan["port"]) - pid = spawn(plan["argv"], str(log_path)) - if not readiness(plan["readiness"], readiness_deadline): - _terminate_pid(pid) - raise ServeError( - "readiness_timeout", - "The server did not answer its readiness endpoint in time.", - "Inspect the serve log at {0}.".format(log_path), - ) - - receipt = { - "schema_version": SERVE_RECEIPT_SCHEMA_VERSION, - "kind": SERVE_RECEIPT_KIND, - "repo": plan["repo"], - "runtime": plan["runtime"], - "port": plan["port"], - "argv": list(plan["argv"]), - "pid": pid, - "log_path": str(log_path), - "readiness": plan["readiness"], - "started_at": now(), - "preview_hash": plan["preview_hash"], - } - content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") - _atomic_in_directory(root, receipt_path.name, content, 0o600) - return {"status": "started", "receipt": receipt} - - -def _read_receipt(path): - try: - value = json.loads(Path(path).read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - return None - if not isinstance(value, dict) or value.get("kind") != SERVE_RECEIPT_KIND: - return None - if not isinstance(value.get("pid"), int) or isinstance(value.get("pid"), bool): - return None - if not isinstance(value.get("argv"), list) or not isinstance(value.get("port"), int): - return None - return value - - -def status_serve(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command): - """Cross-check serve receipts against live processes; read-only.""" - root = receipts_root(receipts_dir) - entries = [] - if not root.is_dir(): - return entries - for path in sorted(root.glob("*.json")): - if len(entries) >= MAX_RECEIPTS: - break - receipt = _read_receipt(path) - if receipt is None: - continue - alive = pid_alive(receipt["pid"]) - command = pid_command(receipt["pid"]) if alive else None - entries.append({ - "receipt": str(path), - "repo": receipt["repo"], - "runtime": receipt["runtime"], - "port": receipt["port"], - "pid": receipt["pid"], - "alive": alive, - "argv_match": _argv_matches(receipt, command) if alive else False, - "log_path": receipt.get("log_path"), - "started_at": receipt.get("started_at"), - }) - return entries - - -def _argv_matches(receipt, command, require_port=True): - if not command: - return False - argv = receipt.get("argv") or [] - if not argv: - return False - executable = Path(str(argv[0])).name - if executable not in command or str(receipt.get("repo")) not in command: - return False - if require_port: - return "--port {0}".format(receipt.get("port")) in command - return True - - -def _terminate_pid(pid, sig=signal.SIGTERM): - try: - os.kill(pid, sig) - except OSError: - return False - return True - - -def stop_serve(port, receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, - terminate=_terminate_pid, clock=time.monotonic, sleep=time.sleep): - """Stop exactly the process a serve receipt owns; never kills by port scan.""" - root = receipts_root(receipts_dir) - receipt_path = root / "{0}.json".format(port) - receipt = _read_receipt(receipt_path) if receipt_path.is_file() else None - if receipt is None: - raise ServeError( - "receipt_not_found", - "No serve receipt exists for port {0}.".format(port), - "serve stop only stops processes that serve started; stop foreign processes yourself.", - ) - pid = receipt["pid"] - if not pid_alive(pid): - receipt_path.unlink() - return {"status": "already_stopped", "port": port, "pid": pid} - command = pid_command(pid) - if not _argv_matches(receipt, command): - raise ServeError( - "pid_argv_mismatch", - "The live pid {0} does not match the serve receipt; refusing to signal it.".format(pid), - "Inspect the process yourself; the receipt is retained at {0}.".format(receipt_path), - ) - terminate(pid, signal.SIGTERM) - deadline = clock() + STOP_DEADLINE_SECONDS - while clock() < deadline: - if not pid_alive(pid): - receipt_path.unlink() - return {"status": "stopped", "port": port, "pid": pid} - sleep(0.2) - terminate(pid, signal.SIGKILL) - if pid_alive(pid): - raise ServeError( - "stop_failed", - "The process did not exit after SIGTERM and SIGKILL.", - "Inspect pid {0} yourself; the receipt is retained at {1}.".format(pid, receipt_path), - ) - receipt_path.unlink() - return {"status": "stopped", "port": port, "pid": pid, "forced": True} - - -def wired_port_claim(roots): - """Build a wired_claims(port, runtime) gate from Wire-managed configs.""" - wired = scan_wired_configs(roots) - claims = {} - for config in wired: - endpoint = config.get("endpoint") - if not endpoint: - continue - try: - parsed = validate_health_endpoint(endpoint) - from urllib.parse import urlsplit - - parts = urlsplit(parsed) - port = parts.port or (443 if parts.scheme == "https" else 80) - except ValueError: - continue - claims.setdefault(port, config["runtime"]) - - def check(port, runtime): - owner = claims.get(port) - return owner is not None and owner != runtime - - return check - - -def default_model_present(hf_cache=None): - from .model_doctor import default_hf_cache, inspect_hf_cache - - cache = Path(hf_cache) if hf_cache is not None else default_hf_cache() - known = {item["id"].casefold() for item in inspect_hf_cache(cache)} - - def check(repo): - return repo.casefold() in known - - return check - - -LAUNCHD_LABEL_PREFIX = "com.mlx-agent.serve." -_LAUNCHD_LABEL = re.compile(r"\A" + LAUNCHD_LABEL_PREFIX.replace(".", r"\.") + r"(\d{1,5})\Z") - - -def launchd_label(port): - return "{0}{1}".format(LAUNCHD_LABEL_PREFIX, port) - - -def render_launchd_plist(plan, log_path=None): - """Render a deterministic launchd plist for a reviewed serve plan.""" - from xml.sax.saxutils import escape - - label = launchd_label(plan["port"]) - log = log_path or str(receipts_root() / "{0}.log".format(plan["port"])) - arguments = "\n".join( - " {0}".format(escape(str(part))) for part in plan["argv"] - ) - return ( - '\n' - '\n' - '\n' - '\n' - ' Label\n' - ' {label}\n' - ' ProgramArguments\n' - ' \n' - '{arguments}\n' - ' \n' - ' RunAtLoad\n' - ' \n' - ' KeepAlive\n' - ' \n' - ' StandardOutPath\n' - ' {log}\n' - ' StandardErrorPath\n' - ' {log}\n' - '\n' - '\n' - ).format(label=escape(label), arguments=arguments, log=escape(str(log))) - - -class LaunchdPlistAdapter: - """Validate the exact launchd plist subset serve renders.""" - - version = "1.0" - runtime = "launchd" - - def __init__(self, path=None): - self.path = Path(path) if path is not None else None - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("plist content must be text") - require_secret_free_config(content) - lines = content.splitlines() - if len(lines) < 8 or not lines[0].startswith("': - raise ValueError("launchd plist must use the managed XML subset") - if lines[-1] != "" or lines[-2] != "": - raise ValueError("launchd plist must close its dict and plist elements") - text = content - for required in ("Label", "ProgramArguments", "RunAtLoad"): - if required not in text: - raise ValueError("launchd plist is missing {0}".format(required)) - label_match = re.search(r"Label\s*\n\s*([^<]+)", text) - if label_match is None or _LAUNCHD_LABEL.fullmatch(label_match.group(1)) is None: - raise ValueError("launchd plist label must use the managed serve prefix") - return True diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/transactions.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/transactions.py deleted file mode 100644 index ee06116..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/transactions.py +++ /dev/null @@ -1,1343 +0,0 @@ -"""Durable, reversible configuration transactions for Wire. - -The transaction root is opened with ``O_NOFOLLOW`` where macOS/Python exposes -it. Stage, backup, receipt, and replace operations then use that directory -file descriptor. A hostile actor can still race an already-open target leaf -between validation and replacement, but ``os.replace`` replaces that leaf and -never follows it; ancestors are verified before opening their directory fd. -""" - -from __future__ import annotations - -import difflib -import errno -import fcntl -import hashlib -import json -import os -import stat -import unicodedata -import uuid -from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from contextlib import contextmanager, nullcontext - -from .wiring import ( - ConfigAdapter, - redact_endpoint, - redact_secrets, - require_secret_free_config, - validate_health_endpoint, -) - - -RECEIPT_SCHEMA_VERSION = "2.1" -MAX_PREVIEW_CHARS = 12000 -_STATUSES = {"pending", "applied", "failed", "rolled_back", "rollback_failed", "recovery_required"} -_HASH = __import__("re").compile(r"^[0-9a-f]{64}$") -_LEGACY_LOCK_NAME = __import__("re").compile(r"^\.mlx-agent-wire-([0-9a-f]{64})\.lock$") -COOPERATIVE_CONCURRENCY_NOTE = "Advisory lock protects accidental/cooperative writers; a malicious process ignoring it can still race the final rename." -LEGACY_LOCK_MIGRATION = "legacy-target-locks-v1" -_LEGACY_LOCK_MARKER = "legacy-lock-migration-v1.json" - - -class ConcurrentTransactionError(ValueError): - """A cooperative writer holds the transaction's advisory lock.""" - - -class LegacyLockError(ConcurrentTransactionError): - """A legacy target-adjacent lock prevents a safe scoped-lock upgrade.""" - - -def _sha256(value): - return hashlib.sha256(value).hexdigest() - - -def _timestamp(): - return datetime.now(timezone.utc).isoformat() - - -def _absolute(path): - value = Path(path) - if _has_parent_reference(value): - raise ValueError("path traversal is not allowed: {0}".format(path)) - return Path(os.path.abspath(str(value))) - - -def _physical_absolute(path): - result = _absolute(path) - # /var is a fixed macOS compatibility alias. Normalize before descriptor - # traversal rather than following a user-controlled symlink component. - if str(result) == "/var" or str(result).startswith("/var/"): - if not os.path.islink("/var") or os.readlink("/var") != "private/var": - raise ValueError("untrusted /var compatibility alias") - result = Path("/private/var") / result.relative_to("/var") - return result - - -def _has_parent_reference(path): - return ".." in Path(path).parts - - -def _walk_directory(path, create=False, component_hook=None): - """Open each absolute component from `/` using dir-fd + O_NOFOLLOW only.""" - if not hasattr(os, "O_NOFOLLOW"): - raise ValueError("platform lacks required no-follow directory traversal") - logical = _absolute(path) - value = _physical_absolute(logical) - flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | os.O_NOFOLLOW - descriptor = os.open("/", flags) - try: - for component in value.parts[1:]: - if component_hook is not None: - component_hook(descriptor, component) - try: - child = os.open(component, flags, dir_fd=descriptor) - except FileNotFoundError: - if not create: - raise ValueError("path component does not exist: {0}".format(component)) - os.mkdir(component, 0o700, dir_fd=descriptor) - os.fsync(descriptor) - child = os.open(component, flags, dir_fd=descriptor) - except OSError as error: - raise ValueError("refusing unsafe directory component {0}: {1}".format(component, error)) - os.close(descriptor) - descriptor = child - return logical, descriptor - except BaseException: - os.close(descriptor) - raise - - -def _assert_safe_directory(path, create=False): - value, descriptor = _walk_directory(path, create=create) - os.close(descriptor) - return value - - -def _target_lock_name(target): - """Return the target-adjacent v1 lock name used by migration checks.""" - return ".mlx-agent-wire-{0}.lock".format(_target_lock_digest(target)) - - -def _target_lock_digest(target): - """Return the path-spelling v1 identity used only for legacy migration.""" - return _sha256(str(_physical_absolute(target)).encode("utf-8")) - - -def _filesystem_target_lock_digest(target, parent_fd): - """Bind a lock to the opened parent filesystem object and canonical leaf.""" - item = os.fstat(parent_fd) - # The stdlib does not expose a stable descriptor-bound filesystem - # case-sensitivity capability. Fold conservatively rather than derive an - # identity from mutable directory contents and risk split alias locks. - leaf = unicodedata.normalize("NFC", _physical_absolute(target).name).casefold() - identity = "{0}:{1}:{2}".format(item.st_dev, item.st_ino, leaf) - return _sha256(identity.encode("utf-8")) - - -def _filesystem_target_lock_name(target, parent_fd): - return ".mlx-agent-wire-{0}.lock".format( - _filesystem_target_lock_digest(target, parent_fd) - ) - - -def _filesystem_parent_digest(directory_fd): - item = os.fstat(directory_fd) - return _sha256("{0}:{1}".format(item.st_dev, item.st_ino).encode("utf-8")) - - -def _legacy_parent_scopes(targets, create=False): - """Open and deduplicate physical target parents without following aliases.""" - scopes = {} - try: - for target in sorted({_physical_absolute(item) for item in targets}, key=str): - try: - _parent, directory_fd = _walk_directory(target.parent, create=create) - except ValueError as error: - if str(error).startswith("path component does not exist:"): - continue - raise - try: - item = os.fstat(directory_fd) - identity = (item.st_dev, item.st_ino) - if identity in scopes: - os.close(directory_fd) - scopes[identity]["targets"].append(target) - continue - scopes[identity] = { - "digest": _filesystem_parent_digest(directory_fd), - "directory_fd": directory_fd, - "identity": identity, - "parent": target.parent, - "targets": [target], - } - except BaseException: - os.close(directory_fd) - raise - return [scopes[key] for key in sorted(scopes)] - except BaseException: - for scope in scopes.values(): - os.close(scope["directory_fd"]) - raise - - -def _close_legacy_parent_scopes(scopes): - for scope in reversed(scopes): - os.close(scope["directory_fd"]) - - -def _merge_legacy_parent_scopes(scopes, additions): - """Merge newly created/opened parents without retaining duplicate fds.""" - existing = {scope["identity"]: scope for scope in scopes} - added = [] - for addition in additions: - current = existing.get(addition["identity"]) - if current is not None: - current["targets"] = sorted(set(current["targets"] + addition["targets"]), key=str) - os.close(addition["directory_fd"]) - continue - existing[addition["identity"]] = addition - scopes.append(addition) - added.append(addition) - scopes.sort(key=lambda scope: scope["identity"]) - return added - - -def _legacy_candidate_names(directory_fd): - """List only exact v1 lock names; unrelated directory entries stay opaque.""" - try: - names = os.listdir(directory_fd) - except OSError as error: - raise ValueError("could not safely enumerate legacy target transaction locks: {0}".format(error)) - candidates = [] - for name in names: - match = _LEGACY_LOCK_NAME.fullmatch(name) - if match is not None: - candidates.append((name, match.group(1))) - return sorted(candidates) - - -def _validate_legacy_candidate_item(item): - if not stat.S_ISREG(item.st_mode): - raise ValueError("legacy target transaction lock is not a regular file") - if hasattr(os, "geteuid") and item.st_uid != os.geteuid(): - raise ValueError("legacy target transaction lock is not owned by the current user") - - -def _open_legacy_candidate(scope, name, digest, writable): - descriptor = None - flags = (os.O_RDWR if writable else os.O_RDONLY) | os.O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) - try: - try: - descriptor = os.open(name, flags, dir_fd=scope["directory_fd"]) - except FileNotFoundError: - return None - except OSError as error: - raise ValueError("could not safely open legacy target transaction lock: {0}".format(error)) - item = os.fstat(descriptor) - _validate_legacy_candidate_item(item) - try: - current = os.stat(name, dir_fd=scope["directory_fd"], follow_symlinks=False) - except FileNotFoundError: - os.close(descriptor) - return None - if (current.st_dev, current.st_ino) != (item.st_dev, item.st_ino): - raise ValueError("legacy target transaction lock changed while it was opened") - return { - "descriptor": descriptor, - "digest": digest, - "identity": (item.st_dev, item.st_ino), - "name": name, - "path": str(scope["parent"] / name), - "scope": scope, - } - except BaseException: - if descriptor is not None: - os.close(descriptor) - raise - - -def _release_legacy_candidates(candidates): - for candidate in reversed(candidates): - try: - fcntl.flock(candidate["descriptor"], fcntl.LOCK_UN) - finally: - os.close(candidate["descriptor"]) - - -def _acquire_legacy_candidates(scopes): - """Acquire a stable snapshot of every strict candidate before mutation.""" - held = [] - current = {} - try: - for _attempt in range(32): - for scope in scopes: - for name, digest in _legacy_candidate_names(scope["directory_fd"]): - key = (scope["digest"], name) - existing = current.get(key) - if existing is not None: - try: - item = os.stat(name, dir_fd=scope["directory_fd"], follow_symlinks=False) - except FileNotFoundError: - current.pop(key, None) - else: - if (item.st_dev, item.st_ino) == existing["identity"]: - continue - candidate = _open_legacy_candidate(scope, name, digest, writable=True) - if candidate is None: - current.pop(key, None) - continue - try: - fcntl.flock(candidate["descriptor"], fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as error: - os.close(candidate["descriptor"]) - if error.errno in (errno.EACCES, errno.EAGAIN): - raise LegacyLockError("legacy_lock_busy: stop older mlx-agent processes before upgrading scoped locks") - raise ValueError("could not safely acquire legacy target transaction lock: {0}".format(error)) - held.append(candidate) - current[key] = candidate - - stable = True - present = set() - for scope in scopes: - for name, _digest in _legacy_candidate_names(scope["directory_fd"]): - key = (scope["digest"], name) - present.add(key) - candidate = current.get(key) - try: - item = os.stat(name, dir_fd=scope["directory_fd"], follow_symlinks=False) - except FileNotFoundError: - stable = False - continue - if candidate is None or (item.st_dev, item.st_ino) != candidate["identity"]: - stable = False - for key in list(current): - if key not in present: - current.pop(key) - stable = False - if stable: - return held, [current[key] for key in sorted(current)] - raise ValueError("legacy target transaction lock set changed repeatedly during migration") - except BaseException: - _release_legacy_candidates(held) - raise - - -def _inspect_legacy_candidates(scopes): - """Open strict candidates no-follow for a read-only doctor snapshot.""" - candidates = [] - for scope in scopes: - for name, digest in _legacy_candidate_names(scope["directory_fd"]): - candidate = _open_legacy_candidate(scope, name, digest, writable=False) - if candidate is None: - continue - os.close(candidate.pop("descriptor")) - candidates.append(candidate) - return candidates - - -def _legacy_scope_was_migrated(scope, candidates, state): - if state["unscoped"]: - return True - if scope["digest"] in state["parents"]: - return True - migrated = state["targets"] - if any(entry.get("parent") == scope["digest"] for entry in migrated.values()): - return True - relevant = {_target_lock_digest(target) for target in scope["targets"]} - relevant.update(candidate["digest"] for candidate in candidates) - if relevant.intersection(migrated): - return True - # Old markers cannot map a one-way spelling digest back to a parent. A - # strict candidate in that ambiguous upgrade state must fail closed. - return any("parent" not in entry for entry in migrated.values()) - - -def _read_legacy_migration_state(lock_root): - """Read validated parent-scoped legacy-lock migration state.""" - try: - _root, directory_fd = _walk_directory(lock_root) - except ValueError as error: - if str(error).startswith("path component does not exist:"): - return {"version": LEGACY_LOCK_MIGRATION, "targets": {}, "parents": {}, "unscoped": False} - raise - descriptor = None - try: - try: - descriptor = os.open(_LEGACY_LOCK_MARKER, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory_fd) - except FileNotFoundError: - return {"version": LEGACY_LOCK_MIGRATION, "targets": {}, "parents": {}, "unscoped": False} - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - raise ValueError("legacy lock migration marker is not a regular file") - value = json.loads(os.read(descriptor, 1048576).decode("utf-8")) - # The prior release-candidate marker had no target map. Treat it as - # an unscoped barrier: it cannot prove which physical parents migrated. - if value == {"version": LEGACY_LOCK_MIGRATION}: - return {"version": LEGACY_LOCK_MIGRATION, "targets": {}, "parents": {}, "unscoped": True} - allowed_fields = ( - {"version", "targets"}, - {"version", "targets", "parents"}, - {"version", "targets", "parents", "unscoped"}, - ) - if not isinstance(value, dict) or set(value) not in allowed_fields or value["version"] != LEGACY_LOCK_MIGRATION or not isinstance(value["targets"], dict): - raise ValueError("legacy lock migration marker is malformed") - for digest, entry in value["targets"].items(): - if not isinstance(digest, str) or not _HASH.fullmatch(digest) or not isinstance(entry, dict) or set(entry) not in ({"migrated_at"}, {"migrated_at", "parent"}) or not isinstance(entry["migrated_at"], str) or ("parent" in entry and (not isinstance(entry["parent"], str) or not _HASH.fullmatch(entry["parent"]))): - raise ValueError("legacy lock migration marker is malformed") - try: - if datetime.fromisoformat(entry["migrated_at"].replace("Z", "+00:00")).tzinfo is None: - raise ValueError("missing timezone") - except ValueError: - raise ValueError("legacy lock migration marker is malformed") - parents = value.get("parents", {}) - unscoped = value.get("unscoped", False) - if not isinstance(parents, dict) or not isinstance(unscoped, bool): - raise ValueError("legacy lock migration marker is malformed") - for digest, entry in parents.items(): - if not isinstance(digest, str) or not _HASH.fullmatch(digest) or not isinstance(entry, dict) or set(entry) != {"migrated_at"} or not isinstance(entry["migrated_at"], str): - raise ValueError("legacy lock migration marker is malformed") - try: - if datetime.fromisoformat(entry["migrated_at"].replace("Z", "+00:00")).tzinfo is None: - raise ValueError("missing timezone") - except ValueError: - raise ValueError("legacy lock migration marker is malformed") - value["parents"] = parents - value["unscoped"] = unscoped - return value - except OSError as error: - raise ValueError("could not safely read legacy lock migration marker: {0}".format(error)) - finally: - if descriptor is not None: - os.close(descriptor) - os.close(directory_fd) - - -def _write_legacy_migration_state(lock_root, state): - _atomic_in_directory( - lock_root, - _LEGACY_LOCK_MARKER, - (json.dumps(state, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8"), - 0o600, - ) - - -@contextmanager -def _legacy_migration_window(lock_root): - """Serialize parent-map upgrades so multi-target writes cannot lose entries.""" - _root, directory_fd = _walk_directory(lock_root, create=True) - descriptor = None - try: - descriptor = os.open(".legacy-lock-migration-v1.state.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600, dir_fd=directory_fd) - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - raise ValueError("legacy lock migration state lock is not a regular file") - try: - fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as error: - if error.errno in (errno.EACCES, errno.EAGAIN): - raise LegacyLockError("legacy_lock_busy: another scoped legacy-lock migration is active") - raise ValueError("could not safely acquire legacy lock migration state lock: {0}".format(error)) - yield - finally: - if descriptor is not None: - try: - fcntl.flock(descriptor, fcntl.LOCK_UN) - finally: - os.close(descriptor) - os.close(directory_fd) - - -@contextmanager -def _migrate_legacy_target_locks(targets, lock_root, create_parents=False): - """Upgrade target-adjacent legacy locks before entering scoped lock storage.""" - canonical = sorted({_physical_absolute(item) for item in targets}, key=str) - with _legacy_migration_window(lock_root): - scopes = _legacy_parent_scopes(canonical) - held = [] - try: - held, candidates = _acquire_legacy_candidates(scopes) - state = _read_legacy_migration_state(lock_root) - migrated = state["targets"] - initial_by_parent = {scope["digest"]: [] for scope in scopes} - for candidate in candidates: - initial_by_parent[candidate["scope"]["digest"]].append(candidate) - if any(initial_by_parent[scope["digest"]] and _legacy_scope_was_migrated(scope, initial_by_parent[scope["digest"]], state) for scope in scopes): - raise LegacyLockError("legacy_lock_recreated: stop older mlx-agent processes and remove recreated legacy locks before continuing") - - opened_targets = {target for scope in scopes for target in scope["targets"]} - missing_targets = [target for target in canonical if target not in opened_targets] - if create_parents and missing_targets: - additions = _legacy_parent_scopes(missing_targets, create=True) - added_scopes = _merge_legacy_parent_scopes(scopes, additions) - added_held, added_candidates = _acquire_legacy_candidates(added_scopes) - held.extend(added_held) - candidates.extend(added_candidates) - added_by_parent = {scope["digest"]: [] for scope in added_scopes} - for candidate in added_candidates: - added_by_parent[candidate["scope"]["digest"]].append(candidate) - if any(added_by_parent[scope["digest"]] and _legacy_scope_was_migrated(scope, added_by_parent[scope["digest"]], state) for scope in added_scopes): - raise LegacyLockError("legacy_lock_recreated: stop older mlx-agent processes and remove recreated legacy locks before continuing") - - by_parent = {scope["digest"]: [] for scope in scopes} - for candidate in candidates: - by_parent[candidate["scope"]["digest"]].append(candidate) - - for candidate in candidates: - try: - item = os.stat(candidate["name"], dir_fd=candidate["scope"]["directory_fd"], follow_symlinks=False) - except FileNotFoundError: - raise ValueError("legacy target transaction lock changed before migration") - if (item.st_dev, item.st_ino) != candidate["identity"]: - raise ValueError("legacy target transaction lock changed before migration") - touched = set() - for candidate in candidates: - os.unlink(candidate["name"], dir_fd=candidate["scope"]["directory_fd"]) - touched.add(candidate["scope"]["digest"]) - for scope in scopes: - if scope["digest"] in touched: - os.fsync(scope["directory_fd"]) - - stamp = _timestamp() - for scope in scopes: - state["parents"].setdefault(scope["digest"], {"migrated_at": stamp}) - for target in scope["targets"]: - digest = _target_lock_digest(target) - entry = migrated.setdefault(digest, {"migrated_at": stamp, "parent": scope["digest"]}) - entry.setdefault("parent", scope["digest"]) - for candidate in by_parent[scope["digest"]]: - entry = migrated.setdefault(candidate["digest"], {"migrated_at": stamp, "parent": scope["digest"]}) - entry.setdefault("parent", scope["digest"]) - _write_legacy_migration_state(lock_root, state) - yield - finally: - _release_legacy_candidates(held) - _close_legacy_parent_scopes(scopes) - - -def legacy_lock_problem(targets, lock_root): - """Describe legacy-lock state for installer doctor without mutating it.""" - state = _read_legacy_migration_state(lock_root) - paths = {"legacy_lock_recreated": [], "legacy_lock_migration_required": []} - scopes = _legacy_parent_scopes(targets) - try: - candidates = _inspect_legacy_candidates(scopes) - by_parent = {scope["digest"]: [] for scope in scopes} - for candidate in candidates: - by_parent[candidate["scope"]["digest"]].append(candidate) - for scope in scopes: - scope_candidates = by_parent[scope["digest"]] - if not scope_candidates: - continue - code = "legacy_lock_recreated" if _legacy_scope_was_migrated(scope, scope_candidates, state) else "legacy_lock_migration_required" - paths[code].extend(candidate["path"] for candidate in scope_candidates) - finally: - _close_legacy_parent_scopes(scopes) - return [ - { - "code": code, - "paths": sorted(set(affected)), - "remediation": "stop older mlx-agent processes; after parent-scoped legacy-lock migration, do not run an older binary and remove only recreated legacy locks.", - } - for code, affected in paths.items() if affected - ] - - -@contextmanager -def _target_locks(targets, create_parents=False, lock_root=None): - """Acquire filesystem-identity locks in canonical order to avoid deadlocks.""" - canonical = sorted({_physical_absolute(target) for target in targets}, key=str) - identities = {} - held = [] - try: - for target in canonical: - _parent, target_parent_fd = _walk_directory( - target.parent, create=create_parents - ) - digest = _filesystem_target_lock_digest(target, target_parent_fd) - if digest in identities: - os.close(target_parent_fd) - continue - identities[digest] = { - "target": target, - "target_parent_fd": target_parent_fd, - } - for digest in sorted(identities): - identity = identities[digest] - target = identity["target"] - target_parent_fd = identity["target_parent_fd"] - if lock_root is None: - directory_fd = target_parent_fd - identity["target_parent_fd"] = None - else: - os.close(target_parent_fd) - identity["target_parent_fd"] = None - _parent, directory_fd = _walk_directory(lock_root, create=True) - lock_fd = None - try: - lock_fd = os.open( - ".mlx-agent-wire-{0}.lock".format(digest), - os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, - 0o600, - dir_fd=directory_fd, - ) - if not stat.S_ISREG(os.fstat(lock_fd).st_mode): - raise ValueError("target transaction lock is not a regular file") - os.fsync(directory_fd) - fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - held.append((lock_fd, directory_fd)) - except OSError as error: - if lock_fd is not None: - os.close(lock_fd) - os.close(directory_fd) - if error.errno in (errno.EACCES, errno.EAGAIN): - raise ConcurrentTransactionError("another cooperative Wire transaction is active for a target") - raise ValueError("could not safely acquire target transaction lock: {0}".format(error)) - except BaseException: - if lock_fd is not None: - os.close(lock_fd) - os.close(directory_fd) - raise - yield canonical - finally: - for lock_fd, directory_fd in reversed(held): - try: - fcntl.flock(lock_fd, fcntl.LOCK_UN) - finally: - os.close(lock_fd) - os.close(directory_fd) - for identity in identities.values(): - if identity["target_parent_fd"] is not None: - os.close(identity["target_parent_fd"]) - - -def _assert_safe_target(path): - value = _absolute(path) - _read_target(value) - return value - - -def _open_directory(path): - _value, descriptor = _walk_directory(path) - return descriptor - - -def _fsync_directory(path_or_fd): - descriptor = path_or_fd if isinstance(path_or_fd, int) else _open_directory(path_or_fd) - close = not isinstance(path_or_fd, int) - try: - os.fsync(descriptor) - finally: - if close: - os.close(descriptor) - - -def _write_all(descriptor, content): - offset = 0 - while offset < len(content): - offset += os.write(descriptor, content[offset:]) - - -def _atomic_in_directory(directory, name, content, mode, validator=None): - """Write and fsync a stage file before atomically replacing ``name``.""" - directory = _assert_safe_directory(directory) - dir_fd = _open_directory(directory) - stage_name = ".mlx-agent-stage-{0}".format(uuid.uuid4().hex) - stage_fd = None - try: - stage_fd = os.open(stage_name, os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0), mode, dir_fd=dir_fd) - _write_all(stage_fd, content) - os.fsync(stage_fd) - os.close(stage_fd) - stage_fd = None - _fsync_directory(dir_fd) - if validator is not None: - read_fd = os.open(stage_name, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), dir_fd=dir_fd) - try: - data = b"" - while True: - chunk = os.read(read_fd, 65536) - if not chunk: - break - data += chunk - finally: - os.close(read_fd) - validator(data.decode("utf-8")) - os.chmod(stage_name, mode, dir_fd=dir_fd, follow_symlinks=False) - os.replace(stage_name, name, src_dir_fd=dir_fd, dst_dir_fd=dir_fd) - _fsync_directory(dir_fd) - except BaseException: - if stage_fd is not None: - os.close(stage_fd) - try: - os.unlink(stage_name, dir_fd=dir_fd) - _fsync_directory(dir_fd) - except OSError: - pass - raise - finally: - os.close(dir_fd) - - -def _read_target(path, component_hook=None): - """Read a regular leaf through its opened parent directory descriptor.""" - value = _absolute(path) - _parent, directory = _walk_directory(value.parent, component_hook=component_hook) - try: - try: - descriptor = os.open(value.name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory) - except FileNotFoundError: - return b"", False, None - try: - item = os.fstat(descriptor) - if not stat.S_ISREG(item.st_mode): - raise ValueError("target is not a regular file: {0}".format(value)) - parts = [] - while True: - chunk = os.read(descriptor, 65536) - if not chunk: - break - parts.append(chunk) - return b"".join(parts), True, item.st_mode & 0o777 - finally: - os.close(descriptor) - except OSError as error: - raise ValueError("refusing unsafe target leaf {0}: {1}".format(value, error)) - finally: - os.close(directory) - - -def _read_regular(path): - return _read_target(path)[0] - - -@dataclass -class Receipt: - schema_version: str - transaction_id: str - adapter_version: str - timestamp: str - transaction_root: str - targets: list - target_roots: dict - target_modes: dict - after_modes: dict - before_hashes: dict - after_hashes: dict - backup_paths: dict - validations: dict - status: str - preview: str - preview_hash: str - lock_root: str = None - lock_migration: str = None - receipt_path: str = field(default="", repr=False, compare=False) - - def to_dict(self): - value = asdict(self) - value.pop("receipt_path", None) - if value.get("lock_root") is None: - value.pop("lock_root") - if value.get("lock_migration") is None: - value.pop("lock_migration") - return value - - @classmethod - def from_dict(cls, value, receipt_path=""): - required = { - "schema_version", "transaction_id", "adapter_version", "timestamp", "transaction_root", - "targets", "target_roots", "target_modes", "after_modes", "before_hashes", "after_hashes", "backup_paths", - "validations", "status", "preview", "preview_hash", - } - if not isinstance(value, dict) or not required.issubset(value) or set(value) - (required | {"lock_root", "lock_migration"}): - raise ValueError("receipt fields are malformed or untrusted") - if value["schema_version"] != RECEIPT_SCHEMA_VERSION or value["status"] not in _STATUSES: - raise ValueError("receipt version or status is unsupported") - try: - uuid.UUID(value["transaction_id"]) - parsed_timestamp = datetime.fromisoformat(value["timestamp"].replace("Z", "+00:00")) - except (TypeError, ValueError, AttributeError): - raise ValueError("receipt ID or timestamp is malformed") - if parsed_timestamp.tzinfo is None: - raise ValueError("receipt timestamp must include a timezone") - if not isinstance(value["adapter_version"], str) or not value["adapter_version"]: - raise ValueError("receipt adapter version is malformed") - root = _assert_safe_directory(value["transaction_root"]) - if Path(value["transaction_root"]) != root: - raise ValueError("receipt transaction root must be normalized") - if receipt_path: - location = _assert_safe_target(receipt_path) - if location.parent != root or location.name != "receipt.json": - raise ValueError("receipt is outside its transaction layout") - lock_root = value.get("lock_root") - if lock_root is not None: - if not isinstance(lock_root, str) or not lock_root: - raise ValueError("receipt lock root is malformed") - safe_lock_root = _assert_safe_directory(lock_root) - if str(safe_lock_root) != lock_root: - raise ValueError("receipt lock root must be normalized") - migration = value.get("lock_migration") - if migration is not None and (lock_root is None or migration != LEGACY_LOCK_MIGRATION): - raise ValueError("receipt lock migration is malformed") - targets = value["targets"] - maps = ( - value["target_roots"], value["target_modes"], value["after_modes"], - value["before_hashes"], value["after_hashes"], value["backup_paths"], - ) - if not isinstance(targets, list) or not targets or not all(isinstance(item, str) for item in targets) or len(targets) != len(set(targets)) or not all(isinstance(item, dict) for item in maps): - raise ValueError("receipt targets are malformed") - if any(set(targets) != set(item) for item in maps): - raise ValueError("receipt target maps do not match") - for index, target_name in enumerate(targets): - target = _assert_safe_target(target_name) - if str(target) != target_name or value["target_roots"][target_name] != str(target.parent): - raise ValueError("receipt target is not rooted safely") - mode = value["target_modes"][target_name] - if mode is not None and (not isinstance(mode, int) or mode < 0 or mode > 0o777): - raise ValueError("receipt mode is malformed") - after_mode = value["after_modes"][target_name] - if after_mode is not None and ( - not isinstance(after_mode, int) or after_mode < 0 or after_mode > 0o777 - ): - raise ValueError("receipt after mode is malformed") - for hashes in (value["before_hashes"], value["after_hashes"]): - if not isinstance(hashes[target_name], str) or not _HASH.fullmatch(hashes[target_name]): - raise ValueError("receipt hash is malformed") - backup = value["backup_paths"][target_name] - expected_backup = root / "backup-{0}.bin".format(index) - if (backup is None) != (mode is None): - raise ValueError("receipt backup presence does not match before existence") - if backup is None: - if value["before_hashes"][target_name] != _sha256(b""): - raise ValueError("missing backup has a non-empty hash") - elif not isinstance(backup, str) or Path(backup) != expected_backup: - raise ValueError("receipt backup is outside its transaction layout") - validations_redacted = False - if isinstance(value["validations"], dict): - try: - validations_redacted = json.loads(redact_secrets(json.dumps(value["validations"]))) == value["validations"] - except (TypeError, ValueError, json.JSONDecodeError): - validations_redacted = False - if not isinstance(value["validations"], dict) or not validations_redacted or not isinstance(value["preview"], str) or len(value["preview"]) > MAX_PREVIEW_CHARS + 32 or redact_secrets(value["preview"]) != value["preview"] or not isinstance(value["preview_hash"], str) or not _HASH.fullmatch(value["preview_hash"]): - raise ValueError("receipt validation or preview is malformed") - return cls(receipt_path=str(receipt_path), **value) - - -class Transaction: - """Create a crash-recoverable journal before any configuration mutation.""" - - def __init__(self, receipts_dir=None, health_checker=None, fault_injector=None, receipt_writer=None, path_race_hook=None, create_target_parents=False, transaction_id=None, lock_root=None): - self.receipts_dir = Path(receipts_dir) if receipts_dir else None - self.lock_root = Path(lock_root) if lock_root is not None else None - self.health_checker = health_checker - self.fault_injector = fault_injector - self.receipt_writer = receipt_writer - self.path_race_hook = path_race_hook - self.create_target_parents = bool(create_target_parents) - if transaction_id is not None: - try: - self.transaction_id = str(uuid.UUID(str(transaction_id))) - except (TypeError, ValueError, AttributeError): - raise ValueError("transaction_id must be a UUID") - else: - self.transaction_id = None - self._changes = [] - self._preview = "" - self._preview_hash = "" - self._expected_current = None - - def preview(self, changes, expected_current=None): - if not isinstance(changes, (list, tuple)) or not changes: - raise ValueError("changes must be a non-empty list") - prepared, diffs, binding = [], [], [] - seen = set() - for change in changes: - if not isinstance(change, dict) or "path" not in change or "content" not in change: - raise ValueError("each change requires path and content") - path = _absolute(change["path"]) - if str(path) in seen: - raise ValueError("each target may appear only once") - seen.add(str(path)) - content = change["content"] - if not isinstance(content, str): - raise TypeError("change content must be text") - adapter = change.get("adapter") or ConfigAdapter.detect(path, runtime=change.get("runtime")) - try: - before, existed, mode = _read_target(path, self.path_race_hook) - except ValueError as error: - if not self.create_target_parents or not str(error).startswith("path component does not exist:"): - raise - before, existed, mode = b"", False, None - before_text = before.decode("utf-8") - if not getattr(adapter, "secret_scan_exempt", False): - require_secret_free_config(before_text) - require_secret_free_config(content) - endpoint = change.get("endpoint") - if endpoint: - validate_health_endpoint(endpoint) - adapter.validate(content) - after = content.encode("utf-8") - diffs.append("".join(difflib.unified_diff( - redact_secrets(before_text).splitlines(True), redact_secrets(content).splitlines(True), - fromfile=str(path), tofile=str(path), lineterm="", - ))) - binding.append({"path": str(path), "before": _sha256(before), "after": _sha256(after), "exists": existed, "mode": mode, "endpoint": _sha256(str(endpoint or "").encode("utf-8"))}) - prepared.append({"path": path, "content": content, "adapter": adapter, "endpoint": endpoint, "before_hash": _sha256(before), "existed": existed, "mode": mode}) - self._changes = prepared - self._expected_current = self._normalize_expected_current(expected_current) - self._preview = self._bounded("\n".join(diffs)) - self._preview_hash = _sha256(json.dumps(binding, sort_keys=True, separators=(",", ":")).encode("utf-8")) - return {"changes": len(prepared), "diff": self._preview, "preview_hash": self._preview_hash, "requires_confirmation": True} - - def apply(self, confirmation, expected_current=None): - if confirmation is not True and confirmation != self._preview_hash: - raise PermissionError("explicit confirmation for this preview is required") - if not self._changes: - raise ValueError("preview changes before applying them") - if expected_current is not None: - expected_current = self._normalize_expected_current(expected_current) - if self._expected_current is not None and expected_current != self._expected_current: - raise ValueError("apply expected_current does not match preview") - self._expected_current = expected_current - with self._advisory_lock(): - return self._apply_locked() - - @contextmanager - def _advisory_lock(self): - lock_root = self._safe_lock_root() if self.lock_root is not None else None - if lock_root is None: - with _target_locks(self._lock_targets(), create_parents=self.create_target_parents): - yield - return - with _migrate_legacy_target_locks( - self._lock_targets(), lock_root, - create_parents=self.create_target_parents, - ): - with _target_locks(self._lock_targets(), create_parents=self.create_target_parents, lock_root=lock_root): - yield - - def _safe_lock_root(self): - root, descriptor = _walk_directory(self.lock_root, create=True) - os.close(descriptor) - return root - - def _lock_targets(self): - return [str(path) for path in sorted({change["path"] for change in self._changes}, key=str)] - - def _apply_locked(self): - if self._expected_current is not None: - for change in self._changes: - current, exists, mode = _read_target(change["path"], self.path_race_hook) - expected = self._expected_current[str(change["path"])] - if _sha256(current) != expected["hash"] or exists != expected["exists"] or mode != expected["mode"]: - raise ValueError("expected current target state changed before mutation") - for change in self._changes: - current, exists, mode = _read_target(change["path"], self.path_race_hook) - if _sha256(current) != change["before_hash"] or exists != change["existed"] or mode != change["mode"]: - raise ValueError("preview is stale; target changed after preview") - receipt = self._prepare_journal() - journal_written = False - mutation_started = False - try: - self._persist(receipt) - journal_written = True - self._fault("after_pending_receipt") - for index, change in enumerate(self._changes): - path = change["path"] - self._fault("before_replace:{0}".format(index)) - current, exists, mode = _read_target(path, self.path_race_hook) - if _sha256(current) != receipt.before_hashes[str(path)] or exists != change["existed"] or mode != receipt.target_modes[str(path)]: - raise ValueError("preview is stale; target changed before replacement") - mutation_started = True - self._replace_target(path, change["content"].encode("utf-8"), change["adapter"], receipt.target_modes[str(path)]) - receipt.after_hashes[str(path)] = _sha256(change["content"].encode("utf-8")) - receipt.after_modes[str(path)] = ( - receipt.target_modes[str(path)] - if receipt.target_modes[str(path)] is not None else 0o600 - ) - self._persist(receipt) - self._fault("after_replace:{0}".format(index)) - change["adapter"].validate(_read_regular(path).decode("utf-8")) - after, after_exists, after_mode = _read_target(path) - if not after_exists or (receipt.target_modes[str(path)] is not None and after_mode != receipt.target_modes[str(path)]): - raise ValueError("replacement mode does not match the reviewed target mode") - receipt.after_hashes[str(path)] = _sha256(after) - receipt.after_modes[str(path)] = after_mode - receipt.validations[str(path)] = {"pre": True, "post": True, "passed": True} - self._persist(receipt) - receipt.validations["health_check"] = self._run_health_checks() - if receipt.validations["health_check"]["passed"]: - receipt.status = "applied" - self._persist(receipt) - else: - self._finish_restore(receipt) - except Exception as error: - receipt.validations["error"] = {"passed": False, "message": redact_secrets(str(error))} - if not journal_written and not mutation_started: - receipt.status = "failed" - return receipt - self._finish_restore(receipt) - return receipt - - def _normalize_expected_current(self, expected_current): - if expected_current is None: - return None - if not isinstance(expected_current, dict): - raise ValueError("expected_current must map every target to its current state") - targets = {str(change["path"]) for change in self._changes} - if set(expected_current) != targets: - raise ValueError("expected_current targets do not match the preview") - normalized = {} - for target in sorted(targets): - state = expected_current[target] - if not isinstance(state, dict) or set(state) != {"hash", "exists", "mode"}: - raise ValueError("expected_current state must include hash, exists, and mode") - if not isinstance(state["hash"], str) or not _HASH.match(state["hash"]): - raise ValueError("expected_current hash is invalid") - if not isinstance(state["exists"], bool) or (state["mode"] is not None and not isinstance(state["mode"], int)): - raise ValueError("expected_current existence or mode is invalid") - normalized[target] = {"hash": state["hash"], "exists": state["exists"], "mode": state["mode"]} - return normalized - - def rollback(self, receipt_path): - return rollback(receipt_path) - - def _prepare_journal(self): - self._fault("before_journal_capture") - captured = [] - for change in self._changes: - before, exists, mode = _read_target(change["path"], self.path_race_hook) - if _sha256(before) != change["before_hash"] or exists != change["existed"] or mode != change["mode"]: - raise ValueError("preview is stale; target changed before journal capture") - captured.append((before, exists, mode)) - self._fault("after_journal_capture") - for index, change in enumerate(self._changes): - before, exists, mode = _read_target(change["path"], self.path_race_hook) - if _sha256(before) != change["before_hash"] or exists != change["existed"] or mode != change["mode"]: - raise ValueError("preview is stale; target changed after journal capture") - captured[index] = (before, exists, mode) - receipts_dir, receipts_fd = _walk_directory(self.receipts_dir or self._changes[0]["path"].parent / ".mlx-agent-receipts", create=True) - transaction_id = self.transaction_id or str(uuid.uuid4()) - root = receipts_dir / transaction_id - try: - self._fault("before_transaction_root_create") - os.mkdir(transaction_id, 0o700, dir_fd=receipts_fd) - os.fsync(receipts_fd) - root_fd = os.open(transaction_id, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | os.O_NOFOLLOW, dir_fd=receipts_fd) - try: - os.fsync(root_fd) - finally: - os.close(root_fd) - finally: - os.close(receipts_fd) - targets = [str(change["path"]) for change in self._changes] - receipt = Receipt( - schema_version=RECEIPT_SCHEMA_VERSION, transaction_id=transaction_id, adapter_version=ConfigAdapter.version, - timestamp=_timestamp(), transaction_root=str(root), targets=targets, - target_roots={str(change["path"]): str(change["path"].parent) for change in self._changes}, - target_modes={str(change["path"]): captured[index][2] for index, change in enumerate(self._changes)}, - after_modes={str(change["path"]): captured[index][2] for index, change in enumerate(self._changes)}, - before_hashes={}, after_hashes={}, backup_paths={}, validations={ - "concurrency": {"scope": "advisory_cooperative_target_scoped", "note": COOPERATIVE_CONCURRENCY_NOTE, "passed": True}, - }, status="pending", - preview=self._preview, preview_hash=self._preview_hash, - lock_root=str(self._safe_lock_root()) if self.lock_root is not None else None, - lock_migration=LEGACY_LOCK_MIGRATION if self.lock_root is not None else None, - receipt_path=str(root / "receipt.json"), - ) - for index, change in enumerate(self._changes): - path = change["path"] - before, exists, mode = captured[index] - target = str(path) - receipt.before_hashes[target] = _sha256(before) - receipt.after_hashes[target] = _sha256(before) - if exists: - backup = root / "backup-{0}.bin".format(index) - _atomic_in_directory(root, backup.name, before, mode or 0o600) - receipt.backup_paths[target] = str(backup) - else: - receipt.backup_paths[target] = None - return receipt - - def _replace_target(self, path, content, adapter, mode): - _assert_safe_target(path) - _atomic_in_directory(path.parent, path.name, content, mode if mode is not None else 0o600, adapter.validate) - - def _persist(self, receipt): - root = _assert_safe_directory(receipt.transaction_root) - if self.receipt_writer is not None: - self.receipt_writer(Path(receipt.receipt_path), receipt.to_dict()) - return - _atomic_in_directory(root, "receipt.json", (json.dumps(receipt.to_dict(), indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600) - - def _finish_restore(self, receipt): - if _restore_receipt(receipt, expected_current=_receipt_after_state(receipt)): - receipt.status = "rolled_back" - else: - receipt.status = "recovery_required" - try: - self._persist(receipt) - except Exception as error: - receipt.status = "recovery_required" - receipt.validations["receipt_write"] = {"passed": False, "message": redact_secrets(str(error))} - - def _run_health_checks(self): - checks = [] - for change in self._changes: - endpoint = change.get("endpoint") - if not endpoint: - continue - passed = bool(self.health_checker(endpoint)) if self.health_checker else ConfigAdapter.health_check(endpoint) - checks.append({"endpoint": redact_endpoint(endpoint), "passed": passed}) - return {"passed": all(item["passed"] for item in checks), "endpoints": checks} - - def _fault(self, point): - if self.fault_injector is not None: - self.fault_injector(point) - - @staticmethod - def _bounded(value): - return value if len(value) <= MAX_PREVIEW_CHARS else value[:MAX_PREVIEW_CHARS] + "\n... [preview truncated]" - - -def _restore_receipt(receipt, expected_current=None): - """Preflight every backup, then restore and prove byte-for-byte hashes.""" - planned = [] - try: - root = _assert_safe_directory(receipt.transaction_root) - if expected_current is not None: - _assert_expected_states( - receipt.targets, expected_current, "rollback target changed before recovery" - ) - for index, target_name in enumerate(receipt.targets): - target = _assert_safe_target(target_name) - expected = receipt.before_hashes[target_name] - backup_name = receipt.backup_paths[target_name] - if backup_name is None: - if expected != _sha256(b""): - raise ValueError("missing backup has a non-empty hash") - planned.append((target, None, receipt.target_modes[target_name], expected)) - continue - backup = _assert_safe_target(backup_name) - backup_data, backup_exists, _backup_mode = _read_target(backup) - if backup.parent != root or backup.name != "backup-{0}.bin".format(index) or not backup_exists: - raise ValueError("backup is missing or outside transaction root") - if _sha256(backup_data) != expected: - raise ValueError("backup hash does not match receipt") - planned.append((target, backup_data, receipt.target_modes[target_name], expected)) - for target, data, mode, expected in planned: - if data is None: - _current, exists, _current_mode = _read_target(target) - if exists: - _remove_target(target) - else: - _atomic_in_directory(target.parent, target.name, data, mode if mode is not None else 0o600) - actual_content, actual_exists, actual_mode = _read_target(target) - actual = _sha256(actual_content) - if actual != expected or actual_exists != (mode is not None) or actual_mode != mode: - raise ValueError("restore hash does not match receipt") - receipt.after_hashes[str(target)] = actual - receipt.after_modes[str(target)] = actual_mode - receipt.validations["rollback"] = {"passed": True, "targets": [str(item[0]) for item in planned]} - return True - except Exception as error: - receipt.validations["rollback"] = {"passed": False, "message": redact_secrets(str(error))} - return False - - -def _remove_target(path): - _assert_safe_target(path) - _content, exists, _mode = _read_target(path) - if not exists: - return - directory = _open_directory(path.parent) - try: - item = os.stat(path.name, dir_fd=directory, follow_symlinks=False) - if not stat.S_ISREG(item.st_mode): - raise ValueError("refusing non-regular target removal") - os.unlink(path.name, dir_fd=directory) - _fsync_directory(directory) - finally: - os.close(directory) - - -def _current_state(target_name): - content, exists, mode = _read_target(target_name) - return {"hash": _sha256(content), "exists": exists, "mode": mode} - - -def _receipt_after_state(receipt): - return { - target: { - "hash": receipt.after_hashes[target], - "exists": receipt.after_modes[target] is not None, - "mode": receipt.after_modes[target], - } - for target in receipt.targets - } - - -def _rollback_restore_plan(receipt): - return [ - { - "path": target, - "root": receipt.target_roots[target], - "hash": receipt.before_hashes[target], - "exists": receipt.target_modes[target] is not None, - "mode": receipt.target_modes[target], - "backup_path": receipt.backup_paths[target], - } - for target in receipt.targets - ] - - -def _rollback_receipt_binding(receipt): - """Return every validated receipt field that can change rollback behavior.""" - return { - "transaction_id": receipt.transaction_id, - "transaction_root": receipt.transaction_root, - "targets": list(receipt.targets), - "restore": _rollback_restore_plan(receipt), - "lock_root": receipt.lock_root, - "lock_migration": receipt.lock_migration, - } - - -def _rollback_receipt_hash(receipt): - return _sha256( - json.dumps( - _rollback_receipt_binding(receipt), - sort_keys=True, - separators=(",", ":"), - ).encode("utf-8") - ) - - -def _assert_expected_states(targets, expected, message): - if not isinstance(expected, dict) or set(expected) != set(targets): - raise ValueError("reviewed rollback state does not match receipt targets") - for target in targets: - if _current_state(target) != expected[target]: - raise ValueError(message) - - -def _rollback_preview_locked(receipt, location): - if receipt.status == "rolled_back": - state = { - target: { - "hash": receipt.before_hashes[target], - "exists": receipt.target_modes[target] is not None, - "mode": receipt.target_modes[target], - } - for target in receipt.targets - } - _assert_expected_states( - receipt.targets, state, "rollback target differs from recorded before-state" - ) - else: - if receipt.status not in {"applied", "rollback_failed", "recovery_required"}: - raise ValueError("normal rollback requires an applied receipt") - state = _receipt_after_state(receipt) - _assert_expected_states( - receipt.targets, state, "rollback target differs from recorded after-state" - ) - restore = _rollback_restore_plan(receipt) - receipt_binding = _rollback_receipt_binding(receipt) - binding = { - "operation": "wire-rollback", - "receipt_path": str(location), - "receipt_status": receipt.status, - "current": state, - "operational_receipt": receipt_binding, - } - return { - "targets": [{"path": target, **state[target]} for target in receipt.targets], - "restore": restore, - "operational_receipt_hash": _rollback_receipt_hash(receipt), - "preview_hash": _sha256( - json.dumps(binding, sort_keys=True, separators=(",", ":")).encode("utf-8") - ), - "requires_confirmation": receipt.status != "rolled_back", - "already_rolled_back": receipt.status == "rolled_back", - } - - -def preview_rollback(receipt_path): - """Return a hash-bound, non-secret snapshot of the current rollback targets.""" - location = _assert_safe_target(receipt_path) - initial = Receipt.from_dict( - json.loads(_read_regular(location).decode("utf-8")), str(location) - ) - lock_context = ( - _migrate_legacy_target_locks(initial.targets, initial.lock_root) - if initial.lock_root is not None else nullcontext() - ) - with lock_context: - with _target_locks(initial.targets, lock_root=initial.lock_root): - receipt = Receipt.from_dict( - json.loads(_read_regular(location).decode("utf-8")), str(location) - ) - return _rollback_preview_locked(receipt, location) - - -def rollback( - receipt_path, - preview_hash=None, - expected_after_hashes=None, - expected_receipt_hash=None, -): - """Restore a receipt through a user preview hash or installer-internal CAS.""" - location = _assert_safe_target(receipt_path) - initial = Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) - lock_context = ( - _migrate_legacy_target_locks(initial.targets, initial.lock_root) - if initial.lock_root is not None else nullcontext() - ) - with lock_context: - with _target_locks(initial.targets, lock_root=initial.lock_root): - receipt = Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) - if ( - expected_receipt_hash is not None - and _rollback_receipt_hash(receipt) != expected_receipt_hash - ): - raise ValueError("rollback receipt changed after the reviewed plan") - if expected_after_hashes is not None: - if not isinstance(expected_after_hashes, dict) or set(expected_after_hashes) != set(receipt.targets): - raise ValueError("reviewed rollback hashes do not match receipt targets") - for target_name in receipt.targets: - current, exists, _mode = _read_target(target_name) - if not exists or _sha256(current) != expected_after_hashes[target_name]: - raise ValueError("receipt target changed after reviewed rollback preview") - else: - if preview_hash is None: - raise PermissionError( - "rollback requires the hash from a reviewed rollback preview" - ) - preview = _rollback_preview_locked(receipt, location) - if preview["already_rolled_back"]: - return receipt - if preview_hash != preview["preview_hash"]: - raise PermissionError( - "rollback preview hash does not match current target state" - ) - if receipt.status == "rolled_back": - current_matches = True - for target_name in receipt.targets: - current, exists, mode = _read_target(target_name) - expected_absent = receipt.backup_paths[target_name] is None - if _sha256(current) != receipt.before_hashes[target_name] or exists == expected_absent or mode != receipt.target_modes[target_name]: - current_matches = False - break - if current_matches: - return receipt - expected_current = _receipt_after_state(receipt) - if _restore_receipt(receipt, expected_current=expected_current): - receipt.status = "rolled_back" - else: - receipt.status = "rollback_failed" - try: - _atomic_in_directory(Path(receipt.transaction_root), "receipt.json", (json.dumps(receipt.to_dict(), indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600) - except Exception as error: - receipt.status = "recovery_required" - receipt.validations["receipt_write"] = {"passed": False, "message": redact_secrets(str(error))} - return receipt diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/verification.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/verification.py deleted file mode 100644 index 65b5e75..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/verification.py +++ /dev/null @@ -1,1163 +0,0 @@ -"""Safe, bounded verification of discovered local-model candidates.""" - -from __future__ import annotations - -import ast -import http.client -import ipaddress -import json -import math -import threading -import time -from copy import deepcopy -from dataclasses import asdict, dataclass, field -from enum import Enum -from types import MappingProxyType -from typing import Any, Dict, Optional -from urllib.parse import urlsplit - -from .probe_assets import VISION_PROBE_TEXT, vision_probe_image_base64 -from .wiring import redact_secrets - - -TOOL_USE_PROBE_ID = "tool-use-v1" -TOOL_USE_TOOL_NAME = "lookup_widget" -TOOL_USE_ARGUMENTS = MappingProxyType({"widget_id": "widget-42"}) -TOOL_USE_PROMPT = ( - "Call lookup_widget exactly once for widget-42. " - "Do not answer directly and do not call any other tool." -) -TOOL_USE_TOOLS = [ - { - "type": "function", - "function": { - "name": TOOL_USE_TOOL_NAME, - "description": "Look up one synthetic widget by its identifier.", - "parameters": { - "type": "object", - "properties": { - "widget_id": { - "type": "string", - "description": "Synthetic widget identifier.", - } - }, - "required": ["widget_id"], - "additionalProperties": False, - }, - }, - } -] -_TOOL_USE_ARGUMENTS_MAX_CHARS = 512 -INVENTORY_RESPONSE_MAX_BYTES = 1024 * 1024 -PROBE_RESPONSE_MAX_BYTES = 256 * 1024 -_HTTP_READ_CHUNK_BYTES = 16 * 1024 - -CODING_PROBE_ID = "coding-v1" -REASONING_PROBE_ID = "reasoning-v1" -VISION_PROBE_ID = "vision-v1" -EMBEDDING_PROBE_ID = "embedding-v1" -ROLE_PROBE_IDS = MappingProxyType({ - "coding": CODING_PROBE_ID, - "reasoning": REASONING_PROBE_ID, - "vision": VISION_PROBE_ID, - "embedding": EMBEDDING_PROBE_ID, -}) - -CODING_PROBE_PROMPT = ( - "Write a Python function named add that takes two arguments a and b " - "and returns their sum. Reply with only the function definition." -) -CODING_PROBE_MAX_TOKENS = 256 -_CODING_PROBE_MAX_CODE_CHARS = 2000 -_CODING_PROBE_BLOCKED_TOKENS = ( - "import", "__", "while", "for", "open", "exec", "eval", "input", - "globals", "locals", "compile", "breakpoint", "exit", "quit", - "getattr", "setattr", "delattr", "vars", "help", "print", -) -REASONING_PROBE_PROMPT = ( - "A farmer has 17 sheep. All but 9 run away. " - "How many sheep does the farmer have left? Reply with only the number." -) -REASONING_PROBE_ANSWER = 9 -REASONING_PROBE_MAX_TOKENS = 64 -VISION_PROBE_PROMPT = ( - "What text is shown in this image? Reply with only the text in the image." -) -VISION_PROBE_MAX_TOKENS = 64 -EMBEDDING_PROBE_SENTENCES = ( - "The cat sat on the warm windowsill.", - "A kitten rested on the sunny ledge.", - "Quantum chromodynamics describes the strong nuclear force.", -) -EMBEDDING_PROBE_MAX_VECTORS = 8 -_EMBEDDING_PROBE_MAX_DIMENSION = 8192 - - -def _role_probe_outcome(reason): - return {"valid": reason == "valid", "reason": reason} - - -def validate_coding_response(response): - """Validate the fixed coding probe without retaining raw model output.""" - try: - content, _hidden = _generation_text(response) - code = _extract_probe_code(content) - if code is None: - return _role_probe_outcome("missing_content") - if len(code) > _CODING_PROBE_MAX_CODE_CHARS: - return _role_probe_outcome("unsafe_content") - lowered = code.casefold() - if any(token in lowered for token in _CODING_PROBE_BLOCKED_TOKENS): - return _role_probe_outcome("unsafe_content") - try: - tree = ast.parse(code) - except (SyntaxError, ValueError, RecursionError, MemoryError): - return _role_probe_outcome("parse_failure") - function_defs = [ - node for node in tree.body - if isinstance(node, ast.FunctionDef) and node.name == "add" - ] - if not function_defs: - return _role_probe_outcome("missing_function") - namespace = {"__builtins__": {}} - try: - exec(compile(tree, "", "exec"), namespace) - add = namespace.get("add") - if not callable(add): - return _role_probe_outcome("missing_function") - if add(2, 3) != 5 or add(-1, 1) != 0: - return _role_probe_outcome("wrong_answer") - except Exception: - return _role_probe_outcome("wrong_answer") - return _role_probe_outcome("valid") - except Exception: - return _role_probe_outcome("invalid_response") - - -def _extract_probe_code(content): - if not isinstance(content, str) or not content.strip(): - return None - text = content.strip() - if "```" in text: - segments = text.split("```") - for segment in segments[1::2]: - body = segment.split("\n", 1) - candidate = body[1] if len(body) == 2 else "" - if candidate.strip(): - return candidate.strip() - return None - return text - - -def validate_reasoning_response(response): - """Validate the fixed arithmetic reasoning probe by exact answer match.""" - try: - content, _hidden = _generation_text(response) - if not isinstance(content, str) or not content.strip(): - return _role_probe_outcome("missing_content") - digits = [] - current = [] - for character in content: - if character.isdigit() or (character == "-" and not current): - current.append(character) - elif current: - digits.append("".join(current)) - current = [] - if current: - digits.append("".join(current)) - if not digits: - return _role_probe_outcome("missing_content") - try: - answer = int(digits[-1]) - except ValueError: - return _role_probe_outcome("missing_content") - if answer != REASONING_PROBE_ANSWER: - return _role_probe_outcome("wrong_answer") - return _role_probe_outcome("valid") - except Exception: - return _role_probe_outcome("invalid_response") - - -def validate_vision_response(response): - """Validate the fixed OCR probe by case-folded substring match.""" - try: - content, _hidden = _generation_text(response) - if not isinstance(content, str) or not content.strip(): - return _role_probe_outcome("missing_content") - normalized = "".join(content.split()).casefold() - if VISION_PROBE_TEXT.casefold() not in normalized: - return _role_probe_outcome("wrong_answer") - return _role_probe_outcome("valid") - except Exception: - return _role_probe_outcome("invalid_response") - - -def validate_embedding_response(response): - """Validate the fixed embedding probe by cosine ordering.""" - try: - vectors = _embedding_vectors(response) - if vectors is None or len(vectors) != len(EMBEDDING_PROBE_SENTENCES): - return _role_probe_outcome("invalid_response") - similar = _cosine_similarity(vectors[0], vectors[1]) - dissimilar = _cosine_similarity(vectors[0], vectors[2]) - if similar is None or dissimilar is None: - return _role_probe_outcome("invalid_response") - if not similar > dissimilar: - return _role_probe_outcome("order_inverted") - return _role_probe_outcome("valid") - except Exception: - return _role_probe_outcome("invalid_response") - - -def _embedding_vectors(response): - if not isinstance(response, dict): - return None - records = response.get("embeddings") - if records is None: - data = response.get("data") - if isinstance(data, list): - records = [ - item.get("embedding") if isinstance(item, dict) else None - for item in sorted( - data, - key=lambda item: item.get("index", 0) if isinstance(item, dict) else 0, - ) - ] - if not isinstance(records, list) or len(records) > EMBEDDING_PROBE_MAX_VECTORS: - return None - vectors = [] - for record in records: - if ( - not isinstance(record, list) - or not record - or len(record) > _EMBEDDING_PROBE_MAX_DIMENSION - or any(not isinstance(value, (int, float)) or isinstance(value, bool) for value in record) - ): - return None - vectors.append([float(value) for value in record]) - return vectors - - -def _cosine_similarity(left, right): - if len(left) != len(right): - return None - dot = sum(a * b for a, b in zip(left, right)) - norm_left = math.sqrt(sum(a * a for a in left)) - norm_right = math.sqrt(sum(b * b for b in right)) - if norm_left == 0 or norm_right == 0: - return None - return dot / (norm_left * norm_right) - - -def normalize_tool_call(response): - """Normalize one synthetic tool call without retaining raw model output.""" - try: - message = _tool_call_message(response) - if message is None: - return _tool_call_result("invalid_response") - - if "tool_calls" not in message or message["tool_calls"] is None: - return _tool_call_result("missing_tool_call") - tool_calls = message["tool_calls"] - if not isinstance(tool_calls, list): - return _tool_call_result("invalid_response") - if not tool_calls: - return _tool_call_result("missing_tool_call") - if len(tool_calls) != 1: - return _tool_call_result("multiple_tool_calls") - - tool_call = tool_calls[0] - if not isinstance(tool_call, dict): - return _tool_call_result("invalid_response") - function = tool_call.get("function") - if not isinstance(function, dict): - return _tool_call_result("invalid_response") - raw_name = function.get("name") - if not isinstance(raw_name, str): - return _tool_call_result("invalid_response") - if raw_name != TOOL_USE_TOOL_NAME: - return _tool_call_result("wrong_tool") - - if "arguments" not in function: - return _tool_call_result("malformed_arguments") - arguments = function["arguments"] - if isinstance(arguments, str): - if len(arguments) > _TOOL_USE_ARGUMENTS_MAX_CHARS: - return _tool_call_result("malformed_arguments") - try: - arguments = json.loads(arguments) - except (json.JSONDecodeError, RecursionError): - return _tool_call_result("malformed_arguments") - if not isinstance(arguments, dict): - return _tool_call_result("malformed_arguments") - if arguments != TOOL_USE_ARGUMENTS: - return _tool_call_result("schema_invalid") - return _tool_call_result("valid") - except Exception: - return _tool_call_result("invalid_response") - - -def _tool_call_message(response): - if not isinstance(response, dict): - return None - if "message" in response: - message = response["message"] - return message if isinstance(message, dict) else None - if "choices" not in response: - return None - choices = response["choices"] - if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict): - return None - message = choices[0].get("message") - return message if isinstance(message, dict) else None - - -def _tool_call_result(reason): - valid = reason == "valid" - return { - "valid": valid, - "tool_name": TOOL_USE_TOOL_NAME if valid else None, - "arguments": dict(TOOL_USE_ARGUMENTS) if valid else None, - "reason": reason, - } - - -class EvidenceStrength(str, Enum): - """Ordered labels describing how directly a candidate was verified.""" - - runtime_measured = "runtime_measured" - runtime_tested = "runtime_tested" - runtime_inventory = "runtime_inventory" - metadata_only = "metadata_only" - heuristic_only = "heuristic_only" - - # Conventional aliases preserve a readable internal style while the exact - # lowercase contract names remain available to API consumers. - RUNTIME_MEASURED = runtime_measured - RUNTIME_TESTED = runtime_tested - RUNTIME_INVENTORY = runtime_inventory - METADATA_ONLY = metadata_only - HEURISTIC_ONLY = heuristic_only - - -class VerificationStatus(str, Enum): - """Outcome labels independent from the source strength of the evidence.""" - - verified = "verified" - metadata_only = "metadata-only" - failed = "failed" - unsupported_runtime = "unsupported-runtime" - - VERIFIED = verified - METADATA_ONLY = metadata_only - FAILED = failed - UNSUPPORTED_RUNTIME = unsupported_runtime - - -@dataclass(frozen=True) -class VerificationEvidence: - """Portable evidence from one candidate verification attempt.""" - - repo: str - role: str - strength: EvidenceStrength - status: VerificationStatus - available_locally: bool - loads: Optional[bool] - reasoning_confirmed: Optional[bool] - runtime: Optional[str] - note: str - details: Dict[str, Any] = field(default_factory=dict) - - def to_dict(self) -> Dict[str, Any]: - value = asdict(self) - value["strength"] = self.strength.value - value["status"] = self.status.value - return value - - -def _http_json_get(url, timeout=3.0): - """Read a local runtime endpoint without installing or starting anything.""" - return _http_json_request( - url, - "GET", - timeout=timeout, - max_response_bytes=INVENTORY_RESPONSE_MAX_BYTES, - ) - - -def _http_json_post(url, payload, timeout=10.0): - """Send the single bounded generation probe used by verification.""" - return _http_json_request( - url, - "POST", - payload=payload, - timeout=timeout, - max_response_bytes=PROBE_RESPONSE_MAX_BYTES, - ) - - -def _http_event_stream(url, payload, timeout=120.0, max_response_bytes=INVENTORY_RESPONSE_MAX_BYTES, - connection_factory=None, clock=time.monotonic): - """Yield bounded JSON events from a validated loopback streaming endpoint. - - Handles both server-sent events (``data: {...}`` lines terminated by - ``data: [DONE]``) and newline-delimited JSON (Ollama-style). - """ - parsed = _validated_loopback_url(url) - deadline = clock() + timeout - port = parsed.port or (443 if parsed.scheme == "https" else 80) - if connection_factory is None: - connection_type = ( - http.client.HTTPSConnection - if parsed.scheme == "https" - else http.client.HTTPConnection - ) - connection = connection_type(parsed.hostname, port, timeout=_deadline_remaining(deadline, clock)) - else: - connection = connection_factory(parsed.scheme, parsed.hostname, port, _deadline_remaining(deadline, clock)) - try: - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - connection.request( - "POST", - parsed.path or "/", - body=json.dumps(payload).encode("utf-8"), - headers={"Content-Type": "application/json"}, - ) - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - response = connection.getresponse() - _deadline_remaining(deadline, clock) - if 300 <= response.status < 400: - raise http.client.HTTPException( - "redirect responses are not allowed for local runtime requests" - ) - if not 200 <= response.status < 300: - raise http.client.HTTPException( - "local runtime returned HTTP status {0}".format(response.status) - ) - buffer = b"" - total = 0 - done = False - while not done: - remaining = _deadline_remaining(deadline, clock) - _set_connection_timeout(connection, remaining) - chunk = response.read(_HTTP_READ_CHUNK_BYTES) - _deadline_remaining(deadline, clock) - if not chunk: - break - total += len(chunk) - if total > max_response_bytes: - raise ValueError("local runtime response exceeds size limit") - buffer += chunk - while b"\n" in buffer: - line, buffer = buffer.split(b"\n", 1) - event, done = _parse_stream_line(line) - if event is not None: - yield event - if done: - break - if not done and buffer.strip(): - event, _done = _parse_stream_line(buffer) - if event is not None: - yield event - finally: - connection.close() - - -def _parse_stream_line(line): - text = line.strip() - if not text: - return None, False - if text.startswith(b"data:"): - text = text[len(b"data:"):].strip() - if text == b"[DONE]": - return None, True - if not text: - return None, False - if not text.startswith(b"{"): - return None, False - try: - return json.loads(text.decode("utf-8")), False - except (UnicodeDecodeError, json.JSONDecodeError): - return None, False - - -def _http_json_request( - url, - method, - payload=None, - timeout=10.0, - max_response_bytes=PROBE_RESPONSE_MAX_BYTES, - connection_factory=None, - clock=time.monotonic, -): - """Exchange bounded JSON with one validated loopback endpoint.""" - parsed = _validated_loopback_url(url) - deadline = clock() + timeout - remaining = _deadline_remaining(deadline, clock) - port = parsed.port or (443 if parsed.scheme == "https" else 80) - if connection_factory is None: - connection_type = ( - http.client.HTTPSConnection - if parsed.scheme == "https" - else http.client.HTTPConnection - ) - connection = connection_type(parsed.hostname, port, timeout=remaining) - else: - connection = connection_factory( - parsed.scheme, - parsed.hostname, - port, - remaining, - ) - body = None - headers = {} - if payload is not None: - body = json.dumps(payload).encode("utf-8") - headers["Content-Type"] = "application/json" - try: - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - connection.request(method, parsed.path or "/", body=body, headers=headers) - _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) - response = connection.getresponse() - _deadline_remaining(deadline, clock) - if 300 <= response.status < 400: - raise http.client.HTTPException( - "redirect responses are not allowed for local runtime requests" - ) - if not 200 <= response.status < 300: - raise http.client.HTTPException( - "local runtime returned HTTP status {0}".format(response.status) - ) - response_body = _read_bounded_response( - response, - connection, - max_response_bytes, - deadline, - clock, - ) - return json.loads(response_body.decode("utf-8")) - finally: - connection.close() - - -def _read_bounded_response(response, connection, limit, deadline, clock): - if not isinstance(limit, int) or limit <= 0: - raise ValueError("response size limit must be a positive integer") - content_length = response.getheader("Content-Length") - if content_length is not None: - try: - declared_length = int(content_length) - except (TypeError, ValueError) as error: - raise ValueError("invalid Content-Length from local runtime") from error - if declared_length < 0: - raise ValueError("invalid Content-Length from local runtime") - if declared_length > limit: - raise ValueError("local runtime response exceeds size limit") - - chunks = [] - total = 0 - while True: - remaining = _deadline_remaining(deadline, clock) - _set_connection_timeout(connection, remaining) - chunk = response.read(min(_HTTP_READ_CHUNK_BYTES, limit - total + 1)) - _deadline_remaining(deadline, clock) - if not chunk: - break - total += len(chunk) - if total > limit: - raise ValueError("local runtime response exceeds size limit") - chunks.append(chunk) - return b"".join(chunks) - - -def _deadline_remaining(deadline, clock): - remaining = deadline - clock() - if remaining <= 0: - raise TimeoutError("local runtime request exceeded its overall deadline") - return remaining - - -def _set_connection_timeout(connection, timeout): - sock = getattr(connection, "sock", None) - if sock is not None: - sock.settimeout(timeout) - - -def _local_runtime_origin(base_url): - """Return a normalized credential-free HTTP(S) loopback origin.""" - parsed = _validated_loopback_url(base_url) - if parsed.path not in ("", "/"): - raise ValueError("runtime base URL must be an origin without a path") - return base_url.rstrip("/") - - -def _validated_loopback_url(url): - if not isinstance(url, str) or url != url.strip(): - raise ValueError("runtime URL must be a trimmed string") - parsed = urlsplit(url) - if parsed.scheme not in ("http", "https"): - raise ValueError("runtime URL must use HTTP(S)") - if parsed.username is not None or parsed.password is not None: - raise ValueError("runtime URL must not contain credentials") - if parsed.query or parsed.fragment: - raise ValueError("runtime URL must not contain a query or fragment") - hostname = parsed.hostname - if hostname is None: - raise ValueError("runtime URL must include a host") - try: - parsed.port - except ValueError as error: - raise ValueError("runtime URL has an invalid port") from error - if hostname.casefold() != "localhost": - try: - address = ipaddress.ip_address(hostname) - except ValueError as error: - raise ValueError("runtime URL must use a loopback host") from error - if not address.is_loopback: - raise ValueError("runtime URL must use a loopback host") - return parsed - - -def _tool_use_payload_tools(): - """Copy the fixed schema into ordinary JSON-serializable containers.""" - return deepcopy(TOOL_USE_TOOLS) - - -class OllamaRuntimeClient: - """Read-only inventory and bounded generation adapter for an existing Ollama.""" - - name = "ollama" - - def __init__(self, http_get=None, http_post=None, base_url="http://127.0.0.1:11434"): - self._http_get = http_get or _http_json_get - self._http_post = http_post or _http_json_post - self._base_url = _local_runtime_origin(base_url) - - def list_models(self): - return self._http_get("{0}/api/tags".format(self._base_url), timeout=3.0) - - def generate(self, model, prompt, max_tokens): - return self._http_post( - "{0}/api/chat".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "stream": False, - "options": {"num_predict": max_tokens}, - }, - timeout=10.0, - ) - - def probe_tool_use(self, model): - return self._http_post( - "{0}/api/chat".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": TOOL_USE_PROMPT}], - "tools": _tool_use_payload_tools(), - "stream": False, - "options": {"num_predict": 64}, - }, - timeout=10.0, - ) - - def embed(self, model, inputs): - return self._http_post( - "{0}/api/embed".format(self._base_url), - {"model": model, "input": list(inputs)}, - timeout=10.0, - ) - - def stream_generate(self, model, prompt, max_tokens, timeout=120.0): - return _http_event_stream( - "{0}/api/chat".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "stream": True, - "options": {"num_predict": max_tokens}, - }, - timeout=timeout, - ) - - -class OpenAICompatibleRuntimeClient: - """Bounded adapter for an existing local OpenAI-compatible runtime.""" - - def __init__(self, name, base_url, http_get=None, http_post=None): - self.name = name - self._http_get = http_get or _http_json_get - self._http_post = http_post or _http_json_post - self._base_url = _local_runtime_origin(base_url) - - def list_models(self): - return self._http_get("{0}/v1/models".format(self._base_url), timeout=3.0) - - def generate(self, model, prompt, max_tokens): - return self._http_post( - "{0}/v1/chat/completions".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - }, - timeout=10.0, - ) - - def probe_tool_use(self, model): - return self._http_post( - "{0}/v1/chat/completions".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": TOOL_USE_PROMPT}], - "tools": _tool_use_payload_tools(), - "tool_choice": "auto", - "max_tokens": 64, - }, - timeout=10.0, - ) - - def embed(self, model, inputs): - return self._http_post( - "{0}/v1/embeddings".format(self._base_url), - {"model": model, "input": list(inputs)}, - timeout=10.0, - ) - - def stream_generate(self, model, prompt, max_tokens, timeout=120.0): - return _http_event_stream( - "{0}/v1/chat/completions".format(self._base_url), - { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "stream": True, - "stream_options": {"include_usage": True}, - }, - timeout=timeout, - ) - - -class LMStudioRuntimeClient(OpenAICompatibleRuntimeClient): - """Backward-compatible adapter for an existing LM Studio runtime.""" - - name = "lmstudio" - - def __init__(self, http_get=None, http_post=None, base_url="http://127.0.0.1:1234"): - super().__init__( - "lmstudio", - base_url, - http_get=http_get, - http_post=http_post, - ) - - -class MLXVLMRuntimeClient(OpenAICompatibleRuntimeClient): - """Bounded adapter for an existing local mlx-vlm vision runtime.""" - - name = "mlx-vlm" - - def __init__(self, http_get=None, http_post=None, base_url="http://127.0.0.1:8083"): - super().__init__( - "mlx-vlm", - base_url, - http_get=http_get, - http_post=http_post, - ) - - def generate_vision(self, model, prompt, image_base64, max_tokens): - return self._http_post( - "{0}/v1/chat/completions".format(self._base_url), - { - "model": model, - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,{0}".format(image_base64) - }, - }, - ], - } - ], - "max_tokens": max_tokens, - }, - timeout=30.0, - ) - - -class Verifier: - """Verify installed candidates without ever installing or downloading them. - - Runtime clients implement ``list_models()`` and ``generate(model, prompt, - max_tokens)``; tool-capable clients also implement ``probe_tool_use(model)``. - Metadata clients implement ``inspect_model(repo)``. Keeping these protocols - implicit allows provider adapters and tests to supply clients without adding - runtime dependencies. - """ - - def __init__(self, runtime_clients=None, metadata_client=None, runtime_client=None): - if runtime_clients is None: - runtime_clients = ( - [] - if runtime_client is not None - else [ - OllamaRuntimeClient(), - LMStudioRuntimeClient(), - OpenAICompatibleRuntimeClient("mlx_lm", "http://127.0.0.1:8080"), - MLXVLMRuntimeClient(), - OpenAICompatibleRuntimeClient("litellm", "http://127.0.0.1:4000"), - ] - ) - if isinstance(runtime_clients, dict): - runtime_clients = list(runtime_clients.values()) - self._runtime_clients = list(runtime_clients) - if runtime_client is not None: - self._runtime_clients.append(runtime_client) - self._metadata_client = metadata_client - self._inventory_cache = {} - self._inventory_cache_lock = threading.Lock() - - def clear_inventory_cache(self): - """Start a new verification phase with fresh runtime inventories.""" - with self._inventory_cache_lock: - self._inventory_cache.clear() - - def runtime_clients(self): - """Return the configured runtime clients without mutating them.""" - return tuple(self._runtime_clients) - - def _runtime_inventory(self, runtime): - cache_key = id(runtime) - with self._inventory_cache_lock: - if cache_key not in self._inventory_cache: - try: - result = (installed_model_ids(runtime.list_models()), None) - except Exception as error: - result = (None, _safe_error(error)) - self._inventory_cache[cache_key] = result - return self._inventory_cache[cache_key] - - def verify(self, candidate, host, allow_network=True) -> VerificationEvidence: - """Return the strongest available evidence without mutating runtimes.""" - del host # Reserved for runtime adapters whose probes depend on host facts. - repo = _candidate_repo(candidate) - role = str(candidate.get("role") or candidate.get("heuristics", {}).get("role") or "general") - inventory_errors = [] - - for runtime in self._runtime_clients: - installed, inventory_error = self._runtime_inventory(runtime) - if inventory_error is not None: - inventory_errors.append( - "{0}: {1}".format(_runtime_name(runtime), inventory_error) - ) - continue - if not _is_installed(repo, installed): - continue - runtime_name = _runtime_name(runtime) - if role == "tool-use": - return _verify_tool_use(repo, role, runtime_name, runtime) - try: - response = runtime.generate( - repo, - "Reply with the single word ready.", - max_tokens=24, - ) - content, hidden_reasoning = _generation_text(response) - reasoning_confirmed, reasoning_evidence = _reasoning_confirmation( - candidate, hidden_reasoning - ) - details = { - "visible_content": bool(content.strip()), - "hidden_reasoning": bool(hidden_reasoning), - "reasoning_evidence": reasoning_evidence, - } - if role in ROLE_PROBE_IDS: - details["probes"] = [_run_role_probe(role, repo, runtime)] - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_TESTED, - status=VerificationStatus.VERIFIED, - available_locally=True, - loads=True, - reasoning_confirmed=reasoning_confirmed, - runtime=runtime_name, - note="Runtime generation completed without downloading the model.", - details=details, - ) - except Exception as error: - safe_error = _safe_error(error) - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_INVENTORY, - status=VerificationStatus.FAILED, - available_locally=True, - loads=False, - reasoning_confirmed=None, - runtime=runtime_name, - note=_bounded( - "Installed runtime inventory found the model, but " - "generation failed: {0}".format(safe_error) - ), - details={"error": safe_error}, - ) - - if allow_network and self._metadata_client is not None: - try: - metadata = self._metadata_client.inspect_model(repo) - reasoning_confirmed, reasoning_evidence = _reasoning_confirmation( - candidate, metadata - ) - details = _bounded_metadata(metadata) - details["reasoning_evidence"] = reasoning_evidence - if role == "tool-use": - details["probe_id"] = TOOL_USE_PROBE_ID - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.METADATA_ONLY, - status=VerificationStatus.METADATA_ONLY, - available_locally=False, - loads=None, - reasoning_confirmed=reasoning_confirmed, - runtime=None, - note="Model is not installed; repository metadata was inspected without downloading it.", - details=details, - ) - except Exception as error: - inventory_errors.append("metadata: {0}".format(_safe_error(error))) - - note = "Model is not installed; only candidate heuristics were available." - if inventory_errors: - note += " Probe errors: {0}".format("; ".join(inventory_errors)) - details = { - "reasoning_heuristic": candidate.get("reasoning"), - "reasoning_source": candidate.get("reason_src"), - } - if role == "tool-use": - details["probe_id"] = TOOL_USE_PROBE_ID - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.HEURISTIC_ONLY, - status=VerificationStatus.METADATA_ONLY, - available_locally=False, - loads=None, - reasoning_confirmed=None, - runtime=None, - note=_bounded(note), - details=details, - ) - - -def _run_role_probe(role, repo, runtime): - """Run the role's deterministic probe; never raises into verification.""" - probe_id = ROLE_PROBE_IDS[role] - try: - if role == "coding": - response = runtime.generate(repo, CODING_PROBE_PROMPT, max_tokens=CODING_PROBE_MAX_TOKENS) - outcome = validate_coding_response(response) - elif role == "reasoning": - response = runtime.generate(repo, REASONING_PROBE_PROMPT, max_tokens=REASONING_PROBE_MAX_TOKENS) - outcome = validate_reasoning_response(response) - elif role == "embedding": - embed = getattr(runtime, "embed", None) - if not callable(embed): - outcome = _role_probe_outcome("unsupported_runtime") - else: - outcome = validate_embedding_response( - embed(repo, list(EMBEDDING_PROBE_SENTENCES)) - ) - elif role == "vision": - generate_vision = getattr(runtime, "generate_vision", None) - if not callable(generate_vision): - outcome = _role_probe_outcome("unsupported_runtime") - else: - response = generate_vision( - repo, - VISION_PROBE_PROMPT, - vision_probe_image_base64(), - VISION_PROBE_MAX_TOKENS, - ) - outcome = validate_vision_response(response) - else: - outcome = _role_probe_outcome("unsupported_runtime") - except Exception as error: - outcome = {"valid": False, "reason": "probe_error", "error": _bounded(_safe_error(error), 160)} - return {"probe_id": probe_id, "outcome": outcome} - - -def _verify_tool_use(repo, role, runtime_name, runtime): - if runtime_name == "mlx-vlm" or not callable(getattr(runtime, "probe_tool_use", None)): - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_INVENTORY, - status=VerificationStatus.UNSUPPORTED_RUNTIME, - available_locally=True, - loads=None, - reasoning_confirmed=None, - runtime=runtime_name, - note=_bounded( - "Installed runtime inventory found the model, but this runtime " - "does not support the bounded tool-use probe." - ), - details={ - "probe_id": TOOL_USE_PROBE_ID, - "outcome": {"reason": "unsupported_runtime"}, - }, - ) - try: - outcome = normalize_tool_call(runtime.probe_tool_use(repo)) - except Exception as error: - error_name = _redacted_error(error) - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_INVENTORY, - status=VerificationStatus.FAILED, - available_locally=True, - loads=False, - reasoning_confirmed=None, - runtime=runtime_name, - note=_bounded( - "Installed runtime inventory found the model, but the bounded " - "tool-use probe failed: {0}".format(error_name) - ), - details={ - "probe_id": TOOL_USE_PROBE_ID, - "error": _bounded(error_name, 160), - }, - ) - if outcome["valid"]: - note = "Runtime returned a schema-valid synthetic tool call." - else: - note = "Runtime responded, but the probe did not return a valid call." - return VerificationEvidence( - repo=repo, - role=role, - strength=EvidenceStrength.RUNTIME_TESTED, - status=( - VerificationStatus.VERIFIED - if outcome["valid"] - else VerificationStatus.FAILED - ), - available_locally=True, - loads=True, - reasoning_confirmed=None, - runtime=runtime_name, - note=note, - details={"probe_id": TOOL_USE_PROBE_ID, "outcome": outcome}, - ) - - -def _redacted_error(error): - name = error.__class__.__name__ - return _bounded(name if name else "runtime error", 160) - - -def _safe_error(error): - name = error.__class__.__name__ or "Error" - message = redact_secrets(str(error)) - value = "{0}: {1}".format(name, message) if message else name - return _bounded(value, 160) - - -def _candidate_repo(candidate): - if not isinstance(candidate, dict): - raise TypeError("candidate must be an object") - repo = candidate.get("repo") or candidate.get("repository") - if not isinstance(repo, str) or not repo: - raise ValueError("candidate.repo must be a non-empty string") - return repo - - -def _runtime_name(runtime): - return str(getattr(runtime, "name", runtime.__class__.__name__)) - - -def installed_model_ids(records) -> set: - """Return model IDs from a supported runtime's read-only inventory response.""" - if isinstance(records, dict): - records = records.get("models", records.get("data", [])) - models = set() - for record in records or []: - if isinstance(record, str): - models.add(record) - continue - if isinstance(record, dict): - value = record.get("repo") or record.get("id") or record.get("name") or record.get("model") - if isinstance(value, str) and value: - models.add(value) - return models - - -def _is_installed(repo, installed): - normalized = repo.casefold() - return any(item.casefold() == normalized for item in installed) - - -def _generation_text(response): - if isinstance(response, str): - return response, "" - if not isinstance(response, dict): - return "", "" - message = response.get("message") - if message is None: - choices = response.get("choices") or [] - if choices and isinstance(choices[0], dict): - message = choices[0].get("message") or choices[0] - if not isinstance(message, dict): - message = response - content = message.get("content") or message.get("response") or "" - hidden = ( - message.get("reasoning_content") - or message.get("thinking") - or message.get("reasoning") - or "" - ) - return str(content), str(hidden) - - -def _reasoning_confirmation(candidate, signal): - """Prefer any positive runtime or metadata signal over an inconclusive probe.""" - if isinstance(signal, str) and signal.strip(): - return True, "runtime_hidden" - if isinstance(signal, dict): - if signal.get("reasoning") is True: - source = signal.get("reason_src") or "field" - return True, "metadata_{0}".format(source) - tags = signal.get("tags") or [] - if any(str(tag).lower() in ("reasoning", "thinking", "chain-of-thought") for tag in tags): - return True, "metadata_tags" - if signal.get("chat_template"): - return True, "metadata_chat_template" - if candidate.get("reasoning") is True and candidate.get("reason_src") in ( - "chat_template", "tags", - ): - return True, "discovery_{0}".format(candidate["reason_src"]) - return False, "none" - - -def _bounded(value, limit=300): - text = " ".join(str(value).split()) - if len(text) <= limit: - return text - return text[: max(0, limit - 3)] + "..." - - -def _bounded_metadata(metadata): - if not isinstance(metadata, dict): - return {} - allowed = ("metadata_available", "gated", "license", "tags", "reasoning", "reason_src", "weight_bytes") - value = {key: metadata[key] for key in allowed if key in metadata} - if isinstance(value.get("tags"), list): - value["tags"] = value["tags"][:20] - return value diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/watch.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/watch.py deleted file mode 100644 index 63519d1..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/watch.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Stateful Hugging Face diff digest focused on models you own.""" - -from __future__ import annotations - -import json -from datetime import datetime, timezone -from pathlib import Path - -from .model_doctor import ( - collect_runtime_inventory, - default_hf_cache, - inspect_hf_cache, - scan_wired_configs, -) -from .models import base_name -from .transactions import _atomic_in_directory - - -WATCH_SCHEMA_VERSION = "1.0" -STATE_FILENAME = "watch-state.json" -MAX_OWNED = 500 -MAX_CANDIDATES = 2000 -MAX_FINDINGS = 100 - - -class WatchError(RuntimeError): - """Classified watch failure safe to surface in a result envelope.""" - - def __init__(self, code, message, remediation): - super().__init__(message) - self.code = code - self.remediation = remediation - - -def _utc_now(): - return datetime.now(timezone.utc).isoformat() - - -def state_file(state_dir): - return Path(state_dir) / "watch" / STATE_FILENAME - - -def collect_owned(runtime_clients, hf_cache=None, wired_roots=None): - """Collect owned model ids from local inventories and wired configs.""" - owned = [] - seen = set() - - def add(model_id, source): - if not isinstance(model_id, str) or not model_id: - return - key = model_id.casefold() - if key in seen or len(owned) >= MAX_OWNED: - return - seen.add(key) - owned.append({"id": model_id, "source": source}) - - cache = Path(hf_cache) if hf_cache is not None else default_hf_cache() - for item in inspect_hf_cache(cache): - add(item["id"], "hf-cache") - runtime_inventory, _errors = collect_runtime_inventory(runtime_clients) - for item in runtime_inventory: - add(item["id"], item["source"]) - for config in scan_wired_configs(wired_roots or [Path.cwd()]): - add(config.get("model"), "wired") - return owned - - -def snapshot_candidates(discovery_data): - """Flatten a discovery result into a bounded repo -> signal mapping.""" - candidates = {} - roles = discovery_data.get("roles") if isinstance(discovery_data, dict) else None - if not isinstance(roles, dict): - return candidates - for records in roles.values(): - if not isinstance(records, list): - continue - for record in records: - if not isinstance(record, dict): - continue - repo = record.get("repo") - if not isinstance(repo, str) or not repo or repo in candidates: - continue - facts = record.get("facts") if isinstance(record.get("facts"), dict) else {} - candidates[repo] = { - "base": record.get("base") or base_name(repo), - "weight_bytes": facts.get("weight_bytes"), - "gated": facts.get("gated"), - "license": record.get("license"), - } - if len(candidates) >= MAX_CANDIDATES: - return candidates - return candidates - - -def build_snapshot(owned, candidates, now=None): - return { - "schema_version": WATCH_SCHEMA_VERSION, - "created_at": now or _utc_now(), - "owned": list(owned)[:MAX_OWNED], - "candidates": dict(candidates), - "previous": None, - } - - -def write_snapshot(state_dir, snapshot): - """Persist the snapshot, rotating any existing one into `previous`.""" - destination = state_file(state_dir) - destination.parent.mkdir(parents=True, exist_ok=True) - existing = None - if destination.is_file(): - try: - existing = json.loads(destination.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - existing = None - if isinstance(existing, dict) and existing.get("schema_version") == WATCH_SCHEMA_VERSION: - snapshot["previous"] = { - key: existing.get(key) - for key in ("schema_version", "created_at", "owned", "candidates") - } - snapshot["previous"]["previous"] = None - content = (json.dumps(snapshot, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") - _atomic_in_directory(destination.parent, destination.name, content, 0o600) - return destination - - -def read_baseline(state_dir): - destination = state_file(state_dir) - if not destination.is_file(): - raise WatchError( - "missing_baseline", - "No watch snapshot exists yet.", - "Run mlx-agent watch snapshot first to record a baseline.", - ) - try: - value = json.loads(destination.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError) as error: - raise WatchError( - "invalid_state", - "The watch state file could not be read: {0}".format(error), - "Remove the corrupted file and run watch snapshot again.", - ) - if not isinstance(value, dict) or value.get("schema_version") != WATCH_SCHEMA_VERSION: - raise WatchError( - "invalid_state", - "The watch state file has an unsupported schema.", - "Run watch snapshot to record a fresh baseline.", - ) - return value - - -def diff_snapshots(baseline, current): - """Classify changes relevant to owned models between two snapshots.""" - findings = [] - owned_ids = {item["id"].casefold() for item in current.get("owned", [])} - owned_bases = {base_name(item["id"]) for item in current.get("owned", [])} - baseline_candidates = baseline.get("candidates") or {} - baseline_owned = {item["id"].casefold() for item in baseline.get("owned", [])} - - for repo, info in sorted((current.get("candidates") or {}).items()): - if len(findings) >= MAX_FINDINGS: - break - base = base_name(info.get("base") or repo) - if repo.casefold() not in owned_ids and base_name(repo) not in owned_bases and base not in owned_bases: - continue - previous = baseline_candidates.get(repo) - if previous is None: - findings.append({ - "code": "new_quant_of_owned", - "repo": repo, - "base": base, - "detail": "new tracked repository matching an owned model", - }) - continue - old_bytes = previous.get("weight_bytes") - new_bytes = info.get("weight_bytes") - if old_bytes is not None and new_bytes is not None and old_bytes != new_bytes: - findings.append({ - "code": "updated_tracked_repo", - "repo": repo, - "base": base, - "detail": "weight bytes changed from {0} to {1}".format(old_bytes, new_bytes), - }) - if previous.get("gated") != info.get("gated"): - findings.append({ - "code": "gated_changed", - "repo": repo, - "base": base, - "detail": "gated changed from {0} to {1}".format( - previous.get("gated"), info.get("gated") - ), - }) - - for item in baseline.get("owned", []): - if len(findings) >= MAX_FINDINGS: - break - if item["id"].casefold() not in owned_ids: - findings.append({ - "code": "owned_missing", - "repo": item["id"], - "base": base_name(item["id"]), - "detail": "previously owned via {0}; no longer present".format( - item.get("source", "unknown") - ), - }) - return findings diff --git a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/wiring.py b/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/wiring.py deleted file mode 100644 index a0e753d..0000000 --- a/packages/codex/mlx-agent/skills/mlx-wire/src/mlx_agent/wiring.py +++ /dev/null @@ -1,353 +0,0 @@ -"""Deterministic, injection-safe configuration renderers for Wire.""" - -from __future__ import annotations - -import ipaddress -import json -import re -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path - - -_RUNTIMES = {"ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"} -_MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,191}$") -_SECRET_NAMES = { - "accesskey", "accesstoken", "apikey", "auth", "authorization", "clientsecret", "credential", - "credentials", "idtoken", "passwd", "password", "privatekey", - "refreshtoken", "secret", "secretkey", "token", -} -_ASSIGNMENT_SECRET = re.compile( - r"(?im)(^|[\s,{&?;])([\"']?(?:" - r"[a-z0-9_-]*(?:token|secret|passw(?:or)?d|authorization|credentials?|auth)|" - r"[a-z0-9_-]*(?:api|access|private)[_-]?key" - r")[\"']?\s*[:=]\s*)" - r"([^\s,}\]\r\n]+|\"(?:\\.|[^\"])*\"|'(?:\\.|[^'])*')" -) -_URL_USERINFO = re.compile(r"(https?://)([^/@\s]+@)", re.I) -_OLLAMA = re.compile(r"\A# MLX_AGENT_WIRE v1\nFROM (" + _MODEL.pattern[1:-1] + r")\n\Z") -_ENV_REFERENCE = re.compile( - r"^(?:os\.environ(?:\[[\"'][A-Z][A-Z0-9_]{0,127}[\"']\]|/[A-Z][A-Z0-9_]{0,127})|" - r"\$\{[A-Z][A-Z0-9_]{0,127}\}|\$[A-Z][A-Z0-9_]{0,127}|env:[A-Z][A-Z0-9_]{0,127})$" -) -_STATE_CHANGING_PATH_PARTS = { - "admin", "create", "delete", "install", "kill", "mutate", "pull", "push", - "reload", "remove", "reset", "restart", "shutdown", "update", "write", -} - - -def _secret_name(name): - raw = str(name).lower() - collapsed = re.sub(r"[^a-z0-9]", "", raw) - if collapsed in _SECRET_NAMES: - return True - segments = [item for item in re.split(r"[^a-z0-9]+", raw) if item] - if any( - item in {"auth", "authorization", "credential", "credentials", "passwd", "password", "secret", "token"} - for item in segments - ): - return True - return collapsed.endswith( - ("accesskey", "apikey", "authorization", "credential", "credentials", "passwd", "password", "privatekey", "secret", "token") - ) - - -def _environment_reference(value): - return isinstance(value, str) and _ENV_REFERENCE.fullmatch(value.strip()) is not None - - -def _json_contains_resolved_secret(value): - if isinstance(value, dict): - for key, item in value.items(): - if _secret_name(key) and not _environment_reference(item): - return True - if _json_contains_resolved_secret(item): - return True - return False - if isinstance(value, list): - return any(_json_contains_resolved_secret(item) for item in value) - if isinstance(value, str): - try: - parsed = urllib.parse.urlsplit(value) - except ValueError: - return False - if parsed.scheme not in {"http", "https"}: - return False - if parsed.username is not None or parsed.password is not None: - return True - return any( - _secret_name(key) and not _environment_reference(item) - for key, item in urllib.parse.parse_qsl(parsed.query, keep_blank_values=True) - ) - return False - - -def contains_resolved_secrets(content): - """Return whether configuration text contains a resolved credential value.""" - if not isinstance(content, str): - raise TypeError("configuration content must be text") - try: - parsed = json.loads(content) - except (TypeError, ValueError, json.JSONDecodeError): - parsed = None - if parsed is not None and _json_contains_resolved_secret(parsed): - return True - if _URL_USERINFO.search(content): - return True - for match in _ASSIGNMENT_SECRET.finditer(content): - raw = match.group(3).strip() - if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {"\"", "'"}: - raw = raw[1:-1] - if not _environment_reference(raw): - return True - return False - - -def require_secret_free_config(content): - """Fail closed without echoing a resolved secret-bearing value.""" - if contains_resolved_secrets(content): - raise ValueError( - "existing configuration contains resolved secret-bearing fields; " - "move secrets to environment references or a secret-free managed fragment" - ) - return content - - -def _redact_json(value): - if isinstance(value, dict): - return {key: "" if _secret_name(key) else _redact_json(item) for key, item in value.items()} - if isinstance(value, list): - return [_redact_json(item) for item in value] - if isinstance(value, str): - return re.sub(r"https?://[^\s\"']+", lambda match: redact_endpoint(match.group(0)), value) - return value - - -def redact_endpoint(endpoint): - """Return only a non-credential origin/path representation of an endpoint.""" - try: - parsed = urllib.parse.urlsplit(str(endpoint)) - if parsed.scheme not in {"http", "https"} or not parsed.hostname: - return "" - host = parsed.hostname - if ":" in host and not host.startswith("["): - host = "[{0}]".format(host) - origin = "{0}://{1}".format(parsed.scheme, host) - if parsed.port is not None: - origin += ":{0}".format(parsed.port) - return origin + (parsed.path or "/") - except ValueError: - return "" - - -def redact_secrets(content): - """Redact common credential forms before they can enter previews or receipts.""" - text = str(content) - try: - parsed = json.loads(text) - except (TypeError, ValueError, json.JSONDecodeError): - parsed = None - if parsed is not None: - return json.dumps(_redact_json(parsed), sort_keys=True, separators=(",", ": ")) - text = _URL_USERINFO.sub(r"\1@", text) - def redact_assignment(match): - value = match.group(3) - replacement = value if "" in value else "" - return match.group(1) + match.group(2) + replacement - text = _ASSIGNMENT_SECRET.sub(redact_assignment, text) - def redact_url(match): - return redact_endpoint(match.group(0)) - text = re.sub(r"https?://[^\s\"']+", redact_url, text) - return text - - -def validate_health_endpoint(endpoint): - """Validate one credential-free, read-only loopback HTTP(S) health URL.""" - if not isinstance(endpoint, str) or not endpoint: - raise ValueError("health endpoint must be a non-empty URL") - try: - parsed = urllib.parse.urlsplit(endpoint) - port = parsed.port - except ValueError as error: - raise ValueError("health endpoint has an invalid port") from error - if parsed.scheme not in {"http", "https"} or not parsed.hostname: - raise ValueError("health endpoint must use HTTP(S)") - if parsed.username is not None or parsed.password is not None or parsed.query or parsed.fragment: - raise ValueError("health endpoint must not contain credentials, query, or fragment") - if port is not None and not 1 <= port <= 65535: - raise ValueError("health endpoint port is outside 1-65535") - hostname = parsed.hostname.lower().rstrip(".") - if hostname != "localhost": - try: - address = ipaddress.ip_address(hostname) - except ValueError as error: - raise ValueError("health endpoint must use localhost or a loopback IP literal") from error - if not address.is_loopback: - raise ValueError("health endpoint must resolve only to loopback") - decoded_path = urllib.parse.unquote(parsed.path or "/").lower() - parts = {part for part in decoded_path.split("/") if part} - if parts & _STATE_CHANGING_PATH_PARTS: - raise ValueError("health endpoint path appears state-changing") - return endpoint - - -def _endpoint_origin(endpoint): - parsed = urllib.parse.urlsplit(validate_health_endpoint(endpoint)) - port = parsed.port or (443 if parsed.scheme == "https" else 80) - return parsed.scheme.lower(), parsed.hostname.lower().rstrip("."), port - - -class _SameLoopbackOriginRedirect(urllib.request.HTTPRedirectHandler): - def __init__(self, origin): - super().__init__() - self.origin = origin - - def redirect_request(self, request, file_pointer, code, message, headers, new_url): - destination = urllib.parse.urljoin(request.full_url, new_url) - if _endpoint_origin(destination) != self.origin: - raise urllib.error.URLError("health redirect left the reviewed loopback origin") - return super().redirect_request( - request, file_pointer, code, message, headers, destination - ) - - -class ConfigAdapter: - """A small format adapter using only documented, exact stdlib-safe subsets.""" - - version = "2.0" - - def __init__(self, runtime, path=None): - if runtime not in _RUNTIMES: - raise ValueError("unsupported runtime: {0}".format(runtime)) - self.runtime = runtime - self.path = Path(path) if path is not None else None - - @classmethod - def detect(cls, path, runtime=None): - location = Path(path) - selected = runtime - if selected is None: - name = location.name.lower() - if name == "modelfile" or location.suffix.lower() == ".modelfile": - selected = "ollama" - elif location.suffix.lower() in {".yaml", ".yml"}: - selected = "litellm" - elif "lmstudio" in name: - selected = "lmstudio" - else: - selected = "mlx_lm" - return cls(selected, location) - - def render(self, model, runtime=None, existing=""): - selected = runtime or self.runtime - if selected != self.runtime: - return ConfigAdapter(selected, self.path).render(model, existing=existing) - self._validate_model(model) - if not isinstance(existing, str): - raise TypeError("existing config content must be text") - require_secret_free_config(existing) - if selected == "ollama": - if existing and existing != self._render_ollama(model): - raise ValueError("Ollama Wire accepts only its exact managed configuration") - return self._render_ollama(model) - if selected == "litellm": - if existing: - self._validate_litellm(existing) - return self._render_litellm(model) - return self._render_json(model, existing) - - @staticmethod - def _validate_model(model): - if not isinstance(model, str) or not _MODEL.fullmatch(model): - raise ValueError("model must be a safe publisher/model identifier") - - def _render_json(self, model, existing): - if existing.strip(): - try: - document = json.loads(existing) - except json.JSONDecodeError as error: - raise ValueError("existing JSON configuration is invalid: {0}".format(error)) - if not isinstance(document, dict): - raise ValueError("existing JSON configuration must be an object") - else: - document = {} - port = 8081 if self.runtime == "mlx-vlm" else 8080 - provider_id = "mlxvlm" if self.runtime == "mlx-vlm" else self.runtime.replace("_", "") - document["mlx_agent_wire"] = { - "marker": "MLX_AGENT_WIRE", - "version": self.version, - "runtime": self.runtime, - "model": model, - "provider": { - "id": provider_id, - "type": "openai", - "base_url": "http://127.0.0.1:{0}/v1".format(port), - "api_key_env": "MLX_AGENT_LOCAL_API_KEY", - }, - } - return json.dumps(document, indent=2, sort_keys=True) + "\n" - - @staticmethod - def _render_ollama(model): - return "# MLX_AGENT_WIRE v1\nFROM {0}\n".format(model) - - @staticmethod - def _render_litellm(model): - short = model.split("/", 1)[1].lower() - return ( - "# MLX_AGENT_WIRE v1\n" - "model_list:\n" - " - model_name: {0}\n" - " litellm_params:\n" - " model: openai/{1}\n" - " api_base: http://127.0.0.1:8080/v1\n" - " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY\n" - ).format(short, model) - - def validate(self, content): - if not isinstance(content, str): - raise TypeError("configuration content must be text") - require_secret_free_config(content) - if self.runtime in {"lmstudio", "mlx_lm", "mlx-vlm"}: - value = json.loads(content) - if not isinstance(value, dict): - raise ValueError("JSON configuration must be an object") - return True - if self.runtime == "ollama": - match = _OLLAMA.fullmatch(content) - if match is None: - raise ValueError("Ollama configuration must match the exact Wire grammar") - self._validate_model(match.group(1)) - return True - self._validate_litellm(content) - return True - - @staticmethod - def _validate_litellm(content): - lines = content.splitlines() - if len(lines) != 7 or lines[0] != "# MLX_AGENT_WIRE v1" or lines[1] != "model_list:": - raise ValueError("LiteLLM configuration must use the supported Wire YAML subset") - model_name = re.fullmatch(r" - model_name: ([A-Za-z0-9][A-Za-z0-9._-]{0,191})", lines[2]) - model = re.fullmatch(r" model: openai/(" + _MODEL.pattern[1:-1] + r")", lines[4]) - if lines[3] != " litellm_params:" or lines[5] != " api_base: http://127.0.0.1:8080/v1" or lines[6] != " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY" or not model_name or not model: - raise ValueError("LiteLLM configuration must use the supported Wire YAML subset") - ConfigAdapter._validate_model(model.group(1)) - if model_name.group(1) != model.group(1).split("/", 1)[1].lower(): - raise ValueError("LiteLLM model_name does not match the model identifier") - - @staticmethod - def health_check(endpoint, timeout=2.0): - if not endpoint: - return True - try: - endpoint = validate_health_endpoint(endpoint) - request = urllib.request.Request(endpoint, method="GET") - opener = urllib.request.build_opener( - urllib.request.ProxyHandler({}), - _SameLoopbackOriginRedirect(_endpoint_origin(endpoint)) - ) - with opener.open(request, timeout=timeout) as response: - return 200 <= response.status < 400 - except (OSError, ValueError, urllib.error.URLError): - return False diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/__init__.py b/packages/codex/mlx-agent/src/mlx_agent/__init__.py similarity index 84% rename from packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/__init__.py rename to packages/codex/mlx-agent/src/mlx_agent/__init__.py index 67dcb84..d887d69 100644 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/__init__.py +++ b/packages/codex/mlx-agent/src/mlx_agent/__init__.py @@ -3,4 +3,4 @@ from .contracts import ResultEnvelope __all__ = ["ResultEnvelope"] -__version__ = "0.4.0" +__version__ = "0.10.0" diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/__main__.py b/packages/codex/mlx-agent/src/mlx_agent/__main__.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/__main__.py rename to packages/codex/mlx-agent/src/mlx_agent/__main__.py diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/__init__.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/__init__.py new file mode 100644 index 0000000..55b62df --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/__init__.py @@ -0,0 +1,24 @@ +__version__ = "0.1.0" + +from .curated_ids import curated_dataset_digest, structured_caption_digest +from .curated_models import ( + CuratedDatasetManifestV1, + CuratedFrameV1, + CuratedSequenceV1, + DatasetSplitPolicyV1, + SequenceTransformRecipeV1, + StructuredCaptionV1, + render_caption_v1, +) + +__all__ = ( + "CuratedDatasetManifestV1", + "CuratedFrameV1", + "CuratedSequenceV1", + "DatasetSplitPolicyV1", + "SequenceTransformRecipeV1", + "StructuredCaptionV1", + "curated_dataset_digest", + "render_caption_v1", + "structured_caption_digest", +) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapter_protocol.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapter_protocol.py new file mode 100644 index 0000000..0c3c9a6 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapter_protocol.py @@ -0,0 +1,152 @@ +import json +import math +from collections.abc import Callable, Mapping, Sequence, Set +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from types import MappingProxyType +from typing import Protocol, runtime_checkable + +from pydantic import Field, field_serializer, field_validator + +from video_lab.capabilities import AdapterCapabilities +from video_lab.domain import FrozenModel, Manifest + + +class GenerationRequest(FrozenModel): + job_id: str + attempt_id: str + manifest: Manifest + scratch_root: Path + + +class GenerationProgress(FrozenModel): + phase: str + completed: int = Field(ge=0) + total: int = Field(gt=0) + message: str + elapsed_seconds: float = Field(ge=0) + + +class GenerationResult(FrozenModel): + video_path: Path + effective_manifest: Manifest + frame_count: int = Field(gt=0) + fps: int = Field(gt=0) + width: int = Field(gt=0) + height: int = Field(gt=0) + adapter_version: str + runtime_versions: Mapping[str, str] + adapter_telemetry: Mapping[str, object] + + @field_validator("runtime_versions") + @classmethod + def freeze_runtime_versions(cls, value: Mapping[str, str]) -> Mapping[str, str]: + return MappingProxyType(dict(value)) + + @field_validator("adapter_telemetry") + @classmethod + def freeze_adapter_telemetry(cls, value: object) -> Mapping[str, object]: + frozen = _freeze_json(value) + if not isinstance(frozen, Mapping): + raise TypeError("adapter telemetry must be a mapping") + return frozen + + @field_serializer("runtime_versions") + def serialize_runtime_versions(self, value: Mapping[str, str]) -> dict[str, str]: + return dict(value) + + @field_serializer("adapter_telemetry") + def serialize_adapter_telemetry(self, value: Mapping[str, object]) -> dict[str, object]: + serialized = _serialize_json(value) + if not isinstance(serialized, dict): + raise TypeError("adapter telemetry must serialize to a mapping") + return serialized + + +def _freeze_json(value: object) -> object: + if isinstance(value, float) and not math.isfinite(value): + raise ValueError("unsupported non-finite JSON number") + if value is None or isinstance(value, str | int | float | bool): + return value + if isinstance(value, Mapping): + if not all(isinstance(key, str) for key in value): + raise TypeError("adapter telemetry mapping keys must be strings") + return MappingProxyType({key: _freeze_json(item) for key, item in value.items()}) + if isinstance(value, Set): + frozen = tuple(_freeze_json(item) for item in value) + return tuple(sorted(frozen, key=_json_order)) + if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): + return tuple(_freeze_json(item) for item in value) + raise TypeError(f"unsupported adapter telemetry value: {type(value).__name__}") + + +def _serialize_json(value: object) -> object: + if isinstance(value, float) and not math.isfinite(value): + raise ValueError("unsupported non-finite JSON number") + if isinstance(value, Mapping): + return {str(key): _serialize_json(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_serialize_json(item) for item in value] + return value + + +def _json_order(value: object) -> str: + return json.dumps(_serialize_json(value), sort_keys=True) + + +class GenerationCancelled(Exception): + """Raised when generation is cancelled before artifact finalization.""" + + +class AdapterExecutionError(Exception): + """Raised when an adapter cannot complete generation.""" + + +ProgressCallback = Callable[[GenerationProgress], None] +CancellationCheck = Callable[[], bool] + + +class ProbePhase(StrEnum): + ENVIRONMENT = "environment" + METADATA = "metadata" + CACHE = "cache" + LOAD = "load" + GENERATE = "generate" + + +@dataclass(frozen=True) +class ProbeReport: + phase: ProbePhase | str + ok: bool + diagnostics: Mapping[str, object] + + +@dataclass(frozen=True) +class ProbeRequest: + generation: GenerationRequest + progress: ProgressCallback + cancelled: CancellationCheck + + +@runtime_checkable +class GenerationAdapter(Protocol): + adapter_id: str + + def capabilities(self) -> AdapterCapabilities: ... + + def probe(self) -> bool: ... + + def generate( + self, + request: GenerationRequest, + progress: ProgressCallback, + cancelled: CancellationCheck, + ) -> GenerationResult: ... + + +@runtime_checkable +class StagedProbeAdapter(GenerationAdapter, Protocol): + def probe_phase( + self, phase: ProbePhase, request: ProbeRequest | None = None + ) -> ProbeReport: ... diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapter_registry.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapter_registry.py new file mode 100644 index 0000000..ef789b2 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapter_registry.py @@ -0,0 +1,300 @@ +from collections.abc import Callable +from typing import Protocol + +from video_lab.adapter_protocol import GenerationAdapter +from video_lab.adapters.animegen_mps import AnimeGenStorageMode +from video_lab.capabilities import AdapterCapabilities +from video_lab.domain import FrozenModel +from video_lab.feasibility_models import SupportVerdict +from video_lab.feasibility_repository import GenerationAuthorizationBundle + +AdapterFactory = Callable[[], GenerationAdapter] +CapabilitiesFactory = Callable[[], AdapterCapabilities] + + +class FeasibilityAuthorizationRepository(Protocol): + def generation_authorization_bundle( + self, adapter_id: str + ) -> GenerationAuthorizationBundle | None: ... + + +class AdapterMetadata(FrozenModel): + adapter_id: str + display_name: str + source_model: str | None = None + frame_rule: str | None = None + declared_profiles: tuple[str, ...] = () + generation_enabled: bool = True + + +MetadataFactory = Callable[[], AdapterMetadata] + + +class _Registration: + def __init__( + self, + factory: AdapterFactory, + metadata_factory: MetadataFactory | None, + capabilities_factory: CapabilitiesFactory | None, + ) -> None: + self.factory = factory + self.metadata_factory = metadata_factory + self.capabilities_factory = capabilities_factory + + +class UnknownAdapterError(LookupError): + """Raised when an exact adapter ID is not registered.""" + + +class AdapterRegistry: + def __init__(self) -> None: + self._registrations: dict[str, _Registration] = {} + + def register( + self, + adapter_id: str, + factory: AdapterFactory, + *, + metadata_factory: MetadataFactory | None = None, + capabilities_factory: CapabilitiesFactory | None = None, + ) -> None: + if adapter_id in self._registrations: + raise ValueError(f"adapter {adapter_id!r} is already registered") + self._registrations[adapter_id] = _Registration( + factory, metadata_factory, capabilities_factory + ) + + def resolve(self, adapter_id: str) -> GenerationAdapter: + try: + registration = self._registrations[adapter_id] + except KeyError as error: + raise UnknownAdapterError(f"unknown adapter: {adapter_id}") from error + return registration.factory() + + def catalog_metadata(self) -> tuple[tuple[AdapterMetadata, AdapterCapabilities], ...]: + catalog: list[tuple[AdapterMetadata, AdapterCapabilities]] = [] + for adapter_id, registration in self._registrations.items(): + if registration.metadata_factory is None or registration.capabilities_factory is None: + continue + metadata = registration.metadata_factory() + capabilities = registration.capabilities_factory() + if metadata.adapter_id != adapter_id: + raise ValueError( + f"catalog metadata does not match registered adapter {adapter_id!r}" + ) + self._validate_capabilities_identity(adapter_id, capabilities) + catalog.append((metadata, capabilities)) + return tuple(catalog) + + def capabilities(self, adapter_id: str) -> AdapterCapabilities: + try: + registration = self._registrations[adapter_id] + except KeyError as error: + raise UnknownAdapterError(f"unknown adapter: {adapter_id}") from error + capabilities = ( + registration.factory().capabilities() + if registration.capabilities_factory is None + else registration.capabilities_factory() + ) + self._validate_capabilities_identity(adapter_id, capabilities) + return capabilities + + @staticmethod + def _validate_capabilities_identity(adapter_id: str, capabilities: AdapterCapabilities) -> None: + if capabilities.adapter_id != adapter_id: + raise ValueError(f"capabilities do not match registered adapter {adapter_id!r}") + + @classmethod + def with_defaults( + cls, + *, + feasibility_repository: FeasibilityAuthorizationRepository | None = None, + animegen_runtime_factory: Callable[[GenerationAuthorizationBundle], object] | None = None, + ) -> "AdapterRegistry": + registry = cls() + + def fake_factory() -> GenerationAdapter: + from video_lab.adapters.fake import FakeVideoAdapter + + return FakeVideoAdapter() + + def fake_capabilities() -> AdapterCapabilities: + return AdapterCapabilities( + adapter_id="video-lab.fake.v1", + model_patterns=("*",), + backends=("cpu",), + dtypes=("uint8",), + quantizations=("none",), + schedulers=("fake",), + min_width=1, + max_width=4096, + width_multiple=1, + min_height=1, + max_height=4096, + height_multiple=1, + min_frames=17, + max_frames=17, + frame_rule="16n+1", + supports_lora=False, + ) + + registry.register( + "video-lab.fake.v1", + fake_factory, + metadata_factory=lambda: AdapterMetadata( + adapter_id="video-lab.fake.v1", display_name="Fake video" + ), + capabilities_factory=fake_capabilities, + ) + + def wan_mps_factory() -> GenerationAdapter: + from video_lab.adapters.wan_mps import WanMpsAdapter + + return WanMpsAdapter() + + def wan_mps_capabilities() -> AdapterCapabilities: + return AdapterCapabilities( + adapter_id="diffusers.wan2.1-t2v-1.3b.mps.v1", + model_patterns=("Wan-AI/Wan2.1-T2V-1.3B-Diffusers",), + backends=("mps",), + dtypes=("bfloat16",), + quantizations=("none",), + schedulers=("unipc",), + min_width=16, + max_width=4096, + width_multiple=16, + min_height=16, + max_height=4096, + height_multiple=16, + min_frames=17, + max_frames=129, + frame_rule="16n+1", + supports_lora=False, + ) + + registry.register( + "diffusers.wan2.1-t2v-1.3b.mps.v1", + wan_mps_factory, + metadata_factory=lambda: AdapterMetadata( + adapter_id="diffusers.wan2.1-t2v-1.3b.mps.v1", display_name="Wan 2.1 T2V 1.3B (MPS)" + ), + capabilities_factory=wan_mps_capabilities, + ) + + animegen_id = "diffusers.animegen-t2v.wan2.2-a14b.mps.v1" + authorization_bundle = ( + None + if feasibility_repository is None + else feasibility_repository.generation_authorization_bundle(animegen_id) + ) + animegen_verdict = ( + SupportVerdict.PENDING if authorization_bundle is None else authorization_bundle.verdict + ) + animegen_profile = ( + None if authorization_bundle is None else authorization_bundle.measured_profile[:3] + ) + + def blocked_animegen_factory() -> GenerationAdapter: + raise UnknownAdapterError( + f"generation is disabled for candidate adapter: {animegen_id}" + ) + + def animegen_capabilities() -> AdapterCapabilities: + if animegen_verdict is SupportVerdict.EXPERIMENTAL_LOCAL_HYBRID: + if animegen_profile is None: + raise ValueError("experimental AnimeGen authorization requires minimum profile") + min_width, min_height, min_frames = animegen_profile + max_width, max_height, max_frames = min_width, min_height, min_frames + elif animegen_verdict is SupportVerdict.STANDARD_LAB_QUALIFIED: + if animegen_profile is None: + raise ValueError("standard AnimeGen authorization requires measured profile") + min_width, min_height, min_frames = 16, 16, 17 + max_width, max_height, max_frames = animegen_profile + else: + min_width, min_height, min_frames = 16, 16, 17 + max_width, max_height, max_frames = 4096, 4096, 129 + return AdapterCapabilities( + adapter_id=animegen_id, + model_patterns=("aidealab/AnimeGen-T2V",), + backends=("mps",), + dtypes=("bfloat16",), + quantizations=tuple(mode.value for mode in AnimeGenStorageMode), + schedulers=("flow-match-euler-shift-3",), + min_width=min_width, + max_width=max_width, + width_multiple=16, + min_height=min_height, + max_height=max_height, + height_multiple=16, + min_frames=min_frames, + max_frames=max_frames, + frame_rule="16n+1", + supports_lora=True, + min_fps=( + 1 if authorization_bundle is None else authorization_bundle.measured_profile[3] + ), + max_fps=( + 120 + if authorization_bundle is None + else authorization_bundle.measured_profile[3] + ), + ) + + registry.register( + animegen_id, + ( + ( + lambda: _authorized_animegen_adapter( + animegen_runtime_factory, authorization_bundle + ) + ) + if animegen_runtime_factory is not None + and authorization_bundle is not None + and animegen_verdict + in ( + SupportVerdict.EXPERIMENTAL_LOCAL_HYBRID, + SupportVerdict.STANDARD_LAB_QUALIFIED, + ) + else blocked_animegen_factory + ), + metadata_factory=lambda: AdapterMetadata( + adapter_id=animegen_id, + display_name="AnimeGen T2V / Wan 2.2 A14B (MPS candidate)", + source_model="aidealab/AnimeGen-T2V", + frame_rule="16n+1", + declared_profiles=("minimum", "standard-512x288-17f-8fps"), + generation_enabled=animegen_verdict + in ( + SupportVerdict.EXPERIMENTAL_LOCAL_HYBRID, + SupportVerdict.STANDARD_LAB_QUALIFIED, + ), + ), + capabilities_factory=animegen_capabilities, + ) + return registry + + +def _authorized_animegen_adapter( + runtime_factory: Callable[[GenerationAuthorizationBundle], object], + authorization: GenerationAuthorizationBundle | None, +) -> GenerationAdapter: + from typing import cast + + from video_lab.adapters.animegen_mps import AnimeGenGenerationRuntime, AnimeGenMpsAdapter + + if authorization is None: + raise UnknownAdapterError("AnimeGen generation is not authorized") + verdict, profile = authorization.verdict, authorization.measured_profile[:3] + return AnimeGenMpsAdapter( + runtime=cast(AnimeGenGenerationRuntime, runtime_factory(authorization)), + authorized_profile=profile, + exact_profile_only=verdict is SupportVerdict.EXPERIMENTAL_LOCAL_HYBRID, + expected_fingerprints=( + authorization.source_fingerprint, + authorization.runtime_fingerprint, + authorization.placement_fingerprint, + ), + authorized_manifest=authorization.authorized_manifest, + authorized_verdict=authorization.verdict.value, + authorized_manifest_hash=authorization.canonical_manifest_hash, + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/__init__.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/__init__.py new file mode 100644 index 0000000..7d112ad --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/__init__.py @@ -0,0 +1 @@ +"""Generation adapter implementations.""" diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/animegen_mps.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/animegen_mps.py new file mode 100644 index 0000000..ea9c347 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/animegen_mps.py @@ -0,0 +1,1242 @@ +from __future__ import annotations + +import gc +import hashlib +import importlib +import json +import re +from collections.abc import Callable, Mapping +from copy import deepcopy +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from time import monotonic +from typing import Any, Protocol, cast + +from video_lab.adapter_protocol import ( + AdapterExecutionError, + CancellationCheck, + GenerationCancelled, + GenerationProgress, + GenerationRequest, + GenerationResult, + ProgressCallback, +) +from video_lab.capabilities import AdapterCapabilities, validate_manifest_support +from video_lab.domain import Manifest, canonical_json +from video_lab.feasibility_models import FeasibilityStage, StageEvidence +from video_lab.feasibility_telemetry import mark_mps_context_ready +from video_lab.runtime_identity import animegen_runtime_versions, runtime_fingerprint +from video_lab.source_inventory import ( + ANIMEGEN_REQUIRED_FILES, + ANIMEGEN_SOURCE_REPOS, + ResolvedSource, + canonical_source_digest, +) + +ADAPTER_ID = "diffusers.animegen-t2v.wan2.2-a14b.mps.v1" +_PLACEMENT_SAMPLE_LIMIT = 16 +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +STANDARD_VARIABLE_FIELDS = frozenset( + { + "prompt", + "negative_prompt", + "seed", + "width", + "height", + "num_frames", + "inference_steps", + "guidance_scale", + } +) + + +class AnimeGenStorageMode(StrEnum): + LAYERWISE_FP8 = "layerwise-fp8" + BF16 = "bf16" + + +def storage_mode_for_manifest(manifest: Manifest) -> AnimeGenStorageMode: + try: + return AnimeGenStorageMode(manifest.quantization) + except ValueError as error: + raise AdapterExecutionError("unsupported transformer storage") from error + + +@dataclass(frozen=True) +class GenerationProfile: + width: int + height: int + num_frames: int + fps: int + + +@dataclass(frozen=True) +class RepeatabilityEvidence: + classification: str + artifact_sha256: tuple[str, str] + decoded_frame_sha256: tuple[tuple[str, ...], tuple[str, ...]] + matching_frame_count: int + differing_frame_count: int + matching_frame_ratio: float + manifest_hash: str + source_fingerprint: str + runtime_fingerprint: str + placement_fingerprint: str + + +@dataclass(frozen=True) +class GenerationInvocation: + pipeline_kwargs: Mapping[str, object] + controls: Mapping[str, object] + + +class AnimeGenGenerationRuntime(Protocol): + def prepare(self, requested: Manifest) -> tuple[object, Mapping[str, object]]: ... + def cpu_generator(self, seed: int) -> object: ... + def invoke(self, pipeline: object, **kwargs: object) -> object: ... + def encode(self, frames: object, path: Path, fps: int) -> None: ... + def runtime_versions(self) -> Mapping[str, str]: ... + + +class AnimeGenMpsAdapter: + adapter_id = ADAPTER_ID + adapter_version = "1" + + def __init__( + self, + *, + runtime: AnimeGenGenerationRuntime, + authorized_profile: tuple[int, int, int], + exact_profile_only: bool = True, + expected_fingerprints: tuple[str, str, str] | None = None, + authorized_manifest: Manifest | None = None, + authorized_verdict: str | None = None, + authorized_manifest_hash: str | None = None, + ) -> None: + self._generation_runtime = runtime + self._authorized_profile = authorized_profile + self._exact_profile_only = exact_profile_only + self._expected_fingerprints = expected_fingerprints + self._authorized_manifest = authorized_manifest + self._authorized_verdict = authorized_verdict + self._authorized_manifest_hash = authorized_manifest_hash + + def capabilities(self) -> AdapterCapabilities: + width, height, frames = self._authorized_profile + return AdapterCapabilities( + adapter_id=self.adapter_id, + model_patterns=("aidealab/AnimeGen-T2V",), + backends=("mps",), + dtypes=("bfloat16",), + quantizations=tuple(mode.value for mode in AnimeGenStorageMode), + schedulers=("flow-match-euler-shift-3",), + min_width=width if self._exact_profile_only else 16, + max_width=width, + width_multiple=16, + min_height=height if self._exact_profile_only else 16, + max_height=height, + height_multiple=16, + min_frames=frames if self._exact_profile_only else 17, + max_frames=frames, + frame_rule="16n+1", + supports_lora=True, + min_fps=(1 if self._authorized_manifest is None else self._authorized_manifest.fps), + max_fps=(120 if self._authorized_manifest is None else self._authorized_manifest.fps), + ) + + def probe(self) -> bool: + return True + + def generate( + self, + request: GenerationRequest, + progress: ProgressCallback, + cancelled: CancellationCheck, + ) -> GenerationResult: + manifest = request.manifest + self._validate_authorized_request(manifest) + violations = validate_manifest_support(manifest, self.capabilities()) + if violations: + raise AdapterExecutionError( + "unsupported manifest: " + + "; ".join(f"{item.field}: {item.reason}" for item in violations) + ) + scratch = request.scratch_root + partial = scratch / "output.partial.mp4" + final = scratch / "output.mp4" + scratch.mkdir(parents=True, exist_ok=True) + partial.unlink(missing_ok=True) + final.unlink(missing_ok=True) + try: + if cancelled(): + raise GenerationCancelled("AnimeGen generation cancelled") + pipeline, observed = self._generation_runtime.prepare(manifest) + self._validate_observed_configuration(manifest, observed) + runtime_versions = self._generation_runtime.runtime_versions() + if dict(runtime_versions) != dict(manifest.runtime_versions): + raise AdapterExecutionError("runtime versions differ from frozen manifest") + observed_runtime_fingerprint = runtime_fingerprint(runtime_versions) + if ( + self._expected_fingerprints is not None + and ( + observed.get("source_fingerprint"), + observed_runtime_fingerprint, + observed.get("placement_fingerprint"), + ) + != self._expected_fingerprints + ): + raise AdapterExecutionError("runtime evidence differs from authorized run") + generator = self._generation_runtime.cpu_generator(manifest.seed) + invocation = build_generation_invocation(manifest, generator=generator) + + def callback(pipe: object, step: int, timestep: object, kwargs: object) -> object: + if cancelled(): + raise GenerationCancelled("AnimeGen generation cancelled") + progress( + GenerationProgress( + phase="generating", + completed=step + 1, + total=manifest.inference_steps, + message=f"completed diffusion step {step + 1}", + elapsed_seconds=0, + ) + ) + return kwargs + + frames = self._generation_runtime.invoke( + pipeline, **invocation.pipeline_kwargs, callback_on_step_end=callback + ) + self._generation_runtime.encode(frames, partial, manifest.fps) + if cancelled(): + raise GenerationCancelled("AnimeGen generation cancelled") + if not partial.is_file() or partial.stat().st_size == 0: + raise AdapterExecutionError("AnimeGen encoder produced no MP4") + partial.replace(final) + return GenerationResult( + video_path=final, + effective_manifest=manifest, + frame_count=manifest.num_frames, + fps=manifest.fps, + width=manifest.width, + height=manifest.height, + adapter_version=self.adapter_version, + runtime_versions=runtime_versions, + adapter_telemetry={ + "generator_device": "cpu", + "execution_placement": observed["execution_placement"], + "observed_configuration": dict(observed), + }, + ) + except (GenerationCancelled, AdapterExecutionError): + partial.unlink(missing_ok=True) + final.unlink(missing_ok=True) + raise + except Exception as error: + partial.unlink(missing_ok=True) + final.unlink(missing_ok=True) + raise AdapterExecutionError( + f"AnimeGen generation failed: {type(error).__name__}: {str(error)[:500]}" + ) from error + + @staticmethod + def _validate_observed_configuration( + manifest: Manifest, observed: Mapping[str, object] + ) -> None: + storage_mode = storage_mode_for_manifest(manifest) + expected = { + "scheduler": manifest.scheduler, + "dtype": manifest.dtype, + "quantization": manifest.quantization, + "compute_dtype": manifest.dtype, + "storage_dtype": ( + "float8_e4m3fn" if storage_mode is AnimeGenStorageMode.LAYERWISE_FP8 else "bfloat16" + ), + "loras": tuple(item.model_dump(mode="json") for item in manifest.loras), + "memory_strategy": manifest.memory_strategy, + "execution_placement": "local_hybrid:mps", + } + if any(observed.get(key) != value for key, value in expected.items()): + raise AdapterExecutionError("observed AnimeGen configuration differs from manifest") + for key in ("source_fingerprint", "placement_fingerprint"): + value = observed.get(key) + if not isinstance(value, str) or _SHA256.fullmatch(value) is None: + raise AdapterExecutionError(f"observed {key} is invalid") + + def _validate_authorized_request(self, manifest: Manifest) -> None: + baseline = self._authorized_manifest + if baseline is None: + return + baseline_hash = hashlib.sha256(canonical_json(baseline).encode()).hexdigest() + if baseline_hash != self._authorized_manifest_hash: + raise AdapterExecutionError("authorized manifest hash is invalid") + if self._authorized_verdict == "experimental_local_hybrid": + if manifest != baseline: + raise AdapterExecutionError("experimental request differs from authorized manifest") + return + requested = manifest.model_dump(mode="json") + authorized = baseline.model_dump(mode="json") + for field in STANDARD_VARIABLE_FIELDS: + requested.pop(field) + authorized.pop(field) + if requested != authorized: + raise AdapterExecutionError("standard request drifts from authorized frozen controls") + + +def build_generation_invocation(manifest: Manifest, *, generator: object) -> GenerationInvocation: + return GenerationInvocation( + pipeline_kwargs={ + "prompt": manifest.prompt, + "negative_prompt": manifest.negative_prompt, + "height": manifest.height, + "width": manifest.width, + "num_frames": manifest.num_frames, + "num_inference_steps": manifest.inference_steps, + "guidance_scale": manifest.guidance_scale, + "generator": generator, + }, + controls={ + "seed": manifest.seed, + "scheduler": manifest.scheduler, + "dtype": manifest.dtype, + "quantization": manifest.quantization, + "loras": tuple(item.model_dump(mode="json") for item in manifest.loras), + "memory_strategy": manifest.memory_strategy, + "execution_placement": "local_hybrid:mps", + "generator_device": "cpu", + }, + ) + + +def select_minimum_profile( + *, + profiles: tuple[GenerationProfile, ...], + width_multiple: int, + height_multiple: int, +) -> GenerationProfile: + valid = tuple( + profile + for profile in profiles + if profile.width > 0 + and profile.height > 0 + and profile.fps > 0 + and profile.width % width_multiple == 0 + and profile.height % height_multiple == 0 + and profile.num_frames >= 17 + and (profile.num_frames - 1) % 16 == 0 + ) + if not valid or len(valid) != len(profiles): + raise ValueError("pipeline configuration has no valid minimum generation profile") + return min(valid, key=lambda item: (item.width * item.height, item.num_frames, item.fps)) + + +def classify_repeatability( + *, + first_artifact_sha256: str, + second_artifact_sha256: str, + first_frame_sha256: tuple[str, ...], + second_frame_sha256: tuple[str, ...], + manifest_hash: str, + source_fingerprint: str, + runtime_fingerprint: str, + placement_fingerprint: str, + second_manifest_hash: str | None = None, + second_source_fingerprint: str | None = None, + second_runtime_fingerprint: str | None = None, + second_placement_fingerprint: str | None = None, +) -> RepeatabilityEvidence: + hashes = ( + first_artifact_sha256, + second_artifact_sha256, + manifest_hash, + source_fingerprint, + runtime_fingerprint, + placement_fingerprint, + *first_frame_sha256, + *second_frame_sha256, + ) + provenance_matches = ( + (second_manifest_hash or manifest_hash) == manifest_hash + and (second_source_fingerprint or source_fingerprint) == source_fingerprint + and (second_runtime_fingerprint or runtime_fingerprint) == runtime_fingerprint + and (second_placement_fingerprint or placement_fingerprint) == placement_fingerprint + ) + valid = all(_SHA256.fullmatch(value) for value in hashes) and bool(first_frame_sha256) + valid = valid and len(first_frame_sha256) == len(second_frame_sha256) and provenance_matches + if not valid: + classification = "invalid" + matching_frame_count = 0 + differing_frame_count = 0 + matching_frame_ratio = 0.0 + else: + matching_frame_count = sum( + left == right + for left, right in zip(first_frame_sha256, second_frame_sha256, strict=True) + ) + differing_frame_count = len(first_frame_sha256) - matching_frame_count + matching_frame_ratio = matching_frame_count / len(first_frame_sha256) + if first_artifact_sha256 == second_artifact_sha256: + classification = "bitwise" + elif first_frame_sha256 == second_frame_sha256: + classification = "decoded_frame" + else: + classification = "numerically_nondeterministic" + return RepeatabilityEvidence( + classification=classification, + artifact_sha256=(first_artifact_sha256, second_artifact_sha256), + decoded_frame_sha256=(first_frame_sha256, second_frame_sha256), + matching_frame_count=matching_frame_count, + differing_frame_count=differing_frame_count, + matching_frame_ratio=matching_frame_ratio, + manifest_hash=manifest_hash, + source_fingerprint=source_fingerprint, + runtime_fingerprint=runtime_fingerprint, + placement_fingerprint=placement_fingerprint, + ) + + +@dataclass(frozen=True) +class AnimeGenAssembly: + animegen_repo: str = "aidealab/AnimeGen-T2V" + wan_repo: str = "Wan-AI/Wan2.2-T2V-A14B-Diffusers" + scheduler_shift: float = 3.0 + compute_dtype: str = "bfloat16" + storage_dtype: str = "float8_e4m3fn" + loras: tuple[tuple[str, str, float], ...] = ( + ( + "lightx2v/Wan2.2-Lightning", + "Wan2.2-T2V-A14B-4steps-lora-250928/high_noise_model.safetensors", + 2.0, + ), + ( + "lightx2v/Wan2.2-Lightning", + "Wan2.2-T2V-A14B-4steps-lora-250928/low_noise_model.safetensors", + 1.0, + ), + ) + + +ANIMEGEN_ASSEMBLY = AnimeGenAssembly() + + +@dataclass(frozen=True) +class CastingObservation: + storage_dtype: str + compute_dtype: str + + +class PlacementPolicyError(RuntimeError): + """Observed execution state does not prove the declared strategy.""" + + +class AnimeGenRuntime(Protocol): + def load_text_stack(self, repo: str, **kwargs: object) -> Mapping[str, object]: ... + def load_vae(self, repo: str, **kwargs: object) -> object: ... + def load_transformer_file(self, path: Path, **kwargs: object) -> object: ... + def load_scheduler_config(self, repo: str, **kwargs: object) -> object: ... + def scheduler_from_config(self, config: object, *, shift: float) -> object: ... + def assemble_pipeline(self, repo: str, **components: object) -> object: ... + def observe_adapters(self, pipeline: object) -> Mapping[str, float]: ... + + +class _DiffusersRuntime: + """Lazy real-library boundary; its signatures mirror upstream public APIs.""" + + def load_text_stack(self, repo: str, **kwargs: object) -> Mapping[str, object]: + transformers = importlib.import_module("transformers") + tokenizer_kwargs = {**kwargs, "use_fast": True} + tokenizer = transformers.AutoTokenizer.from_pretrained( + repo, subfolder="tokenizer", **tokenizer_kwargs + ) + text_encoder = transformers.UMT5EncoderModel.from_pretrained( + repo, subfolder="text_encoder", **kwargs + ) + return {"tokenizer": tokenizer, "text_encoder": text_encoder} + + def load_vae(self, repo: str, **kwargs: object) -> object: + cls = importlib.import_module("diffusers").AutoencoderKLWan + return cls.from_pretrained(repo, **kwargs) + + def load_transformer_file(self, path: Path, **kwargs: object) -> object: + cls = importlib.import_module("diffusers").WanTransformer3DModel + return cls.from_single_file(path, **kwargs) + + def load_scheduler_config(self, repo: str, **kwargs: object) -> object: + cls = importlib.import_module("diffusers").FlowMatchEulerDiscreteScheduler + return cls.load_config(repo, **kwargs) + + def scheduler_from_config(self, config: object, *, shift: float) -> object: + cls = importlib.import_module("diffusers").FlowMatchEulerDiscreteScheduler + return cls.from_config(config, shift=shift) + + def assemble_pipeline(self, repo: str, **components: object) -> object: + cls = importlib.import_module("diffusers").WanPipeline + return cls.from_pretrained(repo, **components) + + def observe_adapters(self, pipeline: object) -> Mapping[str, float]: + peft_pipeline = cast(Any, pipeline) + active = tuple(peft_pipeline.get_active_adapters()) + values: dict[str, set[float]] = {name: set() for name in active} + for component_name in ("transformer", "transformer_2"): + component = getattr(peft_pipeline, component_name, None) + for _name, module in getattr(component, "named_modules", lambda: ())(): + scaling = getattr(module, "scaling", None) + lora_alpha = getattr(module, "lora_alpha", None) + ranks = getattr(module, "r", None) + if isinstance(scaling, Mapping): + for adapter in active: + if ( + adapter in scaling + and isinstance(lora_alpha, Mapping) + and isinstance(ranks, Mapping) + and adapter in lora_alpha + and adapter in ranks + and float(ranks[adapter]) != 0 + ): + base = float(lora_alpha[adapter]) / float(ranks[adapter]) + values[adapter].add(float(scaling[adapter]) / base) + if any(len(scales) != 1 for scales in values.values()): + raise PlacementPolicyError("PEFT adapter scales are not uniquely observable") + return {name: next(iter(scales)) for name, scales in values.items()} + + +class DiffusersAnimeGenGenerationRuntime: + """Lazy production runtime assembled from the frozen Task 4 source inventory.""" + + def __init__( + self, + *, + cache_root: Path, + sources: Mapping[str, ResolvedSource], + runtime: AnimeGenRuntime | None = None, + torch: Any | None = None, + ) -> None: + self._cache_root = cache_root + self._sources = sources + self._runtime = runtime or _DiffusersRuntime() + self._torch = torch + self._executor = AnimeGenStageExecutor( + cache_root=cache_root, + sources=sources, + runtime=self._runtime, + torch=torch, + ) + self._prepared: tuple[str, object, dict[str, object]] | None = None + + def prepare(self, requested: Manifest) -> tuple[object, Mapping[str, object]]: + torch = self._executor._warm_mps() + storage_mode = storage_mode_for_manifest(requested) + if self._executor.storage_mode is not storage_mode: + self._executor = AnimeGenStageExecutor( + cache_root=self._cache_root, + sources=self._sources, + runtime=self._runtime, + torch=self._torch, + storage_mode=storage_mode, + ) + torch = self._executor._warm_mps() + sources = self._executor._validated_sources() + self._validate_manifest_bindings(requested, sources) + compatibility = requested.model_dump(mode="json") + for field in STANDARD_VARIABLE_FIELDS: + compatibility.pop(field) + compatibility_digest = hashlib.sha256( + canonical_json(compatibility).encode() + ).hexdigest() + if self._prepared is not None and self._prepared[0] == compatibility_digest: + return self._prepared[1], deepcopy(self._prepared[2]) + animegen = sources["animegen"] + wan = sources["wan-base"] + wan_snapshot = self._executor._snapshot_dir(wan) + common = {"local_files_only": True, "torch_dtype": torch.bfloat16} + text = self._executor._runtime.load_text_stack(str(wan_snapshot), **common) + vae = self._executor._runtime.load_vae(str(wan_snapshot), subfolder="vae", **common) + high = self._executor._runtime.load_transformer_file( + self._executor._snapshot_file(animegen, "high_noise.safetensors"), + config=str(wan_snapshot), + subfolder="transformer", + local_files_only=True, + torch_dtype=torch.bfloat16, + ) + low = self._executor._runtime.load_transformer_file( + self._executor._snapshot_file(animegen, "low_noise.safetensors"), + config=str(wan_snapshot), + subfolder="transformer", + local_files_only=True, + torch_dtype=torch.bfloat16, + ) + casting_observations: list[CastingObservation] = [] + for transformer in (high, low): + observation = self._executor._configure_transformer_storage(transformer) + if observation is not None: + casting_observations.append(observation) + config = self._executor._runtime.load_scheduler_config( + str(wan_snapshot), subfolder="scheduler", local_files_only=True + ) + scheduler = self._executor._runtime.scheduler_from_config(config, shift=3.0) + pipeline = self._executor._runtime.assemble_pipeline( + str(wan_snapshot), + **dict(text), + vae=vae, + transformer=high, + transformer_2=low, + scheduler=scheduler, + local_files_only=True, + ) + self._executor._apply_and_observe_loras(pipeline) + offload = getattr(pipeline, "enable_model_cpu_offload", None) + if not callable(offload): + raise PlacementPolicyError("model CPU offload is unavailable") + offload(device="mps") + placement = self._executor._observe_placement(pipeline) + transformer_config_file = next( + (item for item in wan.files if item.relative_path == "transformer/config.json"), + None, + ) + if transformer_config_file is None or not ( + transformer_config_sha256 := ( + transformer_config_file.local_sha256 or transformer_config_file.upstream_sha256 + ) + ): + raise AdapterExecutionError("pinned Wan transformer config identity is unavailable") + transformer_config_identity = { + "repo_id": wan.repo_id, + "revision": wan.revision, + "subfolder": "transformer", + "relative_path": "transformer/config.json", + "sha256": transformer_config_sha256, + "local_files_only": True, + } + source_payload: dict[str, object] = { + key: { + "repo": source.repo_id, + "revision": source.revision, + "files": tuple( + ( + item.relative_path, + item.local_sha256 or item.upstream_sha256, + item.size, + ) + for item in source.files + ), + } + for key, source in sorted(sources.items()) + } + source_payload["observed_transformer_config"] = transformer_config_identity + source_fingerprint = hashlib.sha256( + json.dumps(source_payload, sort_keys=True).encode() + ).hexdigest() + placement_fingerprint = hashlib.sha256( + json.dumps(placement, sort_keys=True).encode() + ).hexdigest() + configuration = self._executor._observe_configuration( + pipeline, high, low, tuple(casting_observations) + ) + observed_scales = self._executor._runtime.observe_adapters(pipeline) + observed_loras = tuple( + { + "model_id": sources[source_name].repo_id, + "revision": sources[source_name].revision, + "checksum": canonical_source_digest(sources[source_name]), + "weight": observed_scales[adapter_name], + } + for source_name, (_repo, _filename, _weight), adapter_name in zip( + ("lightning-high", "lightning-low"), + ANIMEGEN_ASSEMBLY.loras, + ("lightning_high", "lightning_low"), + strict=True, + ) + ) + observed = { + "scheduler": "flow-match-euler-shift-3", + "dtype": "bfloat16", + "quantization": configuration["quantization"], + "compute_dtype": configuration["compute_dtype"], + "storage_dtype": configuration["storage_dtype"], + "transformer_config_identity": transformer_config_identity, + "loras": observed_loras, + "memory_strategy": "model_cpu_offload", + "source_fingerprint": source_fingerprint, + "placement_fingerprint": placement_fingerprint, + "execution_placement": "local_hybrid:mps", + } + self._prepared = (compatibility_digest, pipeline, deepcopy(observed)) + return pipeline, observed + + @staticmethod + def _validate_manifest_bindings( + requested: Manifest, sources: Mapping[str, ResolvedSource] + ) -> None: + storage_mode_for_manifest(requested) + if ( + requested.model_id != ANIMEGEN_ASSEMBLY.animegen_repo + or requested.model_revision != sources["animegen"].revision + or requested.scheduler != "flow-match-euler-shift-3" + or requested.dtype != ANIMEGEN_ASSEMBLY.compute_dtype + or requested.memory_strategy != "model_cpu_offload" + ): + raise AdapterExecutionError("manifest differs from frozen AnimeGen assembly") + expected: list[tuple[str, str, str, float]] = [] + expected_source_checksums = { + component: canonical_source_digest(source) for component, source in sources.items() + } + if dict(requested.source_checksums) != expected_source_checksums: + raise AdapterExecutionError("manifest checksums differ from frozen sources") + for source_name, (_repo, _filename, weight) in zip( + ("lightning-high", "lightning-low"), ANIMEGEN_ASSEMBLY.loras, strict=True + ): + source = sources[source_name] + expected.append( + (source.repo_id, source.revision, expected_source_checksums[source_name], weight) + ) + actual = [ + (item.model_id, item.revision, item.checksum, item.weight) for item in requested.loras + ] + if actual != expected: + raise AdapterExecutionError("manifest LoRA bindings differ from frozen sources") + + def cpu_generator(self, seed: int) -> object: + return self._executor._get_torch().Generator(device="cpu").manual_seed(seed) + + def invoke(self, pipeline: object, **kwargs: object) -> object: + output = cast(Any, pipeline)(**kwargs) + frames = getattr(output, "frames", None) + if frames is None or len(frames) == 0 or len(frames[0]) == 0: + raise AdapterExecutionError("AnimeGen pipeline returned no frames") + return frames[0] + + def encode(self, frames: object, path: Path, fps: int) -> None: + importlib.import_module("diffusers.utils").export_to_video(frames, str(path), fps=fps) + + def runtime_versions(self) -> Mapping[str, str]: + return animegen_runtime_versions() + + +class ArtifactBoundary(Protocol): + def stage(self, source: Path, media_type: str) -> object: ... + def commit(self, staged: object, job_id: str | None) -> object: ... + def discard(self, staged: object) -> None: ... + + +GenerationInvoker = Callable[[Callable[[object, int, object, object], object]], Path] + + +class AnimeGenStageExecutor: + def __init__( + self, + *, + cache_root: Path, + torch: Any | None = None, + runtime: AnimeGenRuntime | None = None, + sources: Mapping[str, ResolvedSource] | None = None, + storage_mode: AnimeGenStorageMode = AnimeGenStorageMode.LAYERWISE_FP8, + ) -> None: + self._cache_root = Path(cache_root) + self._torch = torch + self._runtime = runtime or _DiffusersRuntime() + self._sources = sources + self._storage_mode = storage_mode + self._retained: list[object] = [] + + @property + def retained_component_count(self) -> int: + return len(self._retained) + + @property + def storage_mode(self) -> AnimeGenStorageMode: + return self._storage_mode + + def _get_torch(self) -> Any: + if self._torch is None: + self._torch = importlib.import_module("torch") + return self._torch + + def _warm_mps(self) -> Any: + torch = self._get_torch() + if not torch.backends.mps.is_available() or not torch.backends.mps.is_built(): + raise PlacementPolicyError("MPS is unavailable") + torch.empty(1, device="mps") + mark_mps_context_ready(torch) + return torch + + def execute_protocol_stage( + self, stage: FeasibilityStage, *, cancelled: CancellationCheck + ) -> StageEvidence: + """Measure one independently resumable component-load stage.""" + if stage is FeasibilityStage.ASSEMBLE_PIPELINE: + return self.execute(cancelled=cancelled) + loaders: dict[FeasibilityStage, Callable[[], object]] = { + FeasibilityStage.LOAD_TEXT_STACK: self._protocol_text_stack, + FeasibilityStage.LOAD_VAE: self._protocol_vae, + FeasibilityStage.LOAD_HIGH_TRANSFORMER: lambda: self._protocol_transformer( + "high_noise.safetensors" + ), + FeasibilityStage.LOAD_LOW_TRANSFORMER: lambda: self._protocol_transformer( + "low_noise.safetensors" + ), + } + loader = loaders.get(stage) + if loader is None: + raise ValueError(f"unsupported AnimeGen component stage: {stage.value}") + torch = self._warm_mps() + self._checkpoint(cancelled) + started = monotonic() + component = loader() + self._retained.append(component) + try: + self._checkpoint(cancelled) + observation = self._protocol_observation(stage, component) + return StageEvidence( + measurements={"duration_seconds": monotonic() - started}, + details=cast( + Any, + { + "stage": stage.value, + "local_files_only": True, + **observation, + }, + ), + ) + finally: + self._retained.clear() + component = None + gc.collect() + try: + torch.mps.empty_cache() + except (AttributeError, RuntimeError): + pass + + def _protocol_observation( + self, stage: FeasibilityStage, component: object + ) -> dict[str, object]: + placement = dict(self._component_summary(stage.value, component)) + count = placement.get("parameter_count") + counts = placement.get("counts") + if ( + not isinstance(count, int) + or count <= 0 + or not isinstance(counts, Mapping) + or any(device not in {"cpu", "mps"} for device in counts) + ): + raise PlacementPolicyError(f"uninspectable placement for {stage.value}") + evidence: dict[str, object] = { + "placement": placement, + "placement_policy": ( + "declared_cpu_staging" if counts.get("cpu", 0) else "mps_resident" + ), + } + if stage in { + FeasibilityStage.LOAD_HIGH_TRANSFORMER, + FeasibilityStage.LOAD_LOW_TRANSFORMER, + }: + compute = _dtype_name(getattr(component, "dtype", "unknown")) + storage = { + _dtype_name(getattr(parameter, "dtype", "unknown")) + for parameter in getattr(component, "parameters", lambda: ())() + } + if self._storage_mode is AnimeGenStorageMode.LAYERWISE_FP8: + fp8 = storage & {"float8_e4m3fn", "float8_e5m2"} + if ( + compute != "bfloat16" + or not fp8 + or storage + - { + "bfloat16", + "float8_e4m3fn", + "float8_e5m2", + } + ): + raise PlacementPolicyError("isolated transformer ignored bf16/FP8 casting") + storage_dtype = sorted(fp8)[0] + elif compute != "bfloat16" or storage != {"bfloat16"}: + raise PlacementPolicyError("isolated transformer does not retain bf16 storage") + else: + storage_dtype = "bfloat16" + evidence["configuration"] = { + "compute_dtype": compute, + "storage_dtype": storage_dtype, + "casting_applied": self._storage_mode is AnimeGenStorageMode.LAYERWISE_FP8, + } + return evidence + + def _protocol_text_stack(self) -> object: + wan = self._validated_sources()["wan-base"] + return self._runtime.load_text_stack( + str(self._snapshot_dir(wan)), + local_files_only=True, + torch_dtype=self._get_torch().bfloat16, + ) + + def _protocol_vae(self) -> object: + wan = self._validated_sources()["wan-base"] + return self._runtime.load_vae( + str(self._snapshot_dir(wan)), + subfolder="vae", + local_files_only=True, + torch_dtype=self._get_torch().bfloat16, + ) + + def _protocol_transformer(self, filename: str) -> object: + animegen = self._validated_sources()["animegen"] + wan = self._validated_sources()["wan-base"] + transformer = self._runtime.load_transformer_file( + self._snapshot_file(animegen, filename), + config=str(self._snapshot_dir(wan)), + subfolder="transformer", + local_files_only=True, + torch_dtype=self._get_torch().bfloat16, + ) + self._configure_transformer_storage(transformer) + return transformer + + def _configure_transformer_storage(self, transformer: object) -> CastingObservation | None: + if self._storage_mode is AnimeGenStorageMode.BF16: + self._require_bf16_transformer_storage(transformer) + return None + casting = getattr(transformer, "enable_layerwise_casting", None) + if not callable(casting): + raise PlacementPolicyError("layerwise FP8 casting is unsupported") + casting( + storage_dtype=self._get_torch().float8_e4m3fn, + compute_dtype=self._get_torch().bfloat16, + ) + return CastingObservation("float8_e4m3fn", "bfloat16") + + @staticmethod + def _require_bf16_transformer_storage(transformer: object) -> None: + storage = { + _dtype_name(getattr(parameter, "dtype", "unknown")) + for parameter in getattr(transformer, "parameters", lambda: ())() + } + if storage != {"bfloat16"}: + raise PlacementPolicyError("bf16 transformer storage must be exactly bfloat16") + + def execute(self, *, cancelled: CancellationCheck) -> StageEvidence: + torch = self._warm_mps() + common = { + "local_files_only": True, + "torch_dtype": torch.bfloat16, + } + stages: list[dict[str, object]] = [] + sources = self._validated_sources() + animegen = sources["animegen"] + wan = sources["wan-base"] + wan_snapshot = self._snapshot_dir(wan) + self._retained.clear() + try: + text = self._load_stage( + "text_stack", + stages, + cancelled, + lambda: self._runtime.load_text_stack(str(wan_snapshot), **common), + ) + vae = self._load_stage( + "vae", + stages, + cancelled, + lambda: self._runtime.load_vae(str(wan_snapshot), subfolder="vae", **common), + ) + high = self._load_stage( + "transformer_high", + stages, + cancelled, + lambda: self._runtime.load_transformer_file( + self._snapshot_file(animegen, "high_noise.safetensors"), + config=str(wan_snapshot), + subfolder="transformer", + torch_dtype=torch.bfloat16, + ), + ) + low = self._load_stage( + "transformer_low", + stages, + cancelled, + lambda: self._runtime.load_transformer_file( + self._snapshot_file(animegen, "low_noise.safetensors"), + config=str(wan_snapshot), + subfolder="transformer", + torch_dtype=torch.bfloat16, + ), + ) + casting_observations: list[CastingObservation] = [] + for transformer in (high, low): + self._checkpoint(cancelled) + cast_started = monotonic() + observation = self._configure_transformer_storage(transformer) + if observation is not None: + casting_observations.append(observation) + stages.append( + { + "stage": "layerwise_casting", + "duration_seconds": monotonic() - cast_started, + } + ) + self._checkpoint(cancelled) + config_started = monotonic() + config = self._runtime.load_scheduler_config( + str(wan_snapshot), + subfolder="scheduler", + local_files_only=True, + ) + stages.append( + {"stage": "scheduler_config", "duration_seconds": monotonic() - config_started} + ) + self._checkpoint(cancelled) + scheduler_started = monotonic() + scheduler = self._runtime.scheduler_from_config(config, shift=3.0) + stages.append( + { + "stage": "scheduler_construct", + "duration_seconds": monotonic() - scheduler_started, + "configuration": {"shift": 3.0}, + } + ) + self._checkpoint(cancelled) + assembled = monotonic() + pipeline = self._runtime.assemble_pipeline( + str(wan_snapshot), + **dict(cast(Mapping[str, object], text)), + vae=vae, + transformer=high, + transformer_2=low, + scheduler=scheduler, + local_files_only=True, + ) + self._retained.append(pipeline) + stages.append( + {"stage": "assemble_pipeline", "duration_seconds": monotonic() - assembled} + ) + self._checkpoint(cancelled) + lora_started = monotonic() + self._apply_and_observe_loras(pipeline) + stages.append({"stage": "apply_loras", "duration_seconds": monotonic() - lora_started}) + self._checkpoint(cancelled) + offload_started = monotonic() + offload = getattr(pipeline, "enable_model_cpu_offload", None) + if not callable(offload): + raise PlacementPolicyError("model CPU offload is unavailable") + offload(device="mps") + stages.append( + {"stage": "enable_offload", "duration_seconds": monotonic() - offload_started} + ) + placement = self._observe_placement(pipeline) + observed = self._observe_configuration(pipeline, high, low, tuple(casting_observations)) + self._checkpoint(cancelled) + return StageEvidence( + measurements={"component_stage_count": float(len(stages))}, + details=cast( + Any, + {"stages": tuple(stages), "placement": placement, "observed": observed}, + ), + ) + finally: + self._retained.clear() + text = vae = high = low = pipeline = None # type: ignore[assignment,unused-ignore] + gc.collect() + try: + torch.mps.empty_cache() + except (AttributeError, RuntimeError): + pass + + def _validated_sources(self) -> Mapping[str, ResolvedSource]: + if self._sources is None: + raise PlacementPolicyError("frozen source inventory is required") + required = {"animegen", "wan-base", "lightning-high", "lightning-low"} + if set(self._sources) != required: + raise PlacementPolicyError("frozen source inventory is incomplete") + for key, source in self._sources.items(): + if len(source.revision) != 40 or any( + char not in "0123456789abcdef" for char in source.revision + ): + raise PlacementPolicyError("source revision must be an exact 40-character commit") + component = key + repo = ANIMEGEN_SOURCE_REPOS[key] + files = set(ANIMEGEN_REQUIRED_FILES[key]) + actual_files = {item.relative_path for item in source.files} + if source.component != component or source.repo_id != repo or not files <= actual_files: + raise PlacementPolicyError(f"frozen source identity mismatch for {key}") + return self._sources + + def _snapshot_file(self, source: ResolvedSource, filename: str) -> Path: + return self._snapshot_dir(source) / filename + + def _snapshot_dir(self, source: ResolvedSource) -> Path: + repo = source.repo_id.replace("/", "--") + return self._cache_root / f"models--{repo}" / "snapshots" / source.revision + + def _load_stage( + self, + name: str, + stages: list[dict[str, object]], + cancelled: CancellationCheck, + loader: Callable[[], object], + ) -> object: + self._checkpoint(cancelled) + started = monotonic() + component = loader() + self._retained.append(component) + stages.append( + { + "stage": name, + "duration_seconds": monotonic() - started, + "placement": self._component_summary(name, component), + } + ) + return component + + def execute_generation( + self, + *, + invoke: GenerationInvoker, + scratch_root: Path, + cancelled: CancellationCheck, + artifact_store: ArtifactBoundary, + job_id: str, + ) -> object: + partial = scratch_root / "output.partial.mp4" + final = scratch_root / "output.mp4" + partial.unlink(missing_ok=True) + final.unlink(missing_ok=True) + staged: object | None = None + + def callback(_pipe: object, _step: int, _time: object, kwargs: object) -> object: + self._checkpoint(cancelled) + return kwargs + + try: + produced = invoke(callback) + self._checkpoint(cancelled) + if produced != partial or not partial.is_file() or not partial.stat().st_size: + raise RuntimeError("invalid partial artifact") + partial.replace(final) + self._checkpoint(cancelled) + staged = artifact_store.stage(final, "video/mp4") + self._checkpoint(cancelled) + return artifact_store.commit(staged, job_id) + except BaseException: + if staged is not None: + artifact_store.discard(staged) + partial.unlink(missing_ok=True) + final.unlink(missing_ok=True) + raise + finally: + try: + self._get_torch().mps.empty_cache() + except (AttributeError, RuntimeError): + pass + + @staticmethod + def _checkpoint(cancelled: CancellationCheck) -> None: + if cancelled(): + raise GenerationCancelled("AnimeGen execution cancelled") + + def _apply_and_observe_loras(self, pipeline: object) -> None: + load = getattr(pipeline, "load_lora_weights", None) + set_adapters = getattr(pipeline, "set_adapters", None) + if not callable(load) or not callable(set_adapters): + raise PlacementPolicyError("LoRA APIs unavailable") + names = ["lightning_high", "lightning_low"] + source_names = ("lightning-high", "lightning-low") + for (_repo, weight, _), name, source_name in zip( + ANIMEGEN_ASSEMBLY.loras, names, source_names, strict=True + ): + source = self._sources[source_name] # type: ignore[index] + load( + str(self._snapshot_dir(source)), + weight_name=weight, + adapter_name=name, + local_files_only=True, + ) + set_adapters(names, adapter_weights=[2.0, 1.0]) + + @staticmethod + def _component_summary(name: str, component: object) -> Mapping[str, object]: + members = component.values() if isinstance(component, Mapping) else (component,) + counts: dict[str, int] = {} + sample: list[dict[str, str]] = [] + total = 0 + for member in members: + for parameter in getattr(member, "parameters", lambda: ())(): + device = str(getattr(parameter, "device", "unknown")).split(":", 1)[0] + counts[device] = counts.get(device, 0) + 1 + total += 1 + if len(sample) < _PLACEMENT_SAMPLE_LIMIT: + sample.append({"device": device}) + return { + "component": name, + "parameter_count": total, + "counts": counts, + "sample": tuple(sample), + "truncated": total > len(sample), + } + + def _observe_placement(self, pipeline: object) -> Mapping[str, object]: + components = getattr(pipeline, "components", None) + if not isinstance(components, Mapping): + raise PlacementPolicyError("pipeline placement is uninspectable") + sample: list[dict[str, str]] = [] + counts = {"mps": 0, "cpu_offloaded": 0} + for name, component in components.items(): + for index, parameter in enumerate(getattr(component, "parameters", lambda: ())()): + device = str(getattr(parameter, "device", "unknown")) + if device.startswith("mps"): + counts["mps"] += 1 + elif device.startswith("cpu"): + hook = getattr(component, "_hf_hook", None) + execution_device = str(getattr(hook, "execution_device", "")) + if not execution_device.startswith("mps"): + raise PlacementPolicyError( + f"CPU component {name} lacks an MPS offload hook" + ) + counts["cpu_offloaded"] += 1 + else: + raise PlacementPolicyError(f"uninspectable placement for {name}") + if len(sample) < _PLACEMENT_SAMPLE_LIMIT: + sample.append({"name": f"{name}.parameters[{index}]", "device": device}) + return { + "counts": counts, + "sample": tuple(sample), + "truncated": sum(counts.values()) > len(sample), + } + + def _observe_configuration( + self, + pipeline: object, + high: object, + low: object, + casting_observations: tuple[CastingObservation, ...], + ) -> Mapping[str, object]: + compute = {_dtype_name(getattr(item, "dtype", "unknown")) for item in (high, low)} + storage: set[str] = set() + for item in (high, low): + for parameter in getattr(item, "parameters", lambda: ())(): + storage.add(_dtype_name(getattr(parameter, "dtype", "unknown"))) + observed_loras = dict(self._runtime.observe_adapters(pipeline)) + if self._storage_mode is AnimeGenStorageMode.LAYERWISE_FP8: + allowed = {"bfloat16", "float8_e4m3fn", "float8_e5m2"} + fp8 = storage & {"float8_e4m3fn", "float8_e5m2"} + casting_valid = len(casting_observations) == 2 and all( + item == CastingObservation("float8_e4m3fn", "bfloat16") + for item in casting_observations + ) + if not casting_valid or compute != {"bfloat16"} or not fp8 or storage - allowed: + raise PlacementPolicyError("observed compute/storage dtype does not match bf16/FP8") + storage_dtype = sorted(fp8)[0] + elif compute != {"bfloat16"} or storage != {"bfloat16"} or casting_observations: + raise PlacementPolicyError("observed compute/storage dtype does not match bf16 storage") + else: + storage_dtype = "bfloat16" + if observed_loras != {"lightning_high": 2.0, "lightning_low": 1.0}: + raise PlacementPolicyError("active LoRA names or scales do not match") + return { + "quantization": self._storage_mode.value, + "compute_dtype": "bfloat16", + "storage_dtype": storage_dtype, + "active_loras": observed_loras, + "casting_calls": len(casting_observations), + } + + +def _dtype_name(value: object) -> str: + name = str(value).lower().removeprefix("torch.") + return {"bf16": "bfloat16", "fp8": "float8_e4m3fn"}.get(name, name) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/fake.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/fake.py new file mode 100644 index 0000000..29848a4 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/fake.py @@ -0,0 +1,164 @@ +import hashlib +import os +from collections.abc import Callable +from pathlib import Path +from time import monotonic +from typing import Protocol, cast + +import imageio_ffmpeg # type: ignore[import-untyped] +import psutil # type: ignore[import-untyped] + +from video_lab.adapter_protocol import ( + AdapterExecutionError, + CancellationCheck, + GenerationCancelled, + GenerationProgress, + GenerationRequest, + GenerationResult, + ProgressCallback, +) +from video_lab.capabilities import AdapterCapabilities +from video_lab.domain import canonical_json + + +class _Writer(Protocol): + def send(self, frame: bytes | None) -> None: ... + + def close(self) -> None: ... + + +_WriterFactory = Callable[..., _Writer] + + +def _default_writer_factory(*args: object, **kwargs: object) -> _Writer: + return cast(_Writer, imageio_ffmpeg.write_frames(*args, **kwargs)) + + +class FakeVideoAdapter: + adapter_id = "video-lab.fake.v1" + adapter_version = "1" + + def __init__(self, *, _writer_factory: _WriterFactory = _default_writer_factory) -> None: + self._writer_factory = _writer_factory + + def capabilities(self) -> AdapterCapabilities: + return AdapterCapabilities( + adapter_id=self.adapter_id, + model_patterns=("*",), + backends=("cpu",), + dtypes=("uint8",), + quantizations=("none",), + schedulers=("fake",), + min_width=1, + max_width=4096, + width_multiple=1, + min_height=1, + max_height=4096, + height_multiple=1, + min_frames=17, + max_frames=17, + frame_rule="16n+1", + supports_lora=False, + ) + + def probe(self) -> bool: + return bool(imageio_ffmpeg.get_ffmpeg_exe()) + + def generate( + self, + request: GenerationRequest, + progress: ProgressCallback, + cancelled: CancellationCheck, + ) -> GenerationResult: + scratch = request.scratch_root + partial_path = scratch / "output.partial.mp4" + final_path = scratch / "output.mp4" + scratch.mkdir(parents=True, exist_ok=True) + self._remove(partial_path, final_path) + started = monotonic() + manifest = request.manifest + digest = hashlib.sha256(canonical_json(manifest).encode()).digest() + frame = bytes(digest[:3]) * (manifest.width * manifest.height) + writer: _Writer | None = None + primary_error: Exception | None = None + try: + writer = self._writer_factory( + partial_path, + (manifest.width, manifest.height), + fps=manifest.fps, + codec="libx264", + pix_fmt_in="rgb24", + pix_fmt_out="yuv420p", + macro_block_size=1, + ffmpeg_log_level="error", + output_params=[ + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + ], + ) + writer.send(None) + for index in range(17): + if cancelled(): + raise GenerationCancelled("generation cancelled") + writer.send(frame) + progress( + GenerationProgress( + phase="generating", + completed=index + 1, + total=17, + message=f"rendered frame {index + 1} of 17", + elapsed_seconds=monotonic() - started, + ) + ) + except Exception as error: + primary_error = error + finally: + if writer is not None: + try: + writer.close() + except Exception as close_error: + if primary_error is None: + primary_error = close_error + else: + primary_error.add_note(f"writer close failed: {close_error}") + + if primary_error is not None: + self._remove(partial_path, final_path) + if isinstance(primary_error, GenerationCancelled): + raise primary_error + raise AdapterExecutionError("fake adapter generation failed") from primary_error + + try: + with partial_path.open("rb") as artifact: + os.fsync(artifact.fileno()) + if cancelled(): + raise GenerationCancelled("generation cancelled") + partial_path.replace(final_path) + except GenerationCancelled: + self._remove(partial_path, final_path) + raise + except Exception as error: + self._remove(partial_path, final_path) + raise AdapterExecutionError("fake adapter generation failed") from error + + process = psutil.Process() + return GenerationResult( + video_path=final_path, + effective_manifest=manifest, + frame_count=17, + fps=manifest.fps, + width=manifest.width, + height=manifest.height, + adapter_version=self.adapter_version, + runtime_versions={"imageio-ffmpeg": imageio_ffmpeg.__version__}, + adapter_telemetry={"process_rss_bytes": process.memory_info().rss}, + ) + + @staticmethod + def _remove(*paths: Path) -> None: + for path in paths: + path.unlink(missing_ok=True) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/wan_mps.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/wan_mps.py new file mode 100644 index 0000000..14f952f --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adapters/wan_mps.py @@ -0,0 +1,523 @@ +from __future__ import annotations + +import hashlib +import importlib +import math +import os +import platform +import re +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from time import monotonic +from typing import Any, Protocol, cast +from urllib.parse import urlsplit, urlunsplit + +from video_lab.adapter_protocol import ( + AdapterExecutionError, + CancellationCheck, + GenerationCancelled, + GenerationProgress, + GenerationRequest, + GenerationResult, + ProbePhase, + ProbeReport, + ProbeRequest, + ProgressCallback, +) +from video_lab.capabilities import AdapterCapabilities, validate_manifest_support +from video_lab.domain import Manifest + +MODEL_ID = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" +REQUESTED_REVISION = "main" +_MAX_DIAGNOSTIC_LENGTH = 1000 + + +@dataclass(frozen=True) +class _Metadata: + requested_revision: str + resolved_revision: str + file_sizes: Mapping[str, int] + + +class _Pipeline(Protocol): + def to(self, device: str) -> _Pipeline: ... + def __call__(self, **kwargs: object) -> object: ... + + +PipelineLoader = Callable[..., _Pipeline] +Exporter = Callable[[object, str, int], None] +DeviceInspector = Callable[[object], Mapping[str, str]] +VideoInspector = Callable[[Path], Mapping[str, object]] +FrameCounter = Callable[[Path, int], int] + + +def _load_pipeline(model_id: str, **kwargs: object) -> _Pipeline: + diffusers = importlib.import_module("diffusers") + return diffusers.WanPipeline.from_pretrained(model_id, **kwargs) # type: ignore[no-any-return] + + +def _export_video(frames: object, path: str, fps: int) -> None: + importlib.import_module("diffusers.utils").export_to_video(frames, path, fps=fps) + + +def _inspect_video(path: Path) -> Mapping[str, object]: + return cast(Mapping[str, object], importlib.import_module("imageio.v3").immeta(path)) + + +def _count_video_frames(path: Path, limit: int) -> int: + count = 0 + for _frame in importlib.import_module("imageio.v3").imiter(path): + count += 1 + if count >= limit: + break + return count + + +def _sanitize_diagnostic(value: object, sensitive_paths: tuple[Path, ...] = ()) -> str: + text = str(value) + for match in re.findall(r"https?://[^\s]+", text): + parsed = urlsplit(match) + host = parsed.hostname or "redacted-host" + if parsed.port: + host = f"{host}:{parsed.port}" + clean = urlunsplit((parsed.scheme, host, parsed.path, "", "")) + text = text.replace(match, clean) + text = re.sub(r"(?i)\bBearer\s+[^\s,;]+", "Bearer ", text) + text = re.sub(r"\bhf_[A-Za-z0-9_-]+", "hf_", text) + homes = {str(Path.home())} + for variable in ("HF_HOME", "HF_HUB_CACHE"): + if os.environ.get(variable): + homes.add(str(Path(os.environ[variable]).expanduser())) + for home in sorted(homes, key=len, reverse=True): + text = text.replace(home, "") + for path in sorted((str(item.expanduser()) for item in sensitive_paths), key=len, reverse=True): + text = text.replace(path, "") + return text[:_MAX_DIAGNOSTIC_LENGTH] + + +def _error_diagnostics( + error: BaseException, sensitive_paths: tuple[Path, ...] = () +) -> Mapping[str, object]: + return { + "error_type": type(error).__name__, + "error": _sanitize_diagnostic(error, sensitive_paths), + } + + +class WanMpsAdapter: + adapter_id = "diffusers.wan2.1-t2v-1.3b.mps.v1" + adapter_version = "1" + + def __init__( + self, + *, + _torch: Any | None = None, + _pipeline_loader: PipelineLoader = _load_pipeline, + _exporter: Exporter = _export_video, + _hf_api: object | None = None, + _device_inspector: DeviceInspector | None = None, + _cache_root: Path | None = None, + _video_inspector: VideoInspector = _inspect_video, + _frame_counter: FrameCounter = _count_video_frames, + ) -> None: + self._torch = _torch + self._pipeline_loader = _pipeline_loader + self._exporter = _exporter + self._hf_api = _hf_api + self._device_inspector = _device_inspector or self._inspect_devices + self._cache_root = _cache_root + self._video_inspector = _video_inspector + self._frame_counter = _frame_counter + self._metadata: _Metadata | None = None + + def _get_torch(self) -> Any: + if self._torch is None: + self._torch = importlib.import_module("torch") + return self._torch + + def capabilities(self) -> AdapterCapabilities: + return AdapterCapabilities( + adapter_id=self.adapter_id, + model_patterns=(MODEL_ID,), + backends=("mps",), + dtypes=("bfloat16",), + quantizations=("none",), + schedulers=("unipc",), + min_width=16, + max_width=4096, + width_multiple=16, + min_height=16, + max_height=4096, + height_multiple=16, + min_frames=17, + max_frames=129, + frame_rule="16n+1", + supports_lora=False, + ) + + def probe(self) -> bool: + return self.probe_phase("environment").ok + + def probe_phase( + self, phase: ProbePhase | str, request: ProbeRequest | None = None + ) -> ProbeReport: + try: + if phase == "environment": + torch = self._get_torch() + diagnostics = { + "python": platform.python_version(), + "system": platform.system(), + "machine": platform.machine(), + "torch": str(torch.__version__), + "diffusers": self._optional_version("diffusers"), + "mps_available": bool(torch.backends.mps.is_available()), + "mps_built": bool(torch.backends.mps.is_built()), + } + ok = (diagnostics["system"], diagnostics["machine"]) == ("Darwin", "arm64") + return ProbeReport( + phase, + bool(ok and diagnostics["mps_available"] and diagnostics["mps_built"]), + diagnostics, + ) + if phase == "metadata": + metadata = self._resolve_metadata() + return ProbeReport(phase, True, self._metadata_diagnostics(metadata)) + if phase == "cache": + metadata = self._resolve_metadata() + root = self._effective_cache_root() + snapshot = ( + root + / "models--Wan-AI--Wan2.1-T2V-1.3B-Diffusers" + / "snapshots" + / metadata.resolved_revision + ) + missing_files = tuple( + name for name in metadata.file_sizes if not (snapshot / name).is_file() + ) + missing_bytes = sum(metadata.file_sizes[name] for name in missing_files) + return ProbeReport( + phase, + not missing_files, + { + **self._metadata_diagnostics(metadata), + "cache_root": str(root), + "snapshot_path": str(snapshot), + "missing_files": missing_files, + "missing_bytes": missing_bytes, + }, + ) + if phase == "load": + metadata = self._resolve_metadata() + self._load(metadata.resolved_revision) + return ProbeReport( + phase, True, {**self._metadata_diagnostics(metadata), "execution_device": "mps"} + ) + if phase == "generate": + if request is None: + raise RuntimeError("generate probe requires an explicit ProbeRequest") + profile = request.generation.manifest + if (profile.width, profile.height, profile.num_frames, profile.inference_steps) != ( + 832, + 480, + 17, + 4, + ): + raise RuntimeError( + "generate probe requires profile 832x480, 17 frames, 4 steps" + ) + result: GenerationResult | None = None + try: + result = self.generate(request.generation, request.progress, request.cancelled) + manifest = request.generation.manifest + video = self._video_inspector(result.video_path) + size: tuple[object, ...] = tuple(cast(Any, video.get("size", ()))) + frame_count = self._validated_frame_count( + result.video_path, video.get("nframes"), manifest.num_frames + ) + fps = float(cast(Any, video.get("fps", 0))) + valid = ( + result.video_path.is_file() + and result.video_path.stat().st_size > 0 + and size == (manifest.width, manifest.height) + and frame_count == manifest.num_frames + and math.isfinite(fps) + and round(fps) == manifest.fps + ) + if not valid: + raise RuntimeError("generate probe produced invalid MP4 metadata") + return ProbeReport( + phase, + True, + { + "video_path": str(result.video_path), + "width": result.width, + "height": result.height, + "frame_count": result.frame_count, + "fps": result.fps, + "container_metadata": dict(video), + }, + ) + except Exception: + scratch = request.generation.scratch_root + self._remove(scratch / "output.partial.mp4", scratch / "output.mp4") + if result is not None: + self._remove(result.video_path) + raise + raise ValueError(f"unknown probe phase: {phase}") + except Exception as error: + paths = (self._cache_root,) if self._cache_root is not None else () + return ProbeReport(phase, False, _error_diagnostics(error, paths)) + + def _validated_frame_count(self, path: Path, reported: object, expected: int) -> int: + if isinstance(reported, int | float) and not isinstance(reported, bool): + numeric = float(reported) + if math.isfinite(numeric) and numeric > 0 and numeric.is_integer(): + return int(numeric) + return self._frame_counter(path, expected + 1) + + def _resolve_metadata(self, revision: str = REQUESTED_REVISION) -> _Metadata: + if self._metadata is not None and self._metadata.requested_revision == revision: + return self._metadata + api = self._hf_api + if api is None: + api = importlib.import_module("huggingface_hub").HfApi() + info = cast(Any, api).model_info(MODEL_ID, revision=revision, files_metadata=True) + resolved = str(info.sha or "").strip() + if revision != REQUESTED_REVISION and resolved != revision: + raise AdapterExecutionError("Pinned model revision did not resolve exactly.") + if not resolved: + raise RuntimeError("Hub metadata returned an empty resolved revision") + sizes = {str(item.rfilename): int(item.size or 0) for item in list(info.siblings or [])} + if not sizes: + raise RuntimeError("Hub metadata returned no model files") + self._metadata = _Metadata(revision, resolved, sizes) + return self._metadata + + @staticmethod + def _metadata_diagnostics(metadata: _Metadata) -> Mapping[str, object]: + return { + "requested_revision": metadata.requested_revision, + "resolved_revision": metadata.resolved_revision, + "file_sizes": dict(metadata.file_sizes), + "download_bytes": sum(metadata.file_sizes.values()), + } + + def _effective_cache_root(self) -> Path: + if self._cache_root is not None: + return self._cache_root.expanduser() + if os.environ.get("HF_HUB_CACHE"): + return Path(os.environ["HF_HUB_CACHE"]).expanduser() + if os.environ.get("HF_HOME"): + return Path(os.environ["HF_HOME"]).expanduser() / "hub" + try: + return Path(importlib.import_module("huggingface_hub.constants").HF_HUB_CACHE) + except ImportError: + return Path.home() / ".cache" / "huggingface" / "hub" + + @staticmethod + def _optional_version(module: str) -> str: + try: + return str(importlib.import_module(module).__version__) + except (ImportError, AttributeError): + return "not-installed" + + def _load(self, resolved_revision: str) -> _Pipeline: + torch = self._get_torch() + if not torch.backends.mps.is_available() or not torch.backends.mps.is_built(): + raise AdapterExecutionError("MPS is unavailable; CPU fallback is forbidden") + pipeline = self._pipeline_loader( + MODEL_ID, + revision=resolved_revision, + torch_dtype=torch.bfloat16, + local_files_only=True, + trust_remote_code=False, + ) + pipeline.to("mps") + self._assert_execution_device(pipeline) + return pipeline + + @staticmethod + def _inspect_devices(pipeline: object) -> Mapping[str, str]: + devices: dict[str, str] = {} + execution_device = getattr(pipeline, "device", getattr(pipeline, "_execution_device", None)) + if execution_device is not None: + devices["pipeline.device"] = str(execution_device) + for collection in ("parameters", "buffers"): + getter = getattr(pipeline, collection, None) + if callable(getter): + for index, item in enumerate(getter()): + devices[f"pipeline.{collection}[{index}]"] = str( + getattr(item, "device", "unknown") + ) + components = getattr(pipeline, "components", {}) + for name, component in components.items(): + if component is None: + devices[name] = "non-executing" + continue + found = False + explicit = getattr(component, "device", None) + if explicit is not None: + devices[f"{name}.device"] = str(explicit) + found = True + for collection in ("parameters", "buffers"): + getter = getattr(component, collection, None) + if callable(getter): + for index, item in enumerate(getter()): + devices[f"{name}.{collection}[{index}]"] = str( + getattr(item, "device", "unknown") + ) + found = True + if not found: + kind = type(component).__name__.lower() + devices[name] = ( + "non-executing" + if any(label in kind for label in ("tokenizer", "scheduler", "processor")) + else "uninspectable" + ) + if not devices: + devices["pipeline"] = "uninspectable" + return devices + + def _assert_execution_device(self, pipeline: _Pipeline) -> None: + for name, device in self._device_inspector(pipeline).items(): + if device == "non-executing": + continue + if not device.startswith("mps"): + raise AdapterExecutionError( + f"execution-device fallback detected in {name}: {device}" + ) + + def _verify_pinned_weights(self, manifest: Manifest, metadata: _Metadata) -> None: + if manifest.model_revision == REQUESTED_REVISION: + return + root = ( + self._effective_cache_root() + / ("models--" + MODEL_ID.replace("/", "--")) + / "snapshots" + / metadata.resolved_revision + ) + digest = hashlib.sha256() + files = sorted(root.rglob("*.safetensors")) + if not files: + raise AdapterExecutionError("Pinned model weights are unavailable.") + for path in files: + with path.open("rb") as stream: + checksum = hashlib.file_digest(stream, "sha256").hexdigest() + digest.update(str(path.relative_to(root)).encode() + b"\0" + checksum.encode()) + if "sha256:" + digest.hexdigest() != manifest.model_checksum: + raise AdapterExecutionError("Pinned model weight content changed.") + + def generate( + self, request: GenerationRequest, progress: ProgressCallback, cancelled: CancellationCheck + ) -> GenerationResult: + violations = validate_manifest_support(request.manifest, self.capabilities()) + if ( + request.manifest.model_revision != REQUESTED_REVISION + and re.fullmatch(r"[0-9a-f]{40}", request.manifest.model_revision) is None + ): + raise AdapterExecutionError( + "unsupported manifest: model_revision must be a pinned revision " + f"or {REQUESTED_REVISION}" + ) + if violations: + detail = ", ".join(f"{item.field}: {item.reason}" for item in violations) + raise AdapterExecutionError(f"unsupported manifest: {detail}") + scratch = request.scratch_root + partial = scratch / "output.partial.mp4" + final = scratch / "output.mp4" + scratch.mkdir(parents=True, exist_ok=True) + self._remove(partial, final) + started = monotonic() + try: + if cancelled(): + raise GenerationCancelled("generation cancelled") + manifest = request.manifest + if "study-recipes" in manifest.runtime_versions: + from mlx_agent.study_recipes import verify + sizes = verify(manifest, self._effective_cache_root()) + metadata = _Metadata(manifest.model_revision, manifest.model_revision, sizes) + else: + metadata = self._resolve_metadata(manifest.model_revision) + self._verify_pinned_weights(manifest, metadata) + pipeline = self._load(metadata.resolved_revision) + generator = self._get_torch().Generator(device="cpu").manual_seed(manifest.seed) + + def on_step_end(_pipe: object, step: int, _time: object, kwargs: object) -> object: + if cancelled(): + raise GenerationCancelled("generation cancelled") + progress( + GenerationProgress( + phase="generating", + completed=step + 1, + total=manifest.inference_steps, + message=f"completed diffusion step {step + 1}", + elapsed_seconds=monotonic() - started, + ) + ) + self._assert_execution_device(pipeline) + return kwargs + + output = pipeline( + prompt=manifest.prompt, + negative_prompt=manifest.negative_prompt, + height=manifest.height, + width=manifest.width, + num_frames=manifest.num_frames, + num_inference_steps=manifest.inference_steps, + guidance_scale=manifest.guidance_scale, + generator=generator, + callback_on_step_end=on_step_end, + ) + self._assert_execution_device(pipeline) + frames = getattr(output, "frames", None) + if frames is None or len(frames) == 0 or frames[0] is None or len(frames[0]) == 0: + raise AdapterExecutionError("pipeline returned invalid video frames") + self._exporter(frames[0], str(partial), manifest.fps) + if not partial.is_file() or partial.stat().st_size == 0: + raise AdapterExecutionError("video exporter produced an invalid artifact") + with partial.open("rb") as artifact: + os.fsync(artifact.fileno()) + if cancelled(): + raise GenerationCancelled("generation cancelled") + partial.replace(final) + if "study-recipes" in manifest.runtime_versions: + from mlx_agent.study_recipes import verify + verify(manifest, self._effective_cache_root()) + else: + self._verify_pinned_weights(manifest, metadata) + return GenerationResult( + video_path=final, + effective_manifest=manifest, + frame_count=manifest.num_frames, + fps=manifest.fps, + width=manifest.width, + height=manifest.height, + adapter_version=self.adapter_version, + runtime_versions={ + "python": platform.python_version(), + "torch": str(self._get_torch().__version__), + "diffusers": self._optional_version("diffusers"), + }, + adapter_telemetry={ + "execution_device": "mps", + "rng_device": "cpu", + "requested_revision": metadata.requested_revision, + "resolved_revision": metadata.resolved_revision, + }, + ) + except GenerationCancelled: + self._remove(partial, final) + raise + except Exception as error: + self._remove(partial, final) + paths = (self._cache_root,) if self._cache_root is not None else () + sanitized = _sanitize_diagnostic(error, paths) + raise AdapterExecutionError( + f"Wan MPS generation failed: {type(error).__name__}: {sanitized}" + ) from error + + @staticmethod + def _remove(*paths: Path) -> None: + for path in paths: + path.unlink(missing_ok=True) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_study_models.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_study_models.py new file mode 100644 index 0000000..b64f1eb --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_study_models.py @@ -0,0 +1,415 @@ +from __future__ import annotations + +import hashlib +from collections.abc import Mapping, Sequence +from datetime import datetime +from enum import StrEnum +from math import isfinite +from types import MappingProxyType +from typing import Any, Literal, Self + +from pydantic import Field, field_serializer, field_validator, model_validator + +from video_lab.domain import FrozenModel, Manifest, _aware_utc, canonical_json +from video_lab.preference_optimizer import ( + AcquisitionScore, + CandidatePair, + OptimizerConfig, + OptimizerDecision, + PosteriorSummary, + PreferenceObservation, + PreferenceOutcome, + SearchDisposition, + StoppingDecision, + StopReason, +) + +ADAPTIVE_STUDY_SCHEMA_VERSION: Literal["adaptive-study.v1"] = "adaptive-study.v1" +ADAPTIVE_STUDY_INPUT_SCHEMA_VERSION: Literal["adaptive-study-input.v1"] = "adaptive-study-input.v1" +ADAPTIVE_STUDY_EVENT_SCHEMA_VERSION: Literal["adaptive-study-event.v1"] = "adaptive-study-event.v1" +ADAPTIVE_STUDY_OWNER_BINDING_SCHEMA_VERSION: Literal["adaptive-study-owner-binding.v1"] = ( + "adaptive-study-owner-binding.v1" +) +ADAPTIVE_OBSERVATION_SCHEMA_VERSION: Literal["adaptive-observation.v1"] = "adaptive-observation.v1" +ADAPTIVE_OBSERVATION_PROVENANCE_SCHEMA_VERSION: Literal["adaptive-observation-provenance.v1"] = ( + "adaptive-observation-provenance.v1" +) +ADAPTIVE_OPTIMIZER_STEP_SCHEMA_VERSION: Literal["adaptive-optimizer-step.v1"] = ( + "adaptive-optimizer-step.v1" +) + +Digest = str +type JsonScalar = str | int | float | bool | None +type JsonValue = JsonScalar | Mapping[str, JsonValue] | tuple[JsonValue, ...] + + +def _sha256(value: dict[str, Any]) -> Digest: + return hashlib.sha256(canonical_json(value).encode()).hexdigest() + + +def adaptive_observation_provenance_digest( + *, + observation: Mapping[str, Any], + optimizer_output_digest: Digest, + experiment: Mapping[str, Any], + review: Mapping[str, Any], + display_mapping: Mapping[str, Any], +) -> Digest: + return _sha256( + { + "schema_version": ADAPTIVE_OBSERVATION_PROVENANCE_SCHEMA_VERSION, + "observation": dict(observation), + "optimizer_output_digest": optimizer_output_digest, + "experiment": dict(experiment), + "review": dict(review), + "display_mapping": dict(display_mapping), + } + ) + + +def adaptive_study_input_fingerprint( + base_manifest: Manifest, optimizer_config: OptimizerConfig +) -> Digest: + return _sha256( + { + "schema_version": ADAPTIVE_STUDY_INPUT_SCHEMA_VERSION, + "prompt": base_manifest.prompt, + "negative_prompt": base_manifest.negative_prompt or None, + "seed": base_manifest.seed, + "base_manifest": base_manifest.model_dump(mode="json"), + "optimizer_config": optimizer_config.model_dump(mode="json"), + } + ) + + +def adaptive_optimizer_input_digest( + optimizer_config: OptimizerConfig, + observations: tuple[PreferenceObservation, ...], +) -> Digest: + return _sha256( + { + "schema_version": "adaptive-optimizer-input.v1", + "optimizer_config": optimizer_config.model_dump(mode="json"), + "observations": [item.model_dump(mode="json") for item in observations], + } + ) + + +def adaptive_optimizer_output_digest(decision: OptimizerDecision) -> Digest: + return _sha256( + { + "schema_version": "adaptive-optimizer-output.v1", + "posterior": decision.posterior.model_dump(mode="json"), + "acquisition_ledger": [ + item.model_dump(mode="json") for item in decision.candidate_scores + ], + "selected_pair": ( + None + if decision.selected_pair is None + else decision.selected_pair.model_dump(mode="json") + ), + "stopping": decision.stopping.model_dump(mode="json"), + } + ) + + +def _freeze_json(value: Any) -> JsonValue: + if value is None or isinstance(value, (str, bool, int)): + return value + if isinstance(value, float): + if not isfinite(value): + raise ValueError("JSON numbers must be finite") + return value + if isinstance(value, Mapping): + if any(not isinstance(key, str) for key in value): + raise ValueError("JSON object keys must be strings") + return MappingProxyType({key: _freeze_json(item) for key, item in value.items()}) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return tuple(_freeze_json(item) for item in value) + raise ValueError(f"value is not JSON-compatible: {type(value).__name__}") + + +def _thaw_json(value: JsonValue) -> object: + if isinstance(value, Mapping): + return {key: _thaw_json(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thaw_json(item) for item in value] + return value + + +class AdaptiveStudyState(StrEnum): + DRAFT = "draft" + SEARCH_READY = "search_ready" + GENERATING_PAIR = "generating_pair" + AWAITING_PREFERENCE = "awaiting_preference" + UPDATING_POSTERIOR = "updating_posterior" + SEARCH_CONVERGED = "search_converged" + SEARCH_BUDGET_EXHAUSTED = "search_budget_exhausted" + VALIDATING = "validating" + COMPLETED = "completed" + PAUSED = "paused" + OPTIMIZER_FAILED = "optimizer_failed" + + +ADAPTIVE_STUDY_TRANSITIONS: Mapping[AdaptiveStudyState, frozenset[AdaptiveStudyState]] = ( + MappingProxyType( + { + AdaptiveStudyState.DRAFT: frozenset({AdaptiveStudyState.SEARCH_READY}), + AdaptiveStudyState.SEARCH_READY: frozenset( + { + AdaptiveStudyState.GENERATING_PAIR, + AdaptiveStudyState.PAUSED, + AdaptiveStudyState.OPTIMIZER_FAILED, + } + ), + AdaptiveStudyState.GENERATING_PAIR: frozenset( + {AdaptiveStudyState.AWAITING_PREFERENCE, AdaptiveStudyState.PAUSED} + ), + AdaptiveStudyState.AWAITING_PREFERENCE: frozenset( + {AdaptiveStudyState.UPDATING_POSTERIOR, AdaptiveStudyState.PAUSED} + ), + AdaptiveStudyState.UPDATING_POSTERIOR: frozenset( + { + AdaptiveStudyState.GENERATING_PAIR, + AdaptiveStudyState.SEARCH_CONVERGED, + AdaptiveStudyState.SEARCH_BUDGET_EXHAUSTED, + AdaptiveStudyState.PAUSED, + AdaptiveStudyState.OPTIMIZER_FAILED, + } + ), + AdaptiveStudyState.SEARCH_CONVERGED: frozenset({AdaptiveStudyState.VALIDATING}), + AdaptiveStudyState.SEARCH_BUDGET_EXHAUSTED: frozenset({AdaptiveStudyState.VALIDATING}), + AdaptiveStudyState.VALIDATING: frozenset( + {AdaptiveStudyState.COMPLETED, AdaptiveStudyState.PAUSED} + ), + AdaptiveStudyState.COMPLETED: frozenset(), + AdaptiveStudyState.PAUSED: frozenset( + { + AdaptiveStudyState.SEARCH_READY, + AdaptiveStudyState.GENERATING_PAIR, + AdaptiveStudyState.AWAITING_PREFERENCE, + AdaptiveStudyState.UPDATING_POSTERIOR, + AdaptiveStudyState.VALIDATING, + } + ), + AdaptiveStudyState.OPTIMIZER_FAILED: frozenset( + { + AdaptiveStudyState.SEARCH_READY, + AdaptiveStudyState.UPDATING_POSTERIOR, + AdaptiveStudyState.PAUSED, + } + ), + } + ) +) + + +class AdaptiveStudyDefinition(FrozenModel): + id: str = Field(min_length=1) + schema_version: Literal["adaptive-study.v1"] = ADAPTIVE_STUDY_SCHEMA_VERSION + prompt: str = Field(min_length=1) + negative_prompt: str | None = None + seed: int + base_manifest: Manifest + optimizer_config: OptimizerConfig + input_fingerprint: Digest = Field(pattern=r"^[0-9a-f]{64}$") + created_at: datetime + + @field_validator("negative_prompt") + @classmethod + def normalize_negative_prompt(cls, value: str | None) -> str | None: + return value or None + + @field_validator("created_at") + @classmethod + def normalize_created_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + @model_validator(mode="after") + def validate_frozen_inputs(self) -> Self: + if self.prompt != self.base_manifest.prompt: + raise ValueError("study prompt must match the base manifest") + if (self.negative_prompt or "") != self.base_manifest.negative_prompt: + raise ValueError("study negative prompt must match the base manifest") + if self.seed != self.base_manifest.seed: + raise ValueError("study seed must match the base manifest") + expected = adaptive_study_input_fingerprint(self.base_manifest, self.optimizer_config) + if self.input_fingerprint != expected: + raise ValueError("study input fingerprint does not match its immutable inputs") + return self + + +class AdaptiveStudyEvent(FrozenModel): + study_id: str = Field(min_length=1) + schema_version: Literal["adaptive-study-event.v1"] = ADAPTIVE_STUDY_EVENT_SCHEMA_VERSION + sequence: int = Field(gt=0) + previous_state: AdaptiveStudyState + new_state: AdaptiveStudyState + payload: Mapping[str, JsonValue] = Field(default_factory=lambda: MappingProxyType({})) + created_at: datetime + + @field_validator("payload") + @classmethod + def freeze_payload(cls, value: object) -> Mapping[str, JsonValue]: + frozen = _freeze_json(value) + if not isinstance(frozen, Mapping): + raise ValueError("event payload must be a mapping") + return frozen + + @field_serializer("payload") + def serialize_payload(self, value: Mapping[str, JsonValue]) -> dict[str, object]: + return {key: _thaw_json(item) for key, item in value.items()} + + @field_validator("created_at") + @classmethod + def normalize_created_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + @model_validator(mode="after") + def state_must_change(self) -> Self: + if self.previous_state is self.new_state: + raise ValueError("study event must change state") + return self + + +class AdaptiveObservation(FrozenModel): + study_id: str = Field(min_length=1) + schema_version: Literal["adaptive-observation.v1"] = ADAPTIVE_OBSERVATION_SCHEMA_VERSION + ordinal: int = Field(gt=0) + optimizer_step_ordinal: int = Field(gt=0) + pair: CandidatePair + experiment_id: str = Field(min_length=1) + review_id: str = Field(min_length=1) + display_mapping_digest: Digest = Field(pattern=r"^[0-9a-f]{64}$") + outcome: PreferenceOutcome + created_at: datetime + + @field_validator("created_at") + @classmethod + def normalize_created_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + @model_validator(mode="after") + def require_canonical_pair(self) -> Self: + if self.pair.value_a >= self.pair.value_b: + raise ValueError("observation pair must be in ascending canonical order") + if self.optimizer_step_ordinal != self.ordinal: + raise ValueError("observation must consume the optimizer step at the same ordinal") + return self + + @property + def preference_observation(self) -> PreferenceObservation: + return PreferenceObservation( + value_a=self.pair.value_a, + value_b=self.pair.value_b, + outcome=self.outcome, + ) + + +class AdaptiveObservationRequest(FrozenModel): + study_id: str = Field(min_length=1) + ordinal: int = Field(gt=0) + review_id: str = Field(min_length=1) + created_at: datetime + + @field_validator("created_at") + @classmethod + def normalize_created_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + +class AdaptiveOptimizerStep(FrozenModel): + study_id: str = Field(min_length=1) + schema_version: Literal["adaptive-optimizer-step.v1"] = ADAPTIVE_OPTIMIZER_STEP_SCHEMA_VERSION + ordinal: int = Field(gt=0) + input_digest: Digest = Field(pattern=r"^[0-9a-f]{64}$") + output_digest: Digest = Field(pattern=r"^[0-9a-f]{64}$") + posterior: PosteriorSummary + acquisition_ledger: tuple[AcquisitionScore, ...] + selected_pair: CandidatePair | None + stop_reason: StopReason + disposition: SearchDisposition | None = None + comparison_count: int = Field(ge=0) + maximum_information_bits: float = Field(ge=0) + created_at: datetime + + @field_validator("created_at") + @classmethod + def normalize_created_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + @model_validator(mode="after") + def validate_complete_decision(self) -> Self: + if self.ordinal != self.comparison_count + 1: + raise ValueError("optimizer step ordinal must be one greater than comparison count") + if not isfinite(self.maximum_information_bits): + raise ValueError("maximum information must be finite") + posterior = self.posterior + cardinality = len(posterior.grid) + if ( + cardinality == 0 + or len(posterior.mean) != cardinality + or len(posterior.covariance) != cardinality + or any(len(row) != cardinality for row in posterior.covariance) + or len(posterior.argmax_probability) != cardinality + ): + raise ValueError("posterior arrays must match the non-empty guidance grid") + posterior_values = ( + *posterior.grid, + *posterior.mean, + *(value for row in posterior.covariance for value in row), + *posterior.argmax_probability, + posterior.recommendation, + *posterior.concentration_interval, + posterior.concentration_probability, + ) + if any(not isfinite(value) for value in posterior_values): + raise ValueError("posterior values must be finite") + if any(score.pair.value_a >= score.pair.value_b for score in self.acquisition_ledger): + raise ValueError("acquisition ledger pairs must be canonical") + if any(not isfinite(score.expected_information_bits) for score in self.acquisition_ledger): + raise ValueError("acquisition scores must be finite") + ledger_pairs = tuple( + (score.pair.value_a, score.pair.value_b) for score in self.acquisition_ledger + ) + if len(set(ledger_pairs)) != len(ledger_pairs): + raise ValueError("acquisition ledger pairs must be unique") + posterior_digest = self.posterior.content_digest + if not ( + posterior_digest.startswith("sha256:") + and len(posterior_digest) == 71 + and set(posterior_digest[7:]) <= set("0123456789abcdef") + ): + raise ValueError("posterior content digest must use the sha256: form") + if ( + self.selected_pair is not None + and self.selected_pair.value_a >= self.selected_pair.value_b + ): + raise ValueError("selected pair must be in ascending canonical order") + if self.stop_reason is StopReason.CONTINUE and self.selected_pair is None: + raise ValueError("continuing optimizer step must select a pair") + if self.stop_reason is not StopReason.CONTINUE and self.selected_pair is not None: + raise ValueError("stopped optimizer step must not select a pair") + if self.stop_reason is StopReason.SEARCH_BUDGET_EXHAUSTED: + if self.disposition is not SearchDisposition.INCONCLUSIVE_WITHIN_BUDGET: + raise ValueError("budget exhaustion requires inconclusive disposition") + elif self.disposition is not None: + raise ValueError("only budget exhaustion has a search disposition") + if self.output_digest != adaptive_optimizer_output_digest(self.decision): + raise ValueError("optimizer output digest does not match the full decision") + return self + + @property + def decision(self) -> OptimizerDecision: + return OptimizerDecision( + posterior=self.posterior, + candidate_scores=self.acquisition_ledger, + selected_pair=self.selected_pair, + stopping=StoppingDecision( + stop=self.stop_reason is not StopReason.CONTINUE, + reason=self.stop_reason, + disposition=self.disposition, + comparison_count=self.comparison_count, + maximum_information_bits=self.maximum_information_bits, + ), + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_study_repository.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_study_repository.py new file mode 100644 index 0000000..7e4cba0 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_study_repository.py @@ -0,0 +1,1069 @@ +from __future__ import annotations + +import json +import sqlite3 +from datetime import datetime +from typing import Literal, cast + +from pydantic import TypeAdapter + +from video_lab.adaptive_study_models import ( + ADAPTIVE_OBSERVATION_SCHEMA_VERSION, + ADAPTIVE_STUDY_OWNER_BINDING_SCHEMA_VERSION, + ADAPTIVE_STUDY_TRANSITIONS, + AdaptiveObservation, + AdaptiveObservationRequest, + AdaptiveOptimizerStep, + AdaptiveStudyDefinition, + AdaptiveStudyEvent, + AdaptiveStudyState, + adaptive_optimizer_input_digest, +) +from video_lab.database import Database +from video_lab.domain import Manifest, canonical_json, utc_now +from video_lab.preference_optimizer import ( + AcquisitionScore, + CandidatePair, + OptimizerConfig, + PosteriorSummary, + PreferenceOutcome, + SearchDisposition, + StopReason, +) +from video_lab.reviews import ( + ADAPTIVE_REVIEW_BINDING_SCHEMA_VERSION, + adaptive_review_binding_digest, +) + +_ACQUISITION_LEDGER_ADAPTER = TypeAdapter(tuple[AcquisitionScore, ...]) + + +class AdaptiveStudyNotFound(LookupError): + pass + + +class AdaptiveStudyConflict(RuntimeError): + pass + + +class AdaptiveStudyRepository: + def __init__(self, database: Database) -> None: + self.database = database + + def create(self, definition: AdaptiveStudyDefinition) -> AdaptiveStudyDefinition: + validated = AdaptiveStudyDefinition.model_validate(definition.model_dump(round_trip=True)) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + connection.execute( + "INSERT INTO adaptive_studies(" + "id, schema_version, prompt, negative_prompt, seed, base_manifest_json, " + "optimizer_config_json, input_fingerprint, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + validated.id, + validated.schema_version, + validated.prompt, + validated.negative_prompt, + validated.seed, + canonical_json(validated.base_manifest), + canonical_json(validated.optimizer_config), + validated.input_fingerprint, + validated.created_at.isoformat(), + ), + ) + connection.commit() + except Exception: + connection.rollback() + raise + return validated + + def create_or_get_bound( + self, + definition: AdaptiveStudyDefinition, + *, + reviewer_id: str, + ) -> AdaptiveStudyDefinition: + """Atomically create one deterministic study and its immutable owner binding.""" + validated = AdaptiveStudyDefinition.model_validate(definition.model_dump(round_trip=True)) + if not reviewer_id or reviewer_id != reviewer_id.strip(): + raise AdaptiveStudyConflict("adaptive study owner must be a normalized identity") + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + row = connection.execute( + "SELECT * FROM adaptive_studies WHERE id = ?", (validated.id,) + ).fetchone() + if row is None: + connection.execute( + "INSERT INTO adaptive_studies(" + "id, schema_version, prompt, negative_prompt, seed, base_manifest_json, " + "optimizer_config_json, input_fingerprint, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + validated.id, + validated.schema_version, + validated.prompt, + validated.negative_prompt, + validated.seed, + canonical_json(validated.base_manifest), + canonical_json(validated.optimizer_config), + validated.input_fingerprint, + validated.created_at.isoformat(), + ), + ) + connection.execute( + "INSERT INTO adaptive_study_events(" + "study_id, schema_version, sequence, previous_state, new_state, " + "payload_json, created_at) VALUES (?, ?, 1, ?, ?, ?, ?)", + ( + validated.id, + "adaptive-study-event.v1", + AdaptiveStudyState.DRAFT.value, + AdaptiveStudyState.SEARCH_READY.value, + canonical_json( + { + "owner_binding_schema_version": ( + ADAPTIVE_STUDY_OWNER_BINDING_SCHEMA_VERSION + ), + "reviewer_id": reviewer_id, + "input_fingerprint": validated.input_fingerprint, + } + ), + validated.created_at.isoformat(), + ), + ) + persisted = validated + else: + persisted = _definition_from_row(row) + if persisted != validated: + raise AdaptiveStudyConflict( + "adaptive study id already has a different immutable definition" + ) + first = connection.execute( + "SELECT * FROM adaptive_study_events WHERE study_id = ? " + "ORDER BY sequence LIMIT 1", + (validated.id,), + ).fetchone() + if first is None: + raise AdaptiveStudyConflict("adaptive study has no immutable owner binding") + event = _event_from_row(first) + if ( + event.sequence != 1 + or event.previous_state is not AdaptiveStudyState.DRAFT + or event.new_state is not AdaptiveStudyState.SEARCH_READY + or event.payload.get("owner_binding_schema_version") + != ADAPTIVE_STUDY_OWNER_BINDING_SCHEMA_VERSION + or event.payload.get("input_fingerprint") != validated.input_fingerprint + ): + raise AdaptiveStudyConflict("adaptive study owner binding is inconsistent") + if event.payload.get("reviewer_id") != reviewer_id: + raise PermissionError("adaptive study belongs to another reviewer") + connection.commit() + except Exception: + connection.rollback() + raise + return persisted + + def get(self, study_id: str) -> AdaptiveStudyDefinition: + with self.database.connect() as connection: + row = connection.execute( + "SELECT * FROM adaptive_studies WHERE id = ?", (study_id,) + ).fetchone() + if row is None: + raise AdaptiveStudyNotFound(f"adaptive study not found: {study_id}") + return _definition_from_row(row) + + def list(self) -> tuple[AdaptiveStudyDefinition, ...]: + with self.database.connect() as connection: + rows = connection.execute( + "SELECT * FROM adaptive_studies ORDER BY created_at, id" + ).fetchall() + return tuple(_definition_from_row(row) for row in rows) + + def bind_owner( + self, + study_id: str, + *, + input_fingerprint: str, + reviewer_id: str, + ) -> AdaptiveStudyEvent: + if not reviewer_id or reviewer_id != reviewer_id.strip(): + raise AdaptiveStudyConflict("adaptive study owner must be a normalized identity") + definition = self.get(study_id) + if definition.input_fingerprint != input_fingerprint: + raise AdaptiveStudyConflict("adaptive study owner binding fingerprint mismatch") + return self.append_event( + AdaptiveStudyEvent( + study_id=study_id, + sequence=1, + previous_state=AdaptiveStudyState.DRAFT, + new_state=AdaptiveStudyState.SEARCH_READY, + payload={ + "owner_binding_schema_version": ADAPTIVE_STUDY_OWNER_BINDING_SCHEMA_VERSION, + "reviewer_id": reviewer_id, + "input_fingerprint": input_fingerprint, + }, + created_at=utc_now(), + ) + ) + + def owner(self, study_id: str) -> str: + definition = self.get(study_id) + events = self.list_events(study_id) + if not events: + raise AdaptiveStudyConflict("adaptive study has no immutable owner binding") + first = events[0] + expected_keys = { + "owner_binding_schema_version", + "reviewer_id", + "input_fingerprint", + } + if ( + first.sequence != 1 + or first.previous_state is not AdaptiveStudyState.DRAFT + or first.new_state is not AdaptiveStudyState.SEARCH_READY + or set(first.payload) != expected_keys + or first.payload.get("owner_binding_schema_version") + != ADAPTIVE_STUDY_OWNER_BINDING_SCHEMA_VERSION + or first.payload.get("input_fingerprint") != definition.input_fingerprint + ): + raise AdaptiveStudyConflict("adaptive study owner binding is invalid") + reviewer_id = first.payload.get("reviewer_id") + if ( + not isinstance(reviewer_id, str) + or not reviewer_id + or reviewer_id != reviewer_id.strip() + ): + raise AdaptiveStudyConflict("adaptive study owner binding is invalid") + return reviewer_id + + def append_event(self, event: AdaptiveStudyEvent) -> AdaptiveStudyEvent: + validated = AdaptiveStudyEvent.model_validate(event.model_dump(round_trip=True)) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_study(connection, validated.study_id) + latest = connection.execute( + "SELECT sequence, new_state FROM adaptive_study_events " + "WHERE study_id = ? ORDER BY sequence DESC LIMIT 1", + (validated.study_id,), + ).fetchone() + next_sequence = 1 if latest is None else int(latest["sequence"]) + 1 + current_state = ( + AdaptiveStudyState.DRAFT + if latest is None + else AdaptiveStudyState(str(latest["new_state"])) + ) + if validated.sequence != next_sequence: + raise AdaptiveStudyConflict( + f"expected event sequence {next_sequence}, got {validated.sequence}" + ) + if validated.previous_state is not current_state: + raise AdaptiveStudyConflict( + f"expected prior state {current_state.value}, " + f"got {validated.previous_state.value}" + ) + if validated.new_state not in ADAPTIVE_STUDY_TRANSITIONS[current_state]: + raise AdaptiveStudyConflict( + "disallowed adaptive study transition " + f"{current_state.value} -> {validated.new_state.value}" + ) + connection.execute( + "INSERT INTO adaptive_study_events(" + "study_id, schema_version, sequence, previous_state, new_state, " + "payload_json, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + validated.study_id, + validated.schema_version, + validated.sequence, + validated.previous_state.value, + validated.new_state.value, + canonical_json(validated.model_dump(mode="json")["payload"]), + validated.created_at.isoformat(), + ), + ) + connection.commit() + except Exception: + connection.rollback() + raise + return validated + + def append_awaiting_preference_or_verify(self, event: AdaptiveStudyEvent) -> AdaptiveStudyEvent: + """Append one sealed-review transition or return its exact concurrent winner.""" + validated = AdaptiveStudyEvent.model_validate(event.model_dump(round_trip=True)) + expected_keys = { + "binding_schema_version", + "study_id", + "optimizer_step_ordinal", + "optimizer_output_digest", + "experiment_id", + "review_session_id", + "reviewer_id", + "sealed_mapping_digest", + } + if ( + validated.previous_state is not AdaptiveStudyState.GENERATING_PAIR + or validated.new_state is not AdaptiveStudyState.AWAITING_PREFERENCE + or set(validated.payload) != expected_keys + ): + raise AdaptiveStudyConflict("sealed review transition is invalid") + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_study(connection, validated.study_id) + latest_row = connection.execute( + "SELECT * FROM adaptive_study_events WHERE study_id = ? " + "ORDER BY sequence DESC LIMIT 1", + (validated.study_id,), + ).fetchone() + if latest_row is None: + raise AdaptiveStudyConflict("sealed review transition has no prior event") + latest = _event_from_row(latest_row) + if ( + latest.sequence == validated.sequence - 1 + and latest.new_state is AdaptiveStudyState.GENERATING_PAIR + ): + connection.execute( + "INSERT INTO adaptive_study_events(" + "study_id, schema_version, sequence, previous_state, new_state, " + "payload_json, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + validated.study_id, + validated.schema_version, + validated.sequence, + validated.previous_state.value, + validated.new_state.value, + canonical_json(validated.model_dump(mode="json")["payload"]), + validated.created_at.isoformat(), + ), + ) + persisted_row = connection.execute( + "SELECT * FROM adaptive_study_events WHERE study_id = ? AND sequence = ?", + (validated.study_id, validated.sequence), + ).fetchone() + if persisted_row is None: + raise RuntimeError("persisted sealed review event disappeared") + persisted = _event_from_row(persisted_row) + elif ( + latest.sequence == validated.sequence + and latest.schema_version == validated.schema_version + and latest.previous_state is validated.previous_state + and latest.new_state is validated.new_state + and latest.payload == validated.payload + ): + persisted = latest + else: + raise AdaptiveStudyConflict( + "sealed review transition lost an unrelated or mismatched race" + ) + connection.commit() + except Exception: + connection.rollback() + raise + return persisted + + def list_events(self, study_id: str) -> tuple[AdaptiveStudyEvent, ...]: + with self.database.connect() as connection: + self._require_study(connection, study_id) + rows = connection.execute( + "SELECT * FROM adaptive_study_events WHERE study_id = ? ORDER BY sequence", + (study_id,), + ).fetchall() + return tuple(_event_from_row(row) for row in rows) + + def append_observation(self, request: AdaptiveObservationRequest) -> AdaptiveObservation: + observation, _consumed = self._consume_observation(request, verify_retry=False) + return observation + + def consume_observation_or_verify( + self, request: AdaptiveObservationRequest + ) -> tuple[AdaptiveObservation, bool]: + """Atomically consume one optimizer step or verify its exact review retry.""" + return self._consume_observation(request, verify_retry=True) + + def _consume_observation( + self, + request: AdaptiveObservationRequest, + *, + verify_retry: bool, + ) -> tuple[AdaptiveObservation, bool]: + validated = AdaptiveObservationRequest.model_validate(request.model_dump(round_trip=True)) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + study_row = self._require_study(connection, validated.study_id) + definition = _definition_from_row(study_row) + existing_row = connection.execute( + "SELECT * FROM adaptive_observations WHERE study_id = ? AND ordinal = ?", + (validated.study_id, validated.ordinal), + ).fetchone() + if existing_row is not None: + if not verify_retry: + next_ordinal = int( + connection.execute( + "SELECT COALESCE(MAX(ordinal), 0) + 1 " + "FROM adaptive_observations WHERE study_id = ?", + (validated.study_id,), + ).fetchone()[0] + ) + raise AdaptiveStudyConflict( + f"expected observation ordinal {next_ordinal}, got {validated.ordinal}" + ) + existing = _observation_from_row(existing_row) + if existing.review_id != validated.review_id: + raise AdaptiveStudyConflict( + "observation retry does not match persisted review evidence" + ) + connection.commit() + return existing, False + current_state = self._current_state(connection, validated.study_id) + if current_state is not AdaptiveStudyState.AWAITING_PREFERENCE: + raise AdaptiveStudyConflict( + "observation cannot be appended while study is " + f"{current_state.value}; expected awaiting_preference" + ) + next_ordinal = int( + connection.execute( + "SELECT COALESCE(MAX(ordinal), 0) + 1 FROM adaptive_observations " + "WHERE study_id = ?", + (validated.study_id,), + ).fetchone()[0] + ) + if validated.ordinal != next_ordinal: + raise AdaptiveStudyConflict( + f"expected observation ordinal {next_ordinal}, got {validated.ordinal}" + ) + optimizer_row = connection.execute( + "SELECT * FROM adaptive_optimizer_steps WHERE study_id = ? AND ordinal = ?", + (validated.study_id, validated.ordinal), + ).fetchone() + if optimizer_row is None: + raise AdaptiveStudyConflict( + f"optimizer step {validated.ordinal} does not exist" + ) + optimizer_step = _optimizer_step_from_row(optimizer_row) + if ( + optimizer_step.stop_reason is not StopReason.CONTINUE + or optimizer_step.selected_pair is None + ): + raise AdaptiveStudyConflict( + f"optimizer step {validated.ordinal} did not select a comparison pair" + ) + if optimizer_step.comparison_count != validated.ordinal - 1: + raise AdaptiveStudyConflict( + "optimizer step comparison count does not match observation ordinal" + ) + already_consumed = connection.execute( + "SELECT 1 FROM adaptive_observations " + "WHERE study_id = ? AND optimizer_step_ordinal = ?", + (validated.study_id, validated.ordinal), + ).fetchone() + if already_consumed is not None: + raise AdaptiveStudyConflict( + f"optimizer step {validated.ordinal} was already consumed" + ) + observation = _derive_observation_from_review( + connection, + definition=definition, + optimizer_step=optimizer_step, + request=validated, + ) + existing_rows = connection.execute( + "SELECT * FROM adaptive_observations WHERE study_id = ? ORDER BY ordinal", + (validated.study_id,), + ).fetchall() + optimizer_observations = tuple( + _observation_from_row(row).preference_observation for row in existing_rows + ) + (observation.preference_observation,) + definition.optimizer_config.validate_observations(optimizer_observations) + connection.execute( + "INSERT INTO adaptive_observations(" + "study_id, schema_version, ordinal, optimizer_step_ordinal, value_a, value_b, " + "experiment_id, review_id, display_mapping_digest, outcome, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + observation.study_id, + observation.schema_version, + observation.ordinal, + observation.optimizer_step_ordinal, + observation.pair.value_a, + observation.pair.value_b, + observation.experiment_id, + observation.review_id, + observation.display_mapping_digest, + observation.outcome.value, + observation.created_at.isoformat(), + ), + ) + persisted_row = connection.execute( + "SELECT * FROM adaptive_observations WHERE study_id = ? AND ordinal = ?", + (observation.study_id, observation.ordinal), + ).fetchone() + if persisted_row is None: + raise RuntimeError("persisted observation disappeared inside transaction") + persisted = _observation_from_row(persisted_row) + if persisted != observation: + raise RuntimeError("persisted observation is not byte-stable") + connection.commit() + except Exception: + connection.rollback() + raise + return persisted, True + + def list_observations(self, study_id: str) -> tuple[AdaptiveObservation, ...]: + with self.database.connect() as connection: + self._require_study(connection, study_id) + rows = connection.execute( + "SELECT * FROM adaptive_observations WHERE study_id = ? ORDER BY ordinal", + (study_id,), + ).fetchall() + return tuple(_observation_from_row(row) for row in rows) + + def ensure_initial_plan( + self, + step: AdaptiveOptimizerStep, + *, + experiment_id: str, + ) -> AdaptiveStudyEvent: + """Persist or verify the deterministic initial optimizer step and generation event.""" + validated = AdaptiveOptimizerStep.model_validate(step.model_dump(round_trip=True)) + if ( + validated.ordinal != 1 + or validated.comparison_count != 0 + or validated.stop_reason is not StopReason.CONTINUE + or validated.selected_pair is None + or not experiment_id + ): + raise AdaptiveStudyConflict("initial adaptive optimizer plan is invalid") + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + definition = _definition_from_row( + self._require_study(connection, validated.study_id) + ) + expected_input = adaptive_optimizer_input_digest(definition.optimizer_config, ()) + if validated.input_digest != expected_input: + raise AdaptiveStudyConflict( + "initial optimizer input digest does not match persisted definition" + ) + latest_row = connection.execute( + "SELECT * FROM adaptive_study_events WHERE study_id = ? " + "ORDER BY sequence DESC LIMIT 1", + (validated.study_id,), + ).fetchone() + if latest_row is None: + raise AdaptiveStudyConflict("adaptive study has no owner binding") + latest = _event_from_row(latest_row) + existing_row = connection.execute( + "SELECT * FROM adaptive_optimizer_steps WHERE study_id = ? AND ordinal = 1", + (validated.study_id,), + ).fetchone() + if existing_row is None: + if latest.new_state is not AdaptiveStudyState.SEARCH_READY: + raise AdaptiveStudyConflict( + "initial optimizer plan is missing after study advanced" + ) + _insert_optimizer_step_row(connection, validated) + persisted_step = validated + else: + persisted_step = _optimizer_step_from_row(existing_row) + if persisted_step != validated: + raise AdaptiveStudyConflict( + "initial optimizer plan conflicts with persisted evidence" + ) + + payload = { + "optimizer_step_ordinal": persisted_step.ordinal, + "optimizer_output_digest": persisted_step.output_digest, + "experiment_id": experiment_id, + } + if latest.new_state is AdaptiveStudyState.SEARCH_READY: + connection.execute( + "INSERT INTO adaptive_study_events(" + "study_id, schema_version, sequence, previous_state, new_state, " + "payload_json, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + validated.study_id, + "adaptive-study-event.v1", + latest.sequence + 1, + AdaptiveStudyState.SEARCH_READY.value, + AdaptiveStudyState.GENERATING_PAIR.value, + canonical_json(payload), + persisted_step.created_at.isoformat(), + ), + ) + planned_row = connection.execute( + "SELECT * FROM adaptive_study_events WHERE study_id = ? AND sequence = ?", + (validated.study_id, latest.sequence + 1), + ).fetchone() + if planned_row is None: + raise RuntimeError("persisted initial generation plan disappeared") + planned = _event_from_row(planned_row) + elif latest.new_state is AdaptiveStudyState.GENERATING_PAIR: + if latest.payload != payload: + raise AdaptiveStudyConflict( + "initial generation event conflicts with optimizer plan" + ) + planned = latest + else: + raise AdaptiveStudyConflict( + "initial optimizer plan cannot be ensured after study advanced" + ) + connection.commit() + except Exception: + connection.rollback() + raise + return planned + + def append_optimizer_step(self, step: AdaptiveOptimizerStep) -> AdaptiveOptimizerStep: + validated = AdaptiveOptimizerStep.model_validate(step.model_dump(round_trip=True)) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + study_row = self._require_study(connection, validated.study_id) + current_state = self._current_state(connection, validated.study_id) + expected_state = ( + AdaptiveStudyState.SEARCH_READY + if validated.ordinal == 1 + else AdaptiveStudyState.UPDATING_POSTERIOR + ) + if current_state is not expected_state: + raise AdaptiveStudyConflict( + "optimizer step cannot be appended while study is " + f"{current_state.value}; expected {expected_state.value}" + ) + definition = _definition_from_row(study_row) + observation_rows = connection.execute( + "SELECT * FROM adaptive_observations WHERE study_id = ? ORDER BY ordinal", + (validated.study_id,), + ).fetchall() + observations = tuple( + _observation_from_row(row).preference_observation for row in observation_rows + ) + expected_input = adaptive_optimizer_input_digest( + definition.optimizer_config, observations + ) + if validated.input_digest != expected_input: + raise AdaptiveStudyConflict( + "optimizer input digest does not match persisted canonical observations" + ) + next_ordinal = int( + connection.execute( + "SELECT COALESCE(MAX(ordinal), 0) + 1 " + "FROM adaptive_optimizer_steps WHERE study_id = ?", + (validated.study_id,), + ).fetchone()[0] + ) + if validated.ordinal != next_ordinal: + raise AdaptiveStudyConflict( + f"expected optimizer step ordinal {next_ordinal}, got {validated.ordinal}" + ) + if validated.comparison_count != len(observations): + raise AdaptiveStudyConflict( + "optimizer comparison count does not match persisted observations" + ) + _insert_optimizer_step_row(connection, validated) + persisted_row = connection.execute( + "SELECT * FROM adaptive_optimizer_steps WHERE study_id = ? AND ordinal = ?", + (validated.study_id, validated.ordinal), + ).fetchone() + if persisted_row is None: + raise RuntimeError("persisted optimizer step disappeared inside transaction") + persisted = _optimizer_step_from_row(persisted_row) + persisted_input = adaptive_optimizer_input_digest( + definition.optimizer_config, observations + ) + if persisted.input_digest != persisted_input: + raise RuntimeError("persisted optimizer input digest is not reproducible") + connection.commit() + except Exception: + connection.rollback() + raise + return persisted + + def list_optimizer_steps(self, study_id: str) -> tuple[AdaptiveOptimizerStep, ...]: + with self.database.connect() as connection: + definition = _definition_from_row(self._require_study(connection, study_id)) + rows = connection.execute( + "SELECT * FROM adaptive_optimizer_steps WHERE study_id = ? ORDER BY ordinal", + (study_id,), + ).fetchall() + observation_rows = connection.execute( + "SELECT * FROM adaptive_observations WHERE study_id = ? ORDER BY ordinal", + (study_id,), + ).fetchall() + observations = tuple( + _observation_from_row(row).preference_observation for row in observation_rows + ) + steps = tuple(_optimizer_step_from_row(row) for row in rows) + for step in steps: + prefix = observations[: step.comparison_count] + if len(prefix) != step.comparison_count: + raise AdaptiveStudyConflict( + "persisted optimizer step references missing observations" + ) + expected_input = adaptive_optimizer_input_digest(definition.optimizer_config, prefix) + if step.input_digest != expected_input: + raise AdaptiveStudyConflict( + "persisted optimizer input digest does not match reloaded evidence" + ) + return steps + + @staticmethod + def _require_study(connection: sqlite3.Connection, study_id: str) -> sqlite3.Row: + row = connection.execute( + "SELECT * FROM adaptive_studies WHERE id = ?", (study_id,) + ).fetchone() + if row is None: + raise AdaptiveStudyNotFound(f"adaptive study not found: {study_id}") + return cast(sqlite3.Row, row) + + @staticmethod + def _current_state(connection: sqlite3.Connection, study_id: str) -> AdaptiveStudyState: + row = connection.execute( + "SELECT new_state FROM adaptive_study_events WHERE study_id = ? " + "ORDER BY sequence DESC LIMIT 1", + (study_id,), + ).fetchone() + return ( + AdaptiveStudyState.DRAFT if row is None else AdaptiveStudyState(str(row["new_state"])) + ) + + +def _insert_optimizer_step_row(connection: sqlite3.Connection, step: AdaptiveOptimizerStep) -> None: + connection.execute( + "INSERT INTO adaptive_optimizer_steps(" + "study_id, schema_version, ordinal, input_digest, output_digest, " + "posterior_json, acquisition_ledger_json, selected_pair_json, " + "stop_reason, disposition, comparison_count, maximum_information_bits, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + step.study_id, + step.schema_version, + step.ordinal, + step.input_digest, + step.output_digest, + canonical_json(step.posterior), + _canonical_array_json(step.acquisition_ledger), + None if step.selected_pair is None else canonical_json(step.selected_pair), + step.stop_reason.value, + None if step.disposition is None else step.disposition.value, + step.comparison_count, + step.maximum_information_bits, + step.created_at.isoformat(), + ), + ) + + +def _definition_from_row(row: sqlite3.Row) -> AdaptiveStudyDefinition: + return AdaptiveStudyDefinition( + id=str(row["id"]), + schema_version=cast(Literal["adaptive-study.v1"], str(row["schema_version"])), + prompt=str(row["prompt"]), + negative_prompt=(None if row["negative_prompt"] is None else str(row["negative_prompt"])), + seed=int(row["seed"]), + base_manifest=Manifest.model_validate_json(str(row["base_manifest_json"])), + optimizer_config=OptimizerConfig.model_validate_json(str(row["optimizer_config_json"])), + input_fingerprint=str(row["input_fingerprint"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + ) + + +def _event_from_row(row: sqlite3.Row) -> AdaptiveStudyEvent: + payload = json.loads(str(row["payload_json"])) + if not isinstance(payload, dict): + raise ValueError("adaptive study event payload must be an object") + return AdaptiveStudyEvent( + study_id=str(row["study_id"]), + schema_version=cast(Literal["adaptive-study-event.v1"], str(row["schema_version"])), + sequence=int(row["sequence"]), + previous_state=AdaptiveStudyState(str(row["previous_state"])), + new_state=AdaptiveStudyState(str(row["new_state"])), + payload=payload, + created_at=datetime.fromisoformat(str(row["created_at"])), + ) + + +def _observation_from_row(row: sqlite3.Row) -> AdaptiveObservation: + return AdaptiveObservation( + study_id=str(row["study_id"]), + schema_version=cast(Literal["adaptive-observation.v1"], str(row["schema_version"])), + ordinal=int(row["ordinal"]), + optimizer_step_ordinal=int(row["optimizer_step_ordinal"]), + pair=CandidatePair(value_a=float(row["value_a"]), value_b=float(row["value_b"])), + experiment_id=str(row["experiment_id"]), + review_id=str(row["review_id"]), + display_mapping_digest=str(row["display_mapping_digest"]), + outcome=PreferenceOutcome(str(row["outcome"])), + created_at=datetime.fromisoformat(str(row["created_at"])), + ) + + +def _canonical_array_json(values: tuple[AcquisitionScore, ...]) -> str: + return json.dumps( + [value.model_dump(mode="json") for value in values], + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + +def _optimizer_step_from_row(row: sqlite3.Row) -> AdaptiveOptimizerStep: + selected_pair_json = row["selected_pair_json"] + disposition = row["disposition"] + return AdaptiveOptimizerStep( + study_id=str(row["study_id"]), + schema_version=cast(Literal["adaptive-optimizer-step.v1"], str(row["schema_version"])), + ordinal=int(row["ordinal"]), + input_digest=str(row["input_digest"]), + output_digest=str(row["output_digest"]), + posterior=PosteriorSummary.model_validate_json(str(row["posterior_json"])), + acquisition_ledger=_ACQUISITION_LEDGER_ADAPTER.validate_json( + str(row["acquisition_ledger_json"]) + ), + selected_pair=( + None + if selected_pair_json is None + else CandidatePair.model_validate_json(str(selected_pair_json)) + ), + stop_reason=StopReason(str(row["stop_reason"])), + disposition=(None if disposition is None else SearchDisposition(str(disposition))), + comparison_count=int(row["comparison_count"]), + maximum_information_bits=float(row["maximum_information_bits"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + ) + + +def _derive_observation_from_review( + connection: sqlite3.Connection, + *, + definition: AdaptiveStudyDefinition, + optimizer_step: AdaptiveOptimizerStep, + request: AdaptiveObservationRequest, +) -> AdaptiveObservation: + revision = connection.execute( + "SELECT id, session_id, revision FROM review_revisions WHERE id = ?", + (request.review_id,), + ).fetchone() + if revision is None: + raise AdaptiveStudyConflict(f"review does not exist: {request.review_id}") + latest = connection.execute( + "SELECT id FROM review_revisions WHERE session_id = ? ORDER BY revision DESC LIMIT 1", + (str(revision["session_id"]),), + ).fetchone() + if latest is None or str(latest["id"]) != request.review_id: + raise AdaptiveStudyConflict("observation must consume the latest review revision") + row = connection.execute( + "SELECT " + "rr.id AS review_id, rr.session_id, rr.preferred_variant_id, " + "rr.display_preference, rr.notes, rr.created_at AS review_created_at, " + "rr.revision, rr.supersedes_review_id, " + "rs.experiment_id, rs.reviewer_id, rs.created_at AS session_created_at, " + "rs.left_variant_id, rs.right_variant_id, rs.left_job_id, rs.right_job_id, " + "rs.left_artifact_id, rs.right_artifact_id, " + "rs.left_artifact_sha256, rs.right_artifact_sha256, " + "e.schema_version AS experiment_schema_version, e.variable_path, " + "e.changed_paths_json, " + "lv.experiment_id AS left_variant_experiment_id, lv.label AS left_label, " + "lv.manifest_json AS left_manifest_json, " + "rv.experiment_id AS right_variant_experiment_id, rv.label AS right_label, " + "rv.manifest_json AS right_manifest_json, " + "lj.variant_id AS left_job_variant_id, lj.state AS left_job_state, " + "rj.variant_id AS right_job_variant_id, rj.state AS right_job_state, " + "la.sha256 AS left_persisted_sha256, ra.sha256 AS right_persisted_sha256 " + "FROM review_revisions rr " + "JOIN review_sessions rs ON rs.id = rr.session_id " + "JOIN experiments e ON e.id = rs.experiment_id " + "JOIN variants lv ON lv.id = rs.left_variant_id " + "JOIN variants rv ON rv.id = rs.right_variant_id " + "JOIN jobs lj ON lj.id = rs.left_job_id " + "JOIN jobs rj ON rj.id = rs.right_job_id " + "JOIN artifacts la ON la.id = rs.left_artifact_id " + "JOIN artifacts ra ON ra.id = rs.right_artifact_id " + "JOIN job_artifacts lja ON lja.job_id = rs.left_job_id " + "AND lja.artifact_id = rs.left_artifact_id " + "JOIN job_artifacts rja ON rja.job_id = rs.right_job_id " + "AND rja.artifact_id = rs.right_artifact_id " + "WHERE rr.id = ?", + (request.review_id,), + ).fetchone() + if row is None: + raise AdaptiveStudyConflict("review provenance is incomplete") + _require_active_review_binding( + connection, + definition=definition, + optimizer_step=optimizer_step, + request=request, + review_row=row, + ) + if not _review_relations_are_consistent(row): + raise AdaptiveStudyConflict("review provenance does not form one sealed experiment") + try: + changed_paths = json.loads(str(row["changed_paths_json"])) + left_manifest = Manifest.model_validate_json(str(row["left_manifest_json"])) + right_manifest = Manifest.model_validate_json(str(row["right_manifest_json"])) + except (ValueError, TypeError, json.JSONDecodeError) as error: + raise AdaptiveStudyConflict("review provenance contains invalid manifests") from error + if ( + str(row["experiment_schema_version"]) != "manifest.v1" + or str(row["variable_path"]) != "guidance_scale" + or changed_paths != ["guidance_scale"] + or not _matches_study_identity(left_manifest, definition.base_manifest) + or not _matches_study_identity(right_manifest, definition.base_manifest) + ): + raise AdaptiveStudyConflict("review provenance does not match the adaptive study") + pair = CandidatePair( + value_a=min(left_manifest.guidance_scale, right_manifest.guidance_scale), + value_b=max(left_manifest.guidance_scale, right_manifest.guidance_scale), + ) + if pair != optimizer_step.selected_pair: + raise AdaptiveStudyConflict("review comparison pair does not match optimizer-selected pair") + outcome = _preference_outcome(row, left_manifest, right_manifest, pair) + digest_row = connection.execute( + "SELECT adaptive_observation_provenance_digest(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + definition.id, + ADAPTIVE_OBSERVATION_SCHEMA_VERSION, + request.ordinal, + optimizer_step.ordinal, + pair.value_a, + pair.value_b, + str(row["experiment_id"]), + request.review_id, + outcome.value, + request.created_at.isoformat(), + ), + ).fetchone() + digest = None if digest_row is None else digest_row[0] + if not isinstance(digest, str): + raise AdaptiveStudyConflict("observation provenance digest could not be derived") + return AdaptiveObservation( + study_id=definition.id, + ordinal=request.ordinal, + optimizer_step_ordinal=optimizer_step.ordinal, + pair=pair, + experiment_id=str(row["experiment_id"]), + review_id=request.review_id, + display_mapping_digest=digest, + outcome=outcome, + created_at=request.created_at, + ) + + +def _require_active_review_binding( + connection: sqlite3.Connection, + *, + definition: AdaptiveStudyDefinition, + optimizer_step: AdaptiveOptimizerStep, + request: AdaptiveObservationRequest, + review_row: sqlite3.Row, +) -> None: + score_count = int( + connection.execute( + "SELECT COUNT(*) FROM rubric_scores WHERE review_id = ?", (request.review_id,) + ).fetchone()[0] + ) + if ( + int(review_row["revision"]) != 1 + or review_row["supersedes_review_id"] is not None + or review_row["notes"] is not None + or score_count != 0 + ): + raise AdaptiveStudyConflict( + "adaptive observation requires one pristine immutable review revision" + ) + session_row = connection.execute( + "SELECT * FROM review_sessions WHERE id = ?", (str(review_row["session_id"]),) + ).fetchone() + event_row = connection.execute( + "SELECT new_state, payload_json FROM adaptive_study_events " + "WHERE study_id = ? ORDER BY sequence DESC LIMIT 1", + (definition.id,), + ).fetchone() + if ( + session_row is None + or event_row is None + or str(event_row["new_state"]) != "awaiting_preference" + ): + raise AdaptiveStudyConflict("adaptive observation has no active review binding") + try: + payload = json.loads(str(event_row["payload_json"])) + except (TypeError, json.JSONDecodeError) as error: + raise AdaptiveStudyConflict("adaptive active review binding is invalid") from error + expected_keys = { + "binding_schema_version", + "study_id", + "optimizer_step_ordinal", + "optimizer_output_digest", + "experiment_id", + "review_session_id", + "reviewer_id", + "sealed_mapping_digest", + } + if not isinstance(payload, dict) or set(payload) != expected_keys: + raise AdaptiveStudyConflict("adaptive active review binding is incomplete") + expected_digest = adaptive_review_binding_digest( + session_row, + study_id=definition.id, + optimizer_step_ordinal=optimizer_step.ordinal, + optimizer_output_digest=optimizer_step.output_digest, + ) + if ( + payload["binding_schema_version"] != ADAPTIVE_REVIEW_BINDING_SCHEMA_VERSION + or payload["study_id"] != definition.id + or payload["optimizer_step_ordinal"] != optimizer_step.ordinal + or payload["optimizer_output_digest"] != optimizer_step.output_digest + or payload["experiment_id"] != str(session_row["experiment_id"]) + or payload["review_session_id"] != str(session_row["id"]) + or payload["reviewer_id"] != str(session_row["reviewer_id"]) + or payload["sealed_mapping_digest"] != expected_digest + ): + raise AdaptiveStudyConflict("adaptive observation does not match the active review binding") + + +def _review_relations_are_consistent(row: sqlite3.Row) -> bool: + experiment_id = str(row["experiment_id"]) + return all( + ( + str(row["left_variant_experiment_id"]) == experiment_id, + str(row["right_variant_experiment_id"]) == experiment_id, + str(row["left_job_variant_id"]) == str(row["left_variant_id"]), + str(row["right_job_variant_id"]) == str(row["right_variant_id"]), + str(row["left_job_state"]) == "completed", + str(row["right_job_state"]) == "completed", + str(row["left_persisted_sha256"]) == str(row["left_artifact_sha256"]), + str(row["right_persisted_sha256"]) == str(row["right_artifact_sha256"]), + ) + ) + + +def _matches_study_identity(candidate: Manifest, base: Manifest) -> bool: + payload = candidate.model_dump(mode="json") + payload["guidance_scale"] = base.guidance_scale + return payload == base.model_dump(mode="json") + + +def _preference_outcome( + row: sqlite3.Row, + left_manifest: Manifest, + right_manifest: Manifest, + pair: CandidatePair, +) -> PreferenceOutcome: + preference = str(row["display_preference"]) + preferred_variant_id = row["preferred_variant_id"] + if preference == "tie": + if preferred_variant_id is not None: + raise AdaptiveStudyConflict("tie review has a preferred variant") + return PreferenceOutcome.TIE + position = "left" if preference == "left" else "right" if preference == "right" else None + if position is None: + raise AdaptiveStudyConflict("review has an unsupported display preference") + expected_variant_id = str(row[f"{position}_variant_id"]) + if preferred_variant_id is None or str(preferred_variant_id) != expected_variant_id: + raise AdaptiveStudyConflict("review preference does not match sealed display mapping") + preferred_value = ( + left_manifest.guidance_scale if position == "left" else right_manifest.guidance_scale + ) + return PreferenceOutcome.A if preferred_value == pair.value_a else PreferenceOutcome.B diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_study_service.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_study_service.py new file mode 100644 index 0000000..2a49367 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_study_service.py @@ -0,0 +1,810 @@ +from __future__ import annotations + +import hashlib +import time +from collections.abc import Callable, Mapping +from datetime import datetime +from typing import Literal + +from video_lab.adaptive_study_models import ( + AdaptiveObservationRequest, + AdaptiveOptimizerStep, + AdaptiveStudyDefinition, + AdaptiveStudyEvent, + AdaptiveStudyState, + JsonValue, + adaptive_optimizer_input_digest, + adaptive_optimizer_output_digest, +) +from video_lab.adaptive_study_repository import ( + AdaptiveStudyConflict, + AdaptiveStudyRepository, +) +from video_lab.capabilities import AdapterCapabilities +from video_lab.domain import ( + DisplayPreference, + ExperimentDraft, + FrozenModel, + canonical_json, + utc_now, +) +from video_lab.preference_optimizer import ( + OptimizerConfig, + OptimizerDecision, + PreferenceObservation, + StopReason, + optimize_preference, +) +from video_lab.repositories import adaptive_experiment_id +from video_lab.review_service import ReviewService +from video_lab.services import ExperimentService + +AdaptiveNextAction = Literal["complete_generation", "record_preference", "resume", "none"] + + +class AdaptiveStudyProgress(FrozenModel): + completed_comparisons: int + current_comparison: int | None + maximum_comparisons: int + + +class AdaptiveMediaReference(FrozenModel): + choice: Literal["a", "b"] + media_ref: str + + +class AdaptiveStudyView(FrozenModel): + schema_version: Literal["adaptive-study-view.v1"] = "adaptive-study-view.v1" + study_id: str + prompt: str + state: AdaptiveStudyState + next_action: AdaptiveNextAction + comparison_count: int + maximum_comparisons: int + progress: AdaptiveStudyProgress + pair_ref: str | None = None + review_id: str | None = None + media: tuple[AdaptiveMediaReference, ...] = () + stop_reason: StopReason | None = None + disposition: str | None = None + recommendation: float | None = None + concentration_interval: tuple[float, float] | None = None + concentration_probability: float | None = None + evidence_digest: str | None = None + limitations: tuple[str, ...] = ( + "Stage 1 evidence is limited to one fixed prompt and seed.", + "Only one controlled parameter is varied; this is not a complete recipe optimization.", + ) + + +class AdaptiveStudyListItem(FrozenModel): + schema_version: Literal["adaptive-study-list-item.v1"] = "adaptive-study-list-item.v1" + study_id: str + state: AdaptiveStudyState + next_action: AdaptiveNextAction + progress: AdaptiveStudyProgress + + +class AdaptiveStudyService: + def __init__( + self, + studies: AdaptiveStudyRepository, + experiments: ExperimentService, + reviews: ReviewService, + capabilities: AdapterCapabilities, + *, + reviewer_id: str, + optimizer: Callable[ + [OptimizerConfig, tuple[PreferenceObservation, ...]], OptimizerDecision + ] = optimize_preference, + ) -> None: + if not reviewer_id: + raise ValueError("adaptive reviewer id must not be empty") + self.studies = studies + self.experiments = experiments + self.reviews = reviews + self.capabilities = capabilities + self.reviewer_id = reviewer_id + self.optimizer = optimizer + self._verified_optimizer_outputs: set[tuple[str, int, str]] = set() + + def create_study(self, definition: AdaptiveStudyDefinition) -> AdaptiveStudyView: + validated = AdaptiveStudyDefinition.model_validate(definition.model_dump(round_trip=True)) + self._validate_adapter(validated) + self.studies.create_or_get_bound(validated, reviewer_id=self.reviewer_id) + return self.resume(validated.id) + + def show(self, study_id: str) -> AdaptiveStudyView: + definition = self.studies.get(study_id) + events = self.studies.list_events(study_id) + self._validate_reviewer_binding(study_id, events) + observations = self.studies.list_observations(study_id) + steps = self.studies.list_optimizer_steps(study_id) + state = AdaptiveStudyState.DRAFT if not events else events[-1].new_state + terminal = state in { + AdaptiveStudyState.SEARCH_CONVERGED, + AdaptiveStudyState.SEARCH_BUDGET_EXHAUSTED, + AdaptiveStudyState.COMPLETED, + } + nonrecoverable_failure = ( + state is AdaptiveStudyState.OPTIMIZER_FAILED + and events[-1].payload.get("recoverable") is False + ) + latest_step = steps[-1] if steps else None + comparison_count = len(observations) + maximum_comparisons = definition.optimizer_config.maximum_comparisons + optimizer_output_digest = next( + ( + value + for event in reversed(events) + if isinstance((value := event.payload.get("optimizer_output_digest")), str) + ), + None, + ) + pair_ref = ( + self._pair_ref(study_id, optimizer_output_digest) + if optimizer_output_digest is not None + else None + ) + review_id = ( + self._payload_string(events[-1], "review_session_id") + if state is AdaptiveStudyState.AWAITING_PREFERENCE + else None + ) + return AdaptiveStudyView( + study_id=study_id, + prompt=definition.prompt, + state=state, + next_action=( + "complete_generation" + if state is AdaptiveStudyState.GENERATING_PAIR + else "record_preference" + if state is AdaptiveStudyState.AWAITING_PREFERENCE + else "none" + if terminal or nonrecoverable_failure + else "resume" + ), + comparison_count=comparison_count, + maximum_comparisons=maximum_comparisons, + progress=AdaptiveStudyProgress( + completed_comparisons=comparison_count, + current_comparison=( + None if terminal else min(comparison_count + 1, maximum_comparisons) + ), + maximum_comparisons=maximum_comparisons, + ), + pair_ref=pair_ref, + review_id=review_id, + media=( + ( + AdaptiveMediaReference(choice="a", media_ref=f"adaptive-review:{review_id}:a"), + AdaptiveMediaReference(choice="b", media_ref=f"adaptive-review:{review_id}:b"), + ) + if review_id is not None + else () + ), + stop_reason=( + latest_step.stop_reason + if latest_step and latest_step.stop_reason is not StopReason.CONTINUE + else None + ), + disposition=( + latest_step.disposition.value + if latest_step and latest_step.disposition is not None + else None + ), + recommendation=( + latest_step.posterior.recommendation if terminal and latest_step else None + ), + concentration_interval=( + latest_step.posterior.concentration_interval if terminal and latest_step else None + ), + concentration_probability=( + latest_step.posterior.concentration_probability + if terminal and latest_step + else None + ), + evidence_digest=( + "sha256:" + + hashlib.sha256( + canonical_json( + { + "schema_version": "adaptive-study-evidence-chain.v1", + "input_fingerprint": definition.input_fingerprint, + "events": [item.model_dump(mode="json") for item in events], + "observations": [item.model_dump(mode="json") for item in observations], + "optimizer_steps": [item.model_dump(mode="json") for item in steps], + } + ).encode("utf-8") + ).hexdigest() + if terminal and latest_step + else None + ), + ) + + def list(self, *, limit: int = 50) -> tuple[AdaptiveStudyListItem, ...]: + if not 1 <= limit <= 100: + raise ValueError("adaptive study list limit must be between 1 and 100") + items: list[AdaptiveStudyListItem] = [] + for definition in self.studies.list(): + try: + owner = self.studies.owner(definition.id) + except AdaptiveStudyConflict: + continue + if owner != self.reviewer_id: + continue + view = self.show(definition.id) + items.append( + AdaptiveStudyListItem( + study_id=view.study_id, + state=view.state, + next_action=view.next_action, + progress=view.progress, + ) + ) + if len(items) == limit: + break + return tuple(items) + + def resume(self, study_id: str) -> AdaptiveStudyView: + definition = self.studies.get(study_id) + events = self.studies.list_events(study_id) + self._validate_reviewer_binding(study_id, events) + if not events: + raise PermissionError("adaptive study has no immutable owner binding") + if events[-1].new_state is AdaptiveStudyState.SEARCH_READY: + observations = self.studies.list_observations(study_id) + steps = self.studies.list_optimizer_steps(study_id) + if observations: + raise RuntimeError("search-ready study unexpectedly has observations") + if not steps: + try: + decision = self.optimizer(definition.optimizer_config, ()) + except Exception as error: + self._append_event( + study_id, + previous=AdaptiveStudyState.SEARCH_READY, + target=AdaptiveStudyState.OPTIMIZER_FAILED, + payload={ + "comparison_count": 0, + "error_type": type(error).__name__, + }, + ) + return self.show(study_id) + step = self._optimizer_step(definition, decision, created_at=definition.created_at) + persisted_step = False + elif len(steps) == 1: + step = steps[0] + self._verify_optimizer_step(definition, step) + persisted_step = True + else: + raise RuntimeError("search-ready study has duplicate optimizer steps") + if step.stop_reason is not StopReason.CONTINUE or step.selected_pair is None: + if not persisted_step: + step = self.studies.append_optimizer_step(step) + self._remember_optimizer_step(step) + self._append_event( + study_id, + previous=AdaptiveStudyState.SEARCH_READY, + target=AdaptiveStudyState.OPTIMIZER_FAILED, + payload={ + "comparison_count": 0, + "stop_reason": step.stop_reason.value, + "recoverable": False, + }, + ) + return self.show(study_id) + if step.selected_pair != definition.optimizer_config.initial_pair: + if not persisted_step: + step = self.studies.append_optimizer_step(step) + self._remember_optimizer_step(step) + self._append_event( + study_id, + previous=AdaptiveStudyState.SEARCH_READY, + target=AdaptiveStudyState.OPTIMIZER_FAILED, + payload={ + "comparison_count": 0, + "error_code": "initial_pair_mismatch", + "recoverable": False, + }, + ) + return self.show(study_id) + experiment_id = adaptive_experiment_id(study_id, step.output_digest) + planned = self.studies.ensure_initial_plan( + step, + experiment_id=experiment_id, + ) + self._remember_optimizer_step(step) + return self._resume_generating(definition, planned, verify_optimizer=False) + if events[-1].new_state is AdaptiveStudyState.PAUSED: + paused = events[-1] + self.experiments.requeue_adaptive_failures( + study_id, self._payload_string(paused, "experiment_id") + ) + self._append_event( + study_id, + previous=AdaptiveStudyState.PAUSED, + target=AdaptiveStudyState.GENERATING_PAIR, + payload={ + "optimizer_step_ordinal": self._payload_int(paused, "optimizer_step_ordinal"), + "optimizer_output_digest": self._payload_string( + paused, "optimizer_output_digest" + ), + "experiment_id": self._payload_string(paused, "experiment_id"), + }, + ) + return self.resume(study_id) + if events[-1].new_state is AdaptiveStudyState.OPTIMIZER_FAILED: + if events[-1].payload.get("recoverable") is False: + return self.show(study_id) + observations = self.studies.list_observations(study_id) + target = ( + AdaptiveStudyState.UPDATING_POSTERIOR + if observations + else AdaptiveStudyState.SEARCH_READY + ) + self._append_event( + study_id, + previous=AdaptiveStudyState.OPTIMIZER_FAILED, + target=target, + payload={ + "recovery": "explicit_resume", + "comparison_count": len(observations), + }, + ) + return self.resume(study_id) + if events[-1].new_state is AdaptiveStudyState.UPDATING_POSTERIOR: + return self._resume_optimizer_update(definition) + if events[-1].new_state is not AdaptiveStudyState.GENERATING_PAIR: + return self.show(study_id) + return self._resume_generating(definition, events[-1], verify_optimizer=True) + + def _resume_generating( + self, + definition: AdaptiveStudyDefinition, + event: AdaptiveStudyEvent, + *, + verify_optimizer: bool, + ) -> AdaptiveStudyView: + study_id = definition.id + step_ordinal = self._payload_int(event, "optimizer_step_ordinal") + output_digest = self._payload_string(event, "optimizer_output_digest") + expected_experiment_id = self._payload_string(event, "experiment_id") + steps = self.studies.list_optimizer_steps(study_id) + if not 1 <= step_ordinal <= len(steps): + raise RuntimeError("generating event references a missing optimizer step") + step = steps[step_ordinal - 1] + if step.output_digest != output_digest or step.selected_pair is None: + raise RuntimeError("generating event does not match its optimizer step") + if verify_optimizer: + self._verify_optimizer_step(definition, step) + generated = self.experiments.create_adaptive( + self._pair_draft(definition, step.selected_pair.value_a, step.selected_pair.value_b), + self.capabilities, + study_id=study_id, + optimizer_output_digest=output_digest, + ) + if generated.experiment.id != expected_experiment_id: + raise RuntimeError("adaptive experiment identity is not reproducible") + try: + review = self.reviews.start_adaptive(expected_experiment_id, self.reviewer_id) + except ValueError: + return self.show(study_id) + sealed_mapping_digest = self.reviews.adaptive_binding_digest( + review.session_id, + self.reviewer_id, + study_id=study_id, + optimizer_step_ordinal=step.ordinal, + optimizer_output_digest=step.output_digest, + ) + self.studies.append_awaiting_preference_or_verify( + AdaptiveStudyEvent( + study_id=study_id, + sequence=event.sequence + 1, + previous_state=AdaptiveStudyState.GENERATING_PAIR, + new_state=AdaptiveStudyState.AWAITING_PREFERENCE, + payload={ + "binding_schema_version": "adaptive-review-binding.v1", + "study_id": study_id, + "optimizer_step_ordinal": step.ordinal, + "optimizer_output_digest": step.output_digest, + "experiment_id": generated.experiment.id, + "review_session_id": review.session_id, + "reviewer_id": self.reviewer_id, + "sealed_mapping_digest": sealed_mapping_digest, + }, + created_at=utc_now(), + ) + ) + return self.show(study_id) + + def _validate_adapter(self, definition: AdaptiveStudyDefinition) -> None: + initial = definition.optimizer_config.initial_pair + if (initial.value_a, initial.value_b) != (2.0, 4.0): + raise ValueError("adaptive Stage 1 must start with 2.0 versus 4.0") + self.experiments.preview( + self._pair_draft(definition, initial.value_a, initial.value_b), + self.capabilities, + ) + + def pause_generation_failure( + self, + study_id: str, + experiment_id: str, + *, + error_code: str, + ) -> AdaptiveStudyView: + if not error_code: + raise ValueError("generation failure error code must not be empty") + events = self.studies.list_events(study_id) + if not events: + raise ValueError("adaptive study has no generation to pause") + latest = events[-1] + if latest.new_state is AdaptiveStudyState.PAUSED: + if ( + self._payload_string(latest, "experiment_id") == experiment_id + and self._payload_string(latest, "error_code") == error_code + ): + return self.show(study_id) + raise ValueError("adaptive study is already paused for another failure") + if latest.new_state is not AdaptiveStudyState.GENERATING_PAIR: + raise ValueError("adaptive study is not generating a pair") + if self._payload_string(latest, "experiment_id") != experiment_id: + raise ValueError("generation failure does not match the active experiment") + self._append_event( + study_id, + previous=AdaptiveStudyState.GENERATING_PAIR, + target=AdaptiveStudyState.PAUSED, + payload={ + "optimizer_step_ordinal": self._payload_int(latest, "optimizer_step_ordinal"), + "optimizer_output_digest": self._payload_string(latest, "optimizer_output_digest"), + "experiment_id": experiment_id, + "error_code": error_code, + "resume_state": AdaptiveStudyState.GENERATING_PAIR.value, + }, + ) + return self.show(study_id) + + def record_preference( + self, + study_id: str, + review_id: str, + display_choice: DisplayPreference, + ) -> AdaptiveStudyView: + definition = self.studies.get(study_id) + events = self.studies.list_events(study_id) + self._validate_reviewer_binding(study_id, events) + if not events: + raise ValueError("adaptive study is not awaiting a preference") + latest_review = self.reviews.latest_adaptive(review_id, self.reviewer_id) + observations = self.studies.list_observations(study_id) + existing = ( + None + if latest_review is None + else next( + (item for item in observations if item.review_id == latest_review.id), + None, + ) + ) + if existing is not None: + if latest_review is None or latest_review.display_preference is not display_choice: + raise ValueError("adaptive v1 review is immutable") + latest_event = events[-1] + if ( + latest_event.new_state is AdaptiveStudyState.AWAITING_PREFERENCE + and self._payload_string(latest_event, "review_session_id") == review_id + ): + self._append_event( + study_id, + previous=AdaptiveStudyState.AWAITING_PREFERENCE, + target=AdaptiveStudyState.UPDATING_POSTERIOR, + payload={ + "observation_ordinal": existing.ordinal, + "review_id": existing.review_id, + }, + ) + return self.resume(study_id) + if ( + latest_event.new_state is AdaptiveStudyState.UPDATING_POSTERIOR + and self._payload_string(latest_event, "review_id") == existing.review_id + ): + return self.resume(study_id) + return self.show(study_id) + + if events[-1].new_state in { + AdaptiveStudyState.SEARCH_CONVERGED, + AdaptiveStudyState.SEARCH_BUDGET_EXHAUSTED, + AdaptiveStudyState.COMPLETED, + }: + raise ValueError("terminal adaptive study cannot record a preference") + if events[-1].new_state is not AdaptiveStudyState.AWAITING_PREFERENCE: + raise ValueError("adaptive study is not awaiting a preference") + expected_review_id = self._payload_string(events[-1], "review_session_id") + if review_id != expected_review_id: + raise ValueError("review does not match the study's sealed comparison") + revision = self.reviews.submit_adaptive(review_id, self.reviewer_id, display_choice) + observation, consumed = self.studies.consume_observation_or_verify( + AdaptiveObservationRequest( + study_id=study_id, + ordinal=len(observations) + 1, + review_id=revision.id, + created_at=utc_now(), + ) + ) + if not consumed: + return self._await_preference_winner(study_id, observation.review_id) + self._append_event( + study_id, + previous=AdaptiveStudyState.AWAITING_PREFERENCE, + target=AdaptiveStudyState.UPDATING_POSTERIOR, + payload={ + "observation_ordinal": observation.ordinal, + "review_id": observation.review_id, + }, + ) + return self._resume_optimizer_update(definition) + + def _await_preference_winner( + self, + study_id: str, + review_id: str, + *, + timeout_seconds: float = 5.0, + ) -> AdaptiveStudyView: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + view = self.show(study_id) + if view.state not in { + AdaptiveStudyState.AWAITING_PREFERENCE, + AdaptiveStudyState.UPDATING_POSTERIOR, + }: + return view + time.sleep(0.01) + events = self.studies.list_events(study_id) + latest = events[-1] + if ( + latest.new_state is AdaptiveStudyState.AWAITING_PREFERENCE + and self._payload_string(latest, "review_session_id") == review_id + ): + observations = self.studies.list_observations(study_id) + observation = next(item for item in observations if item.review_id == review_id) + self._append_event( + study_id, + previous=AdaptiveStudyState.AWAITING_PREFERENCE, + target=AdaptiveStudyState.UPDATING_POSTERIOR, + payload={ + "observation_ordinal": observation.ordinal, + "review_id": observation.review_id, + }, + ) + return self.resume(study_id) + + def _validate_reviewer_binding( + self, study_id: str, events: tuple[AdaptiveStudyEvent, ...] + ) -> None: + try: + owner = self.studies.owner(study_id) + except AdaptiveStudyConflict as error: + raise PermissionError("adaptive study owner binding is unavailable") from error + if owner != self.reviewer_id: + raise PermissionError("adaptive study belongs to another reviewer") + bound_reviewers = { + reviewer + for event in events + if isinstance((reviewer := event.payload.get("reviewer_id")), str) + } + if bound_reviewers != {self.reviewer_id}: + raise PermissionError("adaptive study reviewer evidence is inconsistent") + + @staticmethod + def _pair_ref(study_id: str, optimizer_output_digest: str) -> str: + digest = hashlib.sha256( + b"adaptive-pair-ref.v1\x00" + + study_id.encode("utf-8") + + b"\x00" + + optimizer_output_digest.encode("ascii") + ).hexdigest() + return f"adaptive-pair-ref.v1:{digest}" + + def _resume_optimizer_update(self, definition: AdaptiveStudyDefinition) -> AdaptiveStudyView: + observations = self.studies.list_observations(definition.id) + optimizer_observations = tuple(item.preference_observation for item in observations) + steps = self.studies.list_optimizer_steps(definition.id) + expected_ordinal = len(observations) + 1 + if len(steps) == expected_ordinal - 1: + try: + decision = self.optimizer(definition.optimizer_config, optimizer_observations) + except Exception as error: + self._append_event( + definition.id, + previous=AdaptiveStudyState.UPDATING_POSTERIOR, + target=AdaptiveStudyState.OPTIMIZER_FAILED, + payload={ + "comparison_count": len(observations), + "error_type": type(error).__name__, + }, + ) + return self.show(definition.id) + if ( + decision.stopping.reason is StopReason.CONTINUE + and len(observations) >= definition.optimizer_config.maximum_comparisons + ): + self._append_event( + definition.id, + previous=AdaptiveStudyState.UPDATING_POSTERIOR, + target=AdaptiveStudyState.OPTIMIZER_FAILED, + payload={ + "comparison_count": len(observations), + "error_code": "optimizer_continued_at_hard_cap", + "recoverable": False, + }, + ) + return self.show(definition.id) + step = self.studies.append_optimizer_step( + self._optimizer_step( + definition, + decision, + optimizer_observations=optimizer_observations, + ) + ) + self._remember_optimizer_step(step) + elif len(steps) == expected_ordinal: + step = steps[-1] + self._verify_optimizer_step(definition, step) + else: + raise RuntimeError("adaptive optimizer evidence has an invalid ordinal gap") + if step.stop_reason is StopReason.CONTINUE: + if len(observations) >= definition.optimizer_config.maximum_comparisons: + self._append_event( + definition.id, + previous=AdaptiveStudyState.UPDATING_POSTERIOR, + target=AdaptiveStudyState.OPTIMIZER_FAILED, + payload={ + "comparison_count": len(observations), + "error_code": "persisted_optimizer_continue_at_hard_cap", + "optimizer_step_ordinal": step.ordinal, + "optimizer_output_digest": step.output_digest, + "recoverable": False, + }, + ) + return self.show(definition.id) + if step.selected_pair is None: + raise RuntimeError("continuing optimizer step has no selected pair") + experiment_id = adaptive_experiment_id(definition.id, step.output_digest) + planned = self._append_event( + definition.id, + previous=AdaptiveStudyState.UPDATING_POSTERIOR, + target=AdaptiveStudyState.GENERATING_PAIR, + payload={ + "optimizer_step_ordinal": step.ordinal, + "optimizer_output_digest": step.output_digest, + "experiment_id": experiment_id, + }, + ) + return self._resume_generating(definition, planned, verify_optimizer=False) + if step.stop_reason not in { + StopReason.SEARCH_CONVERGED, + StopReason.SEARCH_BUDGET_EXHAUSTED, + }: + self._append_event( + definition.id, + previous=AdaptiveStudyState.UPDATING_POSTERIOR, + target=AdaptiveStudyState.OPTIMIZER_FAILED, + payload={ + "comparison_count": len(observations), + "optimizer_step_ordinal": step.ordinal, + "optimizer_output_digest": step.output_digest, + "stop_reason": step.stop_reason.value, + "recoverable": False, + }, + ) + return self.show(definition.id) + target = ( + AdaptiveStudyState.SEARCH_CONVERGED + if step.stop_reason is StopReason.SEARCH_CONVERGED + else AdaptiveStudyState.SEARCH_BUDGET_EXHAUSTED + ) + self._append_event( + definition.id, + previous=AdaptiveStudyState.UPDATING_POSTERIOR, + target=target, + payload={ + "optimizer_step_ordinal": step.ordinal, + "optimizer_output_digest": step.output_digest, + "stop_reason": step.stop_reason.value, + }, + ) + return self.show(definition.id) + + def _verify_optimizer_step( + self, + definition: AdaptiveStudyDefinition, + step: AdaptiveOptimizerStep, + ) -> None: + key = (definition.id, step.ordinal, step.output_digest) + if key in self._verified_optimizer_outputs: + return + observations = self.studies.list_observations(definition.id) + prefix = tuple( + item.preference_observation for item in observations[: step.comparison_count] + ) + if len(prefix) != step.comparison_count: + raise AdaptiveStudyConflict("optimizer replay references missing observations") + try: + replayed = self.optimizer(definition.optimizer_config, prefix) + except Exception as error: + raise AdaptiveStudyConflict("optimizer replay failed") from error + if adaptive_optimizer_output_digest(replayed) != step.output_digest: + raise AdaptiveStudyConflict("optimizer replay output does not match persisted evidence") + self._verified_optimizer_outputs.add(key) + + def _remember_optimizer_step(self, step: AdaptiveOptimizerStep) -> None: + self._verified_optimizer_outputs.add((step.study_id, step.ordinal, step.output_digest)) + + def _append_event( + self, + study_id: str, + *, + previous: AdaptiveStudyState, + target: AdaptiveStudyState, + payload: Mapping[str, JsonValue], + ) -> AdaptiveStudyEvent: + return self.studies.append_event( + AdaptiveStudyEvent( + study_id=study_id, + sequence=len(self.studies.list_events(study_id)) + 1, + previous_state=previous, + new_state=target, + payload=payload, + created_at=utc_now(), + ) + ) + + @staticmethod + def _pair_draft( + definition: AdaptiveStudyDefinition, value_a: float, value_b: float + ) -> ExperimentDraft: + return ExperimentDraft( + baseline=definition.base_manifest.model_copy(update={"guidance_scale": value_a}), + variable_path="guidance_scale", + alternate_value=value_b, + ) + + @staticmethod + def _optimizer_step( + definition: AdaptiveStudyDefinition, + decision: OptimizerDecision, + *, + optimizer_observations: tuple[PreferenceObservation, ...] = (), + created_at: datetime | None = None, + ) -> AdaptiveOptimizerStep: + return AdaptiveOptimizerStep( + study_id=definition.id, + ordinal=len(optimizer_observations) + 1, + input_digest=adaptive_optimizer_input_digest( + definition.optimizer_config, optimizer_observations + ), + output_digest=adaptive_optimizer_output_digest(decision), + posterior=decision.posterior, + acquisition_ledger=decision.candidate_scores, + selected_pair=decision.selected_pair, + stop_reason=decision.stopping.reason, + disposition=decision.stopping.disposition, + comparison_count=decision.stopping.comparison_count, + maximum_information_bits=decision.stopping.maximum_information_bits, + created_at=utc_now() if created_at is None else created_at, + ) + + @staticmethod + def _payload_string(event: AdaptiveStudyEvent, key: str) -> str: + value = event.payload.get(key) + if not isinstance(value, str) or not value: + raise RuntimeError(f"adaptive event payload is missing {key}") + return value + + @staticmethod + def _payload_int(event: AdaptiveStudyEvent, key: str) -> int: + value = event.payload.get(key) + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise RuntimeError(f"adaptive event payload is missing {key}") + return value diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_web_creation_repository.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_web_creation_repository.py new file mode 100644 index 0000000..44c794a --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/adaptive_web_creation_repository.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +import hashlib +import secrets +import sqlite3 +import unicodedata +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from typing import Self + +from pydantic import Field, field_validator, model_validator + +from video_lab.database import Database +from video_lab.domain import FrozenModel, _aware_utc, utc_now + +ADAPTIVE_WEB_CREATION_GRANT_TTL = timedelta(minutes=15) +ADAPTIVE_WEB_CREATION_PER_OWNER_CAP = 8 +ADAPTIVE_WEB_CREATION_TOTAL_CAP = 250 + + +class AdaptiveWebCreationGrantError(PermissionError): + pass + + +class AdaptiveWebCreationCapacityError(RuntimeError): + pass + + +class AdaptiveWebCreationGrant(FrozenModel): + token_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + reviewer_id: str = Field(min_length=1) + created_at: datetime + expires_at: datetime + input_fingerprint: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}$") + study_id: str | None = None + bound_at: datetime | None = None + + @field_validator("reviewer_id") + @classmethod + def normalized_reviewer(cls, value: str) -> str: + if value != value.strip(): + raise ValueError("creation grant reviewer must be normalized") + return value + + @field_validator("created_at", "expires_at", "bound_at") + @classmethod + def normalized_timestamp(cls, value: datetime | None) -> datetime | None: + return None if value is None else _aware_utc(value) + + @model_validator(mode="after") + def valid_lifecycle(self) -> Self: + if self.expires_at <= self.created_at: + raise ValueError("creation grant expiry must follow creation") + binding = (self.input_fingerprint, self.study_id, self.bound_at) + if any(value is None for value in binding) != all(value is None for value in binding): + raise ValueError("creation grant binding fields must transition together") + if self.bound_at is not None and not self.created_at <= self.bound_at < self.expires_at: + raise ValueError("creation grant binding timestamp is outside its lifetime") + return self + + +class AdaptiveWebCreationGrantRepository: + def __init__( + self, + database: Database, + *, + clock: Callable[[], datetime] = utc_now, + ) -> None: + self.database = database + self.clock = clock + + def issue(self, reviewer_id: str) -> str: + reviewer = _normalized_reviewer(reviewer_id) + now = self._now() + expires_at = now + ADAPTIVE_WEB_CREATION_GRANT_TTL + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + _delete_expired(connection, now) + total = int( + connection.execute( + "SELECT COUNT(*) FROM adaptive_web_creation_grants" + ).fetchone()[0] + ) + owner_total = int( + connection.execute( + "SELECT COUNT(*) FROM adaptive_web_creation_grants WHERE reviewer_id = ?", + (reviewer,), + ).fetchone()[0] + ) + if ( + total >= ADAPTIVE_WEB_CREATION_TOTAL_CAP + or owner_total >= ADAPTIVE_WEB_CREATION_PER_OWNER_CAP + ): + connection.commit() + raise AdaptiveWebCreationCapacityError( + "adaptive web creation grant capacity reached" + ) + token = self._insert_random_token( + connection, + reviewer=reviewer, + created_at=now, + expires_at=expires_at, + ) + connection.commit() + return token + except Exception: + connection.rollback() + raise + + def bind( + self, + reviewer_id: str, + token: str, + input_fingerprint: str, + ) -> AdaptiveWebCreationGrant: + reviewer = _normalized_reviewer(reviewer_id) + if ( + len(input_fingerprint) != 64 + or input_fingerprint != input_fingerprint.lower() + or any(character not in "0123456789abcdef" for character in input_fingerprint) + ): + raise ValueError("adaptive creation input fingerprint is invalid") + token_digest = _token_digest(token) + now = self._now() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + _delete_expired(connection, now) + row = connection.execute( + "SELECT * FROM adaptive_web_creation_grants " + "WHERE token_digest = ? AND reviewer_id = ?", + (token_digest, reviewer), + ).fetchone() + if row is None: + raise AdaptiveWebCreationGrantError( + "adaptive web creation grant is unavailable" + ) + current = _grant_from_row(row) + if current.input_fingerprint is not None: + if current.input_fingerprint != input_fingerprint: + raise AdaptiveWebCreationGrantError( + "adaptive web creation grant is already bound" + ) + connection.commit() + return current + + study_id = _study_id(reviewer, input_fingerprint, token) + connection.execute( + "UPDATE adaptive_web_creation_grants " + "SET input_fingerprint = ?, study_id = ?, bound_at = ? " + "WHERE token_digest = ? AND reviewer_id = ? " + "AND input_fingerprint IS NULL AND study_id IS NULL AND bound_at IS NULL", + ( + input_fingerprint, + study_id, + now.isoformat(), + token_digest, + reviewer, + ), + ) + persisted_row = connection.execute( + "SELECT * FROM adaptive_web_creation_grants " + "WHERE token_digest = ? AND reviewer_id = ?", + (token_digest, reviewer), + ).fetchone() + if persisted_row is None: + raise RuntimeError("bound adaptive web creation grant disappeared") + persisted = _grant_from_row(persisted_row) + connection.commit() + return persisted + except Exception: + connection.rollback() + raise + + def _insert_random_token( + self, + connection: sqlite3.Connection, + *, + reviewer: str, + created_at: datetime, + expires_at: datetime, + ) -> str: + for _ in range(8): + token = secrets.token_urlsafe(32) + try: + connection.execute( + "INSERT INTO adaptive_web_creation_grants(" + "token_digest, reviewer_id, created_at, expires_at" + ") VALUES (?, ?, ?, ?)", + ( + _token_digest(token), + reviewer, + created_at.isoformat(), + expires_at.isoformat(), + ), + ) + except sqlite3.IntegrityError: + continue + return token + raise RuntimeError("could not allocate a unique adaptive web creation token") + + def _now(self) -> datetime: + now = self.clock() + if now.tzinfo is None or now.utcoffset() is None: + raise ValueError("adaptive web creation clock must return an aware timestamp") + return now.astimezone(UTC) + + +def _delete_expired(connection: sqlite3.Connection, now: datetime) -> None: + connection.execute( + "DELETE FROM adaptive_web_creation_grants WHERE julianday(expires_at) <= julianday(?)", + (now.isoformat(),), + ) + + +def _token_digest(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def _normalized_reviewer(reviewer_id: str) -> str: + reviewer = unicodedata.normalize("NFKC", reviewer_id).strip() + if not reviewer or reviewer != reviewer_id: + raise ValueError("adaptive creation reviewer must be a normalized identity") + return reviewer + + +def _study_id(reviewer: str, input_fingerprint: str, token: str) -> str: + digest = hashlib.sha256() + digest.update(b"video-lab.adaptive-study-id.v2\x00") + digest.update(reviewer.encode("utf-8")) + digest.update(b"\x00") + digest.update(input_fingerprint.encode("ascii")) + digest.update(b"\x00") + digest.update(token.encode("utf-8")) + return f"study_{digest.hexdigest()}" + + +def _grant_from_row(row: sqlite3.Row) -> AdaptiveWebCreationGrant: + return AdaptiveWebCreationGrant( + token_digest=str(row["token_digest"]), + reviewer_id=str(row["reviewer_id"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + expires_at=datetime.fromisoformat(str(row["expires_at"])), + input_fingerprint=( + None if row["input_fingerprint"] is None else str(row["input_fingerprint"]) + ), + study_id=None if row["study_id"] is None else str(row["study_id"]), + bound_at=None if row["bound_at"] is None else datetime.fromisoformat(str(row["bound_at"])), + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/animegen_feasibility.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/animegen_feasibility.py new file mode 100644 index 0000000..ed3af50 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/animegen_feasibility.py @@ -0,0 +1,596 @@ +from __future__ import annotations + +import json +import platform +import re +import shutil +import subprocess +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Any, cast + +from video_lab.adapter_protocol import AdapterExecutionError +from video_lab.adapters.animegen_mps import ADAPTER_ID as ANIMEGEN_ADAPTER_ID +from video_lab.adapters.animegen_mps import ( + AnimeGenStageExecutor, + AnimeGenStorageMode, + storage_mode_for_manifest, +) +from video_lab.domain import Manifest +from video_lab.feasibility import ( + InvalidEvidenceError, + StageDefinition, + StageExecutionContext, + StageResult, +) +from video_lab.feasibility_models import ( + FeasibilityConfiguration, + FeasibilityStage, + StageEvidence, + StageOutcome, +) +from video_lab.runtime_identity import animegen_runtime_versions +from video_lab.source_inventory import ( + ANIMEGEN_REQUIRED_FILES, + ANIMEGEN_SOURCE_REPOS, + SourceInventory, + SourceInventoryClient, + SourceInventoryError, + SourceSpec, + canonical_source_digest, + inventory_from_evidence, +) + +ANIMEGEN_BF16_PROTOCOL_VERSION = "animegen-feasibility.v2" +ANIMEGEN_FROZEN_PROMPT = "anime character walking" +ANIMEGEN_FROZEN_NEGATIVE_PROMPT = "" +_GIB = 1024**3 +_MEMORY_POLICY = "max(24GiB,20pct_total)" + + +def canonical_animegen_configuration(configuration: FeasibilityConfiguration) -> dict[str, Any]: + if configuration.adapter_id != ANIMEGEN_ADAPTER_ID: + raise ValueError("configuration adapter_id is not the AnimeGen candidate") + if configuration.protocol_version != ANIMEGEN_BF16_PROTOCOL_VERSION: + raise ValueError("configuration protocol_version is not the AnimeGen BF16 protocol") + if set(configuration.sources) != set(ANIMEGEN_SOURCE_REPOS): + raise ValueError("configuration must contain the canonical AnimeGen sources") + for component, repo_id in ANIMEGEN_SOURCE_REPOS.items(): + if configuration.sources[component].id != repo_id: + raise ValueError(f"source repository mismatch: {component}") + profiles = configuration.manifest_profiles.model_dump(mode="json") + minimum = Manifest.model_validate(profiles["minimum"]) + standard = Manifest.model_validate(profiles["standard"]) + for name, manifest in (("minimum", minimum), ("standard", standard)): + animegen = configuration.sources["animegen"] + if ( + manifest.adapter_id != configuration.adapter_id + or manifest.adapter_version != "1" + or manifest.model_id != animegen.id + or manifest.model_revision != animegen.revision + or manifest.model_checksum != animegen.checksum.removeprefix("sha256:") + or dict(manifest.runtime_versions) != dict(configuration.runtime_versions) + or manifest.hardware_fingerprint != configuration.hardware_fingerprint + or manifest.os_fingerprint != configuration.os_fingerprint + or manifest.memory_strategy != configuration.memory_strategy + or manifest.scheduler != "flow-match-euler-shift-3" + or manifest.dtype != "bfloat16" + or manifest.backend != "mps" + or manifest.device != "mps" + or manifest.num_frames < 17 + or (manifest.num_frames - 1) % 16 + or manifest.width % 16 + or manifest.height % 16 + ): + raise ValueError(f"{name} manifest does not match canonical AnimeGen configuration") + try: + storage_mode = storage_mode_for_manifest(manifest) + except AdapterExecutionError as error: + raise ValueError(f"{name} manifest uses unsupported transformer storage") from error + if storage_mode is not AnimeGenStorageMode.BF16: + raise ValueError(f"{name} manifest does not match canonical AnimeGen configuration") + expected_loras = tuple( + ( + configuration.sources[component].id, + configuration.sources[component].revision, + configuration.sources[component].checksum.removeprefix("sha256:"), + weight, + ) + for component, weight in (("lightning-high", 2.0), ("lightning-low", 1.0)) + ) + actual_loras = tuple( + (item.model_id, item.revision, item.checksum, item.weight) for item in manifest.loras + ) + if actual_loras != expected_loras: + raise ValueError(f"{name} manifest LoRA bindings are not canonical") + expected_checksums = { + component: identity.checksum.removeprefix("sha256:") + for component, identity in configuration.sources.items() + } + if dict(manifest.source_checksums) != expected_checksums: + raise ValueError(f"{name} manifest source checksums are not canonical") + frozen_controls = { + "num_frames": 17, + "fps": 8, + "inference_steps": 4, + "guidance_scale": 1.0, + "seed": 7, + "prompt": ANIMEGEN_FROZEN_PROMPT, + "negative_prompt": ANIMEGEN_FROZEN_NEGATIVE_PROMPT, + } + for name, manifest, dimensions in ( + ("minimum", minimum, (128, 80)), + ("standard", standard, (512, 288)), + ): + if (manifest.width, manifest.height) != dimensions or any( + getattr(manifest, field) != value for field, value in frozen_controls.items() + ): + raise ValueError(f"{name} manifest does not match the frozen AnimeGen BF16 recipe") + minimum_identity = minimum.model_dump(mode="json") + standard_identity = standard.model_dump(mode="json") + for field in ("width", "height"): + minimum_identity.pop(field) + standard_identity.pop(field) + if minimum_identity != standard_identity: + raise ValueError("minimum and standard manifests may differ only in dimensions") + payload = configuration.model_dump(mode="json") + payload["manifest_profiles"] = { + "minimum": minimum.model_dump(mode="json"), + "standard": standard.model_dump(mode="json"), + } + return payload + + +def evaluate_preassembly_preflight( + *, + configuration: Mapping[str, object], + source_digests: Mapping[str, str], + storage: Mapping[str, int], + observation: Mapping[str, object], + stage: FeasibilityStage, +) -> StageResult: + """Compare one complete pre-assembly observation with the persisted run identity.""" + configured_sources = configuration.get("sources") + if not isinstance(configured_sources, Mapping): + raise InvalidEvidenceError("persisted preflight source identity is missing") + expected_sources = { + str(component): str(identity.get("checksum", "")).removeprefix("sha256:") + for component, identity in configured_sources.items() + if isinstance(identity, Mapping) + } + expected_runtime = configuration.get("runtime_versions") + observed_runtime = observation.get("runtime_versions") + mps = observation.get("mps") + memory = observation.get("unified_memory") + if ( + not isinstance(expected_runtime, Mapping) + or not isinstance(observed_runtime, Mapping) + or not isinstance(mps, Mapping) + or not isinstance(memory, Mapping) + ): + raise InvalidEvidenceError("preflight observation is incomplete") + requested = { + "runtime_versions": dict(expected_runtime), + "source_digests": expected_sources, + "hardware_fingerprint": configuration.get("hardware_fingerprint"), + "os_fingerprint": configuration.get("os_fingerprint"), + "mps": {"available": True, "built": True}, + "disk": {"required_bytes": storage.get("required_bytes")}, + "unified_memory": { + "required_available_bytes": memory.get("required_available_bytes"), + "policy": memory.get("policy"), + }, + } + observed = { + "runtime_versions": dict(observed_runtime), + "source_digests": dict(source_digests), + "hardware_fingerprint": observation.get("hardware_fingerprint"), + "os_fingerprint": observation.get("os_fingerprint"), + "mps": dict(mps), + "disk": dict(storage), + "unified_memory": dict(memory), + } + mismatches = tuple( + name + for name, expected, actual in ( + ("runtime", requested["runtime_versions"], observed["runtime_versions"]), + ("source", requested["source_digests"], observed["source_digests"]), + ( + "hardware", + requested["hardware_fingerprint"], + observed["hardware_fingerprint"], + ), + ("os", requested["os_fingerprint"], observed["os_fingerprint"]), + ) + if expected != actual + ) + details: dict[str, object] = { + "stage": stage.value, + "requested": requested, + "observed": observed, + "identity_mismatches": mismatches, + } + evidence = StageEvidence(details=cast(Any, {"preassembly_preflight": details})) + if mismatches: + return StageResult( + outcome=StageOutcome.INVALID_EVIDENCE, + evidence=evidence, + diagnostic_code="preflight_identity_drift", + diagnostic_message="persisted and observed preflight identity differ", + ) + if mps.get("available") is not True or mps.get("built") is not True: + return StageResult( + outcome=StageOutcome.INCOMPATIBLE, + evidence=evidence, + diagnostic_code="mps_unavailable", + diagnostic_message="MPS is unavailable or not built", + ) + required_disk = storage.get("required_bytes") + free_disk = storage.get("free_bytes") + if ( + not isinstance(required_disk, int) + or not isinstance(free_disk, int) + or free_disk < required_disk + ): + return StageResult( + outcome=StageOutcome.RESOURCE_EXHAUSTED, + evidence=evidence, + diagnostic_code="insufficient_disk_headroom", + diagnostic_message="insufficient free disk headroom", + ) + required_memory = memory.get("required_available_bytes") + available_memory = memory.get("available_bytes") + if ( + not isinstance(required_memory, int) + or not isinstance(available_memory, int) + or available_memory < required_memory + ): + return StageResult( + outcome=StageOutcome.RESOURCE_EXHAUSTED, + evidence=evidence, + diagnostic_code="insufficient_unified_memory_headroom", + diagnostic_message="insufficient unified-memory headroom", + ) + return StageResult.passed(evidence) + + +def production_system_observation() -> Mapping[str, object]: + """Observe only privacy-safe fields needed by the local M5 qualification identity.""" + import psutil # type: ignore[import-untyped] + import torch + + hardware = json.loads( + subprocess.run( + ["/usr/sbin/system_profiler", "SPHardwareDataType", "-json"], + check=True, + capture_output=True, + text=True, + ).stdout + )["SPHardwareDataType"][0] + chip = re.sub(r"[^a-z0-9]+", "-", str(hardware["chip_type"]).lower()).strip("-") + memory_label = re.sub(r"\s+", "", str(hardware["physical_memory"]).lower()) + cores = re.search(r"\d+", str(hardware["number_processors"])) + if cores is None: + raise RuntimeError("hardware core count is unavailable") + hardware_fingerprint = f"{chip}-{memory_label}-{cores.group()}c" + product_version = subprocess.run( + ["/usr/bin/sw_vers", "-productVersion"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + build_version = subprocess.run( + ["/usr/bin/sw_vers", "-buildVersion"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + os_fingerprint = f"macos-{product_version}-{build_version}-{platform.machine()}" + virtual_memory = psutil.virtual_memory() + required_available = max(24 * _GIB, (int(virtual_memory.total) + 4) // 5) + return { + "runtime_versions": animegen_runtime_versions(), + "hardware_fingerprint": hardware_fingerprint, + "os_fingerprint": os_fingerprint, + "mps": { + "available": bool(torch.backends.mps.is_available()), + "built": bool(torch.backends.mps.is_built()), + }, + "unified_memory": { + "total_bytes": int(virtual_memory.total), + "available_bytes": int(virtual_memory.available), + "required_available_bytes": required_available, + "policy": _MEMORY_POLICY, + }, + } + + +class AnimeGenFeasibilityExecutor: + """Executes source and isolated component stages from persisted evidence.""" + + def __init__( + self, + cache_root: Path | None, + inventory_client: SourceInventoryClient | None, + component_factory: Callable[..., AnimeGenStageExecutor] | None = None, + component_options: Mapping[str, object] | None = None, + inventory_evidence: Mapping[str, object] | None = None, + system_observer: Callable[[], Mapping[str, object]] | None = None, + ) -> None: + self.cache_root = cache_root + self.inventory_client = inventory_client + self.component_factory = component_factory or AnimeGenStageExecutor + self.component_options = dict(component_options or {}) + self.inventory_evidence = inventory_evidence + self.system_observer = system_observer or production_system_observation + + def execute(self, context: StageExecutionContext, stage: StageDefinition) -> StageResult: + current = stage.stage + if current is FeasibilityStage.RESOLVE_SOURCES: + return StageResult.passed( + StageEvidence(details=cast(Any, {"sources": self._source_specs(context)})) + ) + if current is FeasibilityStage.INVENTORY_SOURCES: + client = self._client() + inventory = ( + client.resolve(self._specs(context)) + if self.inventory_evidence is None + else self._validated_imported_inventory(context, client) + ) + return StageResult.passed( + StageEvidence(details=cast(Any, {"inventory": inventory.export_evidence()})) + ) + if current is FeasibilityStage.STORAGE_PREFLIGHT: + return self.preflight(context, stage) + if current is FeasibilityStage.DOWNLOAD_SOURCES: + inventory = self._inventory(context, FeasibilityStage.INVENTORY_SOURCES) + downloaded = ( + inventory.download() + if self.inventory_evidence is None + else self._client().verify_cached(inventory) + ) + digests = { + source.component: canonical_source_digest(source) for source in downloaded.sources + } + return StageResult.passed( + StageEvidence( + details=cast( + Any, + { + "inventory": downloaded.export_evidence(), + "canonical_source_digests": digests, + }, + ) + ) + ) + if current in { + FeasibilityStage.LOAD_TEXT_STACK, + FeasibilityStage.LOAD_VAE, + FeasibilityStage.LOAD_HIGH_TRANSFORMER, + FeasibilityStage.LOAD_LOW_TRANSFORMER, + FeasibilityStage.ASSEMBLE_PIPELINE, + }: + inventory = self.validated_download_inventory(context) + sources = {source.component: source for source in inventory.sources} + executor = self.component_factory( + cache_root=self._cache(), + sources=sources, + runtime=cast(Any, self.component_options.get("runtime")), + torch=self.component_options.get("torch"), + storage_mode=self._storage_mode(context), + ) + return StageResult.passed( + executor.execute_protocol_stage(current, cancelled=lambda: False) + ) + if current is FeasibilityStage.PUBLISH_VERDICT: + return StageResult.passed(StageEvidence(details={"published": True})) + raise SourceInventoryError(f"stage requires generation composition: {current.value}") + + def preflight(self, context: StageExecutionContext, stage: StageDefinition) -> StageResult: + source_stage = ( + FeasibilityStage.INVENTORY_SOURCES + if stage.stage + in {FeasibilityStage.STORAGE_PREFLIGHT, FeasibilityStage.DOWNLOAD_SOURCES} + else FeasibilityStage.DOWNLOAD_SOURCES + ) + inventory = self._inventory(context, source_stage) + if source_stage is FeasibilityStage.DOWNLOAD_SOURCES: + inventory = self._client().verify_cached(inventory) + source_digests = { + source.component: canonical_source_digest(source) for source in inventory.sources + } + cache_root = self._cache() + cache_root.mkdir(parents=True, exist_ok=True) + free_bytes = int(shutil.disk_usage(cache_root).free) + safety_margin = max(20 * _GIB, (inventory.expected_bytes + 4) // 5) + storage = { + "expected_bytes": inventory.expected_bytes, + "safety_margin_bytes": safety_margin, + "required_bytes": inventory.expected_bytes + safety_margin, + "free_bytes": free_bytes, + } + try: + observation = self.system_observer() + except Exception as error: + return StageResult( + outcome=StageOutcome.INVALID_EVIDENCE, + evidence=StageEvidence( + details={ + "preassembly_preflight": { + "stage": stage.stage.value, + "observation_error": type(error).__name__, + } + } + ), + diagnostic_code="preflight_observation_failed", + diagnostic_message="complete preflight observation failed", + ) + return evaluate_preassembly_preflight( + configuration=cast(Mapping[str, object], context.run.configuration), + source_digests=source_digests, + storage=storage, + observation=observation, + stage=stage.stage, + ) + + @staticmethod + def _storage_mode(context: StageExecutionContext) -> AnimeGenStorageMode: + profiles = context.run.configuration.get("manifest_profiles") + if not isinstance(profiles, Mapping): + raise SourceInventoryError("persisted manifest profiles are missing") + modes = { + storage_mode_for_manifest(Manifest.model_validate(profiles.get(name))) + for name in ("minimum", "standard") + } + if len(modes) != 1: + raise SourceInventoryError("manifest profiles use inconsistent transformer storage") + return modes.pop() + + def _client(self) -> SourceInventoryClient: + if self.inventory_client is None: + raise SourceInventoryError("explicit cache is required for source operation") + return self.inventory_client + + def _cache(self) -> Path: + if self.cache_root is None: + raise SourceInventoryError("explicit cache is required for local source operation") + return self.cache_root + + def _source_specs(self, context: StageExecutionContext) -> tuple[dict[str, object], ...]: + return tuple( + { + "component": spec.component, + "repo_id": spec.repo_id, + "revision": spec.revision, + "required_files": spec.required_files, + } + for spec in self._specs(context) + ) + + def _specs(self, context: StageExecutionContext) -> tuple[SourceSpec, ...]: + configured = context.run.configuration.get("sources") + if not isinstance(configured, Mapping): + raise SourceInventoryError("persisted source identities are missing") + specs: list[SourceSpec] = [] + for component, repo_id in ANIMEGEN_SOURCE_REPOS.items(): + identity = configured.get(component) + if not isinstance(identity, Mapping) or identity.get("id") != repo_id: + raise SourceInventoryError(f"persisted source identity mismatch: {component}") + specs.append( + SourceSpec( + component, + repo_id, + str(identity.get("revision", "")), + ANIMEGEN_REQUIRED_FILES[component], + ) + ) + return tuple(specs) + + def _inventory( + self, context: StageExecutionContext, source_stage: FeasibilityStage + ) -> SourceInventory: + attempt = next( + ( + item + for item in reversed(context.attempts) + if item.stage is source_stage and item.evidence is not None + ), + None, + ) + if attempt is None or attempt.evidence is None: + raise SourceInventoryError(f"persisted {source_stage.value} evidence is missing") + evidence = attempt.evidence.details.get("inventory") + if not isinstance(evidence, Mapping): + raise SourceInventoryError(f"persisted {source_stage.value} inventory is missing") + return inventory_from_evidence(evidence, self._client()) + + def _validated_imported_inventory( + self, context: StageExecutionContext, client: SourceInventoryClient + ) -> SourceInventory: + assert self.inventory_evidence is not None + specs = self._specs(context) + inventory = client.refresh_cached( + inventory_from_evidence(self.inventory_evidence, client), specs + ) + if len(inventory.sources) != len(specs): + raise SourceInventoryError("imported inventory source set does not match configuration") + for source, spec in zip(inventory.sources, specs, strict=True): + if ( + source.component != spec.component + or source.repo_id != spec.repo_id + or source.requested_revision != spec.revision + or source.revision != spec.revision + or tuple(item.relative_path for item in source.files) != spec.required_files + or any( + item.upstream_sha256 is None and item.local_sha256 is None + for item in source.files + ) + ): + raise SourceInventoryError( + f"imported inventory does not match configured source: {spec.component}" + ) + return inventory + + def validated_download_inventory(self, context: StageExecutionContext) -> SourceInventory: + inventory = self._inventory(context, FeasibilityStage.DOWNLOAD_SOURCES) + digests = { + source.component: canonical_source_digest(source) for source in inventory.sources + } + configured = context.run.configuration.get("sources") + profiles = context.run.configuration.get("manifest_profiles") + if not isinstance(configured, Mapping) or not isinstance(profiles, Mapping): + raise SourceInventoryError("persisted checksum configuration is missing") + expected: dict[str, str] = {} + for component in ANIMEGEN_SOURCE_REPOS: + identity = configured.get(component) + if not isinstance(identity, Mapping): + raise SourceInventoryError(f"persisted checksum is missing: {component}") + expected[component] = str(identity.get("checksum", "")).removeprefix("sha256:") + if digests != expected: + raise InvalidEvidenceError("downloaded canonical source checksum mismatch") + for name in ("minimum", "standard"): + manifest = Manifest.model_validate(profiles.get(name)) + if dict(manifest.source_checksums) != digests: + raise InvalidEvidenceError(f"{name} manifest source checksum mismatch") + if manifest.model_checksum != digests["animegen"]: + raise InvalidEvidenceError(f"{name} manifest model checksum mismatch") + expected_loras = tuple( + (ANIMEGEN_SOURCE_REPOS[component], digests[component]) + for component in ("lightning-high", "lightning-low") + ) + actual_loras = tuple((item.model_id, item.checksum) for item in manifest.loras) + if actual_loras != expected_loras: + raise InvalidEvidenceError(f"{name} manifest LoRA checksum mismatch") + return inventory + + +def production_inventory_client(cache_root: Path) -> SourceInventoryClient: + """Build Hub seams lazily; snapshot acquisition remains download-stage-only.""" + from huggingface_hub import HfApi, snapshot_download + + def download(**kwargs: object) -> str: + download_impl = cast(Any, snapshot_download) + return str(download_impl(**kwargs)) + + return SourceInventoryClient(cache_root, HfApi().model_info, download) + + +def production_component_options() -> Mapping[str, object]: + return {} + + +def production_generation_runtime(cache_root: Path, sources: Mapping[str, object]) -> object: + from video_lab.adapters.animegen_mps import DiffusersAnimeGenGenerationRuntime + + return DiffusersAnimeGenGenerationRuntime(cache_root=cache_root, sources=cast(Any, sources)) + + +__all__ = [ + "AnimeGenFeasibilityExecutor", + "canonical_animegen_configuration", + "evaluate_preassembly_preflight", + "production_component_options", + "production_generation_runtime", + "production_inventory_client", + "production_system_observation", +] diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/animegen_pair_contract.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/animegen_pair_contract.py new file mode 100644 index 0000000..193a245 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/animegen_pair_contract.py @@ -0,0 +1,6 @@ +from __future__ import annotations + +ANIMEGEN_CONTROLLED_PAIR_SEED = 8 +ANIMEGEN_CONTROLLED_GUIDANCE_VALUES = (1.0, 3.5) + +__all__ = ["ANIMEGEN_CONTROLLED_GUIDANCE_VALUES", "ANIMEGEN_CONTROLLED_PAIR_SEED"] diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/artifacts.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/artifacts.py new file mode 100644 index 0000000..e4a549a --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/artifacts.py @@ -0,0 +1,988 @@ +from __future__ import annotations + +import fcntl +import hashlib +import os +import shutil +import sqlite3 +import stat +import uuid +from collections.abc import Callable, Iterator +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass, replace +from pathlib import Path +from typing import TypeVar + +from video_lab.database import Database +from video_lab.domain import utc_now + +_HASH_CHUNK_SIZE = 1024 * 1024 +_T = TypeVar("_T") + + +class ArtifactError(Exception): + """Base error for artifact storage failures.""" + + +class ArtifactIntegrityError(ArtifactError): + """Stored or staged bytes do not match their registered digest.""" + + +class ArtifactNotFoundError(ArtifactError): + """An artifact id has not been registered.""" + + +@dataclass(frozen=True) +class StagedArtifact: + path: Path + sha256: str + byte_size: int + media_type: str + + +@dataclass(frozen=True) +class Artifact: + id: str + job_id: str | None + sha256: str + byte_size: int + media_type: str + relative_path: str + created_at: str + + +@dataclass(frozen=True) +class VerifiedArtifactHandle: + """Path-free immutable descriptor held for the caller transaction lifetime.""" + + artifact: Artifact + descriptor: int + relative_parts: tuple[str, ...] + component_identities: tuple[tuple[int, int], ...] + identity: tuple[int, int, int, int] + + def read(self, size: int = -1) -> bytes: + return os.read(self.descriptor, self.artifact.byte_size if size < 0 else size) + + +class ArtifactStore: + def __init__(self, database: Database, root: Path) -> None: + self.database = database + self.root = root.resolve() + self.staging_root = self.root / ".staging" + + def stage(self, source: Path, media_type: str) -> StagedArtifact: + self.staging_root.mkdir(parents=True, exist_ok=True) + staged_path = self.staging_root / uuid.uuid4().hex + digest = hashlib.sha256() + byte_size = 0 + try: + with source.open("rb") as source_file, staged_path.open("xb") as staged_file: + while chunk := source_file.read(_HASH_CHUNK_SIZE): + staged_file.write(chunk) + digest.update(chunk) + byte_size += len(chunk) + except Exception: + staged_path.unlink(missing_ok=True) + raise + return StagedArtifact(staged_path, digest.hexdigest(), byte_size, media_type) + + def stage_bytes(self, data: bytes, media_type: str) -> StagedArtifact: + """Stage already-produced bytes without exposing a mutable caller path.""" + self.staging_root.mkdir(parents=True, exist_ok=True) + staged_path = self.staging_root / uuid.uuid4().hex + try: + with staged_path.open("xb") as staged_file: + staged_file.write(data) + staged_file.flush() + os.fsync(staged_file.fileno()) + except Exception: + staged_path.unlink(missing_ok=True) + raise + return StagedArtifact(staged_path, hashlib.sha256(data).hexdigest(), len(data), media_type) + + def commit_bytes_at_retained_root( + self, + data: bytes, + media_type: str, + root_descriptor: int, + job_id: str | None = None, + *, + reverify: Callable[[], None], + ) -> Artifact: + """Publish through a retained root or compensate every visible side effect.""" + return self._commit_retained_bytes_transaction( + data=data, + media_type=media_type, + root_descriptor=root_descriptor, + job_id=job_id, + reverify=reverify, + ) + + def _commit_retained_bytes_transaction( + self, + *, + data: bytes, + media_type: str, + root_descriptor: int, + job_id: str | None, + reverify: Callable[[], None], + ) -> Artifact: + """Keep retained bytes and database registration recoverable until verified.""" + + if ( + type(data) is not bytes + or type(media_type) is not str + or not media_type + or not callable(reverify) + ): + raise ArtifactIntegrityError("retained artifact input is invalid") + self._reverify_retained_root(reverify) + try: + root_info = os.fstat(root_descriptor) + except OSError: + raise ArtifactIntegrityError("retained artifact root is invalid") from None + if not stat.S_ISDIR(root_info.st_mode): + raise ArtifactIntegrityError("retained artifact root is invalid") + digest = hashlib.sha256(data).hexdigest() + relative_path = str(Path(digest[:2]) / digest[2:4] / digest) + with ExitStack() as descriptors: + locks = self._open_retained_directory(root_descriptor, ".locks", descriptors) + lock_descriptor = os.open( + f"{digest}.lock", + os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), + 0o600, + dir_fd=locks, + ) + descriptors.callback(os.close, lock_descriptor) + fcntl.flock(lock_descriptor, fcntl.LOCK_EX) + descriptors.callback(fcntl.flock, lock_descriptor, fcntl.LOCK_UN) + first = self._open_retained_directory(root_descriptor, digest[:2], descriptors) + second = self._open_retained_directory(first, digest[2:4], descriptors) + destination_created = False + artifact_created = False + association_created = False + commit_attempted = False + temporary_name = f".{digest}.{uuid.uuid4().hex}.tmp" + artifact: Artifact + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + rollback_errors: list[BaseException] = [] + try: + row = connection.execute( + "SELECT * FROM artifacts WHERE sha256 = ? AND media_type = ?", + (digest, media_type), + ).fetchone() + if row is not None: + artifact = _artifact_from_row(row) + self._verify_retained_file(second, digest, artifact) + else: + try: + existing = os.open( + digest, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + dir_fd=second, + ) + except FileNotFoundError: + existing = -1 + if existing >= 0: + try: + payload = os.pread(existing, len(data) + 1, 0) + finally: + os.close(existing) + if payload != data: + raise ArtifactIntegrityError( + "content-addressed destination integrity check failed" + ) + else: + temporary = os.open( + temporary_name, + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_NOFOLLOW", 0), + 0o600, + dir_fd=second, + ) + try: + view = memoryview(data) + while view: + written = os.write(temporary, view) + if written <= 0: + raise ArtifactIntegrityError( + "retained artifact short write" + ) + view = view[written:] + os.fsync(temporary) + finally: + os.close(temporary) + os.rename( + temporary_name, + digest, + src_dir_fd=second, + dst_dir_fd=second, + ) + destination_created = True + os.fsync(second) + artifact = Artifact( + id=f"art_{uuid.uuid4().hex}", + job_id=None, + sha256=digest, + byte_size=len(data), + media_type=media_type, + relative_path=relative_path, + created_at=utc_now().isoformat(), + ) + connection.execute( + "INSERT INTO artifacts(id, job_id, sha256, byte_size, media_type, " + "relative_path, created_at) VALUES (?, NULL, ?, ?, ?, ?, ?)", + ( + artifact.id, + artifact.sha256, + artifact.byte_size, + artifact.media_type, + artifact.relative_path, + artifact.created_at, + ), + ) + artifact_created = True + association_created = self._associate_retained_with_connection( + connection, artifact.id, job_id + ) + self._reverify_retained_root(reverify) + self._before_database_commit() + commit_attempted = True + connection.commit() + except BaseException as primary: + try: + connection.rollback() + except BaseException as rollback_error: + rollback_errors.append(rollback_error) + if commit_attempted: + try: + self._compensate_retained_database( + artifact_id=artifact.id, + job_id=job_id, + artifact_created=artifact_created, + association_created=association_created, + ) + except BaseException as rollback_error: + rollback_errors.append(rollback_error) + rollback_errors.extend( + self._remove_retained_publication( + second, + digest, + temporary_name, + destination_created=destination_created, + ) + ) + if rollback_errors: + self._raise_retained_rollback_failure(primary, rollback_errors) + raise + try: + self._reverify_retained_root(reverify) + except ArtifactIntegrityError as primary: + rollback_errors = [] + try: + self._compensate_retained_database( + artifact_id=artifact.id, + job_id=job_id, + artifact_created=artifact_created, + association_created=association_created, + ) + except BaseException as rollback_error: + rollback_errors.append(rollback_error) + rollback_errors.extend( + self._remove_retained_publication( + second, + digest, + temporary_name, + destination_created=destination_created, + ) + ) + if rollback_errors: + self._raise_retained_rollback_failure(primary, rollback_errors) + raise + return replace(artifact, job_id=job_id) + + @staticmethod + def _reverify_retained_root(reverify: Callable[[], None]) -> None: + try: + reverify() + except Exception: + raise ArtifactIntegrityError( + "retained artifact authority verification failed" + ) from None + + @staticmethod + def _associate_retained_with_connection( + connection: sqlite3.Connection, artifact_id: str, job_id: str | None + ) -> bool: + if job_id is None: + return False + cursor = connection.execute( + "INSERT OR IGNORE INTO job_artifacts(job_id, artifact_id, created_at) " + "VALUES (?, ?, ?)", + (job_id, artifact_id, utc_now().isoformat()), + ) + return cursor.rowcount == 1 + + def _compensate_retained_database( + self, + *, + artifact_id: str, + job_id: str | None, + artifact_created: bool, + association_created: bool, + ) -> None: + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + if association_created and job_id is not None: + connection.execute( + "DELETE FROM job_artifacts WHERE job_id = ? AND artifact_id = ?", + (job_id, artifact_id), + ) + if artifact_created: + connection.execute( + "DELETE FROM job_artifacts WHERE artifact_id = ?", (artifact_id,) + ) + connection.execute("DELETE FROM artifacts WHERE id = ?", (artifact_id,)) + connection.commit() + except BaseException: + connection.rollback() + raise + + @staticmethod + def _remove_retained_publication( + parent: int, + digest: str, + temporary_name: str, + *, + destination_created: bool, + ) -> list[BaseException]: + errors: list[BaseException] = [] + names = ((digest, destination_created), (temporary_name, True)) + removed = False + for name, should_remove in names: + if not should_remove: + continue + try: + os.unlink(name, dir_fd=parent) + removed = True + except FileNotFoundError: + pass + except BaseException as error: + errors.append(error) + if removed: + try: + os.fsync(parent) + except BaseException as error: + errors.append(error) + return errors + + @staticmethod + def _raise_retained_rollback_failure( + primary: BaseException, errors: list[BaseException] + ) -> None: + normalized_primary = ( + ArtifactIntegrityError(str(primary)) + if isinstance(primary, ArtifactIntegrityError) + else ArtifactIntegrityError("retained artifact publication failed") + ) + normalized_rollbacks = tuple( + ArtifactIntegrityError("retained artifact rollback operation failed") + for _error in errors + ) + raise ArtifactIntegrityError("retained artifact rollback failed") from BaseExceptionGroup( + "retained artifact rollback errors", + (normalized_primary, *normalized_rollbacks), + ) + + def _before_database_commit(self) -> None: + """Test seam immediately after pre-commit authority verification.""" + + @staticmethod + def _open_retained_directory( + parent: int, name: str, stack: ExitStack + ) -> int: + try: + os.mkdir(name, mode=0o700, dir_fd=parent) + except FileExistsError: + pass + descriptor = os.open( + name, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=parent, + ) + stack.callback(os.close, descriptor) + if not stat.S_ISDIR(os.fstat(descriptor).st_mode): + raise ArtifactIntegrityError("retained artifact directory is invalid") + return descriptor + + @staticmethod + def _verify_retained_file( + parent: int, name: str, artifact: Artifact + ) -> None: + descriptor = os.open( + name, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + dir_fd=parent, + ) + try: + info = os.fstat(descriptor) + payload = os.pread(descriptor, artifact.byte_size + 1, 0) + finally: + os.close(descriptor) + if ( + not stat.S_ISREG(info.st_mode) + or len(payload) != artifact.byte_size + or hashlib.sha256(payload).hexdigest() != artifact.sha256 + ): + raise ArtifactIntegrityError("retained artifact integrity check failed") + + def discard(self, staged: StagedArtifact) -> None: + try: + path = staged.path.resolve(strict=False) + root = self.staging_root.resolve(strict=False) + except OSError as error: + raise ArtifactIntegrityError("staged artifact path cannot be verified") from error + if not path.is_relative_to(root) or path.parent != root: + raise ArtifactIntegrityError("staged artifact path is outside staging root") + path.unlink(missing_ok=True) + + def commit(self, staged: StagedArtifact, job_id: str | None) -> Artifact: + if not staged.path.resolve().is_relative_to(self.staging_root.resolve()): + raise ArtifactIntegrityError("staged artifact path is outside staging root") + actual_sha256, actual_size = _hash_file(staged.path) + if actual_sha256 != staged.sha256 or actual_size != staged.byte_size: + raise ArtifactIntegrityError("staged artifact integrity check failed") + + locks_root = self.root / ".locks" + locks_root.mkdir(parents=True, exist_ok=True) + lock_path = locks_root / f"{staged.sha256}.lock" + with lock_path.open("a+b") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + return self._commit_locked(staged, job_id) + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + def commit_batch( + self, + staged_artifacts: tuple[StagedArtifact, ...], + persist: Callable[[sqlite3.Connection, tuple[Artifact, ...]], _T], + *, + verify_artifacts: tuple[Artifact, ...] = (), + ) -> _T: + """Atomically register content and persist its owning domain records.""" + for staged in staged_artifacts: + self._verify_staged(staged) + + with ExitStack() as locks: + for sha256 in sorted({staged.sha256 for staged in staged_artifacts}): + locks.enter_context(self._digest_lock(sha256)) + created_destinations: list[Path] = [] + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + artifacts = [] + for staged in staged_artifacts: + artifact, destination_created = self._register_with_connection( + connection, staged + ) + artifacts.append(artifact) + if destination_created is not None: + created_destinations.append(destination_created) + with ExitStack() as verified: + handles = tuple( + verified.enter_context( + self.open_verified_with_connection( + connection, + artifact.id, + sha256=artifact.sha256, + byte_size=artifact.byte_size, + media_type=artifact.media_type, + ) + ) + for artifact in (*artifacts, *verify_artifacts) + ) + result = persist(connection, tuple(artifacts)) + for handle in handles: + self.reverify_open_handle(connection, handle) + for handle in handles: + self.reverify_open_handle(connection, handle) + connection.commit() + except Exception: + connection.rollback() + for destination in created_destinations: + destination.unlink(missing_ok=True) + raise + + for staged in staged_artifacts: + staged.path.unlink(missing_ok=True) + return result + + def _verify_staged(self, staged: StagedArtifact) -> None: + if not staged.path.resolve().is_relative_to(self.staging_root.resolve()): + raise ArtifactIntegrityError("staged artifact path is outside staging root") + actual_sha256, actual_size = _hash_file(staged.path) + if actual_sha256 != staged.sha256 or actual_size != staged.byte_size: + raise ArtifactIntegrityError("staged artifact integrity check failed") + + @contextmanager + def _digest_lock(self, sha256: str) -> Iterator[None]: + locks_root = self.root / ".locks" + locks_root.mkdir(parents=True, exist_ok=True) + lock_path = locks_root / f"{sha256}.lock" + with lock_path.open("a+b") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + def _register_with_connection( + self, connection: sqlite3.Connection, staged: StagedArtifact + ) -> tuple[Artifact, Path | None]: + row = connection.execute( + "SELECT * FROM artifacts WHERE sha256 = ? AND media_type = ?", + (staged.sha256, staged.media_type), + ).fetchone() + if row is not None: + artifact = _artifact_from_row(row) + self._verify_file(self._safe_path(artifact.relative_path), artifact) + return artifact, None + + relative_path = str(Path(staged.sha256[:2]) / staged.sha256[2:4] / staged.sha256) + destination = self._safe_path(relative_path) + destination.parent.mkdir(parents=True, exist_ok=True) + destination_created: Path | None = None + if destination.exists(): + destination_sha256, destination_size = _hash_file(destination) + if destination_sha256 != staged.sha256 or destination_size != staged.byte_size: + raise ArtifactIntegrityError("content-addressed destination integrity check failed") + else: + placement_temp = destination.parent / f".{staged.sha256}.{uuid.uuid4().hex}.tmp" + try: + shutil.copyfile(staged.path, placement_temp) + placement_sha256, placement_size = _hash_file(placement_temp) + if placement_sha256 != staged.sha256 or placement_size != staged.byte_size: + raise ArtifactIntegrityError("placement copy integrity check failed") + os.replace(placement_temp, destination) + destination_created = destination + finally: + placement_temp.unlink(missing_ok=True) + + artifact = Artifact( + id=f"art_{uuid.uuid4().hex}", + job_id=None, + sha256=staged.sha256, + byte_size=staged.byte_size, + media_type=staged.media_type, + relative_path=relative_path, + created_at=utc_now().isoformat(), + ) + try: + connection.execute( + "INSERT INTO artifacts(id, job_id, sha256, byte_size, media_type, " + "relative_path, created_at) VALUES (?, NULL, ?, ?, ?, ?, ?)", + ( + artifact.id, + artifact.sha256, + artifact.byte_size, + artifact.media_type, + artifact.relative_path, + artifact.created_at, + ), + ) + except Exception: + if destination_created is not None: + destination_created.unlink(missing_ok=True) + raise + return artifact, destination_created + + def _commit_locked(self, staged: StagedArtifact, job_id: str | None) -> Artifact: + existing = self._find_by_content(staged.sha256, staged.media_type) + if existing is not None: + self._verify_file(self._safe_path(existing.relative_path), existing) + self._associate(existing.id, job_id) + staged.path.unlink(missing_ok=True) + return replace(existing, job_id=job_id) + + relative_path = str(Path(staged.sha256[:2]) / staged.sha256[2:4] / staged.sha256) + destination = self._safe_path(relative_path) + destination.parent.mkdir(parents=True, exist_ok=True) + destination_created = False + if destination.exists(): + destination_sha256, destination_size = _hash_file(destination) + if destination_sha256 != staged.sha256 or destination_size != staged.byte_size: + raise ArtifactIntegrityError("content-addressed destination integrity check failed") + else: + placement_temp = destination.parent / f".{staged.sha256}.{uuid.uuid4().hex}.tmp" + try: + shutil.copyfile(staged.path, placement_temp) + placement_sha256, placement_size = _hash_file(placement_temp) + if placement_sha256 != staged.sha256 or placement_size != staged.byte_size: + raise ArtifactIntegrityError("placement copy integrity check failed") + os.replace(placement_temp, destination) + destination_created = True + finally: + placement_temp.unlink(missing_ok=True) + + artifact = Artifact( + id=f"art_{uuid.uuid4().hex}", + job_id=None, + sha256=staged.sha256, + byte_size=staged.byte_size, + media_type=staged.media_type, + relative_path=relative_path, + created_at=utc_now().isoformat(), + ) + try: + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + connection.execute( + """INSERT INTO artifacts( + id, job_id, sha256, byte_size, media_type, relative_path, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?)""", + ( + artifact.id, + artifact.job_id, + artifact.sha256, + artifact.byte_size, + artifact.media_type, + artifact.relative_path, + artifact.created_at, + ), + ) + self._associate_with_connection(connection, artifact.id, job_id) + connection.commit() + except sqlite3.IntegrityError: + existing = self._find_by_content(staged.sha256, staged.media_type) + if existing is None: + if destination_created: + destination.unlink(missing_ok=True) + raise + self._associate(existing.id, job_id) + staged.path.unlink(missing_ok=True) + return replace(existing, job_id=job_id) + except Exception: + if destination_created: + destination.unlink(missing_ok=True) + raise + staged.path.unlink(missing_ok=True) + return replace(artifact, job_id=job_id) + + def list_for_job(self, job_id: str) -> tuple[Artifact, ...]: + with self.database.connect() as connection: + rows = connection.execute( + "SELECT artifacts.* FROM artifacts " + "JOIN job_artifacts ON job_artifacts.artifact_id = artifacts.id " + "WHERE job_artifacts.job_id = ? ORDER BY job_artifacts.created_at, artifacts.id", + (job_id,), + ).fetchall() + return tuple(replace(_artifact_from_row(row), job_id=job_id) for row in rows) + + def get(self, artifact_id: str) -> Artifact: + """Return registered artifact metadata without bypassing integrity verification.""" + with self.database.connect() as connection: + row = connection.execute( + "SELECT * FROM artifacts WHERE id = ?", (artifact_id,) + ).fetchone() + if row is None: + raise ArtifactNotFoundError(f"artifact not found: {artifact_id}") + artifact = _artifact_from_row(row) + self.resolve(artifact.id) + return artifact + + def _associate(self, artifact_id: str, job_id: str | None) -> None: + if job_id is None: + return + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + self._associate_with_connection(connection, artifact_id, job_id) + connection.commit() + + @staticmethod + def _associate_with_connection( + connection: sqlite3.Connection, artifact_id: str, job_id: str | None + ) -> None: + if job_id is not None: + connection.execute( + "INSERT OR IGNORE INTO job_artifacts(job_id, artifact_id, created_at) " + "VALUES (?, ?, ?)", + (job_id, artifact_id, utc_now().isoformat()), + ) + + def resolve(self, artifact_id: str) -> Path: + with self.database.connect() as connection: + row = connection.execute( + "SELECT * FROM artifacts WHERE id = ?", (artifact_id,) + ).fetchone() + if row is None: + raise ArtifactNotFoundError("artifact not found") + artifact = _artifact_from_row(row) + return self.resolve_with_connection( + connection, + artifact_id, + sha256=artifact.sha256, + byte_size=artifact.byte_size, + media_type=artifact.media_type, + ) + + def resolve_with_connection( + self, + connection: sqlite3.Connection, + artifact_id: str, + *, + sha256: str, + byte_size: int, + media_type: str, + ) -> Path: + """Resolve verified content without taking ownership of the caller's transaction.""" + self._require_owned_connection(connection) + row = connection.execute("SELECT * FROM artifacts WHERE id = ?", (artifact_id,)).fetchone() + if row is None: + raise ArtifactNotFoundError("artifact not found") + artifact = _artifact_from_row(row) + expected = (sha256, byte_size, media_type) + if _artifact_descriptor(artifact) != expected: + raise ArtifactIntegrityError("artifact descriptor mismatch") + + path = self._safe_path(artifact.relative_path) + self._verify_file(path, artifact) + + stable_row = connection.execute( + "SELECT * FROM artifacts WHERE id = ?", (artifact_id,) + ).fetchone() + if stable_row is None or _artifact_descriptor(_artifact_from_row(stable_row)) != expected: + raise ArtifactIntegrityError("artifact descriptor changed during verification") + return path + + @contextmanager + def open_verified_with_connection( + self, + connection: sqlite3.Connection, + artifact_id: str, + *, + sha256: str, + byte_size: int, + media_type: str, + cancelled: Callable[[], bool] | None = None, + ) -> Iterator[VerifiedArtifactHandle]: + """Hold the exact verified inode open without owning the caller transaction.""" + self._require_owned_connection(connection) + row = connection.execute("SELECT * FROM artifacts WHERE id=?", (artifact_id,)).fetchone() + if row is None: + raise ArtifactNotFoundError("artifact not found") + artifact = _artifact_from_row(row) + if _artifact_descriptor(artifact) != (sha256, byte_size, media_type): + raise ArtifactIntegrityError("artifact descriptor mismatch") + descriptor = -1 + try: + descriptor, relative_parts, component_identities = self._open_canonical_artifact( + artifact.relative_path + ) + opened = os.fstat(descriptor) + identity = (opened.st_dev, opened.st_ino, opened.st_size, opened.st_mtime_ns) + if not stat.S_ISREG(opened.st_mode): + raise ArtifactIntegrityError("artifact is not a regular file") + handle = VerifiedArtifactHandle( + artifact, descriptor, relative_parts, component_identities, identity + ) + if cancelled is None: + self.reverify_open_handle(connection, handle) + else: + self.reverify_open_handle(connection, handle, cancelled=cancelled) + yield handle + except OSError: + raise ArtifactIntegrityError("artifact cannot be opened safely") from None + finally: + if descriptor >= 0: + os.close(descriptor) + + def reverify_open_handle( + self, + connection: sqlite3.Connection, + handle: VerifiedArtifactHandle, + *, + cancelled: Callable[[], bool] | None = None, + ) -> None: + """Rehash a held descriptor and its canonical database binding.""" + self._require_owned_connection(connection) + current = os.fstat(handle.descriptor) + identity = (current.st_dev, current.st_ino, current.st_size, current.st_mtime_ns) + callback = cancelled or (lambda: False) + digest, size = _hash_descriptor(handle.descriptor, cancelled=callback) + canonical_descriptor = -1 + try: + canonical_descriptor, relative_parts, component_identities = ( + self._open_canonical_artifact(handle.artifact.relative_path) + ) + canonical = os.fstat(canonical_descriptor) + canonical_identity = ( + canonical.st_dev, + canonical.st_ino, + canonical.st_size, + canonical.st_mtime_ns, + ) + canonical_digest, canonical_size = _hash_descriptor( + canonical_descriptor, cancelled=callback + ) + except OSError: + raise ArtifactIntegrityError("artifact canonical path changed") from None + finally: + if canonical_descriptor >= 0: + os.close(canonical_descriptor) + row = connection.execute( + "SELECT * FROM artifacts WHERE id=?", (handle.artifact.id,) + ).fetchone() + if ( + identity != handle.identity + or relative_parts != handle.relative_parts + or component_identities != handle.component_identities + or canonical_identity != handle.identity + or digest != handle.artifact.sha256 + or size != handle.artifact.byte_size + or canonical_digest != handle.artifact.sha256 + or canonical_size != handle.artifact.byte_size + or row is None + or _artifact_descriptor(_artifact_from_row(row)) + != _artifact_descriptor(handle.artifact) + ): + raise ArtifactIntegrityError("artifact changed during verification") + + def _open_canonical_artifact( + self, relative_path: str + ) -> tuple[int, tuple[str, ...], tuple[tuple[int, int], ...]]: + relative = Path(relative_path) + if ( + relative.is_absolute() + or not relative.parts + or any(part in {"", ".", ".."} for part in relative.parts) + ): + raise ArtifactIntegrityError("artifact path is invalid") + current = -1 + leaf = -1 + try: + current = os.open( + self.root, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + root_stat = os.fstat(current) + identities = [(root_stat.st_dev, root_stat.st_ino)] + for part in relative.parts[:-1]: + next_descriptor = os.open( + part, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + dir_fd=current, + ) + opened = os.fstat(next_descriptor) + identities.append((opened.st_dev, opened.st_ino)) + os.close(current) + current = next_descriptor + leaf = os.open( + relative.parts[-1], + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + dir_fd=current, + ) + opened_leaf = os.fstat(leaf) + identities.append((opened_leaf.st_dev, opened_leaf.st_ino)) + os.close(current) + current = -1 + return leaf, relative.parts, tuple(identities) + except OSError: + if leaf >= 0: + os.close(leaf) + if current >= 0: + os.close(current) + raise ArtifactIntegrityError("artifact canonical path changed") from None + + def _require_owned_connection(self, connection: sqlite3.Connection) -> None: + try: + rows = connection.execute("PRAGMA database_list").fetchall() + except sqlite3.Error: + raise ArtifactIntegrityError("connection is closed") from None + main = next((str(row[2]) for row in rows if str(row[1]) == "main"), "") + if not main or Path(main).resolve() != self.database.path.resolve(): + raise ArtifactIntegrityError("connection belongs to a foreign database") + + def _find_by_content(self, sha256: str, media_type: str) -> Artifact | None: + with self.database.connect() as connection: + row = connection.execute( + "SELECT * FROM artifacts WHERE sha256 = ? AND media_type = ?", + (sha256, media_type), + ).fetchone() + return None if row is None else _artifact_from_row(row) + + def _safe_path(self, relative_path: str) -> Path: + path = (self.root / relative_path).resolve() + if not path.is_relative_to(self.root): + raise ArtifactIntegrityError("artifact path escapes artifact root") + internal_roots = ((self.root / ".locks").resolve(), (self.root / ".staging").resolve()) + if any(path.is_relative_to(internal_root) for internal_root in internal_roots): + raise ArtifactIntegrityError("database references internal artifact path") + return path + + @staticmethod + def _verify_file(path: Path, artifact: Artifact) -> None: + try: + sha256, byte_size = _hash_file(path) + except FileNotFoundError as error: + raise ArtifactIntegrityError("registered artifact file is missing") from error + if sha256 != artifact.sha256 or byte_size != artifact.byte_size: + raise ArtifactIntegrityError("registered artifact integrity check failed") + + +def _hash_file(path: Path) -> tuple[str, int]: + digest = hashlib.sha256() + byte_size = 0 + with path.open("rb") as file: + while chunk := file.read(_HASH_CHUNK_SIZE): + digest.update(chunk) + byte_size += len(chunk) + return digest.hexdigest(), byte_size + + +def _hash_descriptor( + descriptor: int, *, cancelled: Callable[[], bool] = lambda: False +) -> tuple[str, int]: + digest = hashlib.sha256() + size = 0 + os.lseek(descriptor, 0, os.SEEK_SET) + while True: + if cancelled(): + raise ArtifactIntegrityError("artifact verification cancelled") + chunk = os.read(descriptor, _HASH_CHUNK_SIZE) + if not chunk: + break + digest.update(chunk) + size += len(chunk) + os.lseek(descriptor, 0, os.SEEK_SET) + return digest.hexdigest(), size + + +def _artifact_from_row(row: sqlite3.Row) -> Artifact: + return Artifact( + id=str(row["id"]), + job_id=None if row["job_id"] is None else str(row["job_id"]), + sha256=str(row["sha256"]), + byte_size=int(row["byte_size"]), + media_type=str(row["media_type"]), + relative_path=str(row["relative_path"]), + created_at=str(row["created_at"]), + ) + + +def _artifact_descriptor(artifact: Artifact) -> tuple[str, int, str]: + return artifact.sha256, artifact.byte_size, artifact.media_type diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/capabilities.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/capabilities.py new file mode 100644 index 0000000..b89a801 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/capabilities.py @@ -0,0 +1,82 @@ +from fnmatch import fnmatch +from typing import Literal + +from pydantic import Field + +from video_lab.domain import FrozenModel, Manifest + + +class AdapterCapabilities(FrozenModel): + adapter_id: str + model_patterns: tuple[str, ...] + backends: tuple[str, ...] + dtypes: tuple[str, ...] + quantizations: tuple[str, ...] + schedulers: tuple[str, ...] + min_width: int = Field(gt=0) + max_width: int = Field(gt=0) + width_multiple: int = Field(gt=0) + min_height: int = Field(gt=0) + max_height: int = Field(gt=0) + height_multiple: int = Field(gt=0) + min_frames: int = Field(gt=0) + max_frames: int = Field(gt=0) + frame_rule: Literal["any", "16n+1"] + supports_lora: bool + min_fps: int = Field(default=1, gt=0) + max_fps: int = Field(default=120, gt=0) + + +class CapabilityViolation(FrozenModel): + field: str + value: object + reason: str + + +def validate_manifest_support( + manifest: Manifest, capabilities: AdapterCapabilities +) -> tuple[CapabilityViolation, ...]: + violations: list[CapabilityViolation] = [] + + def reject(field: str, value: object, reason: str) -> None: + violations.append(CapabilityViolation(field=field, value=value, reason=reason)) + + if manifest.adapter_id != capabilities.adapter_id: + reject("adapter_id", manifest.adapter_id, f"expected adapter {capabilities.adapter_id}") + if not any(fnmatch(manifest.model_id, pattern) for pattern in capabilities.model_patterns): + reject("model_id", manifest.model_id, "model is not supported by this adapter") + for field, setting, allowed in ( + ("backend", manifest.backend, capabilities.backends), + ("dtype", manifest.dtype, capabilities.dtypes), + ("quantization", manifest.quantization, capabilities.quantizations), + ("scheduler", manifest.scheduler, capabilities.schedulers), + ): + if setting not in allowed: + reject(field, setting, f"expected one of {allowed}") + for axis, dimension, low, high, multiple in ( + ( + "width", + manifest.width, + capabilities.min_width, + capabilities.max_width, + capabilities.width_multiple, + ), + ( + "height", + manifest.height, + capabilities.min_height, + capabilities.max_height, + capabilities.height_multiple, + ), + ): + if not low <= dimension <= high or dimension % multiple: + reject(axis, dimension, f"expected {low}..{high} in multiples of {multiple}") + if not capabilities.min_frames <= manifest.num_frames <= capabilities.max_frames: + reject("num_frames", manifest.num_frames, "frame count outside adapter range") + if capabilities.frame_rule == "16n+1" and (manifest.num_frames - 1) % 16: + reject("num_frames", manifest.num_frames, "frame count must equal 16n+1") + if manifest.loras and not capabilities.supports_lora: + reject("loras", manifest.loras, "adapter does not support LoRA bindings") + if not capabilities.min_fps <= manifest.fps <= capabilities.max_fps: + reject("fps", manifest.fps, f"expected {capabilities.min_fps}..{capabilities.max_fps}") + return tuple(violations) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_analysis.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_analysis.py new file mode 100644 index 0000000..eb1bd01 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_analysis.py @@ -0,0 +1,283 @@ +from __future__ import annotations + +from collections import Counter + +import numpy as np + +from video_lab.character_benchmark_models import ( + CHARACTER_POSTERIOR_DRAWS, + CHARACTER_POSTERIOR_SEED, + BaselineDisposition, + CharacterBenchmarkEvidence, + CharacterBenchmarkResult, + CharacterContextGroup, + ContextIdentityResult, + ContextReversal, + GroupIdentityResult, + IdentityChoice, + IdentityCounts, + IdentityDisposition, + IdentityObservation, + IdentityPosterior, + SecondaryChoice, + SecondaryCounts, + SecondaryDimension, + SecondaryDimensionResult, + SecondaryObservation, + SecondarySummary, +) +from video_lab.character_benchmark_protocol import schedule_character_trials + +_IDENTITY_QUALIFICATION_THRESHOLD = 0.90 +_CONTEXT_REVERSAL_THRESHOLD = 0.80 +_MINIMUM_REPEAT_AGREEMENTS = 3 + + +def analyze_character_benchmark( + evidence: CharacterBenchmarkEvidence, +) -> CharacterBenchmarkResult: + complete = _has_complete_integrity(evidence) + identity_by_trial = _identity_by_trial(evidence.identity_observations) + primary_trials = tuple(trial for trial in evidence.trials if not trial.hidden_repeat) + primary_observations = tuple( + identity_by_trial[trial.id] for trial in primary_trials if trial.id in identity_by_trial + ) + posterior = identity_posterior(_identity_counts(primary_observations)) + identity_disposition = ( + _identity_disposition(posterior) if complete else IdentityDisposition.INCONCLUSIVE + ) + repeat_agreements = _repeat_agreements(evidence, identity_by_trial) + if not complete: + baseline_disposition = BaselineDisposition.INSUFFICIENT_EVIDENCE + elif repeat_agreements < _MINIMUM_REPEAT_AGREEMENTS: + baseline_disposition = BaselineDisposition.EVALUATOR_UNRELIABLE + else: + baseline_disposition = BaselineDisposition.BASELINE_ESTABLISHED + + context_results = _context_results(evidence, identity_by_trial) + group_results = _group_results(context_results) + primary_counts = posterior.counts + denominator = primary_counts.total + return CharacterBenchmarkResult( + benchmark_id=evidence.definition.id, + baseline_disposition=baseline_disposition, + identity_disposition=identity_disposition, + posterior=posterior, + both_rate=primary_counts.both / denominator if denominator else 0.0, + neither_rate=primary_counts.neither / denominator if denominator else 0.0, + context_results=context_results, + group_results=group_results, + context_reversals=_context_reversals(context_results, identity_disposition), + secondary=_secondary_summary(evidence, identity_by_trial), + repeat_agreements=repeat_agreements, + identity_break_contexts=tuple( + item.context.name for item in context_results if item.posterior.counts.neither > 0 + ), + ) + + +def identity_posterior(counts: IdentityCounts) -> IdentityPosterior: + samples = np.random.default_rng(CHARACTER_POSTERIOR_SEED).dirichlet( + (counts.a + 1, counts.b + 1, counts.both + 1, counts.neither + 1), + CHARACTER_POSTERIOR_DRAWS, + ) + return IdentityPosterior( + counts=counts, + control_superiority_probability=float(np.mean(samples[:, 0] > samples[:, 1])), + identity_recipe_superiority_probability=float(np.mean(samples[:, 1] > samples[:, 0])), + ) + + +def _has_complete_integrity(evidence: CharacterBenchmarkEvidence) -> bool: + if not evidence.media_integrity_complete or not evidence.provenance_complete: + return False + expected_trials = schedule_character_trials(evidence.definition) + if evidence.trials != expected_trials: + return False + if len(evidence.trials) != 40: + return False + if sum(not trial.hidden_repeat for trial in evidence.trials) != 36: + return False + if sum(trial.hidden_repeat for trial in evidence.trials) != 4: + return False + trial_ids = {trial.id for trial in evidence.trials} + if len(trial_ids) != len(evidence.trials): + return False + observations = evidence.identity_observations + if len(observations) != len(evidence.trials): + return False + if {observation.benchmark_id for observation in observations} != {evidence.definition.id}: + return False + return Counter(observation.trial_id for observation in observations) == Counter(trial_ids) + + +def _identity_by_trial( + observations: tuple[IdentityObservation, ...], +) -> dict[str, IdentityObservation]: + counts = Counter(observation.trial_id for observation in observations) + return { + observation.trial_id: observation + for observation in observations + if counts[observation.trial_id] == 1 + } + + +def _identity_counts(observations: tuple[IdentityObservation, ...]) -> IdentityCounts: + counts = Counter(observation.choice for observation in observations) + return IdentityCounts( + a=counts[IdentityChoice.A], + b=counts[IdentityChoice.B], + both=counts[IdentityChoice.BOTH], + neither=counts[IdentityChoice.NEITHER], + ) + + +def _identity_disposition(posterior: IdentityPosterior) -> IdentityDisposition: + if posterior.identity_recipe_superiority_probability >= _IDENTITY_QUALIFICATION_THRESHOLD: + return IdentityDisposition.IDENTITY_RECIPE_QUALIFIED + if posterior.control_superiority_probability >= _IDENTITY_QUALIFICATION_THRESHOLD: + return IdentityDisposition.CONTROL_QUALIFIED + return IdentityDisposition.INCONCLUSIVE + + +def _repeat_agreements( + evidence: CharacterBenchmarkEvidence, + identity_by_trial: dict[str, IdentityObservation], +) -> int: + agreements = 0 + for trial in evidence.trials: + if not trial.hidden_repeat or trial.reuses_trial_id is None: + continue + repeat = identity_by_trial.get(trial.id) + source = identity_by_trial.get(trial.reuses_trial_id) + if repeat is not None and source is not None and repeat.choice is source.choice: + agreements += 1 + return agreements + + +def _context_results( + evidence: CharacterBenchmarkEvidence, + identity_by_trial: dict[str, IdentityObservation], +) -> tuple[ContextIdentityResult, ...]: + results: list[ContextIdentityResult] = [] + for context in evidence.definition.panel_input.contexts: + observations = tuple( + identity_by_trial[trial.id] + for trial in evidence.trials + if not trial.hidden_repeat + and trial.context == context + and trial.id in identity_by_trial + ) + counts = _identity_counts(observations) + results.append( + ContextIdentityResult( + context=context, + posterior=identity_posterior(counts), + both_rate=counts.both / counts.total if counts.total else 0.0, + neither_rate=counts.neither / counts.total if counts.total else 0.0, + ) + ) + return tuple(results) + + +def _group_results( + contexts: tuple[ContextIdentityResult, ...], +) -> tuple[GroupIdentityResult, ...]: + results: list[GroupIdentityResult] = [] + for group in CharacterContextGroup: + grouped = tuple(item for item in contexts if item.context.group is group) + counts = IdentityCounts( + a=sum(item.posterior.counts.a for item in grouped), + b=sum(item.posterior.counts.b for item in grouped), + both=sum(item.posterior.counts.both for item in grouped), + neither=sum(item.posterior.counts.neither for item in grouped), + ) + results.append( + GroupIdentityResult( + group=group, + posterior=identity_posterior(counts), + both_rate=counts.both / counts.total if counts.total else 0.0, + neither_rate=counts.neither / counts.total if counts.total else 0.0, + ) + ) + return tuple(results) + + +def _context_reversals( + contexts: tuple[ContextIdentityResult, ...], + identity_disposition: IdentityDisposition, +) -> tuple[ContextReversal, ...]: + if identity_disposition is IdentityDisposition.IDENTITY_RECIPE_QUALIFIED: + return tuple( + ContextReversal( + context=item.context, + disfavored_recipe="control", + superiority_probability=item.posterior.control_superiority_probability, + ) + for item in contexts + if item.posterior.control_superiority_probability >= _CONTEXT_REVERSAL_THRESHOLD + ) + if identity_disposition is IdentityDisposition.CONTROL_QUALIFIED: + return tuple( + ContextReversal( + context=item.context, + disfavored_recipe="identity", + superiority_probability=item.posterior.identity_recipe_superiority_probability, + ) + for item in contexts + if item.posterior.identity_recipe_superiority_probability >= _CONTEXT_REVERSAL_THRESHOLD + ) + return () + + +def _secondary_summary( + evidence: CharacterBenchmarkEvidence, + identity_by_trial: dict[str, IdentityObservation], +) -> SecondarySummary: + primary_trial_ids = {trial.id for trial in evidence.trials if not trial.hidden_repeat} + eligible_ids = { + trial_id + for trial_id, observation in identity_by_trial.items() + if trial_id in primary_trial_ids and observation.choice is not IdentityChoice.NEITHER + } + supplied = tuple( + observation + for observation in evidence.secondary_observations + if observation.benchmark_id == evidence.definition.id + and observation.trial_id in eligible_ids + ) + key_counts = Counter((item.trial_id, item.dimension) for item in supplied) + valid = tuple(item for item in supplied if key_counts[(item.trial_id, item.dimension)] == 1) + by_dimension = { + dimension: _secondary_dimension_result( + tuple(item for item in valid if item.dimension is dimension) + ) + for dimension in SecondaryDimension + } + return SecondarySummary( + prompt_adherence=by_dimension[SecondaryDimension.PROMPT_ADHERENCE], + motion_quality=by_dimension[SecondaryDimension.MOTION_QUALITY], + temporal_consistency=by_dimension[SecondaryDimension.TEMPORAL_CONSISTENCY], + visual_quality=by_dimension[SecondaryDimension.VISUAL_QUALITY], + ) + + +def _secondary_dimension_result( + observations: tuple[SecondaryObservation, ...], +) -> SecondaryDimensionResult: + choice_counts = Counter(item.choice for item in observations) + counts = SecondaryCounts( + a=choice_counts[SecondaryChoice.A], + b=choice_counts[SecondaryChoice.B], + tie=choice_counts[SecondaryChoice.TIE], + ) + samples = np.random.default_rng(CHARACTER_POSTERIOR_SEED).dirichlet( + (counts.a + 1, counts.b + 1, counts.tie + 1), + CHARACTER_POSTERIOR_DRAWS, + ) + return SecondaryDimensionResult( + counts=counts, + observation_count=counts.total, + control_superiority_probability=float(np.mean(samples[:, 0] > samples[:, 1])), + identity_recipe_superiority_probability=float(np.mean(samples[:, 1] > samples[:, 0])), + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_models.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_models.py new file mode 100644 index 0000000..fb67b31 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_models.py @@ -0,0 +1,264 @@ +from __future__ import annotations + +from enum import StrEnum +from typing import Literal, Self + +from pydantic import Field, model_validator + +from video_lab.domain import FrozenModel + +CHARACTER_BENCHMARK_SCHEMA_VERSION: Literal["character-benchmark.v1"] = "character-benchmark.v1" +CHARACTER_BENCHMARK_PROTOCOL_VERSION: Literal["character-panel.v1"] = "character-panel.v1" +CHARACTER_BENCHMARK_ANALYSIS_VERSION: Literal["character-dirichlet.v1"] = "character-dirichlet.v1" +CHARACTER_POSTERIOR_DRAWS = 65_536 +CHARACTER_POSTERIOR_SEED = 73_021 + +Candidate = Literal["a", "b"] +CandidatePosition = Literal["left", "right"] + + +class CharacterContextName(StrEnum): + NEUTRAL_FACE_PORTRAIT = "neutral_face_portrait" + THREE_QUARTER_PORTRAIT = "three_quarter_portrait" + PROFILE_VIEW = "profile_view" + FULL_BODY_NEUTRAL_POSE = "full_body_neutral_pose" + FULL_BODY_ACTION = "full_body_action" + STRONG_EXPRESSION = "strong_expression" + CAMERA_MOTION = "camera_motion" + CHARACTER_OBJECT_INTERACTION = "character_object_interaction" + STRONG_DIRECTIONAL_LIGHTING = "strong_directional_lighting" + PARTIAL_FACIAL_OCCLUSION = "partial_facial_occlusion" + FORESHORTENED_OR_DIFFICULT_ANGLE = "foreshortened_or_difficult_angle" + BUSY_ENVIRONMENT_WITH_COMPETING_SUBJECTS = "busy_environment_with_competing_subjects" + + +class CharacterContextGroup(StrEnum): + CORE = "core" + STRESS = "stress" + + +class IdentityChoice(StrEnum): + A = "a" + B = "b" + BOTH = "both" + NEITHER = "neither" + + +class SecondaryChoice(StrEnum): + A = "a" + B = "b" + TIE = "tie" + + +class SecondaryDimension(StrEnum): + PROMPT_ADHERENCE = "prompt_adherence" + MOTION_QUALITY = "motion_quality" + TEMPORAL_CONSISTENCY = "temporal_consistency" + VISUAL_QUALITY = "visual_quality" + + +class BaselineDisposition(StrEnum): + BASELINE_ESTABLISHED = "baseline_established" + EVALUATOR_UNRELIABLE = "evaluator_unreliable" + INSUFFICIENT_EVIDENCE = "insufficient_evidence" + + +class IdentityDisposition(StrEnum): + IDENTITY_RECIPE_QUALIFIED = "identity_recipe_qualified" + CONTROL_QUALIFIED = "control_qualified" + INCONCLUSIVE = "inconclusive" + + +class CharacterBenchmarkContext(FrozenModel): + ordinal: int = Field(ge=1, le=12) + name: CharacterContextName + group: CharacterContextGroup + prompt: str = Field(min_length=1) + negative_prompt: str | None = None + + +class CharacterBenchmarkPanelInput(FrozenModel): + contexts: tuple[CharacterBenchmarkContext, ...] + seeds: tuple[int, int, int] + schedule_seed: int + + @model_validator(mode="after") + def require_frozen_context_matrix(self) -> Self: + expected_names = tuple(CharacterContextName) + if len(self.contexts) != len(expected_names): + raise ValueError("character panel must contain exactly twelve contexts") + if tuple(context.ordinal for context in self.contexts) != tuple(range(1, 13)): + raise ValueError("character panel contexts must retain ordinals one through twelve") + if tuple(context.name for context in self.contexts) != expected_names: + raise ValueError("character panel contexts must retain the frozen order and names") + expected_groups = (CharacterContextGroup.CORE,) * 8 + (CharacterContextGroup.STRESS,) * 4 + if tuple(context.group for context in self.contexts) != expected_groups: + raise ValueError( + "character panel contexts one through eight are core and nine through twelve stress" + ) + if len(set(self.seeds)) != 3: + raise ValueError("character panel seeds must be distinct") + return self + + +class CharacterRecipePair(FrozenModel): + control_digest: str = Field(min_length=1) + identity_digest: str = Field(min_length=1) + + @model_validator(mode="after") + def require_distinct_recipes(self) -> Self: + if self.control_digest == self.identity_digest: + raise ValueError("control and identity recipes must be distinct") + return self + + +class CharacterBenchmarkDefinition(FrozenModel): + id: str = Field(pattern=r"^character_benchmark_[0-9a-f]{64}$") + schema_version: Literal["character-benchmark.v1"] = CHARACTER_BENCHMARK_SCHEMA_VERSION + protocol_version: Literal["character-panel.v1"] = CHARACTER_BENCHMARK_PROTOCOL_VERSION + analysis_version: Literal["character-dirichlet.v1"] = CHARACTER_BENCHMARK_ANALYSIS_VERSION + character_pack_digest: str = Field(min_length=1) + stage_2_manifest_digest: str = Field(min_length=1) + stage_2_evidence_digest: str = Field(min_length=1) + recipes: CharacterRecipePair + panel_input: CharacterBenchmarkPanelInput + + +class CharacterGenerationSpec(FrozenModel): + id: str = Field(pattern=r"^character_generation_[0-9a-f]{64}$") + benchmark_id: str = Field(pattern=r"^character_benchmark_[0-9a-f]{64}$") + context: CharacterBenchmarkContext + seed: int + candidate: Candidate + recipe_digest: str = Field(min_length=1) + + +class CharacterTrial(FrozenModel): + id: str = Field(pattern=r"^character_trial_[0-9a-f]{64}$") + benchmark_id: str = Field(pattern=r"^character_benchmark_[0-9a-f]{64}$") + ordinal: int = Field(ge=1, le=40) + context: CharacterBenchmarkContext + seed: int + hidden_repeat: bool = False + reuses_trial_id: str | None = None + generation_specs: tuple[CharacterGenerationSpec, ...] + candidate_a_position: CandidatePosition + + @model_validator(mode="after") + def require_repeat_media_reuse(self) -> Self: + if self.hidden_repeat: + if self.reuses_trial_id is None: + raise ValueError("hidden repeats must identify their primary source trial") + if self.generation_specs: + raise ValueError("hidden repeats reuse media and cannot create generation specs") + if self.reuses_trial_id == self.id: + raise ValueError("a hidden repeat cannot reuse itself") + return self + if self.reuses_trial_id is not None: + raise ValueError("primary trials cannot reuse another trial") + if tuple(spec.candidate for spec in self.generation_specs) != ("a", "b"): + raise ValueError("primary trials require sealed candidate a and b generation specs") + if any( + spec.context != self.context or spec.seed != self.seed for spec in self.generation_specs + ): + raise ValueError("generation specs must match their trial context and seed") + return self + + +class IdentityObservation(FrozenModel): + benchmark_id: str = Field(pattern=r"^character_benchmark_[0-9a-f]{64}$") + trial_id: str = Field(pattern=r"^character_trial_[0-9a-f]{64}$") + choice: IdentityChoice + + +class SecondaryObservation(FrozenModel): + benchmark_id: str = Field(pattern=r"^character_benchmark_[0-9a-f]{64}$") + trial_id: str = Field(pattern=r"^character_trial_[0-9a-f]{64}$") + dimension: SecondaryDimension + choice: SecondaryChoice + + +class CharacterBenchmarkEvidence(FrozenModel): + definition: CharacterBenchmarkDefinition + trials: tuple[CharacterTrial, ...] + identity_observations: tuple[IdentityObservation, ...] + secondary_observations: tuple[SecondaryObservation, ...] = () + media_integrity_complete: bool = True + provenance_complete: bool = True + + +class IdentityCounts(FrozenModel): + a: int = Field(ge=0) + b: int = Field(ge=0) + both: int = Field(ge=0) + neither: int = Field(ge=0) + + @property + def total(self) -> int: + return self.a + self.b + self.both + self.neither + + +class IdentityPosterior(FrozenModel): + counts: IdentityCounts + control_superiority_probability: float = Field(ge=0, le=1) + identity_recipe_superiority_probability: float = Field(ge=0, le=1) + + +class ContextIdentityResult(FrozenModel): + context: CharacterBenchmarkContext + posterior: IdentityPosterior + both_rate: float = Field(ge=0, le=1) + neither_rate: float = Field(ge=0, le=1) + + +class GroupIdentityResult(FrozenModel): + group: CharacterContextGroup + posterior: IdentityPosterior + both_rate: float = Field(ge=0, le=1) + neither_rate: float = Field(ge=0, le=1) + + +class ContextReversal(FrozenModel): + context: CharacterBenchmarkContext + disfavored_recipe: Literal["control", "identity"] + superiority_probability: float = Field(ge=0, le=1) + + +class SecondaryCounts(FrozenModel): + a: int = Field(ge=0) + b: int = Field(ge=0) + tie: int = Field(ge=0) + + @property + def total(self) -> int: + return self.a + self.b + self.tie + + +class SecondaryDimensionResult(FrozenModel): + counts: SecondaryCounts + observation_count: int = Field(ge=0) + control_superiority_probability: float = Field(ge=0, le=1) + identity_recipe_superiority_probability: float = Field(ge=0, le=1) + + +class SecondarySummary(FrozenModel): + prompt_adherence: SecondaryDimensionResult + motion_quality: SecondaryDimensionResult + temporal_consistency: SecondaryDimensionResult + visual_quality: SecondaryDimensionResult + + +class CharacterBenchmarkResult(FrozenModel): + benchmark_id: str = Field(pattern=r"^character_benchmark_[0-9a-f]{64}$") + analysis_version: Literal["character-dirichlet.v1"] = CHARACTER_BENCHMARK_ANALYSIS_VERSION + baseline_disposition: BaselineDisposition + identity_disposition: IdentityDisposition + posterior: IdentityPosterior + both_rate: float = Field(ge=0, le=1) + neither_rate: float = Field(ge=0, le=1) + context_results: tuple[ContextIdentityResult, ...] + group_results: tuple[GroupIdentityResult, ...] + context_reversals: tuple[ContextReversal, ...] + secondary: SecondarySummary + repeat_agreements: int = Field(ge=0, le=4) + identity_break_contexts: tuple[CharacterContextName, ...] diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_protocol.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_protocol.py new file mode 100644 index 0000000..e215c75 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_protocol.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +import hashlib +import random +from typing import Any + +from video_lab.character_benchmark_models import ( + Candidate, + CandidatePosition, + CharacterBenchmarkContext, + CharacterBenchmarkDefinition, + CharacterBenchmarkPanelInput, + CharacterContextGroup, + CharacterContextName, + CharacterGenerationSpec, + CharacterRecipePair, + CharacterTrial, +) +from video_lab.domain import canonical_json + +DEFAULT_CHARACTER_CONTEXTS = ( + CharacterBenchmarkContext( + ordinal=1, + name=CharacterContextName.NEUTRAL_FACE_PORTRAIT, + group=CharacterContextGroup.CORE, + prompt="Neutral face portrait.", + ), + CharacterBenchmarkContext( + ordinal=2, + name=CharacterContextName.THREE_QUARTER_PORTRAIT, + group=CharacterContextGroup.CORE, + prompt="Three-quarter portrait.", + ), + CharacterBenchmarkContext( + ordinal=3, + name=CharacterContextName.PROFILE_VIEW, + group=CharacterContextGroup.CORE, + prompt="Profile view.", + ), + CharacterBenchmarkContext( + ordinal=4, + name=CharacterContextName.FULL_BODY_NEUTRAL_POSE, + group=CharacterContextGroup.CORE, + prompt="Full-body neutral pose.", + ), + CharacterBenchmarkContext( + ordinal=5, + name=CharacterContextName.FULL_BODY_ACTION, + group=CharacterContextGroup.CORE, + prompt="Full-body action.", + ), + CharacterBenchmarkContext( + ordinal=6, + name=CharacterContextName.STRONG_EXPRESSION, + group=CharacterContextGroup.CORE, + prompt="Strong expression.", + ), + CharacterBenchmarkContext( + ordinal=7, + name=CharacterContextName.CAMERA_MOTION, + group=CharacterContextGroup.CORE, + prompt="Camera motion.", + ), + CharacterBenchmarkContext( + ordinal=8, + name=CharacterContextName.CHARACTER_OBJECT_INTERACTION, + group=CharacterContextGroup.CORE, + prompt="Character-object interaction.", + ), + CharacterBenchmarkContext( + ordinal=9, + name=CharacterContextName.STRONG_DIRECTIONAL_LIGHTING, + group=CharacterContextGroup.STRESS, + prompt="Strong directional lighting.", + ), + CharacterBenchmarkContext( + ordinal=10, + name=CharacterContextName.PARTIAL_FACIAL_OCCLUSION, + group=CharacterContextGroup.STRESS, + prompt="Partial facial occlusion.", + ), + CharacterBenchmarkContext( + ordinal=11, + name=CharacterContextName.FORESHORTENED_OR_DIFFICULT_ANGLE, + group=CharacterContextGroup.STRESS, + prompt="Foreshortened or difficult angle.", + ), + CharacterBenchmarkContext( + ordinal=12, + name=CharacterContextName.BUSY_ENVIRONMENT_WITH_COMPETING_SUBJECTS, + group=CharacterContextGroup.STRESS, + prompt="Busy environment with competing subjects.", + ), +) + + +def build_character_panel( + *, + character_pack_digest: str, + stage_2_manifest_digest: str, + stage_2_evidence_digest: str, + recipes: CharacterRecipePair, + panel_input: CharacterBenchmarkPanelInput, +) -> CharacterBenchmarkDefinition: + payload = { + "schema_version": "character-benchmark.v1", + "protocol_version": "character-panel.v1", + "analysis_version": "character-dirichlet.v1", + "character_pack_digest": character_pack_digest, + "stage_2_manifest_digest": stage_2_manifest_digest, + "stage_2_evidence_digest": stage_2_evidence_digest, + "recipes": recipes.model_dump(mode="json"), + "panel_input": panel_input.model_dump(mode="json"), + } + return CharacterBenchmarkDefinition( + id=_content_id("character_benchmark", payload), + character_pack_digest=character_pack_digest, + stage_2_manifest_digest=stage_2_manifest_digest, + stage_2_evidence_digest=stage_2_evidence_digest, + recipes=recipes, + panel_input=panel_input, + ) + + +def schedule_character_trials( + definition: CharacterBenchmarkDefinition, +) -> tuple[CharacterTrial, ...]: + primary = tuple( + _primary_trial(definition, context, seed) + for context in definition.panel_input.contexts + for seed in definition.panel_input.seeds + ) + repeats = _hidden_repeats(definition, primary) + ordered = _schedule_order(definition, primary, repeats) + positions = _balanced_candidate_a_positions(definition, ordered) + return tuple( + trial.model_copy( + update={ + "ordinal": ordinal, + "candidate_a_position": positions[trial.id], + } + ) + for ordinal, trial in enumerate(ordered, start=1) + ) + + +def _primary_trial( + definition: CharacterBenchmarkDefinition, + context: CharacterBenchmarkContext, + seed: int, +) -> CharacterTrial: + candidates: tuple[Candidate, Candidate] = ("a", "b") + generation_specs = tuple( + _generation_spec(definition, context, seed, candidate) for candidate in candidates + ) + payload = { + "benchmark_id": definition.id, + "context": context.model_dump(mode="json"), + "seed": seed, + "hidden_repeat": False, + "generation_spec_ids": [spec.id for spec in generation_specs], + } + return CharacterTrial( + id=_content_id("character_trial", payload), + benchmark_id=definition.id, + ordinal=1, + context=context, + seed=seed, + generation_specs=generation_specs, + candidate_a_position="left", + ) + + +def _generation_spec( + definition: CharacterBenchmarkDefinition, + context: CharacterBenchmarkContext, + seed: int, + candidate: Candidate, +) -> CharacterGenerationSpec: + recipe_digest = ( + definition.recipes.control_digest + if candidate == "a" + else definition.recipes.identity_digest + ) + payload = { + "benchmark_id": definition.id, + "context": context.model_dump(mode="json"), + "seed": seed, + "candidate": candidate, + "recipe_digest": recipe_digest, + } + return CharacterGenerationSpec( + id=_content_id("character_generation", payload), + benchmark_id=definition.id, + context=context, + seed=seed, + candidate=candidate, + recipe_digest=recipe_digest, + ) + + +def _hidden_repeats( + definition: CharacterBenchmarkDefinition, + primary: tuple[CharacterTrial, ...], +) -> tuple[CharacterTrial, ...]: + generator = random.Random(_domain_seed("character-repeat-selection.v1", definition)) + selected_contexts = generator.sample(list(definition.panel_input.contexts), 4) + sources = tuple( + generator.choice(tuple(trial for trial in primary if trial.context == context)) + for context in selected_contexts + ) + repeats: list[CharacterTrial] = [] + for source in sources: + payload = { + "benchmark_id": definition.id, + "hidden_repeat": True, + "reuses_trial_id": source.id, + "context": source.context.model_dump(mode="json"), + "seed": source.seed, + } + repeats.append( + CharacterTrial( + id=_content_id("character_trial", payload), + benchmark_id=definition.id, + ordinal=1, + context=source.context, + seed=source.seed, + hidden_repeat=True, + reuses_trial_id=source.id, + generation_specs=(), + candidate_a_position="left", + ) + ) + return tuple(repeats) + + +def _schedule_order( + definition: CharacterBenchmarkDefinition, + primary: tuple[CharacterTrial, ...], + repeats: tuple[CharacterTrial, ...], +) -> tuple[CharacterTrial, ...]: + generator = random.Random(_domain_seed("character-order.v1", definition)) + ordered = list(primary) + generator.shuffle(ordered) + pending_repeats = list(repeats) + generator.shuffle(pending_repeats) + for repeat in pending_repeats: + valid_positions = [ + position + for position in range(len(ordered) + 1) + if _can_insert_repeat(ordered, position, repeat) + ] + ordered.insert(generator.choice(valid_positions), repeat) + return tuple(ordered) + + +def _can_insert_repeat( + ordered: list[CharacterTrial], position: int, repeat: CharacterTrial +) -> bool: + source_id = repeat.reuses_trial_id + left_source = _source_id(ordered[position - 1]) if position > 0 else None + right_source = _source_id(ordered[position]) if position < len(ordered) else None + return source_id not in {left_source, right_source} + + +def _source_id(trial: CharacterTrial) -> str: + return trial.reuses_trial_id or trial.id + + +def _balanced_candidate_a_positions( + definition: CharacterBenchmarkDefinition, + trials: tuple[CharacterTrial, ...], +) -> dict[str, CandidatePosition]: + generator = random.Random(_domain_seed("character-display.v1", definition)) + positions: dict[str, CandidatePosition] = {} + left: CandidatePosition = "left" + right: CandidatePosition = "right" + primary_by_context = [ + [trial for trial in trials if not trial.hidden_repeat and trial.context == context] + for context in definition.panel_input.contexts + ] + extra_labels: list[CandidatePosition] = [left] * 6 + [right] * 6 + generator.shuffle(extra_labels) + for group, extra_label in zip(primary_by_context, extra_labels, strict=True): + labels = [left, right, extra_label] + generator.shuffle(labels) + for trial, label in zip(group, labels, strict=True): + positions[trial.id] = label + repeats = [trial for trial in trials if trial.hidden_repeat] + repeat_labels: list[CandidatePosition] = [left, left, right, right] + generator.shuffle(repeat_labels) + for trial, label in zip(repeats, repeat_labels, strict=True): + positions[trial.id] = label + return positions + + +def _domain_seed(domain: str, definition: CharacterBenchmarkDefinition) -> int: + payload = f"{domain}:{definition.panel_input.schedule_seed}:{definition.id}".encode() + return int.from_bytes(hashlib.sha256(payload).digest()[:8], "big") + + +def _content_id(prefix: str, payload: dict[str, Any]) -> str: + digest = hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() + return f"{prefix}_{digest}" diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_repository.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_repository.py new file mode 100644 index 0000000..364be43 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_repository.py @@ -0,0 +1,919 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +import unicodedata +from datetime import datetime +from enum import StrEnum +from typing import Literal, cast + +from pydantic import field_validator + +from video_lab.character_benchmark_models import ( + CharacterBenchmarkDefinition, + CharacterBenchmarkResult, + CharacterTrial, + IdentityObservation, + SecondaryObservation, +) +from video_lab.comparison import ControlledPair +from video_lab.database import Database, character_benchmark_reviewer_id +from video_lab.domain import FrozenModel, JobState, Manifest, _aware_utc, canonical_json + + +class CharacterBenchmarkNotFound(LookupError): + pass + + +class CharacterBenchmarkConflict(RuntimeError): + pass + + +class CharacterBenchmarkState(StrEnum): + DRAFT = "draft" + PREREGISTERED = "preregistered" + GENERATING = "generating" + READY = "ready" + JUDGING = "judging" + ANALYZING = "analyzing" + COMPLETED = "completed" + + +class CharacterBenchmarkEvent(FrozenModel): + campaign_id: str + sequence: int + previous_state: CharacterBenchmarkState + new_state: CharacterBenchmarkState + payload_digest: str + created_at: datetime + + @field_validator("created_at") + @classmethod + def normalize_created_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + +class CharacterBenchmarkCampaign(FrozenModel): + definition: CharacterBenchmarkDefinition + reviewer_id: str + character_pack_id: str + character_pack_storage_id: str + stage_2_campaign_id: str + stage_2_manifest: Manifest + identity_prompt: str + content_digest: str + created_at: datetime + state: CharacterBenchmarkState + + @field_validator("created_at") + @classmethod + def normalize_created_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + +class CharacterIdentityJudgment(FrozenModel): + observation: IdentityObservation + review_id: str + display_choice: Literal["left", "right", "both", "neither"] + content_digest: str + created_at: datetime + + +class CharacterSecondaryJudgment(FrozenModel): + observation: SecondaryObservation + review_id: str + content_digest: str + created_at: datetime + + +class CharacterExperimentBinding(FrozenModel): + campaign_id: str + trial_id: str + experiment_id: str + + +class CharacterReviewBinding(FrozenModel): + campaign_id: str + trial_id: str + experiment_id: str + review_id: str + session_id: str + reviewer_alias: str + reused_from_trial_id: str | None = None + display_mapping: dict[str, dict[str, str]] + + +class CharacterBenchmarkRepository: + """Owner-bound append-only storage for character benchmark evidence.""" + + def __init__(self, database: Database, *, reviewer_id: str) -> None: + normalized = unicodedata.normalize("NFKC", reviewer_id).strip() + if not normalized or normalized != reviewer_id: + raise ValueError("reviewer_id must be normalized non-empty text") + self.database = database + self.reviewer_id = normalized + + def create_or_get_bound( + self, + definition: CharacterBenchmarkDefinition, + *, + character_pack_id: str, + character_pack_storage_id: str, + stage_2_campaign_id: str, + stage_2_manifest: Manifest, + identity_prompt: str, + content_digest: str, + created_at: datetime, + ) -> CharacterBenchmarkCampaign: + validated = CharacterBenchmarkDefinition.model_validate(definition.model_dump()) + manifest = Manifest.model_validate(stage_2_manifest.model_dump()) + prompt = unicodedata.normalize("NFKC", identity_prompt).strip() + if not prompt or prompt != identity_prompt: + raise ValueError("identity prompt must be normalized non-empty text") + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + row = connection.execute( + "SELECT * FROM character_benchmark_campaigns WHERE id=?", (validated.id,) + ).fetchone() + by_digest = connection.execute( + "SELECT id FROM character_benchmark_campaigns WHERE content_digest=?", + (content_digest,), + ).fetchone() + if row is None and by_digest is not None: + raise CharacterBenchmarkConflict( + "character benchmark content digest already belongs to another campaign" + ) + if row is None: + self._require_dependency_owners( + connection, character_pack_storage_id, stage_2_campaign_id + ) + connection.execute( + "INSERT INTO character_benchmark_campaigns(" + "id, reviewer_id, content_digest, definition_json, character_pack_id, " + "character_pack_storage_id, stage_2_campaign_id, stage_2_manifest_json, " + "identity_prompt, created_at) VALUES (?,?,?,?,?,?,?,?,?,?)", + ( + validated.id, + self.reviewer_id, + content_digest, + canonical_json(validated), + character_pack_id, + character_pack_storage_id, + stage_2_campaign_id, + canonical_json(manifest), + prompt, + _aware_utc(created_at).isoformat(), + ), + ) + row = connection.execute( + "SELECT * FROM character_benchmark_campaigns WHERE id=?", (validated.id,) + ).fetchone() + assert row is not None + persisted = self._campaign_from_row(connection, row) + expected = CharacterBenchmarkCampaign( + definition=validated, + reviewer_id=self.reviewer_id, + character_pack_id=character_pack_id, + character_pack_storage_id=character_pack_storage_id, + stage_2_campaign_id=stage_2_campaign_id, + stage_2_manifest=manifest, + identity_prompt=prompt, + content_digest=content_digest, + created_at=created_at, + state=persisted.state, + ) + if persisted != expected: + if persisted.reviewer_id != self.reviewer_id: + raise CharacterBenchmarkConflict( + "character benchmark belongs to another reviewer" + ) + raise CharacterBenchmarkConflict( + "character benchmark retry does not match immutable content" + ) + connection.commit() + return persisted + except sqlite3.IntegrityError as error: + connection.rollback() + raise CharacterBenchmarkConflict(str(error)) from error + except Exception: + connection.rollback() + raise + + def get(self, campaign_id: str) -> CharacterBenchmarkCampaign: + with self.database.connect() as connection: + row = connection.execute( + "SELECT * FROM character_benchmark_campaigns WHERE id=?", (campaign_id,) + ).fetchone() + if row is None: + raise CharacterBenchmarkNotFound("character benchmark not found") + campaign = self._campaign_from_row(connection, row) + if campaign.reviewer_id != self.reviewer_id: + raise CharacterBenchmarkConflict("character benchmark belongs to another reviewer") + return campaign + + def preregister_or_verify( + self, + campaign_id: str, + trials: tuple[CharacterTrial, ...], + *, + created_at: datetime, + ) -> CharacterBenchmarkEvent: + validated = tuple(CharacterTrial.model_validate(item.model_dump()) for item in trials) + payload_digest = _digest([item.model_dump(mode="json") for item in validated]) + event = CharacterBenchmarkEvent( + campaign_id=campaign_id, + sequence=1, + previous_state=CharacterBenchmarkState.DRAFT, + new_state=CharacterBenchmarkState.PREREGISTERED, + payload_digest=payload_digest, + created_at=created_at, + ) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_owned(connection, campaign_id) + rows = connection.execute( + "SELECT trial_json, expected_reviewer_id " + "FROM character_benchmark_trials WHERE campaign_id=? " + "ORDER BY ordinal", + (campaign_id,), + ).fetchall() + if rows: + persisted = tuple( + CharacterTrial.model_validate_json(str(row["trial_json"])) for row in rows + ) + expected_reviewers = tuple( + character_benchmark_reviewer_id(campaign_id, trial.id, self.reviewer_id) + for trial in validated + ) + existing = self._event_at(connection, campaign_id, 1) + if ( + persisted != validated + or tuple(str(row["expected_reviewer_id"]) for row in rows) + != expected_reviewers + or existing != event + ): + raise CharacterBenchmarkConflict( + "character benchmark preregistration retry does not match evidence" + ) + connection.commit() + return existing + self._insert_event(connection, event) + connection.executemany( + "INSERT INTO character_benchmark_trials(" + "campaign_id, trial_id, ordinal, trial_json, hidden_repeat, " + "reuses_trial_id, candidate_a_position, expected_reviewer_id) " + "VALUES (?,?,?,?,?,?,?,?)", + [ + ( + campaign_id, + trial.id, + trial.ordinal, + canonical_json(trial), + int(trial.hidden_repeat), + trial.reuses_trial_id, + trial.candidate_a_position, + character_benchmark_reviewer_id( + campaign_id, trial.id, self.reviewer_id + ), + ) + for trial in validated + ], + ) + connection.commit() + return event + except sqlite3.IntegrityError as error: + connection.rollback() + raise CharacterBenchmarkConflict( + "character benchmark preregistration violates persistence" + ) from error + except Exception: + connection.rollback() + raise + + def append_event_or_verify(self, event: CharacterBenchmarkEvent) -> CharacterBenchmarkEvent: + validated = CharacterBenchmarkEvent.model_validate(event.model_dump()) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_owned(connection, validated.campaign_id) + existing = self._event_at( + connection, validated.campaign_id, validated.sequence, required=False + ) + if existing is not None: + if existing != validated: + raise CharacterBenchmarkConflict( + "character benchmark event retry does not match evidence" + ) + connection.commit() + return existing + self._insert_event(connection, validated) + connection.commit() + return validated + except sqlite3.IntegrityError as error: + connection.rollback() + raise CharacterBenchmarkConflict(str(error)) from error + except Exception: + connection.rollback() + raise + + def list_trials(self, campaign_id: str) -> tuple[CharacterTrial, ...]: + with self.database.connect() as connection: + self._require_owned(connection, campaign_id) + rows = connection.execute( + "SELECT trial_json FROM character_benchmark_trials WHERE campaign_id=? " + "ORDER BY ordinal", + (campaign_id,), + ).fetchall() + return tuple(CharacterTrial.model_validate_json(str(row["trial_json"])) for row in rows) + + def list_events(self, campaign_id: str) -> tuple[CharacterBenchmarkEvent, ...]: + with self.database.connect() as connection: + self._require_owned(connection, campaign_id) + rows = connection.execute( + "SELECT * FROM character_benchmark_events WHERE campaign_id=? ORDER BY sequence", + (campaign_id,), + ).fetchall() + return tuple(_event_from_row(row) for row in rows) + + def bind_experiment_or_verify( + self, + campaign_id: str, + trial: CharacterTrial, + pair: ControlledPair, + *, + created_at: datetime, + ) -> CharacterExperimentBinding: + if trial.hidden_repeat or len(trial.generation_specs) != 2: + raise CharacterBenchmarkConflict("only primary character trials create experiments") + experiment_id = _record_id("exp", campaign_id, trial.id) + variant_ids = tuple(spec.id for spec in trial.generation_specs) + job_ids = tuple(_record_id("job", campaign_id, spec.id) for spec in trial.generation_specs) + binding = CharacterExperimentBinding( + campaign_id=campaign_id, + trial_id=trial.id, + experiment_id=experiment_id, + ) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_owned(connection, campaign_id) + existing = connection.execute( + "SELECT * FROM character_benchmark_experiment_bindings " + "WHERE campaign_id=? AND trial_id=?", + (campaign_id, trial.id), + ).fetchone() + if existing is None: + connection.execute( + "INSERT INTO experiments(id, schema_version, variable_path, " + "changed_paths_json, created_at) VALUES (?,?,?,?,?)", + ( + experiment_id, + "character-controlled.v1", + pair.variable_path, + json.dumps( + list(pair.changed_paths), + sort_keys=True, + separators=(",", ":"), + ), + _aware_utc(created_at).isoformat(), + ), + ) + connection.executemany( + "INSERT INTO variants(id, experiment_id, label, manifest_json) " + "VALUES (?,?,?,?)", + [ + (variant_ids[0], experiment_id, "A", canonical_json(pair.variant_a)), + (variant_ids[1], experiment_id, "B", canonical_json(pair.variant_b)), + ], + ) + connection.executemany( + "INSERT INTO jobs(id, variant_id, state, created_at, updated_at) " + "VALUES (?,?,?,?,?)", + [ + ( + job_id, + variant_id, + JobState.QUEUED.value, + _aware_utc(created_at).isoformat(), + _aware_utc(created_at).isoformat(), + ) + for job_id, variant_id in zip(job_ids, variant_ids, strict=True) + ], + ) + connection.execute( + "INSERT INTO character_benchmark_experiment_bindings VALUES (?,?,?)", + (campaign_id, trial.id, experiment_id), + ) + else: + persisted = _experiment_binding_from_row(existing) + if persisted != binding: + raise CharacterBenchmarkConflict( + "character benchmark experiment retry does not match evidence" + ) + experiment = connection.execute( + "SELECT * FROM experiments WHERE id=?", (experiment_id,) + ).fetchone() + variants = connection.execute( + "SELECT * FROM variants WHERE experiment_id=? ORDER BY label", + (experiment_id,), + ).fetchall() + jobs = connection.execute( + "SELECT job.id, job.variant_id FROM jobs AS job JOIN variants AS variant " + "ON variant.id=job.variant_id WHERE variant.experiment_id=? " + "ORDER BY variant.label", + (experiment_id,), + ).fetchall() + if ( + experiment is None + or str(experiment["schema_version"]) != "character-controlled.v1" + or str(experiment["variable_path"]) != pair.variable_path + or tuple(json.loads(str(experiment["changed_paths_json"]))) + != pair.changed_paths + or len(variants) != 2 + or tuple(str(row["id"]) for row in variants) != variant_ids + or tuple( + Manifest.model_validate_json(str(row["manifest_json"])) + for row in variants + ) + != (pair.variant_a, pair.variant_b) + or tuple(str(row["id"]) for row in jobs) != job_ids + or tuple(str(row["variant_id"]) for row in jobs) != variant_ids + ): + raise CharacterBenchmarkConflict( + "character benchmark experiment retry does not match evidence" + ) + connection.commit() + return binding + except sqlite3.IntegrityError as error: + connection.rollback() + raise CharacterBenchmarkConflict( + "character benchmark experiment violates persistence" + ) from error + except Exception: + connection.rollback() + raise + + def list_experiment_bindings(self, campaign_id: str) -> tuple[CharacterExperimentBinding, ...]: + with self.database.connect() as connection: + self._require_owned(connection, campaign_id) + rows = connection.execute( + "SELECT binding.* FROM character_benchmark_experiment_bindings AS binding " + "JOIN character_benchmark_trials AS trial ON trial.campaign_id=binding.campaign_id " + "AND trial.trial_id=binding.trial_id WHERE binding.campaign_id=? " + "ORDER BY trial.ordinal", + (campaign_id,), + ).fetchall() + return tuple(_experiment_binding_from_row(row) for row in rows) + + def bind_review_or_verify(self, binding: CharacterReviewBinding) -> CharacterReviewBinding: + validated = CharacterReviewBinding.model_validate(binding.model_dump()) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_owned(connection, validated.campaign_id) + row = connection.execute( + "SELECT * FROM character_benchmark_review_bindings " + "WHERE campaign_id=? AND trial_id=?", + (validated.campaign_id, validated.trial_id), + ).fetchone() + if row is None: + connection.execute( + "INSERT INTO character_benchmark_review_bindings VALUES (?,?,?,?,?,?,?,?)", + ( + validated.campaign_id, + validated.trial_id, + validated.experiment_id, + validated.review_id, + validated.session_id, + validated.reviewer_alias, + validated.reused_from_trial_id, + canonical_json(validated.display_mapping), + ), + ) + persisted = validated + else: + persisted = _review_binding_from_row(row) + if persisted != validated: + raise CharacterBenchmarkConflict( + "character benchmark review retry does not match evidence" + ) + connection.commit() + return persisted + except sqlite3.IntegrityError as error: + connection.rollback() + raise CharacterBenchmarkConflict( + "character benchmark review violates persistence" + ) from error + except Exception: + connection.rollback() + raise + + def list_review_bindings(self, campaign_id: str) -> tuple[CharacterReviewBinding, ...]: + with self.database.connect() as connection: + self._require_owned(connection, campaign_id) + rows = connection.execute( + "SELECT binding.* FROM character_benchmark_review_bindings AS binding " + "JOIN character_benchmark_trials AS trial ON trial.campaign_id=binding.campaign_id " + "AND trial.trial_id=binding.trial_id WHERE binding.campaign_id=? " + "ORDER BY trial.ordinal", + (campaign_id,), + ).fetchall() + return tuple(_review_binding_from_row(row) for row in rows) + + def record_identity_or_verify( + self, + observation: IdentityObservation, + *, + review_id: str, + display_choice: Literal["left", "right", "both", "neither"], + created_at: datetime, + ) -> CharacterIdentityJudgment: + validated = IdentityObservation.model_validate(observation.model_dump()) + payload = { + "observation": validated.model_dump(mode="json"), + "review_id": review_id, + "display_choice": display_choice, + } + judgment = CharacterIdentityJudgment( + observation=validated, + review_id=review_id, + display_choice=display_choice, + content_digest=_digest(payload), + created_at=_aware_utc(created_at), + ) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_owned(connection, validated.benchmark_id) + row = connection.execute( + "SELECT * FROM character_benchmark_identity_judgments " + "WHERE campaign_id=? AND trial_id=?", + (validated.benchmark_id, validated.trial_id), + ).fetchone() + if row is None: + connection.execute( + "INSERT INTO character_benchmark_identity_judgments VALUES " + "(?,?,?,?,?,?,?,?)", + ( + validated.benchmark_id, + validated.trial_id, + review_id, + validated.choice.value, + display_choice, + canonical_json(validated), + judgment.content_digest, + judgment.created_at.isoformat(), + ), + ) + persisted = judgment + else: + persisted = _identity_from_row(row) + if persisted != judgment: + raise CharacterBenchmarkConflict( + "character benchmark identity retry does not match evidence" + ) + connection.commit() + return persisted + except sqlite3.IntegrityError as error: + connection.rollback() + raise CharacterBenchmarkConflict(str(error)) from error + except Exception: + connection.rollback() + raise + + def record_secondary_or_verify( + self, observation: SecondaryObservation, *, created_at: datetime + ) -> CharacterSecondaryJudgment: + validated = SecondaryObservation.model_validate(observation.model_dump()) + digest = _digest(validated.model_dump(mode="json")) + normalized_created_at = _aware_utc(created_at) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_owned(connection, validated.benchmark_id) + identity = connection.execute( + "SELECT review_id FROM character_benchmark_identity_judgments " + "WHERE campaign_id=? AND trial_id=?", + (validated.benchmark_id, validated.trial_id), + ).fetchone() + if identity is None: + raise CharacterBenchmarkConflict( + "character benchmark secondary requires identity evidence" + ) + review_id = str(identity["review_id"]) + judgment = CharacterSecondaryJudgment( + observation=validated, + review_id=review_id, + content_digest=digest, + created_at=normalized_created_at, + ) + row = connection.execute( + "SELECT * FROM character_benchmark_secondary_judgments WHERE " + "campaign_id=? AND trial_id=? AND dimension=?", + (validated.benchmark_id, validated.trial_id, validated.dimension.value), + ).fetchone() + if row is None: + connection.execute( + "INSERT INTO character_benchmark_secondary_judgments(" + "campaign_id,trial_id,dimension,choice,observation_json," + "content_digest,created_at,review_id) VALUES (?,?,?,?,?,?,?,?)", + ( + validated.benchmark_id, + validated.trial_id, + validated.dimension.value, + validated.choice.value, + canonical_json(validated), + digest, + judgment.created_at.isoformat(), + review_id, + ), + ) + persisted = judgment + else: + persisted = _secondary_from_row(row) + if persisted != judgment: + raise CharacterBenchmarkConflict( + "character benchmark secondary retry does not match evidence" + ) + connection.commit() + return persisted + except sqlite3.IntegrityError as error: + connection.rollback() + raise CharacterBenchmarkConflict(str(error)) from error + except Exception: + connection.rollback() + raise + + def list_identity(self, campaign_id: str) -> tuple[CharacterIdentityJudgment, ...]: + with self.database.connect() as connection: + self._require_owned(connection, campaign_id) + rows = connection.execute( + "SELECT judgment.* FROM character_benchmark_identity_judgments AS judgment " + "JOIN character_benchmark_trials AS trial " + "ON trial.campaign_id=judgment.campaign_id " + "AND trial.trial_id=judgment.trial_id WHERE judgment.campaign_id=? " + "ORDER BY trial.ordinal", + (campaign_id,), + ).fetchall() + return tuple(_identity_from_row(row) for row in rows) + + def list_secondary(self, campaign_id: str) -> tuple[CharacterSecondaryJudgment, ...]: + with self.database.connect() as connection: + self._require_owned(connection, campaign_id) + rows = connection.execute( + "SELECT judgment.* FROM character_benchmark_secondary_judgments AS judgment " + "JOIN character_benchmark_trials AS trial " + "ON trial.campaign_id=judgment.campaign_id " + "AND trial.trial_id=judgment.trial_id WHERE judgment.campaign_id=? " + "ORDER BY trial.ordinal, judgment.dimension", + (campaign_id,), + ).fetchall() + return tuple(_secondary_from_row(row) for row in rows) + + def append_result_or_verify( + self, result: CharacterBenchmarkResult, *, created_at: datetime + ) -> CharacterBenchmarkResult: + validated = CharacterBenchmarkResult.model_validate(result.model_dump()) + digest = _digest(validated.model_dump(mode="json")) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_owned(connection, validated.benchmark_id) + row = connection.execute( + "SELECT * FROM character_benchmark_results WHERE campaign_id=?", + (validated.benchmark_id,), + ).fetchone() + if row is None: + connection.execute( + "INSERT INTO character_benchmark_results VALUES (?,?,?,?)", + ( + validated.benchmark_id, + canonical_json(validated), + digest, + _aware_utc(created_at).isoformat(), + ), + ) + persisted = validated + else: + persisted = CharacterBenchmarkResult.model_validate_json( + str(row["result_json"]) + ) + if persisted != validated or str(row["content_digest"]) != digest: + raise CharacterBenchmarkConflict( + "character benchmark result retry does not match evidence" + ) + connection.commit() + return persisted + except sqlite3.IntegrityError as error: + connection.rollback() + raise CharacterBenchmarkConflict(str(error)) from error + except Exception: + connection.rollback() + raise + + def get_result(self, campaign_id: str) -> CharacterBenchmarkResult | None: + with self.database.connect() as connection: + self._require_owned(connection, campaign_id) + row = connection.execute( + "SELECT result_json FROM character_benchmark_results WHERE campaign_id=?", + (campaign_id,), + ).fetchone() + return ( + None + if row is None + else CharacterBenchmarkResult.model_validate_json(str(row["result_json"])) + ) + + def _require_dependency_owners( + self, connection: sqlite3.Connection, pack_storage_id: str, stage_2_campaign_id: str + ) -> None: + pack = connection.execute( + "SELECT reviewer_id, state FROM character_packs WHERE storage_id=?", + (pack_storage_id,), + ).fetchone() + stage_2 = connection.execute( + "SELECT reviewer_id FROM validation_campaigns WHERE id=?", (stage_2_campaign_id,) + ).fetchone() + if ( + pack is None + or str(pack["reviewer_id"]) != self.reviewer_id + or str(pack["state"]) != "preregistered" + or stage_2 is None + or str(stage_2["reviewer_id"]) != self.reviewer_id + ): + raise CharacterBenchmarkConflict( + "character benchmark dependencies do not belong to reviewer" + ) + + def _require_owned(self, connection: sqlite3.Connection, campaign_id: str) -> sqlite3.Row: + row = connection.execute( + "SELECT * FROM character_benchmark_campaigns WHERE id=?", (campaign_id,) + ).fetchone() + if row is None: + raise CharacterBenchmarkNotFound("character benchmark not found") + if str(row["reviewer_id"]) != self.reviewer_id: + raise CharacterBenchmarkConflict("character benchmark belongs to another reviewer") + return cast(sqlite3.Row, row) + + def _campaign_from_row( + self, connection: sqlite3.Connection, row: sqlite3.Row + ) -> CharacterBenchmarkCampaign: + latest = connection.execute( + "SELECT new_state FROM character_benchmark_events WHERE campaign_id=? " + "ORDER BY sequence DESC LIMIT 1", + (row["id"],), + ).fetchone() + return CharacterBenchmarkCampaign( + definition=CharacterBenchmarkDefinition.model_validate_json( + str(row["definition_json"]) + ), + reviewer_id=str(row["reviewer_id"]), + character_pack_id=str(row["character_pack_id"]), + character_pack_storage_id=str(row["character_pack_storage_id"]), + stage_2_campaign_id=str(row["stage_2_campaign_id"]), + stage_2_manifest=Manifest.model_validate_json(str(row["stage_2_manifest_json"])), + identity_prompt=str(row["identity_prompt"]), + content_digest=str(row["content_digest"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + state=CharacterBenchmarkState(str(latest["new_state"]) if latest else "draft"), + ) + + @staticmethod + def _insert_event(connection: sqlite3.Connection, event: CharacterBenchmarkEvent) -> None: + connection.execute( + "INSERT INTO character_benchmark_events VALUES (?,?,?,?,?,?)", + ( + event.campaign_id, + event.sequence, + event.previous_state.value, + event.new_state.value, + event.payload_digest, + event.created_at.isoformat(), + ), + ) + + @staticmethod + def _event_at( + connection: sqlite3.Connection, + campaign_id: str, + sequence: int, + *, + required: bool = True, + ) -> CharacterBenchmarkEvent | None: + row = connection.execute( + "SELECT * FROM character_benchmark_events WHERE campaign_id=? AND sequence=?", + (campaign_id, sequence), + ).fetchone() + if row is None: + if required: + raise CharacterBenchmarkConflict("character benchmark event is missing") + return None + return _event_from_row(row) + + +def _digest(value: object) -> str: + payload = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + default=lambda item: item.model_dump(mode="json"), + ) + return "sha256:" + hashlib.sha256(payload.encode()).hexdigest() + + +def _event_from_row(row: sqlite3.Row) -> CharacterBenchmarkEvent: + return CharacterBenchmarkEvent( + campaign_id=str(row["campaign_id"]), + sequence=int(row["sequence"]), + previous_state=CharacterBenchmarkState(str(row["previous_state"])), + new_state=CharacterBenchmarkState(str(row["new_state"])), + payload_digest=str(row["payload_digest"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + ) + + +def _identity_from_row(row: sqlite3.Row) -> CharacterIdentityJudgment: + display_choice = str(row["display_choice"]) + if display_choice not in {"left", "right", "both", "neither"}: + raise CharacterBenchmarkConflict("character benchmark display choice is invalid") + observation = IdentityObservation.model_validate_json(str(row["observation_json"])) + expected_digest = _digest( + { + "observation": observation.model_dump(mode="json"), + "review_id": str(row["review_id"]), + "display_choice": display_choice, + } + ) + if ( + str(row["campaign_id"]) != observation.benchmark_id + or str(row["trial_id"]) != observation.trial_id + or str(row["choice"]) != observation.choice.value + or str(row["content_digest"]) != expected_digest + ): + raise CharacterBenchmarkConflict( + "character benchmark identity judgment content is inconsistent" + ) + return CharacterIdentityJudgment( + observation=observation, + review_id=str(row["review_id"]), + display_choice=cast(Literal["left", "right", "both", "neither"], display_choice), + content_digest=str(row["content_digest"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + ) + + +def _secondary_from_row(row: sqlite3.Row) -> CharacterSecondaryJudgment: + observation = SecondaryObservation.model_validate_json(str(row["observation_json"])) + expected_digest = _digest(observation.model_dump(mode="json")) + if ( + str(row["campaign_id"]) != observation.benchmark_id + or str(row["trial_id"]) != observation.trial_id + or str(row["dimension"]) != observation.dimension.value + or str(row["choice"]) != observation.choice.value + or str(row["content_digest"]) != expected_digest + ): + raise CharacterBenchmarkConflict( + "character benchmark secondary judgment content is inconsistent" + ) + return CharacterSecondaryJudgment( + observation=observation, + review_id=str(row["review_id"]), + content_digest=str(row["content_digest"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + ) + + +def _record_id(kind: str, campaign_id: str, identity: str) -> str: + payload = { + "domain": f"video-lab.character-benchmark.{kind}", + "campaign_id": campaign_id, + "identity": identity, + } + return f"character_{kind}_{hashlib.sha256(canonical_json(payload).encode()).hexdigest()}" + + +def _experiment_binding_from_row(row: sqlite3.Row) -> CharacterExperimentBinding: + return CharacterExperimentBinding( + campaign_id=str(row["campaign_id"]), + trial_id=str(row["trial_id"]), + experiment_id=str(row["experiment_id"]), + ) + + +def _review_binding_from_row(row: sqlite3.Row) -> CharacterReviewBinding: + return CharacterReviewBinding( + campaign_id=str(row["campaign_id"]), + trial_id=str(row["trial_id"]), + experiment_id=str(row["experiment_id"]), + review_id=str(row["review_id"]), + session_id=str(row["session_id"]), + reviewer_alias=str(row["reviewer_alias"]), + reused_from_trial_id=( + None if row["reused_from_trial_id"] is None else str(row["reused_from_trial_id"]) + ), + display_mapping=json.loads(str(row["display_mapping_json"])), + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_service.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_service.py new file mode 100644 index 0000000..c18464e --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_benchmark_service.py @@ -0,0 +1,1062 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from collections.abc import Callable +from typing import Any, Literal, cast + +from video_lab.artifacts import ArtifactError +from video_lab.character_benchmark_analysis import analyze_character_benchmark +from video_lab.character_benchmark_models import ( + CharacterBenchmarkDefinition, + CharacterBenchmarkEvidence, + CharacterBenchmarkPanelInput, + CharacterBenchmarkResult, + CharacterRecipePair, + CharacterTrial, + IdentityChoice, + IdentityObservation, + SecondaryDimension, + SecondaryObservation, +) +from video_lab.character_benchmark_protocol import build_character_panel, schedule_character_trials +from video_lab.character_benchmark_repository import ( + CharacterBenchmarkCampaign, + CharacterBenchmarkConflict, + CharacterBenchmarkEvent, + CharacterBenchmarkNotFound, + CharacterBenchmarkRepository, + CharacterBenchmarkState, + CharacterReviewBinding, +) +from video_lab.database import character_benchmark_reviewer_id +from video_lab.domain import ( + DisplayPreference, + ExperimentDraft, + FrozenModel, + JobState, + Manifest, + canonical_json, + utc_now, +) + + +class CharacterBenchmarkProgress(FrozenModel): + completed_identity: int + total_identity: Literal[40] = 40 + generated_primary: int + total_primary: Literal[36] = 36 + + +class CharacterBenchmarkPreview(FrozenModel): + definition: CharacterBenchmarkDefinition + primary_count: Literal[36] = 36 + repeat_count: Literal[4] = 4 + + +class CharacterBenchmarkMedia(FrozenModel): + position: Literal["left", "right"] + media_ref: str + + +class CharacterBenchmarkView(FrozenModel): + campaign_id: str + state: CharacterBenchmarkState + next_action: str + progress: CharacterBenchmarkProgress + trial_id: str | None = None + review_id: str | None = None + media: tuple[CharacterBenchmarkMedia, ...] = () + result: CharacterBenchmarkResult | None = None + + +class CharacterBenchmarkService: + def __init__( + self, + campaigns: CharacterBenchmarkRepository, + character_packs: Any, + stage_2: Any, + experiments: Any, + reviews: Any, + adapters: Any, + *, + reviewer_id: str, + clock: Callable[[], Any] = utc_now, + ) -> None: + self.campaigns = campaigns + self.character_packs = character_packs + self.stage_2 = stage_2 + self.experiments = experiments + self.reviews = reviews + self.adapters = adapters + self.reviewer_id = reviewer_id + self.clock = clock + + def create( + self, + *, + character_pack_id: str, + stage_2_campaign_id: str, + identity_prompt: str, + panel_input: CharacterBenchmarkPanelInput, + ) -> CharacterBenchmarkView: + panel = CharacterBenchmarkPanelInput.model_validate(panel_input.model_dump()) + pack_export = self._pack_export(character_pack_id) + stage_2_evidence, base_manifest = self._stage_2_evidence(stage_2_campaign_id) + if base_manifest.loras: + raise CharacterBenchmarkConflict("character benchmark forbids LoRA bindings") + pack_digest = _digest(pack_export.pack.model_dump(mode="json")) + manifest_digest = _digest(base_manifest.model_dump(mode="json")) + evidence_digest = _digest(stage_2_evidence) + recipes = _recipe_pair(identity_prompt) + definition = build_character_panel( + character_pack_digest=pack_digest, + stage_2_manifest_digest=manifest_digest, + stage_2_evidence_digest=evidence_digest, + recipes=recipes, + panel_input=panel, + ) + with self.campaigns.database.connect() as connection: + pack = connection.execute( + "SELECT storage_id, id, reviewer_id, state FROM character_packs WHERE id=?", + (character_pack_id,), + ).fetchone() + if ( + pack is None + or str(pack["reviewer_id"]) != self.reviewer_id + or str(pack["state"]) != "preregistered" + or str(pack["id"]) != character_pack_id + ): + raise CharacterBenchmarkConflict("character pack is not owned and frozen") + try: + created_at = self.campaigns.get(definition.id).created_at + except CharacterBenchmarkNotFound: + created_at = self.clock() + content_digest = _digest( + { + "definition": definition.model_dump(mode="json"), + "character_pack_id": character_pack_id, + "character_pack_storage_id": str(pack["storage_id"]), + "stage_2_campaign_id": stage_2_campaign_id, + "stage_2_manifest": base_manifest.model_dump(mode="json"), + "identity_prompt": identity_prompt, + } + ) + campaign = self.campaigns.create_or_get_bound( + definition, + character_pack_id=character_pack_id, + character_pack_storage_id=str(pack["storage_id"]), + stage_2_campaign_id=stage_2_campaign_id, + stage_2_manifest=base_manifest, + identity_prompt=identity_prompt, + content_digest=content_digest, + created_at=created_at, + ) + return self._view(campaign) + + def preview(self, campaign_id: str) -> CharacterBenchmarkPreview: + campaign = self.campaigns.get(campaign_id) + if campaign.state is not CharacterBenchmarkState.DRAFT: + raise PermissionError("character benchmark schedule is sealed after preregistration") + return CharacterBenchmarkPreview(definition=campaign.definition) + + def preregister(self, campaign_id: str) -> CharacterBenchmarkView: + campaign = self.campaigns.get(campaign_id) + state = campaign.state + if state is CharacterBenchmarkState.DRAFT: + self._validate_bound_inputs(campaign) + self.campaigns.preregister_or_verify( + campaign_id, + schedule_character_trials(campaign.definition), + created_at=self.clock(), + ) + state = CharacterBenchmarkState.PREREGISTERED + if state is CharacterBenchmarkState.PREREGISTERED: + self.campaigns.append_event_or_verify( + self._event(campaign_id, 2, state, CharacterBenchmarkState.GENERATING, "generation") + ) + state = CharacterBenchmarkState.GENERATING + if state is not CharacterBenchmarkState.GENERATING: + return self.show(campaign_id) + capabilities = self.adapters.capabilities(campaign.stage_2_manifest.adapter_id) + for trial in self.campaigns.list_trials(campaign_id): + if trial.hidden_repeat: + continue + pair = self.experiments.preview(self._draft(campaign, trial), capabilities) + if pair.changed_paths != ("prompt",) or pair.variant_a.loras or pair.variant_b.loras: + raise CharacterBenchmarkConflict( + "character benchmark candidate recipe must be prompt-only without LoRAs" + ) + self.campaigns.bind_experiment_or_verify( + campaign_id, trial, pair, created_at=self.clock() + ) + return self.show(campaign_id) + + def show(self, campaign_id: str) -> CharacterBenchmarkView: + return self._view(self.campaigns.get(campaign_id)) + + def resume(self, campaign_id: str) -> CharacterBenchmarkView: + campaign = self.campaigns.get(campaign_id) + state = campaign.state + if state is CharacterBenchmarkState.DRAFT: + return self._view(campaign) + if state in {CharacterBenchmarkState.PREREGISTERED, CharacterBenchmarkState.GENERATING}: + self.preregister(campaign_id) + bindings = self.campaigns.list_experiment_bindings(campaign_id) + jobs = tuple( + job + for binding in bindings + for job in self.experiments.get(binding.experiment_id).jobs + ) + if ( + len(bindings) != 36 + or len(jobs) != 72 + or any(job.state is not JobState.COMPLETED for job in jobs) + ): + return self.show(campaign_id) + self.campaigns.append_event_or_verify( + self._event( + campaign_id, + 3, + CharacterBenchmarkState.GENERATING, + CharacterBenchmarkState.READY, + "ready", + ) + ) + state = CharacterBenchmarkState.READY + if state is CharacterBenchmarkState.READY: + self.campaigns.append_event_or_verify( + self._event(campaign_id, 4, state, CharacterBenchmarkState.JUDGING, "judging") + ) + state = CharacterBenchmarkState.JUDGING + if state is CharacterBenchmarkState.JUDGING: + trial = self._next_incomplete_trial(campaign_id) + if trial is None: + return self.analyze(campaign_id) + return self._judging_view(campaign_id, trial) + if state in {CharacterBenchmarkState.ANALYZING, CharacterBenchmarkState.COMPLETED}: + return self.analyze(campaign_id) + return self.show(campaign_id) + + def record_identity( + self, campaign_id: str, review_id: str, observation: IdentityObservation + ) -> CharacterBenchmarkView: + campaign = self.campaigns.get(campaign_id) + validated = IdentityObservation.model_validate(observation.model_dump()) + if validated.benchmark_id != campaign_id: + raise CharacterBenchmarkConflict("identity observation belongs to another benchmark") + existing_binding = self._review_binding_for_session(campaign_id, review_id) + if existing_binding is not None: + bound_trial = self._trial(campaign_id, existing_binding.trial_id) + if validated.trial_id != bound_trial.id: + raise CharacterBenchmarkConflict( + "identity observation retry does not match bound review" + ) + existing_identity = next( + ( + item + for item in self.campaigns.list_identity(campaign_id) + if item.observation.trial_id == bound_trial.id + ), + None, + ) + if existing_identity is not None: + self._validate_bound_identity_review( + campaign, + bound_trial, + existing_binding, + existing_identity.observation, + ) + if existing_identity.observation != validated: + raise CharacterBenchmarkConflict( + "identity observation retry does not match immutable evidence" + ) + return self.show(campaign_id) + trial = self._next_incomplete_trial(campaign_id) + if trial is None or trial.id != validated.trial_id: + existing = next( + ( + item + for item in self.campaigns.list_identity(campaign_id) + if item.observation.trial_id == validated.trial_id + ), + None, + ) + if existing is None: + raise CharacterBenchmarkConflict("identity observation is not next in schedule") + session, experiment_id, reviewer_alias = self._authorized_session( + campaign, trial or self._trial(campaign_id, validated.trial_id), review_id + ) + bound_trial = trial or self._trial(campaign_id, validated.trial_id) + mapping = self._display_mapping(bound_trial, session) + display_choice = _display_choice(validated.choice, bound_trial.candidate_a_position) + preference = _underlying_preference(display_choice, mapping) + try: + revision = self.reviews.reviews.submit_adaptive(review_id, preference) + except (LookupError, ValueError) as error: + raise CharacterBenchmarkConflict( + "character benchmark review retry does not match evidence" + ) from error + self.campaigns.bind_review_or_verify( + CharacterReviewBinding( + campaign_id=campaign_id, + trial_id=bound_trial.id, + experiment_id=experiment_id, + review_id=revision.id, + session_id=review_id, + reviewer_alias=reviewer_alias, + reused_from_trial_id=bound_trial.reuses_trial_id, + display_mapping=mapping, + ) + ) + self.campaigns.record_identity_or_verify( + validated, + review_id=revision.id, + display_choice=display_choice, + created_at=self.clock(), + ) + if self._requires_secondary(bound_trial, validated.choice): + secondary = { + item.observation.dimension + for item in self.campaigns.list_secondary(campaign_id) + if item.observation.trial_id == bound_trial.id + } + if secondary != set(SecondaryDimension): + return self._judging_view(campaign_id, bound_trial) + return self.resume(campaign_id) + + def record_secondary( + self, campaign_id: str, review_id: str, observation: SecondaryObservation + ) -> CharacterBenchmarkView: + campaign = self.campaigns.get(campaign_id) + validated = SecondaryObservation.model_validate(observation.model_dump()) + if validated.benchmark_id != campaign_id: + raise CharacterBenchmarkConflict("secondary observation belongs to another benchmark") + binding = self._review_binding_for_session(campaign_id, review_id) + if binding is None or binding.trial_id != validated.trial_id: + raise CharacterBenchmarkConflict("secondary observation is not bound to review") + trial = self._trial(campaign_id, binding.trial_id) + identity = next( + ( + item + for item in self.campaigns.list_identity(campaign_id) + if item.observation.trial_id == trial.id + ), + None, + ) + if identity is None or not self._requires_secondary(trial, identity.observation.choice): + raise CharacterBenchmarkConflict("secondary observation is not eligible") + self._validate_bound_identity_review(campaign, trial, binding, identity.observation) + existing = next( + ( + item.observation + for item in self.campaigns.list_secondary(campaign_id) + if item.observation.trial_id == trial.id + and item.observation.dimension is validated.dimension + ), + None, + ) + if existing is not None: + if existing != validated: + raise CharacterBenchmarkConflict( + "secondary observation retry does not match immutable evidence" + ) + return self.show(campaign_id) + if self._next_incomplete_trial(campaign_id) != trial: + raise CharacterBenchmarkConflict("secondary observation is not next in schedule") + self.campaigns.record_secondary_or_verify(validated, created_at=self.clock()) + supplied = { + item.observation.dimension + for item in self.campaigns.list_secondary(campaign_id) + if item.observation.trial_id == trial.id + } + if supplied == set(SecondaryDimension): + return self.resume(campaign_id) + return self._judging_view(campaign_id, trial) + + def analyze(self, campaign_id: str) -> CharacterBenchmarkView: + campaign = self.campaigns.get(campaign_id) + evidence = self._validated_evidence(campaign, require_terminal=True) + state = campaign.state + if state is CharacterBenchmarkState.JUDGING: + self.campaigns.append_event_or_verify( + self._event(campaign_id, 5, state, CharacterBenchmarkState.ANALYZING, "analysis") + ) + state = CharacterBenchmarkState.ANALYZING + if state not in {CharacterBenchmarkState.ANALYZING, CharacterBenchmarkState.COMPLETED}: + raise CharacterBenchmarkConflict("character benchmark cannot enter analysis") + calculated = analyze_character_benchmark(evidence) + persisted = self.campaigns.get_result(campaign_id) + if persisted is None: + persisted = self.campaigns.append_result_or_verify(calculated, created_at=self.clock()) + elif persisted != calculated: + raise CharacterBenchmarkConflict("character benchmark result digest is inconsistent") + if state is CharacterBenchmarkState.ANALYZING: + self.campaigns.append_event_or_verify( + self._event(campaign_id, 6, state, CharacterBenchmarkState.COMPLETED, "completed") + ) + return self.show(campaign_id) + + def export_evidence(self, campaign_id: str) -> CharacterBenchmarkEvidence: + campaign = self.campaigns.get(campaign_id) + if campaign.state is not CharacterBenchmarkState.COMPLETED: + if self.campaigns.get_result(campaign_id) is not None: + self._validated_evidence(campaign, require_terminal=True) + raise CharacterBenchmarkConflict( + "character benchmark terminal lifecycle is inconsistent" + ) + raise PermissionError("character benchmark evidence remains sealed") + return self._validated_evidence(campaign, require_terminal=True) + + def _pack_export(self, pack_id: str) -> Any: + try: + exported = self.character_packs.export_pack(pack_id) + except ( + LookupError, + PermissionError, + RuntimeError, + ValueError, + ArtifactError, + OSError, + ) as error: + raise CharacterBenchmarkConflict("character pack evidence is not canonical") from error + if getattr(exported.pack, "pack_id", None) != pack_id: + raise CharacterBenchmarkConflict("character pack version does not match binding") + return exported + + def _stage_2_evidence(self, campaign_id: str) -> tuple[dict[str, object], Manifest]: + try: + view = self.stage_2.show(campaign_id) + state = getattr(view.state, "value", view.state) + evidence = self.stage_2.export_evidence(campaign_id) + except (LookupError, PermissionError, RuntimeError, ValueError) as error: + raise CharacterBenchmarkConflict( + "Stage 2 evidence is not owned and terminal" + ) from error + if state != "qualified" or not isinstance(evidence, dict): + raise CharacterBenchmarkConflict("Stage 2 evidence is not qualified") + try: + result = cast(dict[str, object], evidence["result"]) + campaign = cast(dict[str, object], evidence["campaign"]) + base = Manifest.model_validate(campaign["base_manifest"]) + except (KeyError, TypeError, ValueError) as error: + raise CharacterBenchmarkConflict("Stage 2 evidence is incomplete") from error + if result.get("disposition") != "qualified": + raise CharacterBenchmarkConflict("Stage 2 evidence is not qualified") + return evidence, base + + def _validate_bound_inputs(self, campaign: CharacterBenchmarkCampaign) -> None: + exported = self._pack_export(campaign.character_pack_id) + stage_evidence, manifest = self._stage_2_evidence(campaign.stage_2_campaign_id) + expected_definition = build_character_panel( + character_pack_digest=_digest(exported.pack.model_dump(mode="json")), + stage_2_manifest_digest=_digest(manifest.model_dump(mode="json")), + stage_2_evidence_digest=_digest(stage_evidence), + recipes=_recipe_pair(campaign.identity_prompt), + panel_input=campaign.definition.panel_input, + ) + expected_content_digest = _digest( + { + "definition": campaign.definition.model_dump(mode="json"), + "character_pack_id": campaign.character_pack_id, + "character_pack_storage_id": campaign.character_pack_storage_id, + "stage_2_campaign_id": campaign.stage_2_campaign_id, + "stage_2_manifest": manifest.model_dump(mode="json"), + "identity_prompt": campaign.identity_prompt, + } + ) + if ( + expected_definition != campaign.definition + or manifest != campaign.stage_2_manifest + or expected_content_digest != campaign.content_digest + ): + raise CharacterBenchmarkConflict("character benchmark bound evidence digest changed") + + def _validated_evidence( + self, campaign: CharacterBenchmarkCampaign, *, require_terminal: bool + ) -> CharacterBenchmarkEvidence: + self._validate_bound_inputs(campaign) + trials = self.campaigns.list_trials(campaign.definition.id) + if trials != schedule_character_trials(campaign.definition): + raise CharacterBenchmarkConflict("character benchmark trial binding is inconsistent") + experiment_bindings = self.campaigns.list_experiment_bindings(campaign.definition.id) + by_trial = {item.trial_id: item for item in experiment_bindings} + primary = tuple(item for item in trials if not item.hidden_repeat) + if len(experiment_bindings) != 36 or set(by_trial) != {item.id for item in primary}: + raise CharacterBenchmarkConflict( + "character benchmark experiment provenance is incomplete" + ) + artifact_sets: dict[str, frozenset[tuple[str, str]]] = {} + for trial in primary: + experiment_binding = by_trial[trial.id] + pair = self.experiments.preview( + self._draft(campaign, trial), + self.adapters.capabilities(campaign.stage_2_manifest.adapter_id), + ) + view = self.experiments.get(experiment_binding.experiment_id) + expected_jobs = { + _record_id("job", campaign.definition.id, spec.id): spec.id + for spec in trial.generation_specs + } + if ( + experiment_binding.experiment_id + != _record_id("exp", campaign.definition.id, trial.id) + or view.experiment.schema_version != "character-controlled.v1" + or view.experiment.variable_path != "prompt" + or view.experiment.changed_paths != ("prompt",) + or tuple(item.id for item in view.experiment.variants) + != tuple(item.id for item in trial.generation_specs) + or tuple(item.manifest for item in view.experiment.variants) + != (pair.variant_a, pair.variant_b) + or len(view.jobs) != 2 + or {item.id: item.variant_id for item in view.jobs} != expected_jobs + or any(item.state is not JobState.COMPLETED for item in view.jobs) + ): + raise CharacterBenchmarkConflict( + "character benchmark experiment recipe is inconsistent" + ) + artifacts: set[tuple[str, str]] = set() + for job in view.jobs: + registered = self.experiments.artifacts.list_for_job(job.id) + if len(registered) != 1 or registered[0].media_type != "video/mp4": + raise CharacterBenchmarkConflict( + "character benchmark artifact ownership is inconsistent" + ) + artifact = registered[0] + try: + self.experiments.artifacts.resolve(artifact.id) + except (ArtifactError, OSError) as error: + raise CharacterBenchmarkConflict( + "character benchmark artifact bytes are not canonical" + ) from error + artifacts.add((artifact.id, artifact.sha256)) + if len(artifacts) != 2: + raise CharacterBenchmarkConflict( + "character benchmark variant artifacts are not distinct" + ) + artifact_sets[trial.id] = frozenset(artifacts) + identities = self.campaigns.list_identity(campaign.definition.id) + secondary = self.campaigns.list_secondary(campaign.definition.id) + reviews = self.campaigns.list_review_bindings(campaign.definition.id) + if len(identities) != 40 or len(reviews) != 40: + raise CharacterBenchmarkConflict("character benchmark review evidence is incomplete") + identity_by_trial = {item.observation.trial_id: item for item in identities} + review_by_trial = {item.trial_id: item for item in reviews} + if set(identity_by_trial) != {item.id for item in trials} or set(review_by_trial) != set( + identity_by_trial + ): + raise CharacterBenchmarkConflict( + "character benchmark identity judgment relation is inconsistent" + ) + with self.campaigns.database.connect() as connection: + revision_rows = connection.execute( + "SELECT revision.* FROM review_revisions AS revision WHERE session_id IN " + "(SELECT session_id FROM character_benchmark_review_bindings " + "WHERE campaign_id=?)", + (campaign.definition.id,), + ).fetchall() + result_row = connection.execute( + "SELECT result_json, content_digest FROM character_benchmark_results " + "WHERE campaign_id=?", + (campaign.definition.id,), + ).fetchone() + events = self.campaigns.list_events(campaign.definition.id) + expected_event_count = 6 if result_row is not None else 4 + expected_states = ( + (CharacterBenchmarkState.DRAFT, CharacterBenchmarkState.PREREGISTERED), + (CharacterBenchmarkState.PREREGISTERED, CharacterBenchmarkState.GENERATING), + (CharacterBenchmarkState.GENERATING, CharacterBenchmarkState.READY), + (CharacterBenchmarkState.READY, CharacterBenchmarkState.JUDGING), + (CharacterBenchmarkState.JUDGING, CharacterBenchmarkState.ANALYZING), + (CharacterBenchmarkState.ANALYZING, CharacterBenchmarkState.COMPLETED), + ) + expected_event_digests = ( + _digest([item.model_dump(mode="json") for item in trials]), + *( + _digest( + { + "domain": domain, + "campaign_id": campaign.definition.id, + "sequence": sequence, + } + ) + for sequence, domain in enumerate( + ("generation", "ready", "judging", "analysis", "completed"), + start=2, + ) + ), + ) + if ( + len(events) != expected_event_count + or tuple(item.sequence for item in events) != tuple(range(1, expected_event_count + 1)) + or tuple((item.previous_state, item.new_state) for item in events) + != expected_states[:expected_event_count] + or tuple(item.payload_digest for item in events) + != expected_event_digests[:expected_event_count] + ): + raise CharacterBenchmarkConflict( + "character benchmark event cardinality is inconsistent" + ) + revisions_by_session = {str(row["session_id"]): row for row in revision_rows} + if ( + len(revision_rows) != 40 + or len(revisions_by_session) != 40 + or set(revisions_by_session) != {item.session_id for item in reviews} + or {str(row["id"]) for row in revision_rows} != {item.review_id for item in reviews} + ): + raise CharacterBenchmarkConflict( + "character benchmark review revision cardinality is inconsistent" + ) + expected_secondary: set[tuple[str, SecondaryDimension]] = set() + for trial in trials: + review_binding = review_by_trial[trial.id] + source_id = trial.reuses_trial_id or trial.id + if ( + review_binding.experiment_id != by_trial[source_id].experiment_id + or review_binding.reused_from_trial_id != trial.reuses_trial_id + ): + raise CharacterBenchmarkConflict( + "character benchmark repeat source is inconsistent" + ) + session = self._session(review_binding.session_id) + if ( + str(session["reviewer_id"]) + != _reviewer_alias(campaign.definition.id, trial.id, self.reviewer_id) + or str(session["experiment_id"]) != review_binding.experiment_id + or review_binding.display_mapping != self._display_mapping(trial, session) + ): + raise CharacterBenchmarkConflict( + "character benchmark display mapping is inconsistent" + ) + revision = revisions_by_session.get(review_binding.session_id) + if ( + revision is None + or str(revision["id"]) != review_binding.review_id + or int(revision["revision"]) != 1 + or revision["supersedes_review_id"] is not None + ): + raise CharacterBenchmarkConflict( + "character benchmark review revision is inconsistent" + ) + if trial.hidden_repeat: + if artifact_sets[source_id] != self._session_artifacts(session): + raise CharacterBenchmarkConflict( + "character benchmark repeat media reuse is inconsistent" + ) + identity = identity_by_trial[trial.id] + if identity.review_id != review_binding.review_id: + raise CharacterBenchmarkConflict( + "character benchmark identity judgment review is inconsistent" + ) + expected_display = _display_choice( + identity.observation.choice, trial.candidate_a_position + ) + expected_identity_digest = _digest( + { + "observation": identity.observation.model_dump(mode="json"), + "review_id": identity.review_id, + "display_choice": expected_display, + } + ) + underlying_preference = _underlying_preference( + expected_display, review_binding.display_mapping + ) + expected_preferred_variant_id = ( + None + if underlying_preference is DisplayPreference.TIE + else str(session[f"{underlying_preference.value}_variant_id"]) + ) + if ( + identity.display_choice != expected_display + or identity.content_digest != expected_identity_digest + or str(revision["display_preference"]) != underlying_preference.value + or revision["preferred_variant_id"] != expected_preferred_variant_id + ): + raise CharacterBenchmarkConflict( + "character benchmark identity judgment content is inconsistent" + ) + if self._requires_secondary(trial, identity.observation.choice): + expected_secondary.update((trial.id, dimension) for dimension in SecondaryDimension) + actual_secondary = { + (item.observation.trial_id, item.observation.dimension) for item in secondary + } + if actual_secondary != expected_secondary: + raise CharacterBenchmarkConflict( + "character benchmark secondary judgment relation is inconsistent" + ) + if any( + item.review_id != identity_by_trial[item.observation.trial_id].review_id + for item in secondary + ): + raise CharacterBenchmarkConflict( + "character benchmark secondary judgment review is inconsistent" + ) + if any( + item.content_digest != _digest(item.observation.model_dump(mode="json")) + for item in secondary + ): + raise CharacterBenchmarkConflict( + "character benchmark secondary judgment content is inconsistent" + ) + evidence = CharacterBenchmarkEvidence( + definition=campaign.definition, + trials=trials, + identity_observations=tuple(item.observation for item in identities), + secondary_observations=tuple(item.observation for item in secondary), + ) + if require_terminal and result_row is not None: + calculated = analyze_character_benchmark(evidence) + if ( + str(result_row["content_digest"]) != _digest(calculated.model_dump(mode="json")) + or CharacterBenchmarkResult.model_validate_json(str(result_row["result_json"])) + != calculated + ): + raise CharacterBenchmarkConflict( + "character benchmark result digest is inconsistent" + ) + return evidence + + def _draft( + self, campaign: CharacterBenchmarkCampaign, trial: CharacterTrial + ) -> ExperimentDraft: + context_prompt = trial.context.prompt + control = campaign.stage_2_manifest.model_copy( + update={ + "prompt": context_prompt, + "negative_prompt": trial.context.negative_prompt + or campaign.stage_2_manifest.negative_prompt, + "seed": trial.seed, + "loras": (), + } + ) + identity = f"{context_prompt} {campaign.identity_prompt}" + return ExperimentDraft(baseline=control, variable_path="prompt", alternate_value=identity) + + def _next_incomplete_trial(self, campaign_id: str) -> CharacterTrial | None: + identities = { + item.observation.trial_id: item for item in self.campaigns.list_identity(campaign_id) + } + secondary = self.campaigns.list_secondary(campaign_id) + dimensions = { + trial_id: { + item.observation.dimension + for item in secondary + if item.observation.trial_id == trial_id + } + for trial_id in identities + } + for trial in self.campaigns.list_trials(campaign_id): + identity = identities.get(trial.id) + if identity is None: + return trial + if self._requires_secondary(trial, identity.observation.choice) and dimensions.get( + trial.id, set() + ) != set(SecondaryDimension): + return trial + return None + + def _judging_view(self, campaign_id: str, trial: CharacterTrial) -> CharacterBenchmarkView: + campaign = self.campaigns.get(campaign_id) + existing = next( + ( + item + for item in self.campaigns.list_review_bindings(campaign_id) + if item.trial_id == trial.id + ), + None, + ) + source_id = trial.reuses_trial_id or trial.id + experiment_id = next( + item.experiment_id + for item in self.campaigns.list_experiment_bindings(campaign_id) + if item.trial_id == source_id + ) + alias = _reviewer_alias(campaign_id, trial.id, self.reviewer_id) + session_id = ( + existing.session_id + if existing is not None + else self.reviews.start(experiment_id, alias).session_id + ) + session = self._session(session_id) + mapping = self._display_mapping(trial, session) + update = { + "trial_id": trial.id, + "review_id": session_id, + "media": tuple( + CharacterBenchmarkMedia( + position=cast(Literal["left", "right"], position), + media_ref=f"/reviews/{session_id}/artifacts/{item['underlying_position']}", + ) + for position, item in sorted(mapping.items()) + ), + "next_action": ( + "record_secondary" + if trial.id + in {item.observation.trial_id for item in self.campaigns.list_identity(campaign_id)} + else "record_identity" + ), + } + return self._view(campaign).model_copy(update=update) + + def _authorized_session( + self, campaign: CharacterBenchmarkCampaign, trial: CharacterTrial, session_id: str + ) -> tuple[sqlite3.Row, str, str]: + session = self._session(session_id) + source_id = trial.reuses_trial_id or trial.id + experiment_id = next( + item.experiment_id + for item in self.campaigns.list_experiment_bindings(campaign.definition.id) + if item.trial_id == source_id + ) + alias = _reviewer_alias(campaign.definition.id, trial.id, self.reviewer_id) + if str(session["experiment_id"]) != experiment_id or str(session["reviewer_id"]) != alias: + raise CharacterBenchmarkConflict("character benchmark review does not belong to trial") + with self.campaigns.database.connect() as connection: + for position in ("left", "right"): + job_id = str(session[f"{position}_job_id"]) + variant_id = str(session[f"{position}_variant_id"]) + artifact_id = str(session[f"{position}_artifact_id"]) + row = connection.execute( + "SELECT job.state, job.variant_id, link.artifact_id FROM jobs AS job " + "JOIN job_artifacts AS link ON link.job_id=job.id WHERE job.id=?", + (job_id,), + ).fetchone() + artifact = self.experiments.artifacts.get(artifact_id) + if ( + row is None + or str(row["state"]) != "completed" + or str(row["variant_id"]) != variant_id + or str(row["artifact_id"]) != artifact_id + or artifact.sha256 != str(session[f"{position}_artifact_sha256"]) + ): + raise CharacterBenchmarkConflict( + "character benchmark review artifact ownership is inconsistent" + ) + try: + self.experiments.artifacts.resolve(artifact_id) + except (ArtifactError, OSError) as error: + raise CharacterBenchmarkConflict( + "character benchmark review artifact bytes are not canonical" + ) from error + return session, experiment_id, alias + + def _review_binding_for_session( + self, campaign_id: str, session_id: str + ) -> CharacterReviewBinding | None: + matches = tuple( + item + for item in self.campaigns.list_review_bindings(campaign_id) + if item.session_id == session_id + ) + if len(matches) > 1: + raise CharacterBenchmarkConflict( + "character benchmark review session has multiple bindings" + ) + return matches[0] if matches else None + + def _validate_bound_identity_review( + self, + campaign: CharacterBenchmarkCampaign, + trial: CharacterTrial, + binding: CharacterReviewBinding, + observation: IdentityObservation, + ) -> None: + session, experiment_id, reviewer_alias = self._authorized_session( + campaign, trial, binding.session_id + ) + mapping = self._display_mapping(trial, session) + display_choice = _display_choice(observation.choice, trial.candidate_a_position) + preference = _underlying_preference(display_choice, mapping) + expected_preferred_variant_id = ( + None + if preference is DisplayPreference.TIE + else str(session[f"{preference.value}_variant_id"]) + ) + with self.campaigns.database.connect() as connection: + revision = connection.execute( + "SELECT * FROM review_revisions WHERE id=? AND session_id=?", + (binding.review_id, binding.session_id), + ).fetchone() + if ( + binding.campaign_id != campaign.definition.id + or binding.trial_id != trial.id + or binding.experiment_id != experiment_id + or binding.reviewer_alias != reviewer_alias + or binding.reused_from_trial_id != trial.reuses_trial_id + or binding.display_mapping != mapping + or revision is None + or int(revision["revision"]) != 1 + or revision["supersedes_review_id"] is not None + or str(revision["display_preference"]) != preference.value + or revision["preferred_variant_id"] != expected_preferred_variant_id + ): + raise CharacterBenchmarkConflict( + "character benchmark bound review is not immutable evidence" + ) + + def _display_mapping( + self, trial: CharacterTrial, session: sqlite3.Row + ) -> dict[str, dict[str, str]]: + if trial.hidden_repeat and trial.reuses_trial_id is not None: + source = self._trial(trial.benchmark_id, trial.reuses_trial_id) + else: + source = trial + candidate_a_underlying = ( + "left" if str(session["left_variant_id"]) == source.generation_specs[0].id else "right" + ) + mapping: dict[str, dict[str, str]] = {} + for public in ("left", "right"): + candidate = "a" if public == trial.candidate_a_position else "b" + underlying = ( + candidate_a_underlying if candidate == "a" else _opposite(candidate_a_underlying) + ) + mapping[public] = { + "candidate": candidate, + "underlying_position": underlying, + "variant_id": str(session[f"{underlying}_variant_id"]), + "job_id": str(session[f"{underlying}_job_id"]), + "artifact_id": str(session[f"{underlying}_artifact_id"]), + "artifact_sha256": str(session[f"{underlying}_artifact_sha256"]), + } + return mapping + + def _session(self, session_id: str) -> sqlite3.Row: + with self.campaigns.database.connect() as connection: + row = connection.execute( + "SELECT * FROM review_sessions WHERE id=?", (session_id,) + ).fetchone() + if row is None: + raise CharacterBenchmarkConflict("character benchmark review session is missing") + return cast(sqlite3.Row, row) + + @staticmethod + def _session_artifacts(session: sqlite3.Row) -> frozenset[tuple[str, str]]: + return frozenset( + (str(session[f"{position}_artifact_id"]), str(session[f"{position}_artifact_sha256"])) + for position in ("left", "right") + ) + + def _trial(self, campaign_id: str, trial_id: str) -> CharacterTrial: + try: + return next( + item for item in self.campaigns.list_trials(campaign_id) if item.id == trial_id + ) + except StopIteration as error: + raise CharacterBenchmarkConflict("character benchmark trial is missing") from error + + @staticmethod + def _requires_secondary(trial: CharacterTrial, choice: IdentityChoice) -> bool: + return not trial.hidden_repeat and choice is not IdentityChoice.NEITHER + + def _view(self, campaign: CharacterBenchmarkCampaign) -> CharacterBenchmarkView: + generated = ( + len(self.campaigns.list_experiment_bindings(campaign.definition.id)) + if campaign.state is not CharacterBenchmarkState.DRAFT + else 0 + ) + completed = ( + len(self.campaigns.list_identity(campaign.definition.id)) + if campaign.state is not CharacterBenchmarkState.DRAFT + else 0 + ) + next_action = { + CharacterBenchmarkState.DRAFT: "preregister", + CharacterBenchmarkState.PREREGISTERED: "complete_generation", + CharacterBenchmarkState.GENERATING: "complete_generation", + CharacterBenchmarkState.READY: "record_identity", + CharacterBenchmarkState.JUDGING: "record_identity", + CharacterBenchmarkState.ANALYZING: "analyze", + CharacterBenchmarkState.COMPLETED: "none", + }[campaign.state] + return CharacterBenchmarkView( + campaign_id=campaign.definition.id, + state=campaign.state, + next_action=next_action, + progress=CharacterBenchmarkProgress( + completed_identity=completed, + generated_primary=generated, + ), + result=self.campaigns.get_result(campaign.definition.id), + ) + + def _event( + self, + campaign_id: str, + sequence: int, + previous: CharacterBenchmarkState, + new: CharacterBenchmarkState, + domain: str, + ) -> CharacterBenchmarkEvent: + return CharacterBenchmarkEvent( + campaign_id=campaign_id, + sequence=sequence, + previous_state=previous, + new_state=new, + payload_digest=_digest( + {"domain": domain, "campaign_id": campaign_id, "sequence": sequence} + ), + created_at=self.clock(), + ) + + +def _digest(value: object) -> str: + payload = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + default=lambda item: item.model_dump(mode="json"), + ) + return "sha256:" + hashlib.sha256(payload.encode()).hexdigest() + + +def _recipe_pair(identity_prompt: str) -> CharacterRecipePair: + return CharacterRecipePair( + control_digest=_digest( + {"schema_version": "character-prompt-recipe.v1", "template": "{context}"} + ), + identity_digest=_digest( + { + "schema_version": "character-prompt-recipe.v1", + "template": "{context} {identity_prompt}", + "identity_prompt": identity_prompt, + } + ), + ) + + +def _reviewer_alias(campaign_id: str, trial_id: str, reviewer_id: str) -> str: + return character_benchmark_reviewer_id(campaign_id, trial_id, reviewer_id) + + +def _record_id(kind: str, campaign_id: str, identity: str) -> str: + payload = { + "domain": f"video-lab.character-benchmark.{kind}", + "campaign_id": campaign_id, + "identity": identity, + } + return f"character_{kind}_{hashlib.sha256(canonical_json(payload).encode()).hexdigest()}" + + +def _display_choice( + choice: IdentityChoice, candidate_a_position: Literal["left", "right"] +) -> Literal["left", "right", "both", "neither"]: + if choice is IdentityChoice.A: + return candidate_a_position + if choice is IdentityChoice.B: + return _opposite(candidate_a_position) + return cast(Literal["both", "neither"], choice.value) + + +def _underlying_preference( + display_choice: str, mapping: dict[str, dict[str, str]] +) -> DisplayPreference: + if display_choice in {"both", "neither"}: + return DisplayPreference.TIE + underlying = mapping[display_choice]["underlying_position"] + return DisplayPreference(underlying) + + +def _opposite(value: str) -> Literal["left", "right"]: + return "right" if value == "left" else "left" diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_images.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_images.py new file mode 100644 index 0000000..d6f614f --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_images.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal, cast + +from PIL import Image, ImageOps + +DUPLICATE_HASH_ALGORITHM_VERSION: Final[Literal["dhash64-exif-transpose-lanczos.v1"]] = ( + "dhash64-exif-transpose-lanczos.v1" +) + + +@dataclass(frozen=True) +class DecodedCharacterImage: + width: int + height: int + media_type: Literal["image/jpeg", "image/png", "image/webp"] + dhash_hex: str + + +def decode_character_image(path: Path) -> DecodedCharacterImage: + with Image.open(path) as opened: + image = ImageOps.exif_transpose(opened).convert("RGB") + image.load() + return DecodedCharacterImage( + width=image.width, + height=image.height, + media_type=_media_type(opened.format), + dhash_hex=f"{difference_hash(image):016x}", + ) + + +def difference_hash(image: Image.Image) -> int: + luminance = image.convert("L").resize((9, 8), Image.Resampling.LANCZOS) + hash_value = 0 + for row in range(8): + for column in range(8): + left = cast(int, luminance.getpixel((column, row))) + right = cast(int, luminance.getpixel((column + 1, row))) + hash_value = (hash_value << 1) | int(left > right) + return hash_value + + +def hamming_distance(left: int, right: int) -> int: + return (left ^ right).bit_count() + + +def is_near_duplicate(left: int, right: int, *, threshold: int = 5) -> bool: + return hamming_distance(left, right) <= threshold + + +def _media_type(image_format: str | None) -> Literal["image/jpeg", "image/png", "image/webp"]: + match image_format: + case "JPEG": + return "image/jpeg" + case "PNG": + return "image/png" + case "WEBP": + return "image/webp" + case _: + raise ValueError(f"unsupported character image format: {image_format!r}") diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_models.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_models.py new file mode 100644 index 0000000..3cff9f3 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_models.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import re +from enum import StrEnum +from typing import Literal + +from pydantic import Field, field_validator, model_validator + +from video_lab.domain import FrozenModel + + +class Framing(StrEnum): + PORTRAIT = "portrait" + FULL_BODY = "full_body" + + +class HeadAngle(StrEnum): + FRONT = "front" + THREE_QUARTER = "three_quarter" + PROFILE = "profile" + + +class BodyVisibility(StrEnum): + CLEAR_FACE = "clear_face" + PARTIAL_BODY = "partial_body" + FULL_BODY = "full_body" + + +class Expression(StrEnum): + NEUTRAL = "neutral" + SMILE = "smile" + SERIOUS = "serious" + ANGRY = "angry" + SURPRISED = "surprised" + + +class Lighting(StrEnum): + ORDINARY = "ordinary" + DIRECTIONAL = "directional" + + +class Occlusion(StrEnum): + NONE = "none" + PARTIAL = "partial" + HEAVY = "heavy" + + +class CuratorDisposition(StrEnum): + ACTIVE = "active" + REJECTED = "rejected" + + +class WarningResolution(StrEnum): + UNRESOLVED = "unresolved" + ACCEPTED = "accepted" + DISMISSED = "dismissed" + + +class CharacterAnnotation(FrozenModel): + framing: Framing + head_angle: HeadAngle + body_visibility: BodyVisibility + expression: Expression + lighting: Lighting + occlusion: Occlusion = Occlusion.NONE + identity_confirmed: bool = False + + +class CharacterReferenceInput(FrozenModel): + reference_id: str = Field(min_length=1) + image_sha256: str = Field(min_length=1) + media_type: Literal["image/jpeg", "image/png", "image/webp"] + dhash_hex: str + disposition: CuratorDisposition = CuratorDisposition.ACTIVE + annotation: CharacterAnnotation + + @field_validator("dhash_hex") + @classmethod + def validate_dhash_hex(cls, value: str) -> str: + normalized = value.lower() + if re.fullmatch(r"[0-9a-f]{16}", normalized) is None: + raise ValueError("dhash_hex must be exactly 16 lowercase hexadecimal characters") + return normalized + + +class DuplicateWarning(FrozenModel): + left_reference_id: str = Field(min_length=1) + right_reference_id: str = Field(min_length=1) + distance: int = Field(ge=0, le=64) + resolution: WarningResolution = WarningResolution.UNRESOLVED + + @model_validator(mode="after") + def validate_distinct_references(self) -> DuplicateWarning: + if self.left_reference_id == self.right_reference_id: + raise ValueError("duplicate warnings must refer to two distinct references") + return self + + +class CharacterPackDraft(FrozenModel): + character_id: str = Field(min_length=1) + references: tuple[CharacterReferenceInput, ...] + duplicate_warnings: tuple[DuplicateWarning, ...] = () + supersedes_pack_id: str | None = Field(default=None, min_length=1) + + @model_validator(mode="after") + def validate_reference_ids(self) -> CharacterPackDraft: + reference_ids = tuple(reference.reference_id for reference in self.references) + if len(reference_ids) != len(set(reference_ids)): + raise ValueError("reference_id values must be unique") + return self diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_pack.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_pack.py new file mode 100644 index 0000000..2a02b26 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_pack.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +from hashlib import sha256 +from itertools import combinations +from typing import Final, Literal + +from video_lab.character_images import ( + DUPLICATE_HASH_ALGORITHM_VERSION, + hamming_distance, + is_near_duplicate, +) +from video_lab.character_models import ( + BodyVisibility, + CharacterPackDraft, + CharacterReferenceInput, + CuratorDisposition, + DuplicateWarning, + Expression, + Framing, + HeadAngle, + Lighting, + Occlusion, + WarningResolution, +) +from video_lab.domain import FrozenModel, canonical_json + +MIN_REFERENCE_COUNT = 12 +MAX_REFERENCE_COUNT = 20 +NEAR_DUPLICATE_THRESHOLD: Final[Literal[5]] = 5 + + +class CharacterCoverage(FrozenModel): + ready: bool + missing: tuple[str, ...] + duplicate_warnings: tuple[DuplicateWarning, ...] + + +class CharacterPack(FrozenModel): + pack_id: str + character_id: str + duplicate_hash_algorithm_version: Literal["dhash64-exif-transpose-lanczos.v1"] + near_duplicate_threshold_inclusive: Literal[5] + references: tuple[CharacterReferenceInput, ...] + duplicate_warnings: tuple[DuplicateWarning, ...] + supersedes_pack_id: str | None + + +class CharacterPackNotReady(ValueError): + def __init__(self, reason_codes: tuple[str, ...]) -> None: + self.reason_codes = reason_codes + super().__init__(",".join(reason_codes)) + + +def evaluate_pack(draft: CharacterPackDraft) -> CharacterCoverage: + active_references = tuple( + sorted( + ( + reference + for reference in draft.references + if reference.disposition is CuratorDisposition.ACTIVE + ), + key=lambda reference: reference.reference_id, + ) + ) + warnings = _duplicate_warnings(active_references, draft.duplicate_warnings) + missing = _missing_coverage(active_references, warnings) + return CharacterCoverage( + ready=not missing, + missing=tuple(missing), + duplicate_warnings=warnings, + ) + + +def character_pack_id(draft: CharacterPackDraft) -> str: + coverage = evaluate_pack(draft) + payload = CharacterPack( + pack_id="", + character_id=draft.character_id, + duplicate_hash_algorithm_version=DUPLICATE_HASH_ALGORITHM_VERSION, + near_duplicate_threshold_inclusive=NEAR_DUPLICATE_THRESHOLD, + references=tuple(sorted(draft.references, key=lambda reference: reference.reference_id)), + duplicate_warnings=coverage.duplicate_warnings, + supersedes_pack_id=draft.supersedes_pack_id, + ).model_dump(mode="json", exclude={"pack_id"}) + digest = sha256(canonical_json(payload).encode("utf-8")).hexdigest() + return f"character_pack_{digest}" + + +def freeze_pack(draft: CharacterPackDraft) -> CharacterPack: + coverage = evaluate_pack(draft) + if not coverage.ready: + raise CharacterPackNotReady(coverage.missing) + return CharacterPack( + pack_id=character_pack_id(draft), + character_id=draft.character_id, + duplicate_hash_algorithm_version=DUPLICATE_HASH_ALGORITHM_VERSION, + near_duplicate_threshold_inclusive=NEAR_DUPLICATE_THRESHOLD, + references=tuple(sorted(draft.references, key=lambda reference: reference.reference_id)), + duplicate_warnings=coverage.duplicate_warnings, + supersedes_pack_id=draft.supersedes_pack_id, + ) + + +def _duplicate_warnings( + references: tuple[CharacterReferenceInput, ...], + supplied_warnings: tuple[DuplicateWarning, ...], +) -> tuple[DuplicateWarning, ...]: + resolutions = { + _warning_key(warning.left_reference_id, warning.right_reference_id): warning.resolution + for warning in supplied_warnings + } + warnings: list[DuplicateWarning] = [] + for left, right in combinations(references, 2): + distance = hamming_distance(int(left.dhash_hex, 16), int(right.dhash_hex, 16)) + if is_near_duplicate( + int(left.dhash_hex, 16), int(right.dhash_hex, 16), threshold=NEAR_DUPLICATE_THRESHOLD + ): + left_reference_id, right_reference_id = _warning_key( + left.reference_id, right.reference_id + ) + warnings.append( + DuplicateWarning( + left_reference_id=left_reference_id, + right_reference_id=right_reference_id, + distance=distance, + resolution=resolutions.get( + (left_reference_id, right_reference_id), WarningResolution.UNRESOLVED + ), + ) + ) + return tuple(warnings) + + +def _missing_coverage( + references: tuple[CharacterReferenceInput, ...], warnings: tuple[DuplicateWarning, ...] +) -> list[str]: + missing: list[str] = [] + if len(references) < MIN_REFERENCE_COUNT: + missing.append(f"reference_count>={MIN_REFERENCE_COUNT}") + if len(references) > MAX_REFERENCE_COUNT: + missing.append(f"reference_count<={MAX_REFERENCE_COUNT}") + for head_angle in HeadAngle: + if not any(reference.annotation.head_angle is head_angle for reference in references): + missing.append(f"head_angle:{head_angle.value}") + if sum(reference.annotation.framing is Framing.PORTRAIT for reference in references) < 3: + missing.append("framing:portrait>=3") + if sum(reference.annotation.framing is Framing.FULL_BODY for reference in references) < 3: + missing.append("framing:full_body>=3") + if not any(reference.annotation.expression is Expression.NEUTRAL for reference in references): + missing.append("expression:neutral") + non_neutral_expressions = { + reference.annotation.expression + for reference in references + if reference.annotation.expression is not Expression.NEUTRAL + } + if len(non_neutral_expressions) < 2: + missing.append("expression:non_neutral>=2") + for lighting in Lighting: + if not any(reference.annotation.lighting is lighting for reference in references): + missing.append(f"lighting:{lighting.value}") + clear_faces = sum( + reference.annotation.body_visibility is BodyVisibility.CLEAR_FACE + and reference.annotation.occlusion is Occlusion.NONE + for reference in references + ) + if clear_faces < 2: + missing.append("clear_faces>=2") + if any(not reference.annotation.identity_confirmed for reference in references): + missing.append("identity_confirmation") + if any(warning.resolution is WarningResolution.UNRESOLVED for warning in warnings): + missing.append("duplicate_warning:unresolved") + return missing + + +def _warning_key(left_reference_id: str, right_reference_id: str) -> tuple[str, str]: + first, second = sorted((left_reference_id, right_reference_id)) + return first, second diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_repository.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_repository.py new file mode 100644 index 0000000..1d6f25b --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_repository.py @@ -0,0 +1,691 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +import unicodedata +from collections.abc import Mapping +from datetime import datetime +from pathlib import PurePosixPath +from types import MappingProxyType +from typing import Literal, cast + +from pydantic import Field, field_serializer, field_validator + +from video_lab.character_models import ( + CharacterAnnotation, + CharacterPackDraft, + CharacterReferenceInput, + CuratorDisposition, + DuplicateWarning, + WarningResolution, +) +from video_lab.character_pack import CharacterPack, character_pack_id, evaluate_pack, freeze_pack +from video_lab.database import Database +from video_lab.domain import FrozenModel, canonical_json, utc_now + + +class CharacterNotFound(LookupError): + pass + + +class CharacterConflict(RuntimeError): + pass + + +class CharacterRecord(FrozenModel): + character_id: str + name: str + reviewer_id: str + created_at: datetime + + +class CharacterPackRecord(FrozenModel): + pack_id: str + character_id: str + reviewer_id: str + state: Literal["draft", "preregistered"] + supersedes_pack_id: str | None + created_at: datetime + preregistered_at: datetime | None = None + + +class CuratedReference(FrozenModel): + reference: CharacterReferenceInput + artifact_id: str = Field(min_length=1) + source_name: str = Field(min_length=1) + + @field_validator("source_name") + @classmethod + def validate_source_name(cls, value: str) -> str: + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts or value != path.as_posix(): + raise ValueError("source_name must be a normalized relative path") + return value + + +class CharacterPackEvent(FrozenModel): + pack_id: str + sequence: int + reviewer_id: str + event_type: Literal["draft_created", "preregistered"] + payload: Mapping[str, object] + created_at: datetime + + @field_validator("payload", mode="after") + @classmethod + def freeze_payload(cls, value: Mapping[str, object]) -> Mapping[str, object]: + return cast(Mapping[str, object], _deep_freeze(value)) + + @field_serializer("payload") + def serialize_payload(self, value: Mapping[str, object]) -> dict[str, object]: + return cast(dict[str, object], _deep_thaw(value)) + + +class PersistedCharacterPack(FrozenModel): + record: CharacterPackRecord + references: tuple[CuratedReference, ...] + warnings: tuple[DuplicateWarning, ...] + events: tuple[CharacterPackEvent, ...] + frozen_pack: CharacterPack | None = None + + @property + def draft(self) -> CharacterPackDraft: + return CharacterPackDraft( + character_id=self.record.character_id, + references=tuple(curated.reference for curated in self.references), + duplicate_warnings=self.warnings, + supersedes_pack_id=self.record.supersedes_pack_id, + ) + + +class CharacterRepository: + def __init__(self, database: Database, *, reviewer_id: str) -> None: + reviewer = unicodedata.normalize("NFKC", reviewer_id).strip() + if not reviewer or reviewer != reviewer_id: + raise ValueError("reviewer_id must be normalized non-empty text") + self.database = database + self.reviewer_id = reviewer + + def create_character(self, name: str) -> CharacterRecord: + normalized = unicodedata.normalize("NFKC", name).strip() + if not normalized or normalized != name: + raise ValueError("character name must be normalized non-empty text") + character_id = _content_id( + "character", + { + "schema_version": "character-identity.v1", + "reviewer_id": self.reviewer_id, + "name": name, + }, + ) + created_at = utc_now() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + row = connection.execute( + "SELECT * FROM characters WHERE id = ?", (character_id,) + ).fetchone() + if row is None: + connection.execute( + "INSERT INTO characters(id, reviewer_id, name, created_at) " + "VALUES (?, ?, ?, ?)", + (character_id, self.reviewer_id, name, created_at.isoformat()), + ) + record = CharacterRecord( + character_id=character_id, + name=name, + reviewer_id=self.reviewer_id, + created_at=created_at, + ) + else: + record = _character_from_row(row) + self._require_owner(record.reviewer_id) + connection.commit() + except Exception: + connection.rollback() + raise + return record + + def get_character(self, character_id: str) -> CharacterRecord: + with self.database.connect() as connection: + row = connection.execute( + "SELECT * FROM characters WHERE id = ?", (character_id,) + ).fetchone() + if row is None: + raise CharacterNotFound("character not found") + record = _character_from_row(row) + self._require_owner(record.reviewer_id) + return record + + def create_pack( + self, + pack: CharacterPackDraft, + *, + curated_references: tuple[CuratedReference, ...] | None = None, + connection: sqlite3.Connection | None = None, + ) -> CharacterPackRecord: + if connection is not None: + return self._create_pack_with_connection(connection, pack, curated_references) + with self.database.connect() as owned_connection: + owned_connection.execute("BEGIN IMMEDIATE") + try: + record = self._create_pack_with_connection( + owned_connection, pack, curated_references + ) + owned_connection.commit() + return record + except Exception: + owned_connection.rollback() + raise + + def _create_pack_with_connection( + self, + connection: sqlite3.Connection, + pack: CharacterPackDraft, + curated_references: tuple[CuratedReference, ...] | None, + ) -> CharacterPackRecord: + validated = CharacterPackDraft.model_validate(pack.model_dump(round_trip=True)) + character_row = connection.execute( + "SELECT * FROM characters WHERE id=?", (validated.character_id,) + ).fetchone() + if character_row is None: + raise CharacterNotFound("character not found") + self._require_owner(str(character_row["reviewer_id"])) + supersedes_storage_id: str | None = None + if validated.supersedes_pack_id is not None: + predecessor = self._load_pack_with_connection(connection, validated.supersedes_pack_id) + if predecessor.record.character_id != validated.character_id: + raise CharacterConflict("successor pack must belong to the same character") + if predecessor.record.state != "preregistered": + raise CharacterConflict("successor pack requires a preregistered predecessor") + validated = validated.model_copy( + update={"supersedes_pack_id": predecessor.record.pack_id} + ) + supersedes_storage_id = self._pack_storage_id(connection, predecessor.record.pack_id) + pack_id = character_pack_id(validated) + curated = curated_references or () + if ( + validated.references + and tuple(item.reference for item in curated) != validated.references + ): + raise CharacterConflict("curated references do not match character pack draft") + coverage = evaluate_pack(validated) + created_at = utc_now() + try: + row = self._find_pack_row(connection, pack_id) + if row is None: + connection.execute( + "INSERT INTO character_packs(" + "storage_id, id, character_id, reviewer_id, state, " + "supersedes_storage_id, manifest_json, created_at, preregistered_at) " + "VALUES (?, ?, ?, ?, 'draft', ?, NULL, ?, NULL)", + ( + pack_id, + pack_id, + validated.character_id, + self.reviewer_id, + supersedes_storage_id, + created_at.isoformat(), + ), + ) + self._insert_references(connection, pack_id, curated) + self._replace_warnings(connection, pack_id, coverage.duplicate_warnings) + self._insert_event( + connection, pack_id, 1, "draft_created", {"pack_id": pack_id}, created_at + ) + record = CharacterPackRecord( + pack_id=pack_id, + character_id=validated.character_id, + reviewer_id=self.reviewer_id, + state="draft", + supersedes_pack_id=validated.supersedes_pack_id, + created_at=created_at, + ) + else: + record = _pack_record_from_row(row) + self._require_owner(record.reviewer_id) + existing = self._load_pack_with_connection(connection, record.pack_id) + if ( + existing.draft.character_id != validated.character_id + or existing.draft.supersedes_pack_id != validated.supersedes_pack_id + or _immutable_reference_evidence(existing.references) + != _immutable_reference_evidence(curated) + ): + raise CharacterConflict( + "character pack replay does not match persisted evidence" + ) + except sqlite3.IntegrityError as error: + raise CharacterConflict(str(error)) from error + return record + + def replace_draft_curation(self, pack_id: str, reference: CuratedReference) -> None: + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + pack = self._require_pack(connection, pack_id) + current_pack_id = pack.pack_id + pack_storage_id = self._pack_storage_id(connection, current_pack_id) + if pack.state != "draft": + raise CharacterConflict("preregistered character packs are immutable") + existing = connection.execute( + "SELECT stored.*, annotation.* FROM character_references AS stored " + "JOIN character_annotations AS annotation USING(pack_id, reference_id) " + "WHERE stored.pack_id=? AND stored.reference_id=?", + (pack_storage_id, reference.reference.reference_id), + ).fetchone() + if existing is None: + raise CharacterNotFound("character reference not found") + persisted_reference = _curated_reference_from_row(existing) + if _immutable_reference_evidence((persisted_reference,)) != ( + _immutable_reference_evidence((reference,)) + ): + raise CharacterConflict("character curation cannot replace artifact evidence") + item = reference.reference + connection.execute( + "UPDATE character_references SET disposition=? " + "WHERE pack_id=? AND reference_id=?", + (item.disposition.value, pack_storage_id, item.reference_id), + ) + annotation = item.annotation + connection.execute( + "UPDATE character_annotations SET framing=?, head_angle=?, body_visibility=?, " + "expression=?, lighting=?, occlusion=?, identity_confirmed=? " + "WHERE pack_id=? AND reference_id=?", + ( + annotation.framing.value, + annotation.head_angle.value, + annotation.body_visibility.value, + annotation.expression.value, + annotation.lighting.value, + annotation.occlusion.value, + int(annotation.identity_confirmed), + pack_storage_id, + item.reference_id, + ), + ) + self._rekey_draft(connection, current_pack_id) + connection.commit() + except Exception: + connection.rollback() + raise + + def resolve_warning( + self, + pack_id: str, + left_reference_id: str, + right_reference_id: str, + resolution: WarningResolution, + ) -> None: + if resolution is WarningResolution.UNRESOLVED: + raise ValueError("warning resolution must be final") + left, right = sorted((left_reference_id, right_reference_id)) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + pack = self._require_pack(connection, pack_id) + current_pack_id = pack.pack_id + pack_storage_id = self._pack_storage_id(connection, current_pack_id) + if pack.state != "draft": + raise CharacterConflict("preregistered character packs are immutable") + row = connection.execute( + "SELECT resolution FROM character_pack_warnings WHERE pack_id=? " + "AND left_reference_id=? AND right_reference_id=?", + (pack_storage_id, left, right), + ).fetchone() + if row is None: + raise CharacterNotFound("character warning not found") + current = WarningResolution(str(row["resolution"])) + if current is resolution: + connection.commit() + return + if current is not WarningResolution.UNRESOLVED: + raise CharacterConflict("resolved character warnings are immutable") + connection.execute( + "UPDATE character_pack_warnings SET resolution=? WHERE pack_id=? " + "AND left_reference_id=? AND right_reference_id=?", + (resolution.value, pack_storage_id, left, right), + ) + self._rekey_draft(connection, current_pack_id) + connection.commit() + except Exception: + connection.rollback() + raise + + def preregister(self, frozen: CharacterPack) -> None: + validated = CharacterPack.model_validate(frozen.model_dump(round_trip=True)) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + persisted = self._load_pack_with_connection(connection, validated.pack_id) + if persisted.record.state == "preregistered": + if persisted.frozen_pack != validated: + raise CharacterConflict("character pack preregistration replay mismatch") + connection.commit() + return + if freeze_pack(persisted.draft) != validated: + raise CharacterConflict("frozen character pack does not match persisted draft") + preregistered_at = utc_now() + manifest_json = canonical_json(validated.model_dump(mode="json")) + storage_id = self._pack_storage_id(connection, validated.pack_id) + connection.execute( + "UPDATE character_packs SET state='preregistered', manifest_json=?, " + "preregistered_at=? WHERE id=?", + (manifest_json, preregistered_at.isoformat(), validated.pack_id), + ) + self._insert_event( + connection, + storage_id, + 2, + "preregistered", + {"manifest_sha256": hashlib.sha256(manifest_json.encode()).hexdigest()}, + preregistered_at, + ) + connection.commit() + except Exception: + connection.rollback() + raise + + def load_pack( + self, pack_id: str, *, connection: sqlite3.Connection | None = None + ) -> PersistedCharacterPack: + if connection is not None: + return self._load_pack_with_connection(connection, pack_id) + with self.database.connect() as owned_connection: + return self._load_pack_with_connection(owned_connection, pack_id) + + def _load_pack_with_connection( + self, connection: sqlite3.Connection, pack_id: str + ) -> PersistedCharacterPack: + record = self._require_pack(connection, pack_id) + current_pack_id = record.pack_id + pack_storage_id = self._pack_storage_id(connection, current_pack_id) + rows = connection.execute( + "SELECT reference.*, annotation.* FROM character_references AS reference " + "JOIN character_annotations AS annotation USING(pack_id, reference_id) " + "WHERE reference.pack_id=? ORDER BY reference.ordinal", + (pack_storage_id,), + ).fetchall() + references = tuple(_curated_reference_from_row(row) for row in rows) + warning_rows = connection.execute( + "SELECT * FROM character_pack_warnings WHERE pack_id=? " + "ORDER BY left_reference_id, right_reference_id", + (pack_storage_id,), + ).fetchall() + warnings = tuple(_warning_from_row(row) for row in warning_rows) + event_rows = connection.execute( + "SELECT * FROM character_pack_events WHERE pack_id=? ORDER BY sequence", + (pack_storage_id,), + ).fetchall() + events = tuple(_event_from_row(row, current_pack_id) for row in event_rows) + manifest_row = connection.execute( + "SELECT manifest_json FROM character_packs WHERE id=?", (current_pack_id,) + ).fetchone() + frozen = None + if manifest_row is not None and manifest_row["manifest_json"] is not None: + frozen = CharacterPack.model_validate_json(str(manifest_row["manifest_json"])) + return PersistedCharacterPack( + record=record, + references=references, + warnings=warnings, + events=events, + frozen_pack=frozen, + ) + + def _require_pack(self, connection: sqlite3.Connection, pack_id: str) -> CharacterPackRecord: + row = self._find_pack_row(connection, pack_id) + if row is None: + raise CharacterNotFound("character pack not found") + record = _pack_record_from_row(row) + self._require_owner(record.reviewer_id) + return record + + @staticmethod + def _find_pack_row(connection: sqlite3.Connection, pack_id: str) -> sqlite3.Row | None: + row = cast( + sqlite3.Row | None, + connection.execute( + "SELECT pack.*, predecessor.id AS supersedes_pack_id " + "FROM character_packs AS pack " + "LEFT JOIN character_packs AS predecessor " + "ON predecessor.storage_id=pack.supersedes_storage_id WHERE pack.id=?", + (pack_id,), + ).fetchone(), + ) + if row is not None: + return row + return cast( + sqlite3.Row | None, + connection.execute( + "SELECT pack.*, predecessor.id AS supersedes_pack_id " + "FROM character_pack_aliases AS alias " + "JOIN character_packs AS pack ON pack.storage_id=alias.pack_storage_id " + "LEFT JOIN character_packs AS predecessor " + "ON predecessor.storage_id=pack.supersedes_storage_id WHERE alias.alias_id=?", + (pack_id,), + ).fetchone(), + ) + + @staticmethod + def _pack_storage_id(connection: sqlite3.Connection, pack_id: str) -> str: + row = connection.execute( + "SELECT storage_id FROM character_packs WHERE id=?", (pack_id,) + ).fetchone() + if row is None: + raise CharacterNotFound("character pack not found") + return str(row["storage_id"]) + + def _require_owner(self, reviewer_id: str) -> None: + if reviewer_id != self.reviewer_id: + raise CharacterConflict("character evidence belongs to another reviewer") + + def _rekey_draft(self, connection: sqlite3.Connection, pack_id: str) -> str: + persisted = self._load_pack_with_connection(connection, pack_id) + new_pack_id = character_pack_id(persisted.draft) + if new_pack_id != pack_id: + storage_id = self._pack_storage_id(connection, pack_id) + connection.execute( + "UPDATE character_packs SET id=? WHERE storage_id=? AND state='draft'", + (new_pack_id, storage_id), + ) + connection.execute( + "INSERT OR IGNORE INTO character_pack_aliases(alias_id, pack_storage_id) " + "VALUES (?, ?)", + (pack_id, storage_id), + ) + return new_pack_id + + @staticmethod + def _insert_references( + connection: sqlite3.Connection, + pack_id: str, + references: tuple[CuratedReference, ...], + ) -> None: + for ordinal, curated in enumerate(references): + reference = curated.reference + connection.execute( + "INSERT INTO character_references(pack_id, reference_id, artifact_id, source_name, " + "image_sha256, media_type, dhash_hex, disposition, ordinal) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + pack_id, + reference.reference_id, + curated.artifact_id, + curated.source_name, + reference.image_sha256, + reference.media_type, + reference.dhash_hex, + reference.disposition.value, + ordinal, + ), + ) + annotation = reference.annotation + connection.execute( + "INSERT INTO character_annotations VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + pack_id, + reference.reference_id, + annotation.framing.value, + annotation.head_angle.value, + annotation.body_visibility.value, + annotation.expression.value, + annotation.lighting.value, + annotation.occlusion.value, + int(annotation.identity_confirmed), + ), + ) + + @staticmethod + def _replace_warnings( + connection: sqlite3.Connection, + pack_id: str, + warnings: tuple[DuplicateWarning, ...], + ) -> None: + for warning in warnings: + connection.execute( + "INSERT INTO character_pack_warnings VALUES (?, ?, ?, ?, ?)", + ( + pack_id, + warning.left_reference_id, + warning.right_reference_id, + warning.distance, + warning.resolution.value, + ), + ) + + def _insert_event( + self, + connection: sqlite3.Connection, + pack_id: str, + sequence: int, + event_type: Literal["draft_created", "preregistered"], + payload: dict[str, object], + created_at: datetime, + ) -> None: + connection.execute( + "INSERT INTO character_pack_events VALUES (?, ?, ?, ?, ?, ?)", + ( + pack_id, + sequence, + self.reviewer_id, + event_type, + canonical_json(payload), + created_at.isoformat(), + ), + ) + + +def _content_id(prefix: str, payload: dict[str, object]) -> str: + return f"{prefix}_{hashlib.sha256(canonical_json(payload).encode()).hexdigest()}" + + +def _immutable_reference_evidence( + references: tuple[CuratedReference, ...], +) -> tuple[tuple[str, str, str, str, str, str], ...]: + return tuple( + ( + curated.reference.reference_id, + curated.reference.image_sha256, + curated.reference.media_type, + curated.reference.dhash_hex, + curated.artifact_id, + curated.source_name, + ) + for curated in references + ) + + +def _character_from_row(row: sqlite3.Row) -> CharacterRecord: + return CharacterRecord( + character_id=str(row["id"]), + name=str(row["name"]), + reviewer_id=str(row["reviewer_id"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + ) + + +def _pack_record_from_row(row: sqlite3.Row) -> CharacterPackRecord: + return CharacterPackRecord( + pack_id=str(row["id"]), + character_id=str(row["character_id"]), + reviewer_id=str(row["reviewer_id"]), + state=cast(Literal["draft", "preregistered"], str(row["state"])), + supersedes_pack_id=None + if row["supersedes_pack_id"] is None + else str(row["supersedes_pack_id"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + preregistered_at=( + None + if row["preregistered_at"] is None + else datetime.fromisoformat(str(row["preregistered_at"])) + ), + ) + + +def _curated_reference_from_row(row: sqlite3.Row) -> CuratedReference: + annotation = CharacterAnnotation.model_validate( + { + "framing": str(row["framing"]), + "head_angle": str(row["head_angle"]), + "body_visibility": str(row["body_visibility"]), + "expression": str(row["expression"]), + "lighting": str(row["lighting"]), + "occlusion": str(row["occlusion"]), + "identity_confirmed": bool(row["identity_confirmed"]), + } + ) + return CuratedReference( + reference=CharacterReferenceInput.model_validate( + { + "reference_id": str(row["reference_id"]), + "image_sha256": str(row["image_sha256"]), + "media_type": str(row["media_type"]), + "dhash_hex": str(row["dhash_hex"]), + "disposition": CuratorDisposition(str(row["disposition"])), + "annotation": annotation, + } + ), + artifact_id=str(row["artifact_id"]), + source_name=str(row["source_name"]), + ) + + +def _warning_from_row(row: sqlite3.Row) -> DuplicateWarning: + return DuplicateWarning( + left_reference_id=str(row["left_reference_id"]), + right_reference_id=str(row["right_reference_id"]), + distance=int(row["distance"]), + resolution=WarningResolution(str(row["resolution"])), + ) + + +def _event_from_row(row: sqlite3.Row, public_pack_id: str) -> CharacterPackEvent: + payload = json.loads(str(row["payload_json"])) + if not isinstance(payload, dict): + raise CharacterConflict("character event payload is invalid") + return CharacterPackEvent( + pack_id=public_pack_id, + sequence=int(row["sequence"]), + reviewer_id=str(row["reviewer_id"]), + event_type=cast(Literal["draft_created", "preregistered"], str(row["event_type"])), + payload=payload, + created_at=datetime.fromisoformat(str(row["created_at"])), + ) + + +def _deep_freeze(value: object) -> object: + if isinstance(value, Mapping): + return MappingProxyType({str(key): _deep_freeze(item) for key, item in value.items()}) + if isinstance(value, (list, tuple)): + return tuple(_deep_freeze(item) for item in value) + if value is None or isinstance(value, (str, int, float, bool)): + return value + raise ValueError("character event payload must contain JSON values") + + +def _deep_thaw(value: object) -> object: + if isinstance(value, Mapping): + return {str(key): _deep_thaw(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_deep_thaw(item) for item in value] + return value diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_service.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_service.py new file mode 100644 index 0000000..d325510 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_service.py @@ -0,0 +1,528 @@ +from __future__ import annotations + +import hashlib +import os +import sqlite3 +import stat +import unicodedata +from collections.abc import Iterator +from dataclasses import dataclass +from io import BytesIO +from pathlib import Path, PurePosixPath +from typing import Literal, cast + +from PIL import Image, ImageOps, UnidentifiedImageError + +from video_lab.artifacts import Artifact, ArtifactError, ArtifactStore, StagedArtifact +from video_lab.character_images import difference_hash +from video_lab.character_models import ( + BodyVisibility, + CharacterAnnotation, + CharacterPackDraft, + CharacterReferenceInput, + CuratorDisposition, + Expression, + Framing, + HeadAngle, + Lighting, + Occlusion, + WarningResolution, +) +from video_lab.character_pack import ( + CharacterCoverage, + CharacterPack, + character_pack_id, + evaluate_pack, + freeze_pack, +) +from video_lab.character_repository import ( + CharacterRecord, + CharacterRepository, + CuratedReference, + PersistedCharacterPack, +) +from video_lab.domain import FrozenModel + +_SUPPORTED_EXTENSIONS = frozenset({".jpg", ".jpeg", ".png", ".webp"}) +_READ_CHUNK_SIZE = 1024 * 1024 + + +class CharacterImportError(ValueError): + pass + + +class ImportReportEntry(FrozenModel): + source_name: str + reason: Literal["unsupported_extension", "exact_duplicate"] + + +@dataclass(frozen=True) +class CharacterImageDiscovery: + candidates: tuple[Path, ...] + ignored: tuple[ImportReportEntry, ...] + + def __iter__(self) -> Iterator[Path]: + return iter(self.candidates) + + +class CharacterReferenceView(FrozenModel): + reference_id: str + artifact_id: str + source_name: str + image_sha256: str + media_type: Literal["image/jpeg", "image/png", "image/webp"] + dhash_hex: str + disposition: CuratorDisposition + annotation: CharacterAnnotation + + +class CharacterPackView(FrozenModel): + pack_id: str + character_id: str + state: Literal["draft", "preregistered"] + supersedes_pack_id: str | None + references: tuple[CharacterReferenceView, ...] + coverage: CharacterCoverage + ignored: tuple[ImportReportEntry, ...] = () + + +class CharacterPackExport(FrozenModel): + pack: CharacterPack + references: tuple[CharacterReferenceView, ...] + + +@dataclass(frozen=True) +class _DecodedImport: + source_name: str + media_type: Literal["image/jpeg", "image/png", "image/webp"] + sha256: str + byte_size: int + dhash_hex: str + staged: StagedArtifact + + +def discover_character_images(source: Path) -> CharacterImageDiscovery: + try: + root_stat = source.lstat() + except OSError as error: + raise CharacterImportError("character source root is unavailable") from error + if stat.S_ISLNK(root_stat.st_mode): + raise CharacterImportError("character source root must not be a symlink") + if not stat.S_ISDIR(root_stat.st_mode): + raise CharacterImportError("character source root must be a directory") + candidates: list[Path] = [] + ignored: list[ImportReportEntry] = [] + + def traversal_error(error: OSError) -> None: + raise CharacterImportError("character source cannot be traversed") from error + + try: + for directory, directory_names, file_names in os.walk( + source, onerror=traversal_error, followlinks=False + ): + current = Path(directory) + for name in tuple(directory_names): + path = current / name + if path.is_symlink(): + raise CharacterImportError("character source contains a symlink") + directory_names.sort(key=_lexical_name) + for name in sorted(file_names, key=_lexical_name): + path = current / name + if path.is_symlink(): + raise CharacterImportError("character source contains a symlink") + relative = path.relative_to(source) + source_name = _normalized_relative_name(relative) + if path.suffix.lower() in _SUPPORTED_EXTENSIONS: + candidates.append(Path(source_name)) + else: + ignored.append( + ImportReportEntry( + source_name=source_name, + reason="unsupported_extension", + ) + ) + except CharacterImportError: + raise + except (OSError, ValueError) as error: + raise CharacterImportError("character source cannot be traversed") from error + candidates.sort(key=lambda path: _lexical_name(path.as_posix())) + ignored.sort(key=lambda entry: _lexical_name(entry.source_name)) + return CharacterImageDiscovery(candidates=tuple(candidates), ignored=tuple(ignored)) + + +class CharacterService: + def __init__(self, repository: CharacterRepository, artifacts: ArtifactStore) -> None: + self.repository = repository + self.artifacts = artifacts + + def create_character(self, name: str) -> CharacterRecord: + return self.repository.create_character(name) + + def import_folder(self, character_id: str, source: Path) -> CharacterPackView: + self.repository.get_character(character_id) + root_descriptor = _open_root_descriptor(source) + prepared: list[_DecodedImport] = [] + ignored: list[ImportReportEntry] = [] + observed_content: dict[tuple[str, str], str] = {} + try: + discovery = discover_character_images(source) + ignored.extend(discovery.ignored) + for candidate in discovery.candidates: + decoded = self._prepare_stable(root_descriptor, candidate) + content_key = (decoded.sha256, decoded.media_type) + if content_key in observed_content: + self._discard_bounded(decoded.staged) + ignored.append( + ImportReportEntry( + source_name=decoded.source_name, + reason="exact_duplicate", + ) + ) + continue + observed_content[content_key] = decoded.source_name + prepared.append(decoded) + except Exception: + for decoded in prepared: + self._discard_bounded(decoded.staged) + raise + finally: + os.close(root_descriptor) + + def persist( + connection: sqlite3.Connection, artifacts: tuple[Artifact, ...] + ) -> PersistedCharacterPack: + curated = tuple( + CuratedReference( + reference=CharacterReferenceInput( + reference_id=f"character_ref_{decoded.sha256}", + image_sha256=decoded.sha256, + media_type=decoded.media_type, + dhash_hex=decoded.dhash_hex, + disposition=CuratorDisposition.ACTIVE, + annotation=_uncurated_annotation(), + ), + artifact_id=artifact.id, + source_name=decoded.source_name, + ) + for decoded, artifact in zip(prepared, artifacts, strict=True) + ) + draft = CharacterPackDraft( + character_id=character_id, + references=tuple(item.reference for item in curated), + ) + record = self.repository.create_pack( + draft, + curated_references=curated, + connection=connection, + ) + return self.repository.load_pack(record.pack_id, connection=connection) + + try: + persisted = self.artifacts.commit_batch( + tuple(decoded.staged for decoded in prepared), persist + ) + except (OSError, ArtifactError) as error: + for decoded in prepared: + self._discard_bounded(decoded.staged) + source_name = prepared[0].source_name if prepared else "artifact" + raise CharacterImportError( + f"character artifact commit failed: {source_name}" + ) from error + except Exception as error: + for decoded in prepared: + self._discard_bounded(decoded.staged) + raise CharacterImportError("character pack persistence failed") from error + + ignored.sort(key=lambda entry: (_lexical_name(entry.source_name), entry.reason)) + return self._view(persisted, ignored=tuple(ignored)) + + def curate_reference( + self, + pack_id: str, + reference_id: str, + *, + annotation: CharacterAnnotation, + disposition: CuratorDisposition, + ) -> CharacterPackView: + persisted = self.repository.load_pack(pack_id) + replacement: CuratedReference | None = None + references: list[CharacterReferenceInput] = [] + for curated in persisted.references: + if curated.reference.reference_id == reference_id: + updated = curated.reference.model_copy( + update={"annotation": annotation, "disposition": disposition} + ) + replacement = curated.model_copy(update={"reference": updated}) + references.append(updated) + else: + references.append(curated.reference) + if replacement is None: + raise LookupError("character reference not found") + updated_draft = CharacterPackDraft( + character_id=persisted.record.character_id, + references=tuple(references), + duplicate_warnings=persisted.warnings, + supersedes_pack_id=persisted.record.supersedes_pack_id, + ) + new_pack_id = character_pack_id(updated_draft) + self.repository.replace_draft_curation(pack_id, replacement) + return self._view(self.repository.load_pack(new_pack_id)) + + def resolve_warning( + self, + pack_id: str, + left_reference_id: str, + right_reference_id: str, + resolution: WarningResolution, + ) -> CharacterPackView: + persisted = self.repository.load_pack(pack_id) + left, right = sorted((left_reference_id, right_reference_id)) + found = False + warnings = [] + for warning in persisted.warnings: + if (warning.left_reference_id, warning.right_reference_id) == (left, right): + found = True + warnings.append(warning.model_copy(update={"resolution": resolution})) + else: + warnings.append(warning) + if not found: + raise LookupError("character warning not found") + updated_draft = persisted.draft.model_copy(update={"duplicate_warnings": tuple(warnings)}) + new_pack_id = character_pack_id(updated_draft) + self.repository.resolve_warning( + pack_id, + left_reference_id, + right_reference_id, + resolution, + ) + return self._view(self.repository.load_pack(new_pack_id)) + + def pack_status(self, pack_id: str) -> CharacterPackView: + return self._view(self.repository.load_pack(pack_id)) + + def preregister_pack(self, pack_id: str) -> CharacterPackView: + persisted = self.repository.load_pack(pack_id) + frozen = freeze_pack(persisted.draft) + if frozen.pack_id != persisted.record.pack_id: + raise RuntimeError("persisted draft identity does not match frozen pack identity") + self._verify_artifacts(persisted) + self.repository.preregister(frozen) + return self._view(self.repository.load_pack(frozen.pack_id)) + + def export_pack(self, pack_id: str) -> CharacterPackExport: + persisted = self.repository.load_pack(pack_id) + if persisted.frozen_pack is None: + raise PermissionError("character pack must be preregistered before export") + self._verify_artifacts(persisted) + return CharacterPackExport( + pack=persisted.frozen_pack, + references=tuple(_reference_view(item) for item in persisted.references), + ) + + def _prepare_stable(self, root_descriptor: int, candidate: Path) -> _DecodedImport: + source_name = _normalized_relative_name(candidate) + directory_descriptor = os.dup(root_descriptor) + file_descriptor: int | None = None + staged: StagedArtifact | None = None + prepared_ok = False + try: + parts = PurePosixPath(source_name).parts + for component in parts[:-1]: + next_descriptor = os.open( + component, + os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0), + dir_fd=directory_descriptor, + ) + opened_directory = os.fstat(next_descriptor) + if not stat.S_ISDIR(opened_directory.st_mode): + os.close(next_descriptor) + raise OSError("source component is not a directory") + os.close(directory_descriptor) + directory_descriptor = next_descriptor + candidate_stat = os.stat(parts[-1], dir_fd=directory_descriptor, follow_symlinks=False) + if not stat.S_ISREG(candidate_stat.st_mode): + raise OSError("source candidate is not a regular file") + file_descriptor = os.open( + parts[-1], + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0), + dir_fd=directory_descriptor, + ) + opened = os.fstat(file_descriptor) + if not stat.S_ISREG(opened.st_mode) or _identity(opened) != _identity(candidate_stat): + raise OSError("source candidate changed before open") + chunks: list[bytes] = [] + digest = hashlib.sha256() + byte_size = 0 + while True: + chunk = os.read(file_descriptor, _READ_CHUNK_SIZE) + if not chunk: + break + chunks.append(chunk) + digest.update(chunk) + byte_size += len(chunk) + after_read = os.fstat(file_descriptor) + if _identity(opened) != _identity(after_read): + raise CharacterImportError(f"character image changed during import: {source_name}") + data = b"".join(chunks) + try: + media_type, dhash_hex = _decode_image_bytes(data) + except (OSError, UnidentifiedImageError, ValueError) as error: + raise CharacterImportError( + f"character image cannot be decoded: {source_name}" + ) from error + os.lseek(file_descriptor, 0, os.SEEK_SET) + try: + staged = self.artifacts.stage(_descriptor_path(file_descriptor), media_type) + except (OSError, ArtifactError) as error: + raise CharacterImportError( + f"character artifact staging failed: {source_name}" + ) from error + after_stage = os.fstat(file_descriptor) + if ( + _identity(opened) != _identity(after_stage) + or staged.sha256 != digest.hexdigest() + or staged.byte_size != byte_size + ): + raise CharacterImportError(f"character image changed during import: {source_name}") + prepared_ok = True + except CharacterImportError: + raise + except OSError as error: + raise CharacterImportError( + f"character image cannot be opened: {source_name}" + ) from error + finally: + os.close(directory_descriptor) + if file_descriptor is not None: + os.close(file_descriptor) + if staged is not None and not prepared_ok: + self._discard_bounded(staged) + assert staged is not None + return _DecodedImport( + source_name=source_name, + media_type=media_type, + sha256=digest.hexdigest(), + byte_size=byte_size, + dhash_hex=dhash_hex, + staged=staged, + ) + + def _discard_bounded(self, staged: StagedArtifact) -> None: + try: + self.artifacts.discard(staged) + except (OSError, ArtifactError): + return + + def _verify_artifacts(self, persisted: PersistedCharacterPack) -> None: + for curated in persisted.references: + path = self.artifacts.resolve(curated.artifact_id) + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != curated.reference.image_sha256: + raise RuntimeError("character artifact digest does not match frozen reference") + + @staticmethod + def _view( + persisted: PersistedCharacterPack, + *, + ignored: tuple[ImportReportEntry, ...] = (), + ) -> CharacterPackView: + return CharacterPackView( + pack_id=persisted.record.pack_id, + character_id=persisted.record.character_id, + state=persisted.record.state, + supersedes_pack_id=persisted.record.supersedes_pack_id, + references=tuple(_reference_view(item) for item in persisted.references), + coverage=evaluate_pack(persisted.draft), + ignored=ignored, + ) + + +def _uncurated_annotation() -> CharacterAnnotation: + return CharacterAnnotation( + framing=Framing.PORTRAIT, + head_angle=HeadAngle.FRONT, + body_visibility=BodyVisibility.CLEAR_FACE, + expression=Expression.NEUTRAL, + lighting=Lighting.ORDINARY, + occlusion=Occlusion.NONE, + identity_confirmed=False, + ) + + +def _reference_view(curated: CuratedReference) -> CharacterReferenceView: + reference = curated.reference + return CharacterReferenceView( + reference_id=reference.reference_id, + artifact_id=curated.artifact_id, + source_name=curated.source_name, + image_sha256=reference.image_sha256, + media_type=reference.media_type, + dhash_hex=reference.dhash_hex, + disposition=reference.disposition, + annotation=reference.annotation, + ) + + +def _decode_image_bytes( + data: bytes, +) -> tuple[Literal["image/jpeg", "image/png", "image/webp"], str]: + with Image.open(BytesIO(data)) as opened: + image = ImageOps.exif_transpose(opened).convert("RGB") + image.load() + media_type = _media_type(opened.format) + return media_type, f"{difference_hash(image):016x}" + + +def _media_type(image_format: str | None) -> Literal["image/jpeg", "image/png", "image/webp"]: + mapping = {"JPEG": "image/jpeg", "PNG": "image/png", "WEBP": "image/webp"} + if image_format is None or image_format not in mapping: + raise ValueError("unsupported character image format") + return cast(Literal["image/jpeg", "image/png", "image/webp"], mapping[image_format]) + + +def _identity(value: os.stat_result) -> tuple[int, int, int, int, int]: + return value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns, value.st_ctime_ns + + +def _open_root_descriptor(source: Path) -> int: + descriptor: int | None = None + try: + before = source.lstat() + except OSError as error: + raise CharacterImportError("character source root is unavailable") from error + if stat.S_ISLNK(before.st_mode): + raise CharacterImportError("character source root must not be a symlink") + if not stat.S_ISDIR(before.st_mode): + raise CharacterImportError("character source root must be a directory") + try: + descriptor = os.open( + source, + os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0), + ) + opened = os.fstat(descriptor) + if not stat.S_ISDIR(opened.st_mode) or _identity(opened) != _identity(before): + raise OSError("character source root changed") + return descriptor + except OSError as error: + if descriptor is not None: + os.close(descriptor) + raise CharacterImportError("character source root cannot be opened") from error + + +def _descriptor_path(descriptor: int) -> Path: + for root in (Path("/dev/fd"), Path("/proc/self/fd")): + if root.is_dir(): + return root / str(descriptor) + raise CharacterImportError("character descriptor staging is unavailable") + + +def _normalized_relative_name(path: Path) -> str: + value = unicodedata.normalize("NFC", PurePosixPath(path).as_posix()) + parsed = PurePosixPath(value) + if parsed.is_absolute() or ".." in parsed.parts or value in {"", "."}: + raise CharacterImportError("character source path is not a safe relative name") + return value + + +def _lexical_name(value: str) -> str: + return unicodedata.normalize("NFC", value) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_web_grant_repository.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_web_grant_repository.py new file mode 100644 index 0000000..6a8dcee --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/character_web_grant_repository.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import hashlib +import re +import secrets +import sqlite3 +import unicodedata +from collections.abc import Callable +from datetime import UTC, datetime, timedelta + +from video_lab.database import Database +from video_lab.domain import FrozenModel, utc_now + +CHARACTER_CURATION_GRANT_TTL = timedelta(minutes=10) +_TOKEN_PATTERN = re.compile(r"^[A-Za-z0-9_-]{32,256}$") + + +class CharacterCurationGrantError(PermissionError): + pass + + +class CharacterCurationClaim(FrozenModel): + reviewer_id: str + character_id: str + pack_storage_id: str + + +class CharacterCurationGrantRepository: + def __init__( + self, + database: Database, + *, + clock: Callable[[], datetime] = utc_now, + ) -> None: + self.database = database + self.clock = clock + + def issue(self, reviewer_id: str, character_id: str, pack_id: str) -> str: + reviewer = _normalized_reviewer(reviewer_id) + character = _normalized_identity(character_id) + pack = _normalized_identity(pack_id) + now = self._now() + expires_at = now + CHARACTER_CURATION_GRANT_TTL + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + _delete_expired(connection, now) + row = connection.execute( + "SELECT character_pack.storage_id FROM character_packs AS character_pack " + "LEFT JOIN character_pack_aliases AS alias " + "ON alias.pack_storage_id=character_pack.storage_id " + "WHERE character_pack.reviewer_id=? AND character_pack.character_id=? " + "AND (character_pack.id=? OR alias.alias_id=?) " + "ORDER BY character_pack.storage_id LIMIT 1", + (reviewer, character, pack, pack), + ).fetchone() + if row is None: + raise CharacterCurationGrantError("character curation evidence is unavailable") + token = self._insert_random_token( + connection, + reviewer=reviewer, + character_id=character, + pack_storage_id=str(row["storage_id"]), + created_at=now, + expires_at=expires_at, + ) + connection.commit() + return token + except Exception: + connection.rollback() + raise + + def consume(self, token: str) -> CharacterCurationClaim: + token_digest = _token_digest(token) + now = self._now() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + _delete_expired(connection, now) + row = connection.execute( + "SELECT grant.reviewer_id, grant.character_id, grant.pack_storage_id " + "FROM character_curation_grants AS grant " + "JOIN characters AS character " + "ON character.id=grant.character_id " + "AND character.reviewer_id=grant.reviewer_id " + "JOIN character_packs AS character_pack " + "ON character_pack.storage_id=grant.pack_storage_id " + "AND character_pack.character_id=grant.character_id " + "AND character_pack.reviewer_id=grant.reviewer_id " + "WHERE grant.token_digest=?", + (token_digest,), + ).fetchone() + if row is None: + connection.commit() + raise CharacterCurationGrantError("character curation grant is unavailable") + deleted = connection.execute( + "DELETE FROM character_curation_grants WHERE token_digest=?", + (token_digest,), + ) + if deleted.rowcount != 1: + raise CharacterCurationGrantError("character curation grant is unavailable") + claim = CharacterCurationClaim( + reviewer_id=str(row["reviewer_id"]), + character_id=str(row["character_id"]), + pack_storage_id=str(row["pack_storage_id"]), + ) + connection.commit() + return claim + except Exception: + connection.rollback() + raise + + def _insert_random_token( + self, + connection: sqlite3.Connection, + *, + reviewer: str, + character_id: str, + pack_storage_id: str, + created_at: datetime, + expires_at: datetime, + ) -> str: + for _ in range(8): + token = secrets.token_urlsafe(32) + try: + connection.execute( + "INSERT INTO character_curation_grants(" + "token_digest, reviewer_id, character_id, pack_storage_id, " + "created_at, expires_at) VALUES (?, ?, ?, ?, ?, ?)", + ( + _token_digest(token), + reviewer, + character_id, + pack_storage_id, + created_at.isoformat(), + expires_at.isoformat(), + ), + ) + except sqlite3.IntegrityError: + continue + return token + raise RuntimeError("could not allocate a unique character curation token") + + def _now(self) -> datetime: + now = self.clock() + if now.tzinfo is None or now.utcoffset() is None: + raise ValueError("character curation clock must return an aware timestamp") + return now.astimezone(UTC) + + +def _delete_expired(connection: sqlite3.Connection, now: datetime) -> None: + connection.execute( + "DELETE FROM character_curation_grants WHERE julianday(expires_at) <= julianday(?)", + (now.isoformat(),), + ) + + +def _token_digest(token: str) -> str: + if _TOKEN_PATTERN.fullmatch(token) is None: + raise CharacterCurationGrantError("character curation grant is unavailable") + return hashlib.sha256(token.encode("ascii")).hexdigest() + + +def _normalized_reviewer(reviewer_id: str) -> str: + reviewer = unicodedata.normalize("NFKC", reviewer_id).strip() + if not reviewer or reviewer != reviewer_id: + raise ValueError("character curation reviewer must be a normalized identity") + return reviewer + + +def _normalized_identity(value: str) -> str: + if not value or value != value.strip(): + raise CharacterCurationGrantError("character curation evidence is unavailable") + return value diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/cli.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/cli.py new file mode 100644 index 0000000..9f77f7d --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/cli.py @@ -0,0 +1,2801 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +import sys +from collections.abc import Callable, Mapping, Sequence +from dataclasses import asdict +from pathlib import Path +from typing import Annotated, Any, Literal, cast + +import typer +from pydantic import ValidationError +from typer import _click as click +from typer.core import TyperGroup + +from video_lab.adapter_protocol import ( + GenerationRequest, + ProbePhase, + ProbeRequest, + StagedProbeAdapter, +) +from video_lab.adapter_registry import AdapterRegistry, UnknownAdapterError +from video_lab.adaptive_study_models import ( + AdaptiveStudyDefinition, + adaptive_study_input_fingerprint, +) +from video_lab.adaptive_study_repository import ( + AdaptiveStudyConflict, + AdaptiveStudyNotFound, + AdaptiveStudyRepository, +) +from video_lab.adaptive_study_service import AdaptiveStudyService +from video_lab.artifacts import ArtifactError, ArtifactStore +from video_lab.capabilities import AdapterCapabilities, validate_manifest_support +from video_lab.character_benchmark_models import ( + CharacterBenchmarkPanelInput, + IdentityChoice, + IdentityObservation, + SecondaryChoice, + SecondaryDimension, + SecondaryObservation, +) +from video_lab.character_benchmark_repository import ( + CharacterBenchmarkConflict, + CharacterBenchmarkNotFound, + CharacterBenchmarkRepository, +) +from video_lab.character_benchmark_service import CharacterBenchmarkService +from video_lab.character_models import ( + BodyVisibility, + CharacterAnnotation, + CuratorDisposition, + Expression, + Framing, + HeadAngle, + Lighting, + Occlusion, + WarningResolution, +) +from video_lab.character_pack import CharacterPackNotReady +from video_lab.character_repository import ( + CharacterConflict, + CharacterNotFound, + CharacterRepository, +) +from video_lab.character_service import CharacterImportError, CharacterService +from video_lab.character_web_grant_repository import ( + CHARACTER_CURATION_GRANT_TTL, + CharacterCurationGrantError, + CharacterCurationGrantRepository, +) +from video_lab.comparison import ComparisonError, build_controlled_pair +from video_lab.database import Database +from video_lab.domain import ( + DisplayPreference, + ExperimentDraft, + FrozenModel, + Manifest, + RubricScore, + utc_now, +) +from video_lab.preference_optimizer import OptimizerConfig +from video_lab.repositories import ExperimentRepository, JobRepository, RecordNotFound +from video_lab.review_service import BlindMedia, BlindReviewView, ReviewService +from video_lab.reviews import ReviewRepository +from video_lab.services import ExperimentService, ExperimentView, ValidationFailure +from video_lab.states import InvalidTransition +from video_lab.validation_models import ( + ValidationChoice, + ValidationIneligible, + ValidationPanelInput, +) +from video_lab.validation_repository import ( + ValidationConflict, + ValidationNotFound, + ValidationRepository, +) +from video_lab.validation_service import ValidationCampaignService +from video_lab.worker import Worker, WorkerOutcome + + +class JsonEnvelope(FrozenModel): + schema_version: str = "cli.v1" + ok: bool + data: object | None = None + error: dict[str, object] | None = None + + +class JsonTyperGroup(TyperGroup): + def main( + self, + args: Sequence[str] | None = None, + prog_name: str | None = None, + complete_var: str | None = None, + standalone_mode: bool = True, + windows_expand_args: bool = True, + **extra: Any, + ) -> Any: + raw_args = list(args) if args is not None else sys.argv[1:] + root_command = self._root_command(raw_args) + study_command = root_command == "study" + validation_command = root_command == "validation" + character_command = root_command == "character" + character_benchmark_command = root_command == "character-benchmark" + json_command = ( + study_command or validation_command or character_command or character_benchmark_command + ) + if (validation_command or character_command or character_benchmark_command) and any( + argument in {"--help", "-h"} for argument in raw_args + ): + command_name = ( + "character-benchmark" + if character_benchmark_command + else "character" + if character_command + else "validation" + ) + help_map = ( + _CHARACTER_BENCHMARK_HELP + if character_benchmark_command + else _CHARACTER_HELP + if character_command + else _VALIDATION_HELP + ) + subcommand = _json_help_subcommand(raw_args, command_name) + if subcommand is not None and subcommand not in help_map: + _emit( + JsonEnvelope( + ok=False, + error={ + "code": "invalid_input", + "message": f"invalid {command_name} command input", + }, + ) + ) + if standalone_mode: + raise SystemExit(2) + raise typer.Exit(2) + help_data = ( + _character_benchmark_help_data(raw_args) + if character_benchmark_command + else _character_help_data(raw_args) + if character_command + else _validation_help_data(raw_args) + ) + _emit(JsonEnvelope(ok=True, data=help_data)) + return None + if json_command and raw_args[-1] == root_command: + _emit( + JsonEnvelope( + ok=False, + error={ + "code": "invalid_input", + "message": ( + "adaptive study command is required" + if study_command + else "character benchmark command is required" + if character_benchmark_command + else "character command is required" + if character_command + else "validation command is required" + ), + }, + ) + ) + if standalone_mode: + raise SystemExit(2) + raise typer.Exit(2) + if "--json" not in raw_args and not json_command: + return super().main( + args=args, + prog_name=prog_name, + complete_var=complete_var, + standalone_mode=standalone_mode, + windows_expand_args=windows_expand_args, + **extra, + ) + try: + result = super().main( + args=args, + prog_name=prog_name, + complete_var=complete_var, + standalone_mode=False, + windows_expand_args=windows_expand_args, + **extra, + ) + if isinstance(result, int) and result != 0: + if standalone_mode: + raise SystemExit(result) + raise typer.Exit(result) + return result + except click.ClickException as error: + _emit( + JsonEnvelope( + ok=False, + error={ + "code": ( + "invalid_validation_input" + if validation_command + else "invalid_character_benchmark_input" + if character_benchmark_command + else "invalid_character_input" + if character_command + else "invalid_input" + ), + "message": ( + "invalid adaptive study command input" + if study_command + else "character benchmark input is invalid" + if character_benchmark_command + else "character input is invalid" + if character_command + else "validation input is invalid" + if validation_command + else error.format_message() + ), + }, + ) + ) + if standalone_mode: + raise SystemExit(2) from error + raise typer.Exit(2) from error + except typer.Exit as error: + if standalone_mode: + raise SystemExit(error.exit_code) from error + raise + + @staticmethod + def _root_command(raw_args: Sequence[str]) -> str | None: + value_options = ("--database", "--artifacts", "--reviewer-id") + index = 0 + while index < len(raw_args): + argument = raw_args[index] + if argument in value_options: + index += 2 + continue + if any(argument.startswith(f"{option}=") for option in value_options): + index += 1 + continue + if argument.startswith("-"): + return None + return argument + return None + + +class CliContext(FrozenModel): + database: Path | None + artifacts: Path | None + reviewer_id: str | None + + +class FeasibilityCommandEnvelope(FrozenModel): + schema_version: Literal["feasibility-command.v1"] = "feasibility-command.v1" + ok: bool + data: object | None = None + error: dict[str, object] | None = None + + +class ReviewInput(FrozenModel): + schema_version: Literal["review.v1"] = "review.v1" + display_preference: DisplayPreference + scores: tuple[RubricScore, ...] + notes: str | None = None + supersedes_review_id: str | None = None + + +class InvalidInput(ValueError): + pass + + +class InvalidReviewInput(ValueError): + pass + + +class InvalidStudyInput(ValueError): + pass + + +class InvalidValidationInput(ValueError): + pass + + +app = typer.Typer(no_args_is_help=True, cls=JsonTyperGroup) +db_app = typer.Typer(no_args_is_help=True) +experiment_app = typer.Typer(no_args_is_help=True) +worker_app = typer.Typer(no_args_is_help=True) +adapter_app = typer.Typer(no_args_is_help=True) +feasibility_app = typer.Typer(no_args_is_help=True) +review_app = typer.Typer(no_args_is_help=True) +study_app = typer.Typer(no_args_is_help=True) +validation_app = typer.Typer(no_args_is_help=True) +character_app = typer.Typer(no_args_is_help=True) +character_benchmark_app = typer.Typer(no_args_is_help=True) +dataset_app = typer.Typer( + no_args_is_help=True, + help=( + "Strictly local curated dataset operations. Stable exits: 0 accepted; " + "64 usage; 65 rejected; 66 needs-review; 67 conflict; 68 in-progress; " + "69 cancelled; 70 integrity/security; 71 internal." + ), +) +app.add_typer(db_app, name="db") +app.add_typer(experiment_app, name="experiment") +app.add_typer(worker_app, name="worker") +app.add_typer(adapter_app, name="adapter") +adapter_app.add_typer(feasibility_app, name="feasibility") +app.add_typer(review_app, name="review") +app.add_typer(study_app, name="study") +app.add_typer(validation_app, name="validation") +app.add_typer(character_app, name="character") +app.add_typer(character_benchmark_app, name="character-benchmark") +app.add_typer(dataset_app, name="dataset") + + +def _dataset_roots(context: typer.Context, json_output: bool) -> tuple[Path, Path]: + from video_lab.dataset_cli import emit_usage_error + + options = _context(context) + if options.database is None or options.artifacts is None: + emit_usage_error( + json_output=json_output, message="database and artifact roots are required" + ) + raise AssertionError("unreachable") + return options.database, options.artifacts + + +@dataset_app.command("validate") +def dataset_validate( + context: typer.Context, + request_root: Annotated[Path, typer.Option("--request-root")], + request_file: Annotated[str, typer.Option("--request-file")], + source_root: Annotated[Path | None, typer.Option("--source-root")] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + from video_lab.dataset_cli import run_command + + database, artifacts = _dataset_roots(context, json_output) + run_command( + command="validate", + database_path=database, + artifact_root=artifacts, + request_root=request_root, + request_file=request_file, + source_root=source_root, + json_output=json_output, + ) + + +def _run_dataset_compilation_command( + command: Literal["compile", "reproduce"], + context: typer.Context, + request_root: Path, + request_file: str, + json_output: bool, + tokenizer_root: Path | None, + text_encoder_root: Path | None, + vae_root: Path | None, + scheduler_root: Path | None, +) -> None: + from video_lab.dataset_cli import run_command + + database, artifacts = _dataset_roots(context, json_output) + values = (tokenizer_root, text_encoder_root, vae_root, scheduler_root) + run_command( + command=command, + database_path=database, + artifact_root=artifacts, + request_root=request_root, + request_file=request_file, + json_output=json_output, + component_roots=values if any(value is not None for value in values) else None, + ) + + +@dataset_app.command("compile") +def dataset_compile( + context: typer.Context, + request_root: Annotated[Path, typer.Option("--request-root")], + request_file: Annotated[str, typer.Option("--request-file")], + tokenizer_root: Annotated[Path | None, typer.Option("--tokenizer-root")] = None, + text_encoder_root: Annotated[Path | None, typer.Option("--text-encoder-root")] = None, + vae_root: Annotated[Path | None, typer.Option("--vae-root")] = None, + scheduler_root: Annotated[Path | None, typer.Option("--scheduler-root")] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + _run_dataset_compilation_command( + "compile", + context, + request_root, + request_file, + json_output, + tokenizer_root, + text_encoder_root, + vae_root, + scheduler_root, + ) + + +@dataset_app.command("inspect") +def dataset_inspect( + context: typer.Context, + request_root: Annotated[Path, typer.Option("--request-root")], + request_file: Annotated[str, typer.Option("--request-file")], + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + from video_lab.dataset_cli import run_command + + database, artifacts = _dataset_roots(context, json_output) + run_command( + command="inspect", + database_path=database, + artifact_root=artifacts, + request_root=request_root, + request_file=request_file, + json_output=json_output, + ) + + +@dataset_app.command("reproduce") +def dataset_reproduce( + context: typer.Context, + request_root: Annotated[Path, typer.Option("--request-root")], + request_file: Annotated[str, typer.Option("--request-file")], + tokenizer_root: Annotated[Path | None, typer.Option("--tokenizer-root")] = None, + text_encoder_root: Annotated[Path | None, typer.Option("--text-encoder-root")] = None, + vae_root: Annotated[Path | None, typer.Option("--vae-root")] = None, + scheduler_root: Annotated[Path | None, typer.Option("--scheduler-root")] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + _run_dataset_compilation_command( + "reproduce", + context, + request_root, + request_file, + json_output, + tokenizer_root, + text_encoder_root, + vae_root, + scheduler_root, + ) + + +_CHARACTER_BENCHMARK_HELP: Mapping[str, dict[str, object]] = { + "create": { + "usage": ( + "create --pack PACK_ID --stage-2-evidence CAMPAIGN_ID " + "--identity-prompt TEXT --panel PATH" + ), + "arguments": [], + "options": ["--pack", "--stage-2-evidence", "--identity-prompt", "--panel"], + "choices": {}, + }, + "preview": { + "usage": "preview CAMPAIGN_ID", + "arguments": ["campaign_id"], + "options": [], + "choices": {}, + }, + "preregister": { + "usage": "preregister CAMPAIGN_ID", + "arguments": ["campaign_id"], + "options": [], + "choices": {}, + }, + "status": { + "usage": "status CAMPAIGN_ID", + "arguments": ["campaign_id"], + "options": [], + "choices": {}, + }, + "resume": { + "usage": "resume CAMPAIGN_ID", + "arguments": ["campaign_id"], + "options": [], + "choices": {}, + }, + "submit-identity": { + "usage": ( + "submit-identity CAMPAIGN_ID --review-id REVIEW_ID " + "--trial-id TRIAL_ID --choice {a|b|both|neither}" + ), + "arguments": ["campaign_id"], + "options": ["--review-id", "--trial-id", "--choice"], + "choices": {"--choice": ["a", "b", "both", "neither"]}, + }, + "submit-secondary": { + "usage": ( + "submit-secondary CAMPAIGN_ID --review-id REVIEW_ID " + "--trial-id TRIAL_ID --dimension DIMENSION --choice {a|b|tie}" + ), + "arguments": ["campaign_id"], + "options": ["--review-id", "--trial-id", "--dimension", "--choice"], + "choices": { + "--dimension": [item.value for item in SecondaryDimension], + "--choice": ["a", "b", "tie"], + }, + }, + "analyze": { + "usage": "analyze CAMPAIGN_ID", + "arguments": ["campaign_id"], + "options": [], + "choices": {}, + }, + "export-evidence": { + "usage": "export-evidence CAMPAIGN_ID --output PATH", + "arguments": ["campaign_id"], + "options": ["--output"], + "choices": {}, + }, +} +_CHARACTER_BENCHMARK_GLOBAL_USAGE = ( + "video-lab --database PATH --artifacts PATH --reviewer-id TEXT character-benchmark" +) + + +_CHARACTER_HELP: Mapping[str, dict[str, object]] = { + "create": { + "usage": "create NAME", + "arguments": ["name"], + "options": [], + "choices": {}, + }, + "import": { + "usage": "import CHARACTER_ID --source SOURCE", + "arguments": ["character_id"], + "options": ["--source"], + "choices": {}, + }, + "status": { + "usage": "status PACK_ID", + "arguments": ["pack_id"], + "options": [], + "choices": {}, + }, + "curate": { + "usage": ( + "curate PACK_ID REFERENCE_ID --framing FRAMING --head-angle HEAD_ANGLE " + "--body-visibility BODY_VISIBILITY --expression EXPRESSION --lighting LIGHTING " + "--occlusion OCCLUSION --identity-confirmed|--identity-not-confirmed " + "--accept|--reject" + ), + "arguments": ["pack_id", "reference_id"], + "options": [ + "--framing", + "--head-angle", + "--body-visibility", + "--expression", + "--lighting", + "--occlusion", + "--identity-confirmed|--identity-not-confirmed", + "--accept|--reject", + ], + "choices": { + "--framing": [item.value for item in Framing], + "--head-angle": [item.value for item in HeadAngle], + "--body-visibility": [item.value for item in BodyVisibility], + "--expression": [item.value for item in Expression], + "--lighting": [item.value for item in Lighting], + "--occlusion": [item.value for item in Occlusion], + }, + }, + "resolve-warning": { + "usage": ( + "resolve-warning PACK_ID LEFT_REFERENCE_ID RIGHT_REFERENCE_ID " + "--resolution {accepted|dismissed}" + ), + "arguments": ["pack_id", "left_reference_id", "right_reference_id"], + "options": ["--resolution"], + "choices": {"--resolution": ["accepted", "dismissed"]}, + }, + "preregister": { + "usage": "preregister PACK_ID", + "arguments": ["pack_id"], + "options": [], + "choices": {}, + }, + "export-pack": { + "usage": "export-pack PACK_ID --output OUTPUT_PATH", + "arguments": ["pack_id"], + "options": ["--output"], + "choices": {}, + }, + "open": { + "usage": "open CHARACTER_ID --pack PACK_ID", + "arguments": ["character_id"], + "options": ["--pack"], + "choices": {}, + }, +} +_CHARACTER_GLOBAL_OPTIONS: list[dict[str, object]] = [ + {"option": "--database", "value": "PATH", "required": True}, + {"option": "--artifacts", "value": "PATH", "required": True}, + {"option": "--reviewer-id", "value": "TEXT", "required": True}, +] +_CHARACTER_GLOBAL_USAGE = "video-lab --database PATH --artifacts PATH --reviewer-id TEXT character" + + +_VALIDATION_HELP: Mapping[str, dict[str, object]] = { + "create": { + "usage": "create --study STUDY_ID --panel PANEL_PATH", + "arguments": [], + "options": ["--study", "--panel"], + "choices": {}, + }, + "preview": { + "usage": "preview CAMPAIGN_ID", + "arguments": ["campaign_id"], + "options": [], + "choices": {}, + }, + "preregister": { + "usage": "preregister CAMPAIGN_ID", + "arguments": ["campaign_id"], + "options": [], + "choices": {}, + }, + "status": { + "usage": "status CAMPAIGN_ID", + "arguments": ["campaign_id"], + "options": [], + "choices": {}, + }, + "resume": { + "usage": "resume CAMPAIGN_ID", + "arguments": ["campaign_id"], + "options": [], + "choices": {}, + }, + "submit": { + "usage": "submit CAMPAIGN_ID --review-id REVIEW_ID --choice {a|b|tie}", + "arguments": ["campaign_id"], + "options": ["--review-id", "--choice"], + "choices": {"--choice": ["a", "b", "tie"]}, + }, + "analyze": { + "usage": "analyze CAMPAIGN_ID", + "arguments": ["campaign_id"], + "options": [], + "choices": {}, + }, + "export-evidence": { + "usage": "export-evidence CAMPAIGN_ID --output OUTPUT_PATH", + "arguments": ["campaign_id"], + "options": ["--output"], + "choices": {}, + }, +} +_VALIDATION_GLOBAL_OPTIONS: list[dict[str, object]] = [ + {"option": "--database", "value": "PATH", "required": True}, + {"option": "--artifacts", "value": "PATH", "required": True}, + {"option": "--reviewer-id", "value": "TEXT", "required": True}, +] +_VALIDATION_GLOBAL_USAGE = ( + "video-lab --database PATH --artifacts PATH --reviewer-id TEXT validation" +) + + +def _json_help_subcommand(raw_args: Sequence[str], command_name: str) -> str | None: + command_index = raw_args.index(command_name) + return next( + (argument for argument in raw_args[command_index + 1 :] if not argument.startswith("-")), + None, + ) + + +def _validation_help_data(raw_args: Sequence[str]) -> dict[str, object]: + command = _json_help_subcommand(raw_args, "validation") + if command is None: + return { + "schema_version": "validation-cli-help.v1", + "command": "validation", + "usage": f"{_VALIDATION_GLOBAL_USAGE} ", + "global_options": _VALIDATION_GLOBAL_OPTIONS, + "required_arguments": [], + "required_options": [], + "choices": {}, + "commands": list(_VALIDATION_HELP), + } + details = _VALIDATION_HELP[command] + return { + "schema_version": "validation-cli-help.v1", + "command": f"validation {command}", + "usage": f"{_VALIDATION_GLOBAL_USAGE} {details['usage']}", + "global_options": _VALIDATION_GLOBAL_OPTIONS, + "required_arguments": details["arguments"], + "required_options": details["options"], + "choices": details["choices"], + } + + +def _character_help_data(raw_args: Sequence[str]) -> dict[str, object]: + command = _json_help_subcommand(raw_args, "character") + if command is None: + return { + "schema_version": "character-cli-help.v1", + "command": "character", + "usage": f"{_CHARACTER_GLOBAL_USAGE} ", + "global_options": _CHARACTER_GLOBAL_OPTIONS, + "required_arguments": [], + "required_options": [], + "choices": {}, + "commands": list(_CHARACTER_HELP), + } + details = _CHARACTER_HELP[command] + return { + "schema_version": "character-cli-help.v1", + "command": f"character {command}", + "usage": f"{_CHARACTER_GLOBAL_USAGE} {details['usage']}", + "global_options": _CHARACTER_GLOBAL_OPTIONS, + "required_arguments": details["arguments"], + "required_options": details["options"], + "choices": details["choices"], + } + + +def _character_benchmark_help_data(raw_args: Sequence[str]) -> dict[str, object]: + command = _json_help_subcommand(raw_args, "character-benchmark") + if command is None: + return { + "schema_version": "character-benchmark-cli-help.v1", + "command": "character-benchmark", + "usage": f"{_CHARACTER_BENCHMARK_GLOBAL_USAGE} ", + "global_options": _CHARACTER_GLOBAL_OPTIONS, + "required_arguments": [], + "required_options": [], + "choices": {}, + "commands": list(_CHARACTER_BENCHMARK_HELP), + } + details = _CHARACTER_BENCHMARK_HELP[command] + return { + "schema_version": "character-benchmark-cli-help.v1", + "command": f"character-benchmark {command}", + "usage": f"{_CHARACTER_BENCHMARK_GLOBAL_USAGE} {details['usage']}", + "global_options": _CHARACTER_GLOBAL_OPTIONS, + "required_arguments": details["arguments"], + "required_options": details["options"], + "choices": details["choices"], + } + + +@app.callback() +def configure( + context: typer.Context, + database: Annotated[Path | None, typer.Option("--database")] = None, + artifacts: Annotated[Path | None, typer.Option("--artifacts")] = None, + reviewer_id: Annotated[ + str | None, typer.Option("--reviewer-id", envvar="VIDEO_LAB_REVIEWER_ID") + ] = None, +) -> None: + context.obj = CliContext(database=database, artifacts=artifacts, reviewer_id=reviewer_id) + + +def _context(context: typer.Context) -> CliContext: + value = context.find_root().obj + if not isinstance(value, CliContext): + raise RuntimeError("CLI paths were not configured") + return value + + +def _service(context: typer.Context) -> ExperimentService: + options = _context(context) + database, artifacts = _required_paths( + (options.database, "database"), (options.artifacts, "artifacts") + ) + return ExperimentService.from_paths(database, artifacts) + + +def _review_service(context: typer.Context) -> ReviewService: + options = _context(context) + database_path, artifact_root = _required_paths( + (options.database, "database"), (options.artifacts, "artifacts") + ) + database = Database(database_path) + database.migrate() + return ReviewService( + ExperimentRepository(database), + JobRepository(database), + ArtifactStore(database, artifact_root), + ReviewRepository(database), + ) + + +def _reviewer_id(context: typer.Context) -> str: + try: + return _required_nonblank_text(_context(context).reviewer_id, "reviewer_id").strip() + except InvalidInput as error: + raise InvalidReviewInput(str(error)) from error + + +def _study_reviewer_id(context: typer.Context) -> str: + try: + return _required_nonblank_text(_context(context).reviewer_id, "reviewer_id").strip() + except InvalidInput as error: + raise InvalidStudyInput(str(error)) from error + + +def _adaptive_study_service( + context: typer.Context, + capabilities: AdapterCapabilities, +) -> AdaptiveStudyService: + options = _context(context) + database_path, artifact_root = _required_paths( + (options.database, "database"), (options.artifacts, "artifacts") + ) + database = Database(database_path) + database.migrate() + artifacts = ArtifactStore(database, artifact_root) + return AdaptiveStudyService( + AdaptiveStudyRepository(database), + ExperimentService( + ExperimentRepository(database), + JobRepository(database), + artifacts, + ), + ReviewService( + ExperimentRepository(database), + JobRepository(database), + artifacts, + ReviewRepository(database), + ), + capabilities, + reviewer_id=_study_reviewer_id(context), + ) + + +def _adaptive_study_service_for_id(context: typer.Context, study_id: str) -> AdaptiveStudyService: + database_path = _required_paths((_context(context).database, "database"))[0] + database = Database(database_path) + database.migrate() + definition = AdaptiveStudyRepository(database).get(study_id) + capabilities = AdapterRegistry.with_defaults().capabilities(definition.base_manifest.adapter_id) + return _adaptive_study_service(context, capabilities) + + +def _validation_reviewer_id(context: typer.Context) -> str: + try: + return _required_nonblank_text(_context(context).reviewer_id, "reviewer_id").strip() + except InvalidInput as error: + raise InvalidValidationInput(str(error)) from error + + +def _character_service(context: typer.Context) -> CharacterService: + options = _context(context) + database_path, artifact_root = _required_paths( + (options.database, "database"), (options.artifacts, "artifacts") + ) + try: + reviewer = _required_nonblank_text(options.reviewer_id, "reviewer_id").strip() + except InvalidInput as error: + raise ValueError("character reviewer is required") from error + database = Database(database_path) + database.migrate() + return CharacterService( + CharacterRepository(database, reviewer_id=reviewer), + ArtifactStore(database, artifact_root), + ) + + +def _character_grants( + context: typer.Context, +) -> tuple[CharacterCurationGrantRepository, str]: + options = _context(context) + database_path, _artifact_root = _required_paths( + (options.database, "database"), (options.artifacts, "artifacts") + ) + reviewer = _required_nonblank_text(options.reviewer_id, "reviewer_id").strip() + database = Database(database_path) + database.migrate() + return CharacterCurationGrantRepository(database), reviewer + + +def _validation_campaign_service_for_study( + context: typer.Context, study_id: str +) -> ValidationCampaignService: + options = _context(context) + database_path, artifact_root = _required_paths( + (options.database, "database"), (options.artifacts, "artifacts") + ) + database = Database(database_path) + database.migrate() + definition = AdaptiveStudyRepository(database).get(study_id) + capabilities = AdapterRegistry.with_defaults().capabilities(definition.base_manifest.adapter_id) + stage_1 = _adaptive_study_service(context, capabilities) + artifacts = ArtifactStore(database, artifact_root) + experiments = ExperimentService( + ExperimentRepository(database), JobRepository(database), artifacts + ) + reviews = ReviewService( + ExperimentRepository(database), + JobRepository(database), + artifacts, + ReviewRepository(database), + ) + return ValidationCampaignService( + ValidationRepository(database), + stage_1, + experiments, + reviews, + AdapterRegistry.with_defaults(), + reviewer_id=_validation_reviewer_id(context), + ) + + +def _validation_campaign_service_for_id( + context: typer.Context, campaign_id: str +) -> ValidationCampaignService: + database_path = _required_paths((_context(context).database, "database"))[0] + database = Database(database_path) + database.migrate() + definition = ValidationRepository(database).get(campaign_id) + return _validation_campaign_service_for_study(context, definition.stage_1_study_id) + + +def _character_benchmark_service( + context: typer.Context, stage_2_campaign_id: str +) -> CharacterBenchmarkService: + options = _context(context) + database_path, artifact_root = _required_paths( + (options.database, "database"), (options.artifacts, "artifacts") + ) + reviewer = _required_nonblank_text(options.reviewer_id, "reviewer_id").strip() + database = Database(database_path) + database.migrate() + artifacts = ArtifactStore(database, artifact_root) + experiments = ExperimentService( + ExperimentRepository(database), JobRepository(database), artifacts + ) + reviews = ReviewService( + ExperimentRepository(database), + JobRepository(database), + artifacts, + ReviewRepository(database), + ) + return CharacterBenchmarkService( + CharacterBenchmarkRepository(database, reviewer_id=reviewer), + CharacterService(CharacterRepository(database, reviewer_id=reviewer), artifacts), + _validation_campaign_service_for_id(context, stage_2_campaign_id), + experiments, + reviews, + AdapterRegistry.with_defaults(), + reviewer_id=reviewer, + ) + + +def _character_benchmark_service_for_id( + context: typer.Context, campaign_id: str +) -> CharacterBenchmarkService: + database_path = _required_paths((_context(context).database, "database"))[0] + reviewer = _required_nonblank_text(_context(context).reviewer_id, "reviewer_id").strip() + database = Database(database_path) + database.migrate() + campaign = CharacterBenchmarkRepository(database, reviewer_id=reviewer).get(campaign_id) + return _character_benchmark_service(context, campaign.stage_2_campaign_id) + + +def _projection_reviewer_id(context: typer.Context) -> str | None: + from video_lab.feasibility_projection import normalize_reviewer_id + + return normalize_reviewer_id(_context(context).reviewer_id) + + +def _required_paths(*values: tuple[Path | None, str]) -> tuple[Path, ...]: + missing = [name for value, name in values if value is None] + if missing: + raise InvalidInput(f"missing required value: {', '.join(missing)}") + return tuple(value for value, _ in values if value is not None) + + +def _required_text(value: str | None, name: str) -> str: + if value is None: + raise InvalidInput(f"missing required value: {name}") + return value + + +def _required_nonblank_text(value: str | None, name: str) -> str: + text = _required_text(value, name) + if not text.strip(): + raise InvalidInput(f"{name} must not be blank") + return text + + +def _json_data(value: object) -> object: + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + if isinstance(value, tuple): + return [_json_data(item) for item in value] + if isinstance(value, list): + return [_json_data(item) for item in value] + return value + + +def _emit(envelope: JsonEnvelope) -> None: + typer.echo(envelope.model_dump_json()) + + +def _emit_feasibility(envelope: FeasibilityCommandEnvelope) -> None: + typer.echo(envelope.model_dump_json()) + + +def _fail(code: str, message: str, exit_code: int, details: object | None = None) -> None: + error: dict[str, object] = {"code": code, "message": message} + if details is not None: + error["details"] = _json_data(details) + _emit(JsonEnvelope(ok=False, error=error)) + raise typer.Exit(exit_code) + + +def _run_json(action: Callable[[], object]) -> None: + try: + _emit(JsonEnvelope(ok=True, data=_json_data(action()))) + except ValidationFailure as error: + _fail("manifest_unsupported", str(error), 2, error.violations) + except ( + InvalidInput, + InvalidTransition, + json.JSONDecodeError, + ValidationError, + ComparisonError, + ) as error: + _fail("invalid_input", str(error), 2) + except UnknownAdapterError as error: + _fail("adapter_not_found", str(error), 3) + except RecordNotFound as error: + _fail("record_not_found", str(error), 3) + except (ArtifactError, sqlite3.Error, OSError) as error: + _fail("persistence_failure", str(error), 4) + except Exception as error: + typer.echo(f"internal error: {error}", err=True) + _fail("internal_error", "unexpected internal error", 1) + + +def _run_human(action: Callable[[], object], render: Callable[[object], str]) -> None: + try: + typer.echo(render(action())) + except ( + ValidationFailure, + InvalidInput, + InvalidTransition, + json.JSONDecodeError, + ValidationError, + ComparisonError, + ) as error: + typer.echo(str(error), err=True) + raise typer.Exit(2) from error + except RecordNotFound as error: + typer.echo(str(error), err=True) + raise typer.Exit(3) from error + except UnknownAdapterError as error: + typer.echo(str(error), err=True) + raise typer.Exit(3) from error + except (ArtifactError, sqlite3.Error, OSError) as error: + typer.echo(str(error), err=True) + raise typer.Exit(4) from error + except Exception as error: + typer.echo(f"internal error: {error}", err=True) + raise typer.Exit(1) from error + + +def _run_character(action: Callable[[], object]) -> None: + try: + _emit(JsonEnvelope(ok=True, data=_json_data(action()))) + except CharacterNotFound: + _fail("character_not_found", "character evidence was not found", 2) + except CharacterCurationGrantError: + _fail("character_not_found", "character evidence was not found", 2) + except CharacterConflict: + _fail("character_not_found", "character evidence was not found", 2) + except CharacterPackNotReady as error: + _fail( + "character_not_ready", + "character pack coverage is incomplete", + 2, + {"missing": list(error.reason_codes)}, + ) + except PermissionError: + _fail("character_conflict", "character action conflicts with frozen evidence", 2) + except (CharacterImportError, InvalidInput, ValidationError, ValueError): + _fail("invalid_character_input", "character input is invalid", 2) + except (ArtifactError, sqlite3.Error, OSError): + _fail("character_persistence_failure", "character persistence failed", 4) + except Exception: + _fail("internal_error", "unexpected internal error", 1) + + +def _run_review_json(action: Callable[[], object]) -> None: + try: + _emit(JsonEnvelope(ok=True, data=_json_data(action()))) + except InvalidReviewInput: + _fail("invalid_review_input", "invalid review input", 2) + except PermissionError as error: + code = ( + "review_reveal_forbidden" + if "submitted before reveal" in str(error) + else "review_not_owned" + ) + _fail(code, "review operation is not permitted", 2) + except (RecordNotFound, LookupError): + _fail("review_not_found", "review record not found", 3) + except ValueError: + _fail("invalid_review_state", "review operation is invalid", 2) + except (ArtifactError, sqlite3.Error, OSError): + _fail("review_persistence_failure", "review persistence failed", 4) + except Exception as error: + typer.echo(f"internal error: {type(error).__name__}", err=True) + _fail("internal_error", "unexpected internal error", 1) + + +def _run_review_human(action: Callable[[], object]) -> None: + try: + typer.echo(json.dumps(_json_data(action()), indent=2)) + except InvalidReviewInput as error: + typer.echo(str(error), err=True) + raise typer.Exit(2) from error + except PermissionError as error: + typer.echo("review operation is not permitted", err=True) + raise typer.Exit(2) from error + except (RecordNotFound, LookupError) as error: + typer.echo("review record not found", err=True) + raise typer.Exit(3) from error + except ValueError as error: + typer.echo("review operation is invalid", err=True) + raise typer.Exit(2) from error + except (ArtifactError, sqlite3.Error, OSError) as error: + typer.echo("review persistence failed", err=True) + raise typer.Exit(4) from error + + +def _run_study_json(action: Callable[[], object]) -> None: + try: + _emit(JsonEnvelope(ok=True, data=_json_data(action()))) + except InvalidStudyInput: + _fail("invalid_study_input", "adaptive study input is invalid", 2) + except InvalidInput: + _fail("invalid_study_input", "adaptive study input is invalid", 2) + except ValidationFailure: + _fail("manifest_unsupported", "manifest is unsupported by the selected adapter", 2) + except ValidationError: + _fail("invalid_study_input", "adaptive study input is invalid", 2) + except UnknownAdapterError: + _fail("adapter_not_found", "adaptive study adapter was not found", 3) + except AdaptiveStudyNotFound: + _fail("study_not_found", "adaptive study was not found", 3) + except AdaptiveStudyConflict: + _fail("study_conflict", "adaptive study evidence conflicts with the requested action", 2) + except PermissionError: + _fail("study_review_not_owned", "adaptive review operation is not permitted", 2) + except LookupError: + _fail("study_review_not_found", "adaptive review was not found", 3) + except ValueError: + _fail("invalid_study_state", "adaptive study action is invalid in its current state", 2) + except (ArtifactError, sqlite3.Error, OSError): + _fail("study_persistence_failure", "adaptive study persistence failed", 4) + except Exception: + _fail("internal_error", "unexpected internal error", 1) + + +def _run_validation_json(action: Callable[[], object]) -> None: + try: + _emit(JsonEnvelope(ok=True, data=_json_data(action()))) + except (InvalidValidationInput, InvalidInput, ValidationError, json.JSONDecodeError): + _fail("invalid_validation_input", "validation input is invalid", 2) + except ValidationIneligible: + _fail("validation_ineligible", "validation evidence is not eligible", 2) + except ValidationNotFound: + _fail("validation_not_found", "validation campaign was not found", 2) + except ValidationConflict: + _fail("validation_conflict", "validation action conflicts with sealed evidence", 2) + except PermissionError as error: + if "sealed after preregistration" in str(error): + _fail( + "validation_preview_sealed", + "validation preview is sealed after preregistration", + 2, + ) + _fail("validation_not_owned", "validation operation is not permitted", 2) + except (ArtifactError, sqlite3.Error, OSError): + _fail("validation_persistence_failure", "validation persistence failed", 4) + except (UnknownAdapterError, AdaptiveStudyNotFound): + _fail("validation_ineligible", "validation evidence is not eligible", 2) + except ValueError: + _fail("invalid_validation_state", "validation action is invalid in its current state", 2) + except Exception: + _fail("internal_error", "unexpected internal error", 1) + + +def _run_character_benchmark_json(action: Callable[[], object]) -> None: + try: + _emit(JsonEnvelope(ok=True, data=_json_data(action()))) + except (InvalidInput, ValidationError, json.JSONDecodeError): + _fail("invalid_character_benchmark_input", "character benchmark input is invalid", 2) + except CharacterBenchmarkNotFound: + _fail("character_benchmark_not_found", "character benchmark was not found", 2) + except CharacterBenchmarkConflict as error: + message = str(error).lower() + if "another reviewer" in message or "belongs to another" in message: + _fail("character_benchmark_not_found", "character benchmark was not found", 2) + elif "character pack" in message: + code = "character_benchmark_incomplete_curation" + elif "stage 2" in message: + code = "character_benchmark_stage_2_ineligible" + elif "cannot enter analysis" in message or "not eligible" in message: + code = "character_benchmark_invalid_phase" + elif any( + marker in message + for marker in ( + "inconsistent", + "canonical", + "corrupt", + "digest", + "immutable", + "ownership", + ) + ): + code = "character_benchmark_corrupt_provenance" + elif "review" in message: + code = "character_benchmark_stale_review" + else: + code = "character_benchmark_conflict" + _fail(code, "character benchmark action conflicts with sealed evidence", 2) + except PermissionError as error: + code = ( + "character_benchmark_preview_sealed" + if "sealed after preregistration" in str(error) + else "character_benchmark_not_owned" + ) + _fail(code, "character benchmark operation is not permitted", 2) + except ArtifactError: + _fail( + "character_benchmark_corrupt_provenance", + "character benchmark action conflicts with sealed evidence", + 2, + ) + except (sqlite3.Error, OSError): + _fail( + "character_benchmark_persistence_failure", "character benchmark persistence failed", 4 + ) + except (UnknownAdapterError, ValidationNotFound, CharacterNotFound): + _fail("character_benchmark_ineligible", "character benchmark evidence is not eligible", 2) + except ValueError: + _fail("invalid_character_benchmark_state", "character benchmark action is invalid", 2) + except Exception: + _fail("internal_error", "unexpected internal error", 1) + + +def _feasibility_service( + context: typer.Context, + cache: Path | None = None, + *, + migrate: bool = True, + run_id: str | None = None, + inventory_evidence: Mapping[str, object] | None = None, +) -> object: + from video_lab.animegen_feasibility import ( + AnimeGenFeasibilityExecutor, + production_component_options, + production_generation_runtime, + production_inventory_client, + ) + from video_lab.artifacts import ArtifactStore + from video_lab.database import Database + from video_lab.feasibility import FeasibilityService, build_generation_feasibility_service + from video_lab.feasibility_repository import FeasibilityRepository + + database_path = _required_paths((_context(context).database, "database"))[0] + database = Database(database_path, read_only=not migrate) + if migrate: + database.migrate() + + safe_cache = None if cache is None else cache.expanduser().resolve() + inventory_client = None if safe_cache is None else production_inventory_client(safe_cache) + executor = AnimeGenFeasibilityExecutor( + safe_cache, + inventory_client, + component_options=production_component_options(), + inventory_evidence=inventory_evidence, + ) + options = _context(context) + artifact_store = ( + None if options.artifacts is None else ArtifactStore(database, options.artifacts) + ) + repository = FeasibilityRepository(database, artifact_store) + delegate = FeasibilityService(repository, executor) + if options.artifacts is None: + return delegate + if run_id is None or inventory_client is None: + return delegate + assert safe_cache is not None + from video_lab.adapters.animegen_mps import ( + ADAPTER_ID, + AnimeGenGenerationRuntime, + AnimeGenMpsAdapter, + ) + from video_lab.feasibility import StageExecutionContext + from video_lab.feasibility_models import FeasibilityStage + + run = repository.get_run(run_id) + attempts = repository.list_attempts(run_id) + generation_stages = { + FeasibilityStage.GENERATE_MINIMUM_1.value, + FeasibilityStage.GENERATE_MINIMUM_2.value, + FeasibilityStage.QUALIFY_STANDARD.value, + FeasibilityStage.VALIDATE_CONTROLLED_PAIR.value, + } + if _next_feasibility_stage(delegate.status(run_id)) not in generation_stages: + return delegate + inventory = executor.validated_download_inventory(StageExecutionContext(run, attempts)) + sources = {source.component: source for source in inventory.sources} + runtime = production_generation_runtime(safe_cache, sources) + candidate = AnimeGenMpsAdapter( + runtime=cast(AnimeGenGenerationRuntime, runtime), + authorized_profile=(512, 288, 17), + exact_profile_only=False, + ) + feasibility_registry = AdapterRegistry() + feasibility_registry.register( + ADAPTER_ID, lambda: candidate, capabilities_factory=candidate.capabilities + ) + service = ExperimentService.from_paths(database_path, options.artifacts) + worker = Worker( + repository=service.jobs, + artifact_store=service.artifacts, + adapter_registry=feasibility_registry, + scratch_root=options.artifacts / ".scratch", + worker_id="feasibility-cli", + ) + return build_generation_feasibility_service( + repository=repository, + delegate=executor, + experiments=service.experiments, + jobs=service.jobs, + artifacts=service.artifacts, + worker=worker, + ) + + +def _next_feasibility_stage(status: Any) -> str | None: + from video_lab.feasibility_projection import next_feasibility_stage + + return next_feasibility_stage(status.attempts) + + +def _feasibility_data( + status: Any, + repository: Any, + *, + reviewer_id: str | None = None, + completed: Any | None = None, +) -> dict[str, object]: + from video_lab.feasibility_projection import FeasibilityEvidenceProjector + + return FeasibilityEvidenceProjector(repository).project( + status, + reviewer_id=reviewer_id, + completed_attempt=completed, + ) + + +def _run_feasibility(action: Callable[[], object], *, json_output: bool) -> None: + from video_lab.feasibility_repository import FeasibilityRunNotFound + from video_lab.source_inventory import SourceInventoryError + + try: + value = action() + if json_output: + _emit_feasibility(FeasibilityCommandEnvelope(ok=True, data=value)) + else: + data = value if isinstance(value, dict) else {} + run = data.get("run", {}) + typer.echo( + f"{run.get('adapter_id', 'feasibility')} {run.get('id', '')}: " + f"{data.get('verdict', 'pending')}" + ) + except UnknownAdapterError as error: + _fail_feasibility("adapter_not_found", str(error), 3, json_output) + except FeasibilityRunNotFound as error: + _fail_feasibility("feasibility_run_not_found", str(error), 3, json_output) + except ( + InvalidInput, + ValidationError, + json.JSONDecodeError, + SourceInventoryError, + ValueError, + ) as error: + code = "cache_required" if "cache is required" in str(error) else "invalid_input" + _fail_feasibility(code, str(error), 2, json_output) + except (sqlite3.Error, OSError) as error: + _fail_feasibility("persistence_failure", str(error), 4, json_output) + except Exception as error: + typer.echo(f"internal error: {type(error).__name__}", err=True) + _fail_feasibility("internal_error", "unexpected internal error", 1, json_output) + + +def _fail_feasibility(code: str, message: str, exit_code: int, json_output: bool) -> None: + from video_lab.feasibility import sanitize_diagnostic + + safe_message = str(sanitize_diagnostic(message)) + if json_output: + _emit_feasibility( + FeasibilityCommandEnvelope(ok=False, error={"code": code, "message": safe_message}) + ) + else: + typer.echo(safe_message, err=True) + raise typer.Exit(exit_code) + + +def _review_input(path: Path) -> ReviewInput: + try: + return _load(path, ReviewInput) + except (InvalidInput, ValidationError, json.JSONDecodeError) as error: + raise InvalidReviewInput("invalid review input") from error + + +def _review_data(key: str, value: object) -> dict[str, object]: + return {"schema_version": "review.v1", key: _json_data(value)} + + +def _load[ModelT: FrozenModel](path: Path, model: type[ModelT]) -> ModelT: + try: + document = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as error: + raise InvalidInput(f"cannot read input document {path}: {error}") from error + return model.model_validate_json(document) + + +def _view_data(view: ExperimentView) -> dict[str, object]: + return { + "experiment": view.experiment.model_dump(mode="json"), + "jobs": [job.model_dump(mode="json") for job in view.jobs], + } + + +def _validate(draft_path: Path, capabilities_path: Path) -> dict[str, list[str]]: + draft = _load(draft_path, ExperimentDraft) + capabilities = _load(capabilities_path, AdapterCapabilities) + violations = validate_manifest_support(draft.baseline, capabilities) + if violations: + raise ValidationFailure(violations) + pair = build_controlled_pair(draft) + violations = validate_manifest_support(pair.variant_b, capabilities) + if violations: + raise ValidationFailure(violations) + return {"changed_paths": list(pair.changed_paths)} + + +def _worker(context: typer.Context, worker_id: str) -> Worker: + service = _service(context) + options = _context(context) + artifact_root = _required_paths((options.artifacts, "artifacts"))[0] + return Worker( + repository=service.jobs, + artifact_store=service.artifacts, + adapter_registry=AdapterRegistry.with_defaults(), + scratch_root=artifact_root / ".scratch", + worker_id=worker_id, + ) + + +def _outcome_data(outcome: WorkerOutcome, service: ExperimentService) -> dict[str, object]: + data: dict[str, object] = {"schema_version": "worker.v1", **asdict(outcome)} + if outcome.error_code is not None and outcome.attempt_id is not None: + attempts = service.jobs.attempts_for_job(outcome.job_id or "") + attempt = next((item for item in attempts if item.id == outcome.attempt_id), None) + data["error_message"] = None if attempt is None else attempt.error_message + return data + + +def _status_data(service: ExperimentService, job_id: str) -> dict[str, object]: + job = service.jobs.get(job_id) + attempts = service.jobs.attempts_for_job(job_id) + events = service.jobs.events_for_job(job_id) + telemetry_count = sum( + len(service.jobs.telemetry_for_attempt(attempt.id)) for attempt in attempts + ) + return { + "schema_version": "worker.v1", + "job": job.model_dump(mode="json"), + "current_attempt": None if not attempts else attempts[-1].model_dump(mode="json"), + "attempts": [attempt.model_dump(mode="json") for attempt in attempts], + "events": [event.model_dump(mode="json") for event in events], + "telemetry_count": telemetry_count, + "artifacts": [asdict(artifact) for artifact in service.artifacts.list_for_job(job_id)], + } + + +@db_app.command("init") +def db_init(context: typer.Context, json_output: bool = typer.Option(False, "--json")) -> None: + def initialize() -> dict[str, bool]: + options = _context(context) + database, _ = _required_paths( + (options.database, "database"), (options.artifacts, "artifacts") + ) + Database(database).migrate() + return {"initialized": True} + + if json_output: + _run_json(initialize) + else: + _run_human(initialize, lambda _: "Database initialized.") + + +@character_app.command("create") +def character_create( + context: typer.Context, + name: Annotated[str | None, typer.Argument()] = None, +) -> None: + def create() -> dict[str, object]: + record = _character_service(context).create_character(_required_nonblank_text(name, "name")) + return { + "character_id": record.character_id, + "name": record.name, + "created_at": record.created_at.isoformat(), + } + + _run_character(create) + + +@character_app.command("import") +def character_import( + context: typer.Context, + character_id: Annotated[str | None, typer.Argument()] = None, + source: Annotated[Path | None, typer.Option("--source")] = None, +) -> None: + def import_images() -> object: + source_path = _required_paths((source, "source"))[0] + return _character_service(context).import_folder( + _required_nonblank_text(character_id, "character_id"), source_path + ) + + _run_character(import_images) + + +@character_app.command("status") +def character_status( + context: typer.Context, + pack_id: Annotated[str | None, typer.Argument()] = None, +) -> None: + _run_character( + lambda: _character_service(context).pack_status(_required_nonblank_text(pack_id, "pack_id")) + ) + + +@character_app.command("open") +def character_open( + context: typer.Context, + character_id: Annotated[str | None, typer.Argument()] = None, + pack_id: Annotated[str | None, typer.Option("--pack")] = None, +) -> None: + def issue() -> dict[str, object]: + grants, reviewer = _character_grants(context) + token = grants.issue( + reviewer, + _required_nonblank_text(character_id, "character_id"), + _required_nonblank_text(pack_id, "pack_id"), + ) + return { + "url": f"/characters/claim#token={token}", + "expires_in_seconds": int(CHARACTER_CURATION_GRANT_TTL.total_seconds()), + } + + _run_character(issue) + + +@character_app.command("curate") +def character_curate( + context: typer.Context, + pack_id: Annotated[str | None, typer.Argument()] = None, + reference_id: Annotated[str | None, typer.Argument()] = None, + framing: Annotated[Framing | None, typer.Option("--framing")] = None, + head_angle: Annotated[HeadAngle | None, typer.Option("--head-angle")] = None, + body_visibility: Annotated[BodyVisibility | None, typer.Option("--body-visibility")] = None, + expression: Annotated[Expression | None, typer.Option("--expression")] = None, + lighting: Annotated[Lighting | None, typer.Option("--lighting")] = None, + occlusion: Annotated[Occlusion | None, typer.Option("--occlusion")] = None, + identity_confirmed: Annotated[ + bool | None, + typer.Option("--identity-confirmed/--identity-not-confirmed"), + ] = None, + accept: Annotated[bool, typer.Option("--accept")] = False, + reject: Annotated[bool, typer.Option("--reject")] = False, +) -> None: + def curate() -> object: + annotations = ( + framing, + head_angle, + body_visibility, + expression, + lighting, + occlusion, + identity_confirmed, + ) + if any(value is None for value in annotations): + raise InvalidInput("every character annotation is required") + if accept == reject: + raise InvalidInput("exactly one of --accept or --reject is required") + assert framing is not None + assert head_angle is not None + assert body_visibility is not None + assert expression is not None + assert lighting is not None + assert occlusion is not None + assert identity_confirmed is not None + annotation = CharacterAnnotation( + framing=framing, + head_angle=head_angle, + body_visibility=body_visibility, + expression=expression, + lighting=lighting, + occlusion=occlusion, + identity_confirmed=identity_confirmed, + ) + return _character_service(context).curate_reference( + _required_nonblank_text(pack_id, "pack_id"), + _required_nonblank_text(reference_id, "reference_id"), + annotation=annotation, + disposition=(CuratorDisposition.ACTIVE if accept else CuratorDisposition.REJECTED), + ) + + _run_character(curate) + + +@character_app.command("resolve-warning") +def character_resolve_warning( + context: typer.Context, + pack_id: Annotated[str | None, typer.Argument()] = None, + left_reference_id: Annotated[str | None, typer.Argument()] = None, + right_reference_id: Annotated[str | None, typer.Argument()] = None, + resolution: Annotated[str | None, typer.Option("--resolution")] = None, +) -> None: + def resolve() -> object: + if resolution not in { + WarningResolution.ACCEPTED.value, + WarningResolution.DISMISSED.value, + }: + raise InvalidInput("warning resolution must be accepted or dismissed") + return _character_service(context).resolve_warning( + _required_nonblank_text(pack_id, "pack_id"), + _required_nonblank_text(left_reference_id, "left_reference_id"), + _required_nonblank_text(right_reference_id, "right_reference_id"), + WarningResolution(resolution), + ) + + _run_character(resolve) + + +@character_app.command("preregister") +def character_preregister( + context: typer.Context, + pack_id: Annotated[str | None, typer.Argument()] = None, +) -> None: + _run_character( + lambda: _character_service(context).preregister_pack( + _required_nonblank_text(pack_id, "pack_id") + ) + ) + + +@character_app.command("export-pack") +def character_export_pack( + context: typer.Context, + pack_id: Annotated[str | None, typer.Argument()] = None, + output: Annotated[Path | None, typer.Option("--output")] = None, +) -> None: + def export() -> dict[str, object]: + output_path = _required_paths((output, "output"))[0] + if output_path.exists() or output_path.is_symlink(): + raise InvalidInput("character pack output already exists") + exported = _character_service(context).export_pack( + _required_nonblank_text(pack_id, "pack_id") + ) + encoded = exported.model_dump_json() + "\n" + try: + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("x", encoding="utf-8") as stream: + stream.write(encoded) + except FileExistsError as error: + raise InvalidInput("character pack output already exists") from error + except (OSError, UnicodeError) as error: + raise InvalidInput("character pack export failed") from error + return { + "pack_id": exported.pack.pack_id, + "sha256": hashlib.sha256(encoded.encode("utf-8")).hexdigest(), + } + + _run_character(export) + + +@experiment_app.command("validate") +def validate_experiment( + context: typer.Context, + draft: Annotated[Path | None, typer.Option("--draft")] = None, + capabilities: Annotated[Path | None, typer.Option("--capabilities")] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def validate() -> dict[str, list[str]]: + draft_path, capabilities_path = _required_paths( + (draft, "draft"), (capabilities, "capabilities") + ) + return _validate(draft_path, capabilities_path) + + if json_output: + _run_json(validate) + else: + _run_human( + validate, + lambda value: ( + f"Valid experiment: {', '.join(value['changed_paths'])}" + if isinstance(value, dict) and isinstance(value.get("changed_paths"), list) + else "Valid experiment." + ), + ) + + +@experiment_app.command("create") +def create_experiment( + context: typer.Context, + draft: Annotated[Path | None, typer.Option("--draft")] = None, + capabilities: Annotated[Path | None, typer.Option("--capabilities")] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def create() -> dict[str, object]: + draft_path, capabilities_path = _required_paths( + (draft, "draft"), (capabilities, "capabilities") + ) + draft_value = _load(draft_path, ExperimentDraft) + capabilities_value = _load(capabilities_path, AdapterCapabilities) + return _view_data(_service(context).create(draft_value, capabilities_value)) + + if json_output: + _run_json(create) + else: + _run_human( + create, + lambda value: ( + f"Created experiment {value['experiment']['id']}." + if isinstance(value, dict) + and isinstance(value.get("experiment"), dict) + and "id" in value["experiment"] + else "Created experiment." + ), + ) + + +@experiment_app.command("get") +def get_experiment( + context: typer.Context, + experiment_id: Annotated[str | None, typer.Argument()] = None, + json_output: bool = typer.Option(False, "--json"), +) -> None: + def get() -> dict[str, object]: + return _view_data(_service(context).get(_required_text(experiment_id, "experiment_id"))) + + if json_output: + _run_json(get) + else: + _run_human(get, lambda value: json.dumps(value, indent=2)) + + +@experiment_app.command("list") +def list_experiments( + context: typer.Context, + limit: int = typer.Option(50, "--limit", min=1), + json_output: bool = typer.Option(False, "--json"), +) -> None: + def list_values() -> list[dict[str, object]]: + return [_view_data(view) for view in _service(context).list(limit)] + + if json_output: + _run_json(list_values) + else: + + def render(values: object) -> str: + if not isinstance(values, list) or not values: + return "No experiments." + return "\n".join( + str(value["experiment"]["id"]) + for value in values + if isinstance(value, dict) and isinstance(value.get("experiment"), dict) + ) + + _run_human(list_values, render) + + +@worker_app.command("run-once") +def worker_run_once( + context: typer.Context, + worker_id: Annotated[str | None, typer.Option("--worker-id")] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def run() -> dict[str, object]: + stable_worker_id = _required_nonblank_text(worker_id, "worker_id") + service = _service(context) + return _outcome_data(_worker(context, stable_worker_id).run_once(), service) + + if json_output: + _run_json(run) + else: + _run_human(run, lambda value: json.dumps(value, indent=2)) + + +@worker_app.command("recover") +def worker_recover( + context: typer.Context, + worker_id: Annotated[str | None, typer.Option("--worker-id")] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def recover() -> dict[str, object]: + stable_worker_id = _required_nonblank_text(worker_id, "worker_id") + recovered = _worker(context, stable_worker_id).recover() + return { + "schema_version": "worker.v1", + "recovered_count": len(recovered), + "jobs": [job.model_dump(mode="json") for job in recovered], + } + + if json_output: + _run_json(recover) + else: + _run_human(recover, lambda value: json.dumps(value, indent=2)) + + +@worker_app.command("cancel") +def worker_cancel( + context: typer.Context, + job_id: Annotated[str | None, typer.Argument()] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def cancel() -> dict[str, object]: + service = _service(context) + job = service.jobs.request_cancellation(_required_text(job_id, "job_id")) + return {"schema_version": "worker.v1", "job": job.model_dump(mode="json")} + + if json_output: + _run_json(cancel) + else: + _run_human(cancel, lambda value: json.dumps(value, indent=2)) + + +@worker_app.command("status") +def worker_status( + context: typer.Context, + job_id: Annotated[str | None, typer.Argument()] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def status() -> dict[str, object]: + return _status_data(_service(context), _required_text(job_id, "job_id")) + + if json_output: + _run_json(status) + else: + _run_human(status, lambda value: json.dumps(value, indent=2)) + + +def _review_show(context: typer.Context, session_id: str) -> dict[str, object]: + service = _review_service(context) + reviewer_id = _reviewer_id(context) + for position in ("left", "right"): + service.artifact_for_display(session_id, position, reviewer_id) + view = BlindReviewView( + session_id=session_id, + media=( + BlindMedia(display_position="left", url=f"/reviews/{session_id}/artifacts/left"), + BlindMedia(display_position="right", url=f"/reviews/{session_id}/artifacts/right"), + ), + prompt=service.prompt_for_session(session_id, reviewer_id), + ) + return _review_data("session", view) + + +@review_app.command("queue") +def review_queue( + context: typer.Context, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def queue() -> dict[str, object]: + return _review_data("items", _review_service(context).queue()) + + if json_output: + _run_review_json(queue) + else: + _run_review_human(queue) + + +@review_app.command("start") +def review_start( + context: typer.Context, + experiment_id: Annotated[str | None, typer.Argument()] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def start() -> dict[str, object]: + try: + exact_id = _required_text(experiment_id, "experiment_id") + except InvalidInput as error: + raise InvalidReviewInput(str(error)) from error + view = _review_service(context).start(exact_id, _reviewer_id(context)) + return _review_data("session", view) + + if json_output: + _run_review_json(start) + else: + _run_review_human(start) + + +@review_app.command("show") +def review_show( + context: typer.Context, + session_id: Annotated[str | None, typer.Argument()] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def show() -> dict[str, object]: + try: + exact_id = _required_text(session_id, "session_id") + except InvalidInput as error: + raise InvalidReviewInput(str(error)) from error + return _review_show(context, exact_id) + + if json_output: + _run_review_json(show) + else: + _run_review_human(show) + + +def _review_submit_action( + context: typer.Context, session_id: str | None, input_path: Path | None, *, correction: bool +) -> dict[str, object]: + try: + exact_session_id = _required_text(session_id, "session_id") + exact_input = _required_paths((input_path, "input"))[0] + except InvalidInput as error: + raise InvalidReviewInput(str(error)) from error + document = _review_input(exact_input) + service = _review_service(context) + reviewer_id = _reviewer_id(context) + if correction: + if document.supersedes_review_id is None: + raise InvalidReviewInput("correction requires supersedes_review_id") + submission = service.correct( + exact_session_id, + reviewer_id, + document.supersedes_review_id, + document.display_preference, + document.scores, + document.notes, + ) + else: + if document.supersedes_review_id is not None: + raise InvalidReviewInput("initial submission cannot supersede a review") + submission = service.submit( + exact_session_id, + reviewer_id, + document.display_preference, + document.scores, + document.notes, + ) + return _review_data("submission", submission) + + +@review_app.command("submit") +def review_submit( + context: typer.Context, + session_id: Annotated[str | None, typer.Argument()] = None, + input_path: Annotated[Path | None, typer.Option("--input")] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def action() -> dict[str, object]: + return _review_submit_action(context, session_id, input_path, correction=False) + + if json_output: + _run_review_json(action) + else: + _run_review_human(action) + + +@review_app.command("correct") +def review_correct( + context: typer.Context, + session_id: Annotated[str | None, typer.Argument()] = None, + input_path: Annotated[Path | None, typer.Option("--input")] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def action() -> dict[str, object]: + return _review_submit_action(context, session_id, input_path, correction=True) + + if json_output: + _run_review_json(action) + else: + _run_review_human(action) + + +@review_app.command("reveal") +def review_reveal( + context: typer.Context, + session_id: Annotated[str | None, typer.Argument()] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def reveal() -> dict[str, object]: + try: + exact_id = _required_text(session_id, "session_id") + except InvalidInput as error: + raise InvalidReviewInput(str(error)) from error + value = _review_service(context).reveal(exact_id, _reviewer_id(context)) + return _review_data("reveal", value) + + if json_output: + _run_review_json(reveal) + else: + _run_review_human(reveal) + + +@feasibility_app.command("start") +def feasibility_start( + context: typer.Context, + adapter_id: Annotated[str | None, typer.Argument()] = None, + input_path: Annotated[Path | None, typer.Option("--input")] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def start() -> dict[str, object]: + from video_lab.animegen_feasibility import canonical_animegen_configuration + from video_lab.feasibility_models import FeasibilityConfiguration + + exact_adapter = _required_nonblank_text(adapter_id, "adapter_id") + AdapterRegistry.with_defaults().capabilities(exact_adapter) + configuration_path = _required_paths((input_path, "input"))[0] + configuration = _load(configuration_path, FeasibilityConfiguration) + if configuration.adapter_id != exact_adapter: + raise InvalidInput("configuration adapter_id does not match requested adapter") + from video_lab.runtime_identity import animegen_runtime_versions + + if dict(configuration.runtime_versions) != animegen_runtime_versions(): + raise InvalidInput( + "configuration runtime identity does not match this video-lab process" + ) + canonical = canonical_animegen_configuration(configuration) + service = cast(Any, _feasibility_service(context)) + run = service.start(canonical) + return _feasibility_data( + service.status(run.id), + service.repository, + reviewer_id=_projection_reviewer_id(context), + ) + + _run_feasibility(start, json_output=json_output) + + +@feasibility_app.command("runtime-identity") +def feasibility_runtime_identity( + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def identity() -> dict[str, object]: + from video_lab.runtime_identity import animegen_runtime_versions, runtime_fingerprint + + versions = animegen_runtime_versions() + return { + "runtime_versions": versions, + "runtime_fingerprint": runtime_fingerprint(versions), + } + + _run_feasibility(identity, json_output=json_output) + + +@feasibility_app.command("status") +def feasibility_status( + context: typer.Context, + run_id: Annotated[str | None, typer.Argument()] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def status() -> dict[str, object]: + service = cast(Any, _feasibility_service(context, migrate=False)) + return _feasibility_data( + service.status(_required_nonblank_text(run_id, "run_id")), + service.repository, + reviewer_id=_projection_reviewer_id(context), + ) + + _run_feasibility(status, json_output=json_output) + + +@feasibility_app.command("resume") +def feasibility_resume( + context: typer.Context, + run_id: Annotated[str | None, typer.Argument()] = None, + cache: Annotated[Path | None, typer.Option("--cache")] = None, + inventory_evidence: Annotated[Path | None, typer.Option("--inventory-evidence")] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def resume() -> dict[str, object]: + from video_lab.feasibility_models import FeasibilityStage + + exact_run_id = _required_nonblank_text(run_id, "run_id") + imported: Mapping[str, object] | None = None + if inventory_evidence is not None: + try: + raw = json.loads(inventory_evidence.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise InvalidInput("inventory evidence is not valid JSON") from error + if not isinstance(raw, Mapping): + raise InvalidInput("inventory evidence must be a JSON object") + imported = cast(Mapping[str, object], raw) + service = cast( + Any, + _feasibility_service(context, cache, run_id=exact_run_id, inventory_evidence=imported), + ) + before = service.status(exact_run_id) + next_stage = _next_feasibility_stage(before) + cache_stages = { + stage.value + for stage in FeasibilityStage + if stage not in {FeasibilityStage.RESOLVE_SOURCES, FeasibilityStage.PUBLISH_VERDICT} + } + if next_stage in cache_stages and cache is None: + raise InvalidInput("cache is required for this feasibility stage") + generation_stages = { + FeasibilityStage.GENERATE_MINIMUM_1.value, + FeasibilityStage.GENERATE_MINIMUM_2.value, + FeasibilityStage.QUALIFY_STANDARD.value, + FeasibilityStage.VALIDATE_CONTROLLED_PAIR.value, + } + if next_stage in generation_stages and _context(context).artifacts is None: + raise InvalidInput("artifacts is required for this feasibility stage") + prior_count = len(before.attempts) + after = service.resume(exact_run_id) + completed = after.attempts[-1] if len(after.attempts) > prior_count else None + return _feasibility_data( + after, + service.repository, + reviewer_id=_projection_reviewer_id(context), + completed=completed, + ) + + _run_feasibility(resume, json_output=json_output) + + +@feasibility_app.command("export-inventory") +def feasibility_export_inventory( + context: typer.Context, + run_id: Annotated[str | None, typer.Argument()] = None, + output: Annotated[Path | None, typer.Option("--output")] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def export() -> dict[str, object]: + from video_lab.feasibility import sanitize_diagnostic + from video_lab.feasibility_models import FeasibilityStage, StageOutcome + + exact_run_id = _required_nonblank_text(run_id, "run_id") + output_path = _required_paths((output, "output"))[0] + if output_path.exists() or output_path.is_symlink(): + raise InvalidInput("inventory evidence output already exists") + status = _feasibility_service(context, migrate=False).status(exact_run_id) # type: ignore[attr-defined] + preferred = ( + FeasibilityStage.DOWNLOAD_SOURCES, + FeasibilityStage.INVENTORY_SOURCES, + ) + attempt = next( + ( + item + for stage in preferred + for item in reversed(status.attempts) + if item.stage is stage + and item.outcome is StageOutcome.PASSED + and item.evidence is not None + and isinstance(item.evidence.details.get("inventory"), Mapping) + ), + None, + ) + if attempt is None or attempt.evidence is None: + raise InvalidInput("run has no passed source inventory evidence") + inventory = sanitize_diagnostic(attempt.evidence.details["inventory"]) + encoded = json.dumps(inventory, sort_keys=True, separators=(",", ":")) + "\n" + try: + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(encoded, encoding="utf-8") + except (OSError, UnicodeError) as error: + raise InvalidInput("inventory evidence export failed") from error + return { + "run_id": exact_run_id, + "source_stage": attempt.stage.value, + "sha256": hashlib.sha256(encoded.encode()).hexdigest(), + } + + _run_feasibility(export, json_output=json_output) + + +@feasibility_app.command("export-evidence") +def feasibility_export_evidence( + context: typer.Context, + run_id: Annotated[str | None, typer.Argument()] = None, + output: Annotated[Path | None, typer.Option("--output")] = None, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + """Export the reviewer-scoped phase ledger without media or cache bytes.""" + + def export() -> dict[str, object]: + exact_run_id = _required_nonblank_text(run_id, "run_id") + output_path = _required_paths((output, "output"))[0] + if output_path.exists() or output_path.is_symlink(): + raise InvalidInput("feasibility evidence output already exists") + service = cast(Any, _feasibility_service(context, migrate=False)) + projection = _feasibility_data( + service.status(exact_run_id), + service.repository, + reviewer_id=_projection_reviewer_id(context), + ) + encoded = json.dumps(projection, sort_keys=True, separators=(",", ":")) + "\n" + try: + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(encoded, encoding="utf-8") + except (OSError, UnicodeError) as error: + raise InvalidInput("feasibility evidence export failed") from error + return { + "run_id": exact_run_id, + "schema_version": projection["schema_version"], + "sha256": hashlib.sha256(encoded.encode()).hexdigest(), + } + + _run_feasibility(export, json_output=json_output) + + +@study_app.command("create") +def study_create( + context: typer.Context, + manifest: Annotated[Path | None, typer.Option("--manifest")] = None, + adapter: Annotated[str | None, typer.Option("--adapter")] = None, + capabilities: Annotated[Path | None, typer.Option("--capabilities")] = None, + prompt: Annotated[str | None, typer.Option("--prompt")] = None, + seed: Annotated[int | None, typer.Option("--seed")] = None, +) -> None: + """Create or resume the content-addressed study for a frozen manifest.""" + + def create() -> object: + manifest_path = _required_paths((manifest, "manifest"))[0] + frozen = _load(manifest_path, Manifest) + if prompt is not None and prompt != frozen.prompt: + raise InvalidStudyInput("prompt override conflicts with the frozen manifest") + if seed is not None and seed != frozen.seed: + raise InvalidStudyInput("seed override conflicts with the frozen manifest") + selected_adapter = frozen.adapter_id if adapter is None else adapter.strip() + if not selected_adapter or selected_adapter != frozen.adapter_id: + raise InvalidStudyInput("adapter selection conflicts with the frozen manifest") + registered_capabilities = AdapterRegistry.with_defaults().capabilities(selected_adapter) + selected_capabilities = ( + registered_capabilities + if capabilities is None + else _load(capabilities, AdapterCapabilities) + ) + if selected_capabilities != registered_capabilities: + raise InvalidStudyInput("capabilities conflict with the registered adapter") + config = OptimizerConfig() + fingerprint = adaptive_study_input_fingerprint(frozen, config) + service = _adaptive_study_service(context, selected_capabilities) + reviewer = _study_reviewer_id(context) + study_digest = hashlib.sha256( + b"video-lab.adaptive-cli-study-id.v1\x00" + + reviewer.encode("utf-8") + + b"\x00" + + fingerprint.encode("ascii") + ).hexdigest() + study_id = f"study_{study_digest[:24]}" + try: + definition = service.studies.get(study_id) + except AdaptiveStudyNotFound: + definition = AdaptiveStudyDefinition( + id=study_id, + prompt=frozen.prompt, + negative_prompt=frozen.negative_prompt, + seed=frozen.seed, + base_manifest=frozen, + optimizer_config=config, + input_fingerprint=fingerprint, + created_at=utc_now(), + ) + try: + return service.create_study(definition) + except ValidationFailure: + raise + except ValueError as error: + raise InvalidStudyInput("adaptive study manifest is invalid") from error + + _run_study_json(create) + + +@study_app.command("list") +def study_list( + context: typer.Context, + limit: Annotated[int, typer.Option("--limit", min=1, max=100)] = 50, +) -> None: + """List bounded sealed study summaries.""" + + def list_values() -> object: + capabilities = AdapterRegistry.with_defaults().capabilities("video-lab.fake.v1") + return _adaptive_study_service(context, capabilities).list(limit=limit) + + _run_study_json(list_values) + + +@study_app.command("show") +def study_show( + context: typer.Context, + study_id: Annotated[str | None, typer.Argument()] = None, +) -> None: + """Show the reviewer-scoped sealed state of one study.""" + + def show() -> object: + exact_study_id = _required_nonblank_text(study_id, "study_id") + return _adaptive_study_service_for_id(context, exact_study_id).show(exact_study_id) + + _run_study_json(show) + + +@study_app.command("resume") +def study_resume( + context: typer.Context, + study_id: Annotated[str | None, typer.Argument()] = None, +) -> None: + """Resume the next deterministic local study action.""" + + def resume() -> object: + exact_study_id = _required_nonblank_text(study_id, "study_id") + return _adaptive_study_service_for_id(context, exact_study_id).resume(exact_study_id) + + _run_study_json(resume) + + +@study_app.command("submit") +def study_submit( + context: typer.Context, + study_id: Annotated[str | None, typer.Argument()] = None, + review_id: Annotated[str | None, typer.Option("--review-id")] = None, + choice: Annotated[Literal["a", "b", "tie"] | None, typer.Option("--choice")] = None, +) -> None: + """Submit a human A/B/tie display choice to its sealed review.""" + + def submit() -> object: + exact_study_id = _required_nonblank_text(study_id, "study_id") + exact_review_id = _required_nonblank_text(review_id, "review_id") + if choice is None: + raise InvalidStudyInput("choice is required") + display_choice = { + "a": DisplayPreference.LEFT, + "b": DisplayPreference.RIGHT, + "tie": DisplayPreference.TIE, + }[choice] + return _adaptive_study_service_for_id(context, exact_study_id).record_preference( + exact_study_id, + exact_review_id, + display_choice, + ) + + _run_study_json(submit) + + +@validation_app.command("create") +def validation_create( + context: typer.Context, + study_id: Annotated[str | None, typer.Option("--study")] = None, + panel: Annotated[Path | None, typer.Option("--panel")] = None, +) -> None: + """Create or replay a reviewer-owned balanced validation campaign.""" + + def create() -> object: + exact_study_id = _required_nonblank_text(study_id, "study_id") + panel_path = _required_paths((panel, "panel"))[0] + panel_input = _load(panel_path, ValidationPanelInput) + return _validation_campaign_service_for_study(context, exact_study_id).create( + stage_1_id=exact_study_id, panel_input=panel_input + ) + + _run_validation_json(create) + + +@validation_app.command("preview") +def validation_preview( + context: typer.Context, + campaign_id: Annotated[str | None, typer.Argument()] = None, +) -> None: + """Show draft-only candidate and panel sizing information.""" + + def preview() -> object: + exact_campaign_id = _required_nonblank_text(campaign_id, "campaign_id") + return _validation_campaign_service_for_id(context, exact_campaign_id).preview( + exact_campaign_id + ) + + _run_validation_json(preview) + + +@validation_app.command("preregister") +def validation_preregister( + context: typer.Context, + campaign_id: Annotated[str | None, typer.Argument()] = None, +) -> None: + """Freeze the balanced panel and materialize its controlled generation jobs.""" + + def preregister() -> object: + exact_campaign_id = _required_nonblank_text(campaign_id, "campaign_id") + return _validation_campaign_service_for_id(context, exact_campaign_id).preregister( + exact_campaign_id + ) + + _run_validation_json(preregister) + + +@validation_app.command("status") +def validation_status( + context: typer.Context, + campaign_id: Annotated[str | None, typer.Argument()] = None, +) -> None: + """Show the reviewer-scoped sealed campaign status.""" + + def status() -> object: + exact_campaign_id = _required_nonblank_text(campaign_id, "campaign_id") + return _validation_campaign_service_for_id(context, exact_campaign_id).show( + exact_campaign_id + ) + + _run_validation_json(status) + + +@validation_app.command("resume") +def validation_resume( + context: typer.Context, + campaign_id: Annotated[str | None, typer.Argument()] = None, +) -> None: + """Advance one deterministic campaign action and return its opaque review session.""" + + def resume() -> object: + exact_campaign_id = _required_nonblank_text(campaign_id, "campaign_id") + service = _validation_campaign_service_for_id(context, exact_campaign_id) + view = service.resume(exact_campaign_id) + if view.review_id is None: + return view + trial_id = service.reviews.validation_trial_for_session( + campaign_id=exact_campaign_id, + session_id=view.review_id, + reviewer_id=service.reviewer_id, + ) + return view.model_copy( + update={ + "media": tuple( + item.model_copy( + update={ + "media_ref": service.reviews.artifacts.resolve( + service.reviews.artifact_for_validation_display( + campaign_id=exact_campaign_id, + trial_id=trial_id, + session_id=view.review_id, + position=item.position, + reviewer_id=service.reviewer_id, + ).id + ) + .resolve() + .as_uri() + } + ) + for item in view.media + ) + } + ) + + _run_validation_json(resume) + + +@validation_app.command("submit") +def validation_submit( + context: typer.Context, + campaign_id: Annotated[str | None, typer.Argument()] = None, + review_id: Annotated[str | None, typer.Option("--review-id")] = None, + choice: Annotated[Literal["a", "b", "tie"] | None, typer.Option("--choice")] = None, +) -> None: + """Submit an A/B/tie judgment to the next opaque validation review.""" + + def submit() -> object: + exact_campaign_id = _required_nonblank_text(campaign_id, "campaign_id") + exact_review_id = _required_nonblank_text(review_id, "review_id") + if choice is None: + raise InvalidValidationInput("choice is required") + return _validation_campaign_service_for_id(context, exact_campaign_id).record_choice( + exact_campaign_id, ValidationChoice(choice), review_id=exact_review_id + ) + + _run_validation_json(submit) + + +@validation_app.command("analyze") +def validation_analyze( + context: typer.Context, + campaign_id: Annotated[str | None, typer.Argument()] = None, +) -> None: + """Run the deterministic terminal analysis after all judgments are persisted.""" + + def analyze() -> object: + exact_campaign_id = _required_nonblank_text(campaign_id, "campaign_id") + return _validation_campaign_service_for_id(context, exact_campaign_id).analyze( + exact_campaign_id + ) + + _run_validation_json(analyze) + + +@validation_app.command("export-evidence") +def validation_export_evidence( + context: typer.Context, + campaign_id: Annotated[str | None, typer.Argument()] = None, + output: Annotated[Path | None, typer.Option("--output")] = None, +) -> None: + """Write terminal campaign evidence once and return a SHA-256 receipt.""" + + def export() -> dict[str, object]: + exact_campaign_id = _required_nonblank_text(campaign_id, "campaign_id") + output_path = _required_paths((output, "output"))[0] + if output_path.exists() or output_path.is_symlink(): + raise InvalidValidationInput("validation evidence output already exists") + evidence = _validation_campaign_service_for_id(context, exact_campaign_id).export_evidence( + exact_campaign_id + ) + encoded = json.dumps(evidence, sort_keys=True, separators=(",", ":")) + "\n" + try: + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("x", encoding="utf-8") as stream: + stream.write(encoded) + except (OSError, UnicodeError) as error: + raise InvalidValidationInput("validation evidence export failed") from error + return { + "campaign_id": exact_campaign_id, + "schema_version": evidence["schema_version"], + "sha256": hashlib.sha256(encoded.encode()).hexdigest(), + } + + _run_validation_json(export) + + +def _character_benchmark_resume_projection(service: Any, campaign_id: str, view: Any) -> object: + custom = getattr(service, "cli_resume_projection", None) + if custom is not None: + return custom(campaign_id, view) + data = view.model_dump(mode="json") + if view.review_id is None or view.trial_id is None: + data["phase"] = view.next_action + return data + campaign = service.campaigns.get(campaign_id) + trial = service._trial(campaign_id, view.trial_id) + session, _experiment_id, _alias = service._authorized_session(campaign, trial, view.review_id) + mapping = service._display_mapping(trial, session) + media = [] + for position in ("left", "right"): + artifact = service.experiments.artifacts.get(mapping[position]["artifact_id"]) + path = service.experiments.artifacts.resolve(artifact.id).resolve() + media.append({"position": position, "media_ref": path.as_uri()}) + pack = service.character_packs.export_pack(campaign.character_pack_id) + references = [ + service.experiments.artifacts.resolve(item.artifact_id).resolve().as_uri() + for item in pack.references + ] + return { + "campaign_id": campaign_id, + "state": view.state.value, + "phase": view.next_action, + "progress": view.progress.model_dump(mode="json"), + "trial_id": view.trial_id, + "review_id": view.review_id, + "prompt": trial.context.prompt, + "references": references, + "media": media, + } + + +@character_benchmark_app.command("create") +def character_benchmark_create( + context: typer.Context, + pack_id: Annotated[str | None, typer.Option("--pack")] = None, + stage_2: Annotated[str | None, typer.Option("--stage-2-evidence")] = None, + identity_prompt: Annotated[str | None, typer.Option("--identity-prompt")] = None, + panel: Annotated[Path | None, typer.Option("--panel")] = None, +) -> None: + def create() -> object: + exact_pack = _required_nonblank_text(pack_id, "pack") + exact_stage_2 = _required_nonblank_text(stage_2, "stage_2_evidence") + exact_prompt = _required_nonblank_text(identity_prompt, "identity_prompt") + panel_input = _load(_required_paths((panel, "panel"))[0], CharacterBenchmarkPanelInput) + return _character_benchmark_service(context, exact_stage_2).create( + character_pack_id=exact_pack, + stage_2_campaign_id=exact_stage_2, + identity_prompt=exact_prompt, + panel_input=panel_input, + ) + + _run_character_benchmark_json(create) + + +def _benchmark_campaign_action( + context: typer.Context, campaign_id: str | None, method: str +) -> object: + exact = _required_nonblank_text(campaign_id, "campaign_id") + return getattr(_character_benchmark_service_for_id(context, exact), method)(exact) + + +@character_benchmark_app.command("preview") +def character_benchmark_preview( + context: typer.Context, campaign_id: Annotated[str | None, typer.Argument()] = None +) -> None: + _run_character_benchmark_json( + lambda: _benchmark_campaign_action(context, campaign_id, "preview") + ) + + +@character_benchmark_app.command("preregister") +def character_benchmark_preregister( + context: typer.Context, campaign_id: Annotated[str | None, typer.Argument()] = None +) -> None: + _run_character_benchmark_json( + lambda: _benchmark_campaign_action(context, campaign_id, "preregister") + ) + + +@character_benchmark_app.command("status") +def character_benchmark_status( + context: typer.Context, campaign_id: Annotated[str | None, typer.Argument()] = None +) -> None: + _run_character_benchmark_json(lambda: _benchmark_campaign_action(context, campaign_id, "show")) + + +@character_benchmark_app.command("resume") +def character_benchmark_resume( + context: typer.Context, campaign_id: Annotated[str | None, typer.Argument()] = None +) -> None: + def resume() -> object: + exact = _required_nonblank_text(campaign_id, "campaign_id") + service = _character_benchmark_service_for_id(context, exact) + return _character_benchmark_resume_projection(service, exact, service.resume(exact)) + + _run_character_benchmark_json(resume) + + +@character_benchmark_app.command("submit-identity") +def character_benchmark_submit_identity( + context: typer.Context, + campaign_id: Annotated[str | None, typer.Argument()] = None, + review_id: Annotated[str | None, typer.Option("--review-id")] = None, + trial_id: Annotated[str | None, typer.Option("--trial-id")] = None, + choice: Annotated[IdentityChoice | None, typer.Option("--choice")] = None, +) -> None: + def submit() -> object: + exact = _required_nonblank_text(campaign_id, "campaign_id") + review = _required_nonblank_text(review_id, "review_id") + trial = _required_nonblank_text(trial_id, "trial_id") + if choice is None: + raise InvalidInput("choice is required") + return _character_benchmark_service_for_id(context, exact).record_identity( + exact, review, IdentityObservation(benchmark_id=exact, trial_id=trial, choice=choice) + ) + + _run_character_benchmark_json(submit) + + +@character_benchmark_app.command("submit-secondary") +def character_benchmark_submit_secondary( + context: typer.Context, + campaign_id: Annotated[str | None, typer.Argument()] = None, + review_id: Annotated[str | None, typer.Option("--review-id")] = None, + trial_id: Annotated[str | None, typer.Option("--trial-id")] = None, + dimension: Annotated[SecondaryDimension | None, typer.Option("--dimension")] = None, + choice: Annotated[SecondaryChoice | None, typer.Option("--choice")] = None, +) -> None: + def submit() -> object: + exact = _required_nonblank_text(campaign_id, "campaign_id") + review = _required_nonblank_text(review_id, "review_id") + trial = _required_nonblank_text(trial_id, "trial_id") + if dimension is None or choice is None: + raise InvalidInput("dimension and choice are required") + return _character_benchmark_service_for_id(context, exact).record_secondary( + exact, + review, + SecondaryObservation( + benchmark_id=exact, trial_id=trial, dimension=dimension, choice=choice + ), + ) + + _run_character_benchmark_json(submit) + + +@character_benchmark_app.command("analyze") +def character_benchmark_analyze( + context: typer.Context, campaign_id: Annotated[str | None, typer.Argument()] = None +) -> None: + _run_character_benchmark_json( + lambda: _benchmark_campaign_action(context, campaign_id, "analyze") + ) + + +@character_benchmark_app.command("export-evidence") +def character_benchmark_export_evidence( + context: typer.Context, + campaign_id: Annotated[str | None, typer.Argument()] = None, + output: Annotated[Path | None, typer.Option("--output")] = None, +) -> None: + def export() -> dict[str, object]: + exact = _required_nonblank_text(campaign_id, "campaign_id") + path = _required_paths((output, "output"))[0] + if path.exists() or path.is_symlink(): + raise InvalidInput("character benchmark evidence output already exists") + evidence = _character_benchmark_service_for_id(context, exact).export_evidence(exact) + document = evidence.model_dump(mode="json") if hasattr(evidence, "model_dump") else evidence + encoded = json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n" + try: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x", encoding="utf-8") as stream: + stream.write(encoded) + except (OSError, UnicodeError) as error: + raise InvalidInput("character benchmark evidence export failed") from error + schema = document.get("schema_version") if isinstance(document, dict) else None + return { + "campaign_id": exact, + "schema_version": schema, + "sha256": hashlib.sha256(encoded.encode()).hexdigest(), + } + + _run_character_benchmark_json(export) + + +@adapter_app.command("probe") +def adapter_probe( + adapter_id: Annotated[str | None, typer.Argument()] = None, + phase: Annotated[ProbePhase | None, typer.Option("--phase")] = None, + manifest: Annotated[Path | None, typer.Option("--manifest")] = None, + scratch_root: Annotated[Path | None, typer.Option("--scratch-root")] = None, + allow_generate: Annotated[bool, typer.Option("--allow-generate")] = False, + json_output: Annotated[bool, typer.Option("--json")] = False, +) -> None: + def probe() -> dict[str, object]: + exact_adapter_id = _required_text(adapter_id, "adapter_id") + adapter = AdapterRegistry.with_defaults().resolve(exact_adapter_id) + if phase is None: + if manifest is not None or scratch_root is not None or allow_generate: + raise InvalidInput("generation options require --phase generate") + return { + "schema_version": "worker.v1", + "adapter_id": exact_adapter_id, + "available": adapter.probe(), + } + if not isinstance(adapter, StagedProbeAdapter): + raise InvalidInput(f"adapter {exact_adapter_id} does not support staged probes") + request: ProbeRequest | None = None + if phase is ProbePhase.GENERATE: + if not allow_generate: + raise InvalidInput("generate probe requires --allow-generate") + manifest_path, requested_scratch = _required_paths( + (manifest, "manifest"), (scratch_root, "scratch_root") + ) + generation_manifest = _load(manifest_path, Manifest) + safe_scratch = requested_scratch.resolve() + try: + safe_scratch.mkdir(parents=True, exist_ok=False) + except FileExistsError as error: + raise InvalidInput("scratch_root must not already exist") from error + request = ProbeRequest( + generation=GenerationRequest( + job_id="cli-probe", + attempt_id="cli-probe", + manifest=generation_manifest, + scratch_root=safe_scratch, + ), + progress=lambda _: None, + cancelled=lambda: False, + ) + elif manifest is not None or scratch_root is not None or allow_generate: + raise InvalidInput("generation options require --phase generate") + report = adapter.probe_phase(phase, request) + return { + "schema_version": "worker.v1", + "adapter_id": exact_adapter_id, + "available": report.ok, + "probe_report": asdict(report), + } + + if json_output: + _run_json(probe) + else: + _run_human(probe, lambda value: json.dumps(value, indent=2)) + + +if __name__ == "__main__": + app() diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/comparison.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/comparison.py new file mode 100644 index 0000000..4c89a76 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/comparison.py @@ -0,0 +1,70 @@ +from typing import Any + +from pydantic import ValidationError + +from video_lab.domain import ExperimentDraft, FrozenModel, Manifest + +_MISSING = object() + + +class ComparisonError(ValueError): + pass + + +class ControlledPair(FrozenModel): + variant_a: Manifest + variant_b: Manifest + variable_path: str + changed_paths: tuple[str, ...] + + +def changed_paths(left: Any, right: Any, prefix: str = "") -> tuple[str, ...]: + if isinstance(left, dict) and isinstance(right, dict): + keys = sorted(set(left) | set(right)) + return tuple( + path + for key in keys + for path in changed_paths( + left[key] if key in left else _MISSING, + right[key] if key in right else _MISSING, + f"{prefix}.{key}".strip("."), + ) + ) + if isinstance(left, list) and isinstance(right, list): + paths: list[str] = [] + for index in range(max(len(left), len(right))): + a = left[index] if index < len(left) else _MISSING + b = right[index] if index < len(right) else _MISSING + paths.extend(changed_paths(a, b, f"{prefix}.{index}")) + return tuple(paths) + return () if left == right else (prefix,) + + +def _replace_path(payload: dict[str, Any], path: str, value: Any) -> None: + parts = path.split(".") + cursor: Any = payload + for part in parts[:-1]: + if not isinstance(cursor, dict) or part not in cursor: + raise ComparisonError(f"unknown variable path: {path}") + cursor = cursor[part] + if not isinstance(cursor, dict) or parts[-1] not in cursor: + raise ComparisonError(f"unknown variable path: {path}") + cursor[parts[-1]] = value + + +def build_controlled_pair(draft: ExperimentDraft) -> ControlledPair: + payload = draft.baseline.model_dump(mode="json") + _replace_path(payload, draft.variable_path, draft.alternate_value) + try: + alternate = Manifest.model_validate(payload) + except ValidationError as error: + raise ComparisonError(str(error)) from error + paths = changed_paths(draft.baseline.model_dump(mode="json"), alternate.model_dump(mode="json")) + if paths != (draft.variable_path,): + raise ComparisonError("alternate value must change exactly one field") + return ControlledPair( + variant_a=draft.baseline, + variant_b=alternate, + variable_path=draft.variable_path, + changed_paths=paths, + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/compilers/__init__.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/compilers/__init__.py new file mode 100644 index 0000000..cdcb554 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/compilers/__init__.py @@ -0,0 +1,5 @@ +"""Trainer-specific dataset compiler adapters.""" + +from .wan_curated import WanCuratedBatchCompiler + +__all__ = ["WanCuratedBatchCompiler"] diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/compilers/wan_curated.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/compilers/wan_curated.py new file mode 100644 index 0000000..c6137ef --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/compilers/wan_curated.py @@ -0,0 +1,473 @@ +"""Strictly local deterministic compiler for curated Wan training batches.""" + +from __future__ import annotations + +import base64 +import hashlib +import importlib +import json +import os +import random +import tempfile +import threading +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal, cast + +import numpy as np +from pydantic import BaseModel, ConfigDict, Field + +from video_lab.curated_models import ( + SequenceTransformRecipeV1, + StructuredCaptionV1, + render_caption_v1, +) +from video_lab.dataset_compiler import ( + BackendOutput, + DatasetCompilationError, + TransformReceipt, + validate_transformed_sequence, +) +from video_lab.domain import canonical_json +from video_lab.trainers.wan_batch import ( + build_wan_batch_manifest, + validate_wan_batch_bytes, +) +from video_lab.trainers.wan_factory import ( + WanPinnedLocalRoot, + WanSnapshotCopyLimits, + require_component_on_mps, + root_manifest_digest, + secure_snapshot_pinned_root, +) + +_RNG_LOCK = threading.Lock() +_HEX = frozenset("0123456789abcdef") + + +@dataclass(frozen=True, slots=True) +class WanCompilerLimits: + max_source_bytes: int = 4 * 1024 * 1024 * 1024 + max_recipe_bytes: int = 1024 * 1024 + max_frames: int = 256 + max_pixels: int = 4096 * 4096 * 256 + max_tensor_bytes: int = 16 * 1024 * 1024 * 1024 + max_snapshot_files: int = 100_000 + max_snapshot_bytes: int = 64 * 1024 * 1024 * 1024 + max_snapshot_depth: int = 32 + + +DEFAULT_WAN_COMPILER_LIMITS = WanCompilerLimits() + + +@dataclass(frozen=True, slots=True) +class WanCompilerSnapshot: + tokenizer: WanPinnedLocalRoot + text_encoder: WanPinnedLocalRoot + vae: WanPinnedLocalRoot + scheduler: WanPinnedLocalRoot + + +@dataclass(frozen=True, slots=True) +class WanCompilerComponents: + tokenizer: object + text_encoder: object + vae: object + scheduler: object + + +class _CompilerRecipe(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + schema_version: Literal["wan-curated-compiler-recipe.v1"] + transform_recipe: SequenceTransformRecipeV1 + timestep: int = Field(ge=0) + timestep_boundary: int = Field(gt=0) + latent_channels: int = Field(gt=0) + vae_scaling_factor: float = Field(gt=0) + + +class _CuratedSource(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + schema_version: Literal["wan-curated-source.v1"] + shape: tuple[int, int, int, int] + dtype: Literal["uint8"] + frames_base64: str + frame_digest: str + transform_receipt: TransformReceipt + transform_receipt_digest: str + caption: StructuredCaptionV1 + caption_digest: str + source_digest: str + validation_evidence: dict[str, str] + selected_sequence_ids: tuple[str, ...] + selected_frame_source_digests: tuple[tuple[str, str], ...] + + +def _sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def encode_wan_curated_source( + frames: np.ndarray, + *, + transform_receipt: TransformReceipt, + caption: StructuredCaptionV1, + source_digest: str, + validation_evidence: Mapping[str, str], + selected_sequence_ids: tuple[str, ...], + selected_frame_source_digests: tuple[tuple[str, str], ...], +) -> bytes: + """Encode the Task 4 transformed array and its exact evidence canonically.""" + array = np.asarray(frames) + if array.dtype != np.uint8 or array.ndim != 4 or array.shape[-1] != 3: + raise ValueError("wan_compiler_source_invalid") + frame_bytes = array.tobytes(order="C") + receipt_bytes = canonical_json(transform_receipt).encode() + caption_bytes = canonical_json(caption).encode() + payload = { + "schema_version": "wan-curated-source.v1", + "shape": list(array.shape), + "dtype": "uint8", + "frames_base64": base64.b64encode(frame_bytes).decode("ascii"), + "frame_digest": _sha(frame_bytes), + "transform_receipt": transform_receipt.model_dump(mode="json"), + "transform_receipt_digest": _sha(receipt_bytes), + "caption": caption.model_dump(mode="json"), + "caption_digest": _sha(caption_bytes), + "source_digest": source_digest, + "validation_evidence": dict(sorted(validation_evidence.items())), + "selected_sequence_ids": list(selected_sequence_ids), + "selected_frame_source_digests": [list(item) for item in selected_frame_source_digests], + } + return canonical_json(payload).encode() + + +class WanCuratedBatchCompiler: + compiler_id = "wan_curated_v1" + + def __init__( + self, + *, + snapshot: WanCompilerSnapshot, + component_loader: Callable[[WanCompilerSnapshot], WanCompilerComponents] | None = None, + importer: Callable[[str], object] = importlib.import_module, + limits: WanCompilerLimits = DEFAULT_WAN_COMPILER_LIMITS, + ) -> None: + self._snapshot = snapshot + self._importer = importer + self._uses_default_loader = component_loader is None + self._loader = component_loader or self._load_components + self._limits = limits + + def validate_recipe(self, recipe_bytes: bytes) -> None: + self._parse_recipe(recipe_bytes) + self._snapshot_evidence(self._snapshot) + + def compile_batch(self, source_bytes: bytes, recipe_bytes: bytes) -> tuple[BackendOutput, ...]: + if len(source_bytes) > self._limits.max_source_bytes: + raise ValueError("wan_compiler_budget_exceeded") + recipe = self._parse_recipe(recipe_bytes) + source, frames = self._parse_source(source_bytes, recipe.transform_recipe) + evidence_before = self._snapshot_evidence(self._snapshot) + vae_seed = _derived_seed(recipe.transform_recipe.seed, "wan-vae-latent.v1") + noise_seed = _derived_seed(recipe.transform_recipe.seed, "wan-flow-noise.v1") + with _fenced_rng(self._importer, seed=vae_seed) as torch: + components = self._loader(self._snapshot) + try: + evidence_after_load = self._snapshot_evidence(self._snapshot) + except ValueError: + raise ValueError("wan_compiler_snapshot_replaced") from None + if evidence_after_load != evidence_before: + raise ValueError("wan_compiler_snapshot_replaced") + require_component_on_mps(components.text_encoder) + require_component_on_mps(components.vae) + timestep_max = self._scheduler_timesteps(components.scheduler) + if not 0 <= recipe.timestep <= timestep_max: + raise ValueError("wan_compiler_timestep_invalid") + if not 0 < recipe.timestep_boundary <= timestep_max: + raise ValueError("wan_compiler_selector_invalid") + actual_scale = float( + getattr(getattr(components.vae, "config", None), "scaling_factor", 0) + ) + if actual_scale != recipe.vae_scaling_factor: + raise ValueError("wan_compiler_vae_scale_invalid") + rendered = render_caption_v1(source.caption) + tokens = cast(Any, components.tokenizer)( + [rendered], padding=True, return_tensors="pt", truncation=True + ) + if not isinstance(tokens, Mapping) or set(tokens) != {"input_ids"}: + raise ValueError("wan_compiler_tokenization_invalid") + token_ids = tokens["input_ids"] + model_token_ids = token_ids.to(device="mps") if self._uses_default_loader else token_ids + encoder_hidden = cast(Any, components.text_encoder)(input_ids=model_token_ids)[0] + video = torch.from_numpy(frames.copy()).permute(3, 0, 1, 2).unsqueeze(0) + video = video.to(dtype=torch.float32).div(127.5).sub(1.0) + if self._uses_default_loader: + video = video.to(device="mps") + clean = cast(Any, components.vae).encode(video).latent_dist.sample() + clean = ( + clean.detach().to(device="cpu", dtype=torch.bfloat16) * recipe.vae_scaling_factor + ) + encoder_hidden = encoder_hidden.detach().to(device="cpu", dtype=torch.bfloat16) + if ( + clean.ndim < 3 + or int(clean.shape[0]) != 1 + or int(clean.shape[1]) != recipe.latent_channels + ): + raise ValueError("wan_compiler_latent_shape_invalid") + generator = torch.Generator(device="cpu").manual_seed(noise_seed) + noise = torch.randn(clean.shape, dtype=clean.dtype, device="cpu", generator=generator) + timestep = torch.tensor([float(recipe.timestep)], dtype=torch.float32) + sigma = (timestep / float(timestep_max)).to(dtype=clean.dtype) + while sigma.ndim < clean.ndim: + sigma = sigma.unsqueeze(-1) + selector = (timestep < recipe.timestep_boundary).to(dtype=torch.int64) + if int(selector.item()) != (1 if recipe.timestep < recipe.timestep_boundary else 0): + raise ValueError("wan_compiler_selector_invalid") + tensors = { + "clean_latents": clean.contiguous(), + "noise_latents": noise.contiguous(), + "hidden_states": (sigma * noise + (1.0 - sigma) * clean).contiguous(), + "target": (noise - clean).contiguous(), + "timestep": timestep.contiguous(), + "transformer_selector": selector.contiguous(), + "encoder_hidden_states": encoder_hidden.to(dtype=torch.bfloat16).contiguous(), + } + self._validate_tensor_budget(tensors) + serializer = cast(Any, self._importer("safetensors.torch")) + batch = cast(bytes, serializer.save({name: tensors[name] for name in sorted(tensors)})) + manifest = build_wan_batch_manifest( + batch, + batch_size=1, + timestep_min=0, + timestep_max=timestep_max, + timestep_boundary=recipe.timestep_boundary, + ) + validate_wan_batch_bytes(manifest, batch, torch_module=torch) + if self._snapshot_evidence(self._snapshot) != evidence_before: + raise ValueError("wan_compiler_snapshot_replaced") + provenance = canonical_json( + { + "schema_version": "wan-curated-batch-provenance.v1", + "source_digest": source.source_digest, + "source_document_digest": _sha(source_bytes), + "frame_digest": source.frame_digest, + "transform_receipt_digest": source.transform_receipt_digest, + "caption_digest": source.caption_digest, + "rendered_caption": rendered, + "token_ids": token_ids.detach().to("cpu").tolist(), + "recipe_digest": _sha(recipe_bytes), + "component_digests": evidence_before, + "seed": recipe.transform_recipe.seed, + "batch_digest": _sha(batch), + "manifest_digest": _sha(manifest), + } + ).encode() + return ( + BackendOutput("batch", batch, "application/vnd.safetensors"), + BackendOutput("manifest", manifest, "application/json"), + BackendOutput("provenance", provenance, "application/json"), + ) + + def _parse_recipe(self, payload: bytes) -> _CompilerRecipe: + if len(payload) > self._limits.max_recipe_bytes: + raise ValueError("wan_compiler_budget_exceeded") + try: + parsed = json.loads(payload) + if canonical_json(parsed).encode() != payload: + raise ValueError + parsed["transform_recipe"] = SequenceTransformRecipeV1.model_validate_json( + canonical_json(parsed["transform_recipe"]) + ) + return _CompilerRecipe.model_validate(parsed) + except (UnicodeError, json.JSONDecodeError, ValueError): + raise ValueError("wan_compiler_recipe_invalid") from None + + def _parse_source( + self, payload: bytes, transform: SequenceTransformRecipeV1 + ) -> tuple[_CuratedSource, np.ndarray]: + try: + parsed = json.loads(payload) + if canonical_json(parsed).encode() != payload: + raise ValueError + parsed["shape"] = tuple(parsed["shape"]) + parsed["selected_sequence_ids"] = tuple(parsed["selected_sequence_ids"]) + parsed["selected_frame_source_digests"] = tuple( + tuple(item) for item in parsed["selected_frame_source_digests"] + ) + parsed["caption"] = StructuredCaptionV1.model_validate_json( + canonical_json(parsed["caption"]) + ) + parsed["transform_receipt"] = TransformReceipt.model_validate_json( + canonical_json(parsed["transform_receipt"]) + ) + source = _CuratedSource.model_validate(parsed) + if len(source.source_digest) != 64 or any(c not in _HEX for c in source.source_digest): + raise ValueError + raw = base64.b64decode(source.frames_base64, validate=True) + if _sha(raw) != source.frame_digest: + raise ValueError + if ( + _sha(canonical_json(source.transform_receipt).encode()) + != source.transform_receipt_digest + ): + raise ValueError + if _sha(canonical_json(source.caption).encode()) != source.caption_digest: + raise ValueError + if ( + source.transform_receipt.recipe_id != transform.recipe_id + or source.transform_receipt.seed != transform.seed + ): + raise ValueError + expected_shape = ( + transform.output_frame_count, + transform.output_height, + transform.output_width, + 3, + ) + if source.shape != expected_shape or len(raw) != int(np.prod(expected_shape)): + raise ValueError + if ( + source.shape[0] > self._limits.max_frames + or int(np.prod(source.shape)) > self._limits.max_pixels * 3 + ): + raise ValueError("budget") + frames = np.frombuffer(raw, dtype=np.uint8).reshape(source.shape).copy() + frames.flags.writeable = False + try: + validate_transformed_sequence(frames, source.transform_receipt, transform) + except DatasetCompilationError: + raise ValueError from None + return source, frames + except ValueError as error: + if str(error) == "budget": + raise ValueError("wan_compiler_budget_exceeded") from None + raise ValueError("wan_compiler_source_invalid") from None + except (UnicodeError, json.JSONDecodeError, TypeError): + raise ValueError("wan_compiler_source_invalid") from None + + def _snapshot_evidence(self, snapshot: WanCompilerSnapshot) -> dict[str, dict[str, str]]: + evidence: dict[str, dict[str, str]] = {} + for name in ("tokenizer", "text_encoder", "vae", "scheduler"): + root = cast(WanPinnedLocalRoot, getattr(snapshot, name)) + if ( + type(root) is not WanPinnedLocalRoot + or len(root.revision) != 40 + or any(c not in _HEX for c in root.revision) + or root.manifest_digest is None + or root.manifest_digest + != root_manifest_digest( + root.path, + max_files=self._limits.max_snapshot_files, + max_bytes=self._limits.max_snapshot_bytes, + max_depth=self._limits.max_snapshot_depth, + ) + ): + raise ValueError("wan_compiler_snapshot_invalid") + evidence[name] = {"revision": root.revision, "manifest_digest": root.manifest_digest} + return evidence + + def _scheduler_timesteps(self, scheduler: object) -> int: + value = getattr(getattr(scheduler, "config", None), "num_train_timesteps", None) + if type(value) is not int or value <= 0: + raise ValueError("wan_compiler_scheduler_invalid") + return value + + def _validate_tensor_budget(self, tensors: Mapping[str, Any]) -> None: + byte_size = sum(int(item.numel()) * int(item.element_size()) for item in tensors.values()) + if byte_size > self._limits.max_tensor_bytes: + raise ValueError("wan_compiler_budget_exceeded") + if any( + not bool(item.isfinite().all().item()) + for name, item in tensors.items() + if name != "transformer_selector" + ): + raise ValueError("wan_compiler_tensor_invalid") + + def _load_components(self, snapshot: WanCompilerSnapshot) -> WanCompilerComponents: + torch = cast(Any, self._importer("torch")) + transformers = cast(Any, self._importer("transformers")) + diffusers = cast(Any, self._importer("diffusers")) + with _private_snapshot(snapshot, self._limits) as local: + common = {"local_files_only": True} + tokenizer = transformers.AutoTokenizer.from_pretrained( + str(local.tokenizer.path), **common + ) + encoder = transformers.UMT5EncoderModel.from_pretrained( + str(local.text_encoder.path), torch_dtype=torch.bfloat16, **common + ).to(device="mps", dtype=torch.bfloat16) + vae = diffusers.AutoencoderKLWan.from_pretrained( + str(local.vae.path), torch_dtype=torch.bfloat16, **common + ).to(device="mps", dtype=torch.bfloat16) + scheduler = diffusers.FlowMatchEulerDiscreteScheduler.from_pretrained( + str(local.scheduler.path), **common + ) + return WanCompilerComponents(tokenizer, encoder, vae, scheduler) + + +def _derived_seed(seed: int, domain: str) -> int: + digest = hashlib.sha256(f"{domain}\0{seed}".encode("ascii")).digest() + return int.from_bytes(digest[:8], "big") & ((1 << 63) - 1) + + +@contextmanager +def _fenced_rng(importer: Callable[[str], object], *, seed: int) -> Iterator[Any]: + torch = cast(Any, importer("torch")) + with _RNG_LOCK: + python_state = random.getstate() + numpy_state = np.random.get_state() + torch_state = torch.get_rng_state().clone() + mps_state = None + try: + if hasattr(torch, "mps") and hasattr(torch.mps, "get_rng_state"): + try: + mps_state = torch.mps.get_rng_state().clone() + except Exception: + mps_state = None + random.seed(seed) + np.random.seed(seed % (2**32)) + torch.manual_seed(seed) + if hasattr(torch, "mps") and hasattr(torch.mps, "manual_seed"): + try: + torch.mps.manual_seed(seed) + except Exception: + pass + yield torch + finally: + random.setstate(python_state) + np.random.set_state(numpy_state) + torch.set_rng_state(torch_state) + if mps_state is not None: + torch.mps.set_rng_state(mps_state) + + +@contextmanager +def _private_snapshot( + snapshot: WanCompilerSnapshot, limits: WanCompilerLimits +) -> Iterator[WanCompilerSnapshot]: + root = Path(tempfile.mkdtemp(prefix="wan-compiler-")).resolve() + try: + roots: dict[str, WanPinnedLocalRoot] = {} + for name in ("tokenizer", "text_encoder", "vae", "scheduler"): + source = cast(WanPinnedLocalRoot, getattr(snapshot, name)) + roots[name] = secure_snapshot_pinned_root( + source, + root / name, + limits=WanSnapshotCopyLimits( + max_files=limits.max_snapshot_files, + max_bytes=limits.max_snapshot_bytes, + max_depth=limits.max_snapshot_depth, + ), + ) + yield WanCompilerSnapshot(**roots) + finally: + for item in root.rglob("*"): + if item.is_dir(): + os.chmod(item, 0o700, follow_symlinks=False) + else: + os.chmod(item, 0o600, follow_symlinks=False) + for item in sorted(root.rglob("*"), reverse=True): + item.rmdir() if item.is_dir() else item.unlink() + root.rmdir() diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_ids.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_ids.py new file mode 100644 index 0000000..7fc903c --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_ids.py @@ -0,0 +1,16 @@ +"""Canonical identities for curated-sequence contracts.""" + +from __future__ import annotations + +from hashlib import sha256 + +from .curated_models import CuratedDatasetManifestV1, StructuredCaptionV1 +from .domain import canonical_json + + +def structured_caption_digest(caption: StructuredCaptionV1) -> str: + return sha256(canonical_json(caption).encode("utf-8")).hexdigest() + + +def curated_dataset_digest(manifest: CuratedDatasetManifestV1) -> str: + return sha256(canonical_json(manifest).encode("utf-8")).hexdigest() diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_metrics.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_metrics.py new file mode 100644 index 0000000..84c0837 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_metrics.py @@ -0,0 +1,86 @@ +"""Bounded deterministic image metrics for curated validation.""" + +from __future__ import annotations + +import hashlib +from decimal import ROUND_HALF_EVEN, Decimal + +import numpy as np +from PIL import Image + +from video_lab.curated_source import DecodedFrame + +METRIC_QUANTUM = Decimal("0.0001") + + +def quantize_metric(value: float | int | Decimal) -> str: + decimal = Decimal(str(value)) + if not decimal.is_finite(): + raise ValueError("metric must be finite") + return format(decimal.quantize(METRIC_QUANTUM, rounding=ROUND_HALF_EVEN), "f") + + +def frame_array(frame: DecodedFrame) -> np.ndarray: + expected = frame.width * frame.height * frame.channels + if len(frame.pixels) != expected or frame.channels not in (3, 4): + raise ValueError("decoded frame byte contract is invalid") + return np.frombuffer(frame.pixels, dtype=np.uint8).reshape( + frame.height, frame.width, frame.channels + ) + + +def perceptual_hash64(frame: DecodedFrame) -> int: + array = frame_array(frame) + image = Image.fromarray(array, mode="RGB" if frame.channels == 3 else "RGBA") + grayscale = image.convert("L").resize((8, 8), resample=Image.Resampling.LANCZOS) + values = np.asarray(grayscale, dtype=np.uint8).reshape(64) + median = int(np.median(values)) + result = 0 + for value in values: + result = (result << 1) | int(int(value) > median) + return result + + +def hamming_distance64(left: int, right: int) -> int: + return (left ^ right).bit_count() + + +def blur_variance(frame: DecodedFrame) -> float: + rgb = frame_array(frame)[..., :3].astype(np.float64) + gray = rgb[..., 0] * 0.299 + rgb[..., 1] * 0.587 + rgb[..., 2] * 0.114 + padded = np.pad(gray, 1, mode="edge") + laplacian = ( + -4.0 * padded[1:-1, 1:-1] + + padded[:-2, 1:-1] + + padded[2:, 1:-1] + + padded[1:-1, :-2] + + padded[1:-1, 2:] + ) + return float(np.var(laplacian, dtype=np.float64)) + + +def histogram_discontinuity(left: DecodedFrame, right: DecodedFrame) -> float: + left_array = frame_array(left)[..., :3] + right_array = frame_array(right)[..., :3] + distances: list[float] = [] + for channel in range(3): + left_hist = np.bincount(left_array[..., channel].reshape(-1), minlength=256) + right_hist = np.bincount(right_array[..., channel].reshape(-1), minlength=256) + left_norm = left_hist / left_hist.sum() + right_norm = right_hist / right_hist.sum() + distances.append(float(np.abs(left_norm - right_norm).sum() / 2.0)) + return sum(distances) / 3.0 + + +def temporal_motion(left: DecodedFrame, right: DecodedFrame) -> float: + left_array = frame_array(left) + right_array = frame_array(right) + if left_array.shape != right_array.shape: + return 1.0 + difference = np.abs(left_array[..., :3].astype(np.int16) - right_array[..., :3]) + return float(np.mean(difference, dtype=np.float64) / 255.0) + + +def metric_policy_digest() -> str: + policy = b"phash64:lanczos-l8-median;blur:laplacian4-edge;hist:rgb256-l1;motion:rgb-mae" + return hashlib.sha256(policy).hexdigest() diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_models.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_models.py new file mode 100644 index 0000000..d1646ed --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_models.py @@ -0,0 +1,269 @@ +"""Frozen, trainer-neutral contracts for curated video-frame sequences.""" + +from __future__ import annotations + +import re +import unicodedata +from collections.abc import Mapping +from copy import deepcopy +from math import gcd +from typing import Annotated, Any, Literal, Self + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +_STABLE_ID = re.compile(r"[a-z][a-z0-9_]*") +_SHA256 = re.compile(r"[0-9a-f]{64}") +_RGB_CHANNEL = Annotated[int, Field(ge=0, le=255)] + + +class _CuratedModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + + def model_copy( + self, + *, + update: Mapping[str, Any] | None = None, + deep: bool = False, + ) -> Self: + dumped = self.model_dump(round_trip=True) + payload = deepcopy(dumped) if deep else dumped + if update is not None: + payload.update(update) + return self.__class__.model_validate(payload) + + +def _normalized_text(value: str) -> str: + normalized = unicodedata.normalize("NFC", value) + if not normalized or normalized != normalized.strip(): + raise ValueError("text must be nonempty and have no surrounding whitespace") + return normalized + + +def _stable_id(value: str) -> str: + normalized = _normalized_text(value) + if _STABLE_ID.fullmatch(normalized) is None: + raise ValueError("must be a lowercase stable identifier") + return normalized + + +def _sha256(value: str) -> str: + normalized = _normalized_text(value) + if _SHA256.fullmatch(normalized) is None: + raise ValueError("must be a lowercase SHA-256 hex digest") + return normalized + + +def _caption_items(value: tuple[str, ...]) -> tuple[str, ...]: + if not value: + return () + return tuple(sorted({_normalized_text(item) for item in value})) + + +class StructuredCaptionV1(_CuratedModel): + schema_version: Literal["structured-caption.v1"] = "structured-caption.v1" + character_token: str + immutable_attributes: tuple[str, ...] = Field(min_length=1) + clothing: tuple[str, ...] = () + action: str + pose: str | None = None + expression: str | None = None + environment: str | None = None + lighting: str | None = None + camera: str | None = None + style: str | None = None + omissions: tuple[str, ...] = () + + @field_validator("character_token") + @classmethod + def validate_character_token(cls, value: str) -> str: + return _stable_id(value) + + @field_validator("immutable_attributes", "clothing", "omissions") + @classmethod + def normalize_caption_items(cls, value: tuple[str, ...]) -> tuple[str, ...]: + return _caption_items(value) + + @field_validator( + "action", + "pose", + "expression", + "environment", + "lighting", + "camera", + "style", + ) + @classmethod + def normalize_caption_text(cls, value: str | None) -> str | None: + return None if value is None else _normalized_text(value) + + +class CuratedFrameV1(_CuratedModel): + schema_version: Literal["curated-frame.v1"] = "curated-frame.v1" + frame_id: str + frame_index: int = Field(ge=0) + timestamp_numerator: int = Field(ge=0) + timestamp_denominator: int = Field(gt=0) + source_path: str + source_digest: str + pixel_digest: str + review_state: Literal["accepted"] + reviewer_evidence: tuple[str, ...] = Field(min_length=1) + + @field_validator("frame_id", "reviewer_evidence") + @classmethod + def validate_stable_ids(cls, value: str | tuple[str, ...]) -> str | tuple[str, ...]: + if isinstance(value, tuple): + return tuple(_stable_id(item) for item in value) + return _stable_id(value) + + @field_validator("source_path") + @classmethod + def validate_source_path(cls, value: str) -> str: + normalized = _normalized_text(value) + components = normalized.split("/") + has_ambiguous_component = any(component in {"", ".", ".."} for component in components) + if normalized.startswith("/") or has_ambiguous_component: + raise ValueError("must be a relative, component-wise source path") + return normalized + + @field_validator("source_digest", "pixel_digest") + @classmethod + def validate_digests(cls, value: str) -> str: + return _sha256(value) + + +class CuratedSequenceV1(_CuratedModel): + schema_version: Literal["curated-sequence.v1"] = "curated-sequence.v1" + sequence_id: str + identity_id: str + source_lineage: str + split: Literal["train", "validation", "holdout"] + caption: StructuredCaptionV1 + frames: tuple[CuratedFrameV1, ...] = Field(min_length=1) + + @field_validator("sequence_id", "identity_id", "source_lineage") + @classmethod + def validate_stable_ids(cls, value: str) -> str: + return _stable_id(value) + + @model_validator(mode="after") + def validate_frames(self) -> CuratedSequenceV1: + if self.identity_id != self.caption.character_token: + raise ValueError("identity ID must exactly match the caption character token") + frame_ids = tuple(frame.frame_id for frame in self.frames) + if len(frame_ids) != len(set(frame_ids)): + raise ValueError("frame IDs must be unique within a sequence") + if tuple(frame.frame_index for frame in self.frames) != tuple(range(len(self.frames))): + raise ValueError("frames must use contiguous, ordered frame indices") + + previous_numerator = -1 + previous_denominator = 1 + for frame in self.frames: + current_time = frame.timestamp_numerator * previous_denominator + previous_time = previous_numerator * frame.timestamp_denominator + if current_time <= previous_time: + raise ValueError("frame timestamps must be strictly increasing rational values") + previous_numerator = frame.timestamp_numerator + previous_denominator = frame.timestamp_denominator + return self + + +class DatasetSplitPolicyV1(_CuratedModel): + schema_version: Literal["dataset-split-policy.v1"] = "dataset-split-policy.v1" + policy_version: str + seed: int = Field(ge=0) + train_numerator: int = Field(gt=0) + validation_numerator: int = Field(gt=0) + holdout_numerator: int = Field(gt=0) + denominator: int = Field(gt=0) + + @field_validator("policy_version") + @classmethod + def validate_policy_version(cls, value: str) -> str: + return _stable_id(value) + + @model_validator(mode="after") + def validate_partition(self) -> DatasetSplitPolicyV1: + total = self.train_numerator + self.validation_numerator + self.holdout_numerator + if total != self.denominator: + raise ValueError("split numerators must exactly sum to the denominator") + ratio_divisor = gcd( + self.train_numerator, + self.validation_numerator, + self.holdout_numerator, + self.denominator, + ) + if ratio_divisor != 1: + raise ValueError("split policy ratios must be reduced") + return self + + +class SequenceTransformRecipeV1(_CuratedModel): + schema_version: Literal["sequence-transform-recipe.v1"] = "sequence-transform-recipe.v1" + recipe_id: str + output_width: int = Field(gt=0) + output_height: int = Field(gt=0) + output_frame_count: int = Field(gt=0) + temporal_stride_numerator: int = Field(gt=0) + temporal_stride_denominator: int = Field(gt=0) + resize_mode: Literal["center_crop", "fit_pad"] = "center_crop" + interpolation: Literal["nearest", "bilinear", "bicubic", "lanczos"] + padding_color: tuple[_RGB_CHANNEL, _RGB_CHANNEL, _RGB_CHANNEL] + seed: int = Field(ge=0) + + @field_validator("recipe_id") + @classmethod + def validate_recipe_id(cls, value: str) -> str: + return _stable_id(value) + + +class CuratedDatasetManifestV1(_CuratedModel): + schema_version: Literal["curated-dataset-manifest.v1"] = "curated-dataset-manifest.v1" + dataset_id: str + sequences: tuple[CuratedSequenceV1, ...] = Field(min_length=1) + split_policy: DatasetSplitPolicyV1 + transform_recipe: SequenceTransformRecipeV1 + + @field_validator("dataset_id") + @classmethod + def validate_dataset_id(cls, value: str) -> str: + return _stable_id(value) + + @model_validator(mode="after") + def validate_sequences(self) -> CuratedDatasetManifestV1: + sequence_ids = tuple(sequence.sequence_id for sequence in self.sequences) + if sequence_ids != tuple(sorted(sequence_ids)): + raise ValueError("sequences must be ordered by sequence ID") + if len(sequence_ids) != len(set(sequence_ids)): + raise ValueError("sequence IDs must be unique") + + frame_ids = tuple( + frame.frame_id for sequence in self.sequences for frame in sequence.frames + ) + if len(frame_ids) != len(set(frame_ids)): + raise ValueError("frame IDs must be globally unique") + return self + + +def render_caption_v1(caption: StructuredCaptionV1) -> str: + sections: list[tuple[str, str]] = [ + ("token", caption.character_token), + ("immutable identity", ", ".join(caption.immutable_attributes)), + ] + if caption.clothing: + sections.append(("clothing", ", ".join(caption.clothing))) + + action_pose = caption.action if caption.pose is None else f"{caption.action}; {caption.pose}" + sections.append(("action/pose", action_pose)) + for label, value in ( + ("expression", caption.expression), + ("environment", caption.environment), + ("lighting", caption.lighting), + ("camera", caption.camera), + ("style", caption.style), + ): + if value is not None: + sections.append((label, value)) + if caption.omissions: + sections.append(("omissions", ", ".join(caption.omissions))) + return "; ".join(f"{label}: {value}" for label, value in sections) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_source.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_source.py new file mode 100644 index 0000000..1e0d976 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_source.py @@ -0,0 +1,468 @@ +from __future__ import annotations + +import hashlib +import io +import os +import stat +import warnings +from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +from PIL import Image + +from video_lab.artifacts import Artifact +from video_lab.curated_models import CuratedFrameV1 + +_HASH_CHUNK_SIZE = 1024 * 1024 +_DIRECTORY_FLAGS = ( + os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) +) +_FILE_FLAGS = ( + os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) +) +_Identity = tuple[int, int, int, int, int, int, int] + + +class CuratedSourceError(Exception): + """A curated source failed closed at its filesystem or decode boundary.""" + + +@dataclass(frozen=True, slots=True) +class CuratedSourceLimits: + max_width: int = 16_384 + max_height: int = 16_384 + max_pixels: int = 64 * 1024 * 1024 + max_source_bytes: int = 256 * 1024 * 1024 + max_decoded_bytes: int = 256 * 1024 * 1024 + + def __post_init__(self) -> None: + if any( + value <= 0 + for value in ( + self.max_width, + self.max_height, + self.max_pixels, + self.max_source_bytes, + self.max_decoded_bytes, + ) + ): + raise ValueError("curated source limits must be positive") + + +@dataclass(frozen=True, slots=True) +class DecodedFrame: + width: int + height: int + channels: int + bit_depth: int + color_space: str + pixel_digest: str + pixels: bytes + + +def decoded_pixel_digest(frame: DecodedFrame, limits: CuratedSourceLimits | None = None) -> str: + """Return the sole canonical digest for a bounded immutable decoded frame.""" + source_limits = limits if limits is not None else CuratedSourceLimits() + if any(type(value) is not int for value in (frame.width, frame.height, frame.channels)): + raise CuratedSourceError("decoded frame dimensions must be exact integers") + if frame.width <= 0 or frame.height <= 0: + raise CuratedSourceError("decoded frame dimensions must be positive") + expected_color_space = {3: "RGB", 4: "RGBA"}.get(frame.channels) + if expected_color_space is None or frame.color_space != expected_color_space: + raise CuratedSourceError("decoded frame channel and color-space metadata disagree") + if type(frame.bit_depth) is not int or frame.bit_depth != 8: + raise CuratedSourceError("decoded frame bit depth must be 8") + if not isinstance(frame.pixels, bytes): + raise CuratedSourceError("decoded frame pixels must be immutable bytes") + pixel_count = frame.width * frame.height + decoded_bytes = pixel_count * frame.channels + if ( + frame.width > source_limits.max_width + or frame.height > source_limits.max_height + or pixel_count > source_limits.max_pixels + or decoded_bytes > source_limits.max_decoded_bytes + ): + raise CuratedSourceError("decoded frame exceeds its resource bounds") + if len(frame.pixels) != decoded_bytes: + raise CuratedSourceError("decoded frame pixel length does not match its metadata") + header = (f"{frame.width}x{frame.height}x{frame.channels}|uint8|{frame.color_space}").encode( + "ascii" + ) + return hashlib.sha256(b"curated-pixels-v1\0" + header + b"\0" + frame.pixels).hexdigest() + + +def validate_decoded_frame(frame: DecodedFrame, limits: CuratedSourceLimits | None = None) -> None: + """Fail closed unless metadata, bytes, and the canonical digest agree exactly.""" + if decoded_pixel_digest(frame, limits) != frame.pixel_digest: + raise CuratedSourceError("decoded frame canonical pixel digest mismatch") + + +@dataclass(frozen=True, slots=True) +class _SourceBinding: + artifact: Artifact + descriptor: int + file_identity: _Identity + ancestor_descriptors: tuple[int, ...] + ancestor_identities: tuple[_Identity, ...] + + +class CuratedSourceLease: + """Retain descriptor-bound PNG sources and revalidate them around decoding.""" + + def __init__( + self, + *, + root: Path, + root_descriptor: int, + root_identity: _Identity, + root_component_descriptors: tuple[int, ...], + root_component_identities: tuple[_Identity, ...], + bindings: tuple[_SourceBinding, ...], + descriptors: list[int], + limits: CuratedSourceLimits, + ) -> None: + self._root = root + self._root_descriptor = root_descriptor + self._root_identity = root_identity + self._root_component_descriptors = root_component_descriptors + self._root_component_identities = root_component_identities + self._bindings = {binding.artifact.relative_path: binding for binding in bindings} + self._descriptors = descriptors + self._limits = limits + self._closed = False + + @classmethod + def open( + cls, + *, + root: Path, + expected: Artifact | Iterable[Artifact], + limits: CuratedSourceLimits | None = None, + ) -> CuratedSourceLease: + if not hasattr(os, "O_NOFOLLOW"): + raise CuratedSourceError("O_NOFOLLOW is required for curated sources") + artifacts = (expected,) if isinstance(expected, Artifact) else tuple(expected) + if not artifacts: + raise CuratedSourceError("at least one source descriptor is required") + source_limits = limits if limits is not None else CuratedSourceLimits() + root_path = Path(os.path.abspath(os.fspath(root))) + descriptors: list[int] = [] + try: + root_descriptor, root_component_descriptors, root_component_identities = ( + _open_absolute_directory(root_path, descriptors) + ) + root_status = os.fstat(root_descriptor) + if not stat.S_ISDIR(root_status.st_mode): + raise CuratedSourceError("curated source root is not a directory") + root_identity = _identity(root_status) + bindings: list[_SourceBinding] = [] + seen_paths: set[str] = set() + seen_inodes: set[tuple[int, int]] = set() + for artifact in artifacts: + if artifact.relative_path in seen_paths: + raise CuratedSourceError("duplicate curated source path") + seen_paths.add(artifact.relative_path) + binding = _open_binding( + root_descriptor=root_descriptor, + artifact=artifact, + limits=source_limits, + descriptors=descriptors, + ) + inode = (binding.file_identity[0], binding.file_identity[1]) + if inode in seen_inodes: + raise CuratedSourceError("duplicate curated source inode") + seen_inodes.add(inode) + bindings.append(binding) + lease = cls( + root=root_path, + root_descriptor=root_descriptor, + root_identity=root_identity, + root_component_descriptors=root_component_descriptors, + root_component_identities=root_component_identities, + bindings=tuple(bindings), + descriptors=descriptors, + limits=source_limits, + ) + lease.reverify() + return lease + except CuratedSourceError: + _close_descriptors(descriptors) + raise + except (OSError, ValueError, TypeError): + _close_descriptors(descriptors) + raise CuratedSourceError("curated sources cannot be opened safely") from None + + def read_frame(self, frame: CuratedFrameV1) -> DecodedFrame: + self._require_open() + binding = self._bindings.get(frame.source_path) + if binding is None or binding.artifact.sha256 != frame.source_digest: + raise CuratedSourceError("frame does not match its retained source descriptor") + self.reverify() + try: + source = os.pread( + binding.descriptor, + binding.artifact.byte_size + 1, + 0, + ) + if len(source) != binding.artifact.byte_size: + raise CuratedSourceError("curated source size changed") + decoded = _decode_png(source, self._limits) + except CuratedSourceError: + raise + except Exception: + raise CuratedSourceError("curated PNG decoding failed") from None + self.reverify() + if decoded.pixel_digest != frame.pixel_digest: + raise CuratedSourceError("decoded pixels do not match the curated frame") + return decoded + + def reverify(self) -> None: + self._require_open() + try: + current_root_components = tuple( + _identity(os.fstat(descriptor)) for descriptor in self._root_component_descriptors + ) + if current_root_components != self._root_component_identities: + raise CuratedSourceError("curated source root ancestry changed") + if _identity(os.fstat(self._root_descriptor)) != self._root_identity: + raise CuratedSourceError("curated source root changed") + for binding in self._bindings.values(): + self._reverify_retained_binding(binding) + self._reverify_canonical_binding(binding) + except CuratedSourceError: + raise + except OSError: + raise CuratedSourceError("curated source changed during verification") from None + + def close(self) -> None: + if self._closed: + return + self._closed = True + descriptors = self._descriptors + self._descriptors = [] + errors = _close_descriptors(descriptors) + if errors: + raise CuratedSourceError("curated source descriptor cleanup failed") + + def _require_open(self) -> None: + if self._closed: + raise CuratedSourceError("curated source lease is closed") + + def _reverify_retained_binding(self, binding: _SourceBinding) -> None: + if tuple(_identity(os.fstat(fd)) for fd in binding.ancestor_descriptors) != ( + binding.ancestor_identities + ): + raise CuratedSourceError("curated source ancestor changed") + if _identity(os.fstat(binding.descriptor)) != binding.file_identity: + raise CuratedSourceError("curated source file changed") + digest, size = _hash_descriptor(binding.descriptor, self._limits.max_source_bytes) + if digest != binding.artifact.sha256 or size != binding.artifact.byte_size: + raise CuratedSourceError("curated source content changed") + + def _reverify_canonical_binding(self, binding: _SourceBinding) -> None: + temporary: list[int] = [] + try: + root_descriptor, _, root_component_identities = _open_absolute_directory( + self._root, temporary + ) + if root_component_identities != self._root_component_identities: + raise CuratedSourceError("curated source root ancestry path changed") + if _identity(os.fstat(root_descriptor)) != self._root_identity: + raise CuratedSourceError("curated source root path changed") + current = root_descriptor + parts = _relative_parts(binding.artifact.relative_path) + canonical_ancestors: list[_Identity] = [] + for part in parts[:-1]: + current = os.open(part, _DIRECTORY_FLAGS, dir_fd=current) + temporary.append(current) + canonical_ancestors.append(_identity(os.fstat(current))) + leaf = os.open(parts[-1], _FILE_FLAGS, dir_fd=current) + temporary.append(leaf) + if tuple(canonical_ancestors) != binding.ancestor_identities: + raise CuratedSourceError("curated source ancestor path changed") + if _identity(os.fstat(leaf)) != binding.file_identity: + raise CuratedSourceError("curated source file path changed") + digest, size = _hash_descriptor(leaf, self._limits.max_source_bytes) + if digest != binding.artifact.sha256 or size != binding.artifact.byte_size: + raise CuratedSourceError("curated source canonical content changed") + finally: + if _close_descriptors(temporary): + raise CuratedSourceError("curated source verification cleanup failed") + + +def _open_absolute_directory( + path: Path, descriptors: list[int] +) -> tuple[int, tuple[int, ...], tuple[_Identity, ...]]: + if not path.is_absolute(): + raise CuratedSourceError("curated source root must be absolute") + current = os.open("/", _DIRECTORY_FLAGS) + descriptors.append(current) + component_descriptors = [current] + component_identities = [_identity(os.fstat(current))] + for part in path.parts[1:]: + if part in {"", ".", ".."}: + raise CuratedSourceError("curated source root path is invalid") + current = os.open(part, _DIRECTORY_FLAGS, dir_fd=current) + descriptors.append(current) + component_descriptors.append(current) + component_identities.append(_identity(os.fstat(current))) + return current, tuple(component_descriptors), tuple(component_identities) + + +def _open_binding( + *, + root_descriptor: int, + artifact: Artifact, + limits: CuratedSourceLimits, + descriptors: list[int], +) -> _SourceBinding: + if artifact.media_type != "image/png": + raise CuratedSourceError("curated source must be described as image/png") + if artifact.byte_size <= 0 or artifact.byte_size > limits.max_source_bytes: + raise CuratedSourceError("curated source byte size exceeds its bound") + parts = _relative_parts(artifact.relative_path) + current = root_descriptor + ancestor_descriptors: list[int] = [] + ancestor_identities: list[_Identity] = [] + for part in parts[:-1]: + current = os.open(part, _DIRECTORY_FLAGS, dir_fd=current) + descriptors.append(current) + opened = os.fstat(current) + if not stat.S_ISDIR(opened.st_mode): + raise CuratedSourceError("curated source ancestor is not a directory") + ancestor_descriptors.append(current) + ancestor_identities.append(_identity(opened)) + descriptor = os.open(parts[-1], _FILE_FLAGS, dir_fd=current) + descriptors.append(descriptor) + opened_file = os.fstat(descriptor) + if not stat.S_ISREG(opened_file.st_mode): + raise CuratedSourceError("curated source is not a regular file") + if opened_file.st_nlink != 1: + raise CuratedSourceError("curated source must have exactly one link") + identity = _identity(opened_file) + digest, size = _hash_descriptor(descriptor, limits.max_source_bytes) + if size != artifact.byte_size or digest != artifact.sha256: + raise CuratedSourceError("curated source descriptor mismatch") + return _SourceBinding( + artifact=artifact, + descriptor=descriptor, + file_identity=identity, + ancestor_descriptors=tuple(ancestor_descriptors), + ancestor_identities=tuple(ancestor_identities), + ) + + +def _relative_parts(relative_path: str) -> tuple[str, ...]: + path = Path(relative_path) + if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts): + raise CuratedSourceError("curated source path is invalid") + return path.parts + + +def _identity(status: os.stat_result) -> _Identity: + return ( + status.st_dev, + status.st_ino, + status.st_mode, + status.st_nlink, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ) + + +def _hash_descriptor(descriptor: int, maximum: int) -> tuple[str, int]: + digest = hashlib.sha256() + size = 0 + while size <= maximum: + chunk = os.pread(descriptor, min(_HASH_CHUNK_SIZE, maximum + 1 - size), size) + if not chunk: + return digest.hexdigest(), size + digest.update(chunk) + size += len(chunk) + raise CuratedSourceError("curated source exceeds its byte bound") + + +def _decode_png(source: bytes, limits: CuratedSourceLimits) -> DecodedFrame: + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(io.BytesIO(source)) as verified: + if verified.format != "PNG": + raise CuratedSourceError("curated source is not a PNG") + verified.verify() + with Image.open(io.BytesIO(source)) as image: + width, height, channels, color_space = _validate_png_metadata(image, limits) + image.load() + pixel_array = np.array(image, dtype=np.uint8, copy=True, order="C") + except CuratedSourceError: + raise + except Exception: + raise CuratedSourceError("curated PNG is invalid") from None + if pixel_array.dtype != np.uint8 or pixel_array.shape != (height, width, channels): + raise CuratedSourceError("curated PNG decoded to an unexpected pixel layout") + if not pixel_array.flags.owndata or not pixel_array.flags.c_contiguous: + pixel_array = np.ascontiguousarray(pixel_array).copy() + pixels = pixel_array.tobytes(order="C") + provisional = DecodedFrame( + width=width, + height=height, + channels=channels, + bit_depth=8, + color_space=color_space, + pixel_digest="0" * 64, + pixels=pixels, + ) + return DecodedFrame( + width=width, + height=height, + channels=channels, + bit_depth=8, + color_space=color_space, + pixel_digest=decoded_pixel_digest(provisional, limits), + pixels=pixels, + ) + + +def _validate_png_metadata( + image: Image.Image, limits: CuratedSourceLimits +) -> tuple[int, int, int, str]: + if image.format != "PNG": + raise CuratedSourceError("curated source is not a PNG") + if bool(getattr(image, "is_animated", False)) or int(getattr(image, "n_frames", 1)) != 1: + raise CuratedSourceError("animated PNG sources are not allowed") + if image.mode not in {"RGB", "RGBA"}: + raise CuratedSourceError("curated PNG pixel mode is not approved") + if image.info.get("icc_profile"): + raise CuratedSourceError("embedded color profiles are not approved") + if "transparency" in image.info: + raise CuratedSourceError("implicit transparency is not approved") + orientation = image.getexif().get(274, 1) + if orientation != 1: + raise CuratedSourceError("implicit image orientation is not approved") + width, height = image.size + channels = 3 if image.mode == "RGB" else 4 + pixels = width * height + if ( + width <= 0 + or height <= 0 + or width > limits.max_width + or height > limits.max_height + or pixels > limits.max_pixels + or pixels * channels > limits.max_decoded_bytes + ): + raise CuratedSourceError("curated PNG exceeds its decoded bounds") + return width, height, channels, image.mode + + +def _close_descriptors(descriptors: list[int]) -> bool: + failed = False + while descriptors: + descriptor = descriptors.pop() + try: + os.close(descriptor) + except OSError: + failed = True + return failed diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_splits.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_splits.py new file mode 100644 index 0000000..1070e2a --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_splits.py @@ -0,0 +1,279 @@ +"""Verified deterministic identity, lineage, and perceptual-aware split assignment.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Callable, Mapping +from itertools import combinations +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from video_lab.curated_ids import curated_dataset_digest +from video_lab.curated_metrics import hamming_distance64, metric_policy_digest, perceptual_hash64 +from video_lab.curated_models import CuratedDatasetManifestV1, DatasetSplitPolicyV1 +from video_lab.curated_source import CuratedSourceError, DecodedFrame, validate_decoded_frame +from video_lab.domain import canonical_json + + +class CuratedSplitError(ValueError): + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +class _SplitModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + + +class PerceptualFrameEvidenceV1(_SplitModel): + schema_version: Literal["perceptual-frame-evidence.v1"] = "perceptual-frame-evidence.v1" + frame_id: str + source_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + pixel_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + perceptual_hash64: int = Field(ge=0, le=(1 << 64) - 1) + + +class PerceptualComparisonEvidenceV1(_SplitModel): + schema_version: Literal["perceptual-comparison-evidence.v1"] = ( + "perceptual-comparison-evidence.v1" + ) + left_frame_id: str + right_frame_id: str + hamming_distance: int = Field(ge=0, le=64) + + @model_validator(mode="after") + def validate_order(self) -> PerceptualComparisonEvidenceV1: + if self.left_frame_id >= self.right_frame_id: + raise ValueError("comparison frame IDs must be canonically ordered") + return self + + +class SplitAssignmentEvidenceV1(_SplitModel): + schema_version: Literal["split-assignment-evidence.v2"] = "split-assignment-evidence.v2" + producer_id: Literal["curated-validator.v1"] = "curated-validator.v1" + manifest_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + metric_policy_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + validation_policy_version: Literal["curated-validation-thresholds.v1"] = ( + "curated-validation-thresholds.v1" + ) + perceptual_hamming_threshold: int = Field(ge=0, le=64) + max_sequences: int = Field(gt=0) + max_frames: int = Field(gt=0) + max_comparisons: int = Field(ge=0) + comparison_universe_count: int = Field(ge=0) + frames: tuple[PerceptualFrameEvidenceV1, ...] + comparisons: tuple[PerceptualComparisonEvidenceV1, ...] + producer_receipt_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + + @model_validator(mode="after") + def validate_canonical_universe(self) -> SplitAssignmentEvidenceV1: + frame_ids = tuple(frame.frame_id for frame in self.frames) + if frame_ids != tuple(sorted(frame_ids)) or len(frame_ids) != len(set(frame_ids)): + raise ValueError("perceptual frames must be unique and canonically ordered") + keys = tuple( + (comparison.left_frame_id, comparison.right_frame_id) for comparison in self.comparisons + ) + if keys != tuple(sorted(keys)) or len(keys) != len(set(keys)): + raise ValueError("comparisons must be unique and canonically ordered") + expected_count = len(frame_ids) * (len(frame_ids) - 1) // 2 + if self.comparison_universe_count != expected_count or len(keys) != expected_count: + raise ValueError("comparison evidence must cover the complete universe") + if expected_count > self.max_comparisons: + raise ValueError("comparison evidence exceeds its declared budget") + return self + + +def produce_split_assignment_evidence( + manifest: CuratedDatasetManifestV1, + decoded_frames: Mapping[str, DecodedFrame], + *, + perceptual_hamming_threshold: int, + max_sequences: int, + max_frames: int, + max_comparisons: int, + cancelled: Callable[[], bool] = lambda: False, +) -> SplitAssignmentEvidenceV1: + """Produce a complete receipt from descriptor-verified decoded frame evidence.""" + if not 0 <= perceptual_hamming_threshold <= 64: + raise CuratedSplitError("split_threshold_invalid") + sequence_count = len(manifest.sequences) + manifest_frames = tuple(frame for sequence in manifest.sequences for frame in sequence.frames) + frame_count = len(manifest_frames) + comparison_count = frame_count * (frame_count - 1) // 2 + if sequence_count > max_sequences: + raise CuratedSplitError("split_sequence_budget_exceeded") + if frame_count > max_frames: + raise CuratedSplitError("split_frame_budget_exceeded") + if comparison_count > max_comparisons: + raise CuratedSplitError("split_comparison_budget_exceeded") + if set(decoded_frames) != {frame.frame_id for frame in manifest_frames}: + raise CuratedSplitError("split_decoded_frame_set_mismatch") + try: + for frame_id in sorted(decoded_frames): + if cancelled(): + raise CuratedSplitError("split_cancelled") + validate_decoded_frame(decoded_frames[frame_id]) + except CuratedSourceError: + raise CuratedSplitError("split_decoded_frame_invalid") from None + + frame_evidence: list[PerceptualFrameEvidenceV1] = [] + for frame in sorted(manifest_frames, key=lambda value: value.frame_id): + if cancelled(): + raise CuratedSplitError("split_cancelled") + decoded = decoded_frames[frame.frame_id] + if decoded.pixel_digest != frame.pixel_digest: + raise CuratedSplitError("split_decoded_digest_mismatch") + frame_evidence.append( + PerceptualFrameEvidenceV1( + frame_id=frame.frame_id, + source_digest=frame.source_digest, + pixel_digest=frame.pixel_digest, + perceptual_hash64=perceptual_hash64(decoded), + ) + ) + comparisons: list[PerceptualComparisonEvidenceV1] = [] + for left, right in combinations(frame_evidence, 2): + if cancelled(): + raise CuratedSplitError("split_cancelled") + comparisons.append(PerceptualComparisonEvidenceV1( + left_frame_id=left.frame_id, + right_frame_id=right.frame_id, + hamming_distance=hamming_distance64(left.perceptual_hash64, right.perceptual_hash64), + )) + comparisons_evidence = tuple(comparisons) + payload: dict[str, object] = { + "schema_version": "split-assignment-evidence.v2", + "producer_id": "curated-validator.v1", + "manifest_digest": curated_dataset_digest(manifest), + "metric_policy_digest": metric_policy_digest(), + "validation_policy_version": "curated-validation-thresholds.v1", + "perceptual_hamming_threshold": perceptual_hamming_threshold, + "max_sequences": max_sequences, + "max_frames": max_frames, + "max_comparisons": max_comparisons, + "comparison_universe_count": comparison_count, + "frames": tuple(frame.model_dump(mode="json") for frame in frame_evidence), + "comparisons": tuple( + comparison.model_dump(mode="json") for comparison in comparisons_evidence + ), + } + receipt = hashlib.sha256(canonical_json(payload).encode()).hexdigest() + return SplitAssignmentEvidenceV1( + manifest_digest=curated_dataset_digest(manifest), + metric_policy_digest=metric_policy_digest(), + perceptual_hamming_threshold=perceptual_hamming_threshold, + max_sequences=max_sequences, + max_frames=max_frames, + max_comparisons=max_comparisons, + comparison_universe_count=comparison_count, + frames=tuple(frame_evidence), + comparisons=comparisons_evidence, + producer_receipt_digest=receipt, + ) + + +class _UnionFind: + def __init__(self, values: tuple[str, ...]) -> None: + self.parent = {value: value for value in values} + + def find(self, value: str) -> str: + parent = self.parent[value] + if parent != value: + self.parent[value] = self.find(parent) + return self.parent[value] + + def union(self, left: str, right: str) -> None: + left_root, right_root = self.find(left), self.find(right) + if left_root != right_root: + self.parent[max(left_root, right_root)] = min(left_root, right_root) + + +def assign_identity_aware_splits( + manifest: CuratedDatasetManifestV1, + policy: DatasetSplitPolicyV1, + *, + decoded_frames: Mapping[str, DecodedFrame] | None = None, + evidence: SplitAssignmentEvidenceV1 | None = None, + enforce_declared: bool = True, + cancelled: Callable[[], bool] = lambda: False, +) -> CuratedDatasetManifestV1: + """Assign splits only after independently reproducing the complete pHash receipt.""" + if manifest.split_policy != policy: + raise CuratedSplitError("split_policy_conflict") + if evidence is None or decoded_frames is None: + raise CuratedSplitError("split_evidence_required") + try: + reproduced = produce_split_assignment_evidence( + manifest, + decoded_frames, + perceptual_hamming_threshold=evidence.perceptual_hamming_threshold, + max_sequences=evidence.max_sequences, + max_frames=evidence.max_frames, + max_comparisons=evidence.max_comparisons, + cancelled=cancelled, + ) + except CuratedSplitError: + raise + if reproduced != evidence: + raise CuratedSplitError("split_evidence_verification_failed") + + sequence_ids = tuple(sequence.sequence_id for sequence in manifest.sequences) + groups = _UnionFind(sequence_ids) + lineage_owner: dict[tuple[str, str], str] = {} + digest_owner: dict[str, str] = {} + frame_owner: dict[str, str] = {} + for sequence in manifest.sequences: + if cancelled(): + raise CuratedSplitError("split_cancelled") + lineage_key = (sequence.identity_id, sequence.source_lineage) + owner = lineage_owner.setdefault(lineage_key, sequence.sequence_id) + groups.union(owner, sequence.sequence_id) + for frame in sequence.frames: + frame_owner[frame.frame_id] = sequence.sequence_id + owner = digest_owner.setdefault(frame.pixel_digest, sequence.sequence_id) + groups.union(owner, sequence.sequence_id) + for comparison in evidence.comparisons: + if cancelled(): + raise CuratedSplitError("split_cancelled") + if comparison.hamming_distance <= evidence.perceptual_hamming_threshold: + groups.union( + frame_owner[comparison.left_frame_id], + frame_owner[comparison.right_frame_id], + ) + + assignments: dict[str, str] = {} + components: dict[str, list[str]] = {} + for sequence_id in sequence_ids: + if cancelled(): + raise CuratedSplitError("split_cancelled") + components.setdefault(groups.find(sequence_id), []).append(sequence_id) + by_id = {sequence.sequence_id: sequence for sequence in manifest.sequences} + for members in components.values(): + if cancelled(): + raise CuratedSplitError("split_cancelled") + representative = min(members) + sequence = by_id[representative] + material = "\x00".join( + (policy.policy_version, str(policy.seed), sequence.identity_id, sequence.source_lineage) + ).encode("utf-8") + bucket = int.from_bytes(hashlib.sha256(material).digest(), "big") % policy.denominator + split = ( + "train" + if bucket < policy.train_numerator + else "validation" + if bucket < policy.train_numerator + policy.validation_numerator + else "holdout" + ) + for member in members: + assignments[member] = split + + if enforce_declared and any( + sequence.split != assignments[sequence.sequence_id] for sequence in manifest.sequences + ): + raise CuratedSplitError("declared_split_conflict") + sequences = tuple( + sequence.model_copy(update={"split": assignments[sequence.sequence_id]}) + for sequence in manifest.sequences + ) + return manifest.model_copy(update={"sequences": sequences}) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_validation.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_validation.py new file mode 100644 index 0000000..3629bd4 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/curated_validation.py @@ -0,0 +1,557 @@ +"""Pure deterministic validation and normalized evidence for curated datasets.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Callable, Mapping +from decimal import Decimal +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from video_lab.curated_ids import curated_dataset_digest +from video_lab.curated_metrics import ( + blur_variance, + hamming_distance64, + histogram_discontinuity, + metric_policy_digest, + perceptual_hash64, + quantize_metric, + temporal_motion, +) +from video_lab.curated_models import CuratedDatasetManifestV1, CuratedFrameV1 +from video_lab.curated_source import CuratedSourceError, DecodedFrame, validate_decoded_frame +from video_lab.curated_splits import ( + SplitAssignmentEvidenceV1, + produce_split_assignment_evidence, +) +from video_lab.domain import canonical_json + +FindingState = Literal["accepted", "rejected", "needs_review"] + + +class _ValidationModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + + +class ValidationThresholdsV1(_ValidationModel): + schema_version: Literal["curated-validation-thresholds.v1"] = "curated-validation-thresholds.v1" + perceptual_duplicate_hamming_max: int = Field(default=4, ge=0, le=64) + blur_variance_min: str = "20.0000" + scene_cut_histogram_max: str = "0.6500" + temporal_motion_max: str = "0.6500" + max_sequences: int = Field(default=512, gt=0) + max_frames: int = Field(default=4096, gt=0) + max_comparisons: int = Field(default=1_000_000, ge=0) + + @field_validator("blur_variance_min", "scene_cut_histogram_max", "temporal_motion_max") + @classmethod + def validate_decimal(cls, value: str) -> str: + decimal = Decimal(value) + if not decimal.is_finite() or decimal < 0: + raise ValueError("threshold must be a finite nonnegative decimal") + return quantize_metric(decimal) + + +class FrameFinding(_ValidationModel): + finding_id: str + state: FindingState + reason_code: str + frame_id: str + sequence_id: str + source_digest: str + pixel_digest: str + observed_pixel_digest: str | None = None + observed_value: str + threshold: str + related_frame_id: str | None = None + related_source_digest: str | None = None + related_pixel_digest: str | None = None + human_disposition: Literal["accepted", "rejected"] | None = None + + +class ImplicatedFrameEvidence(_ValidationModel): + frame_id: str + source_digest: str + pixel_digest: str + + +class SequenceFinding(_ValidationModel): + finding_id: str + state: FindingState + reason_code: str + sequence_id: str + related_sequence_ids: tuple[str, ...] = () + implicated_frames: tuple[ImplicatedFrameEvidence, ...] = () + observed_value: str + threshold: str + human_disposition: Literal["accepted", "rejected"] | None = None + + +class CuratedValidationReport(_ValidationModel): + schema_version: Literal["curated-validation-report.v1"] = "curated-validation-report.v1" + dataset_digest: str + thresholds: ValidationThresholdsV1 + metric_policy_digest: str + split_assignment_evidence: SplitAssignmentEvidenceV1 | None = None + state: FindingState + frame_findings: tuple[FrameFinding, ...] + sequence_findings: tuple[SequenceFinding, ...] + + +def _finding_id(payload: Mapping[str, object]) -> str: + return "finding_" + hashlib.sha256(canonical_json(dict(payload)).encode()).hexdigest() + + +def _validation_policy_digest(thresholds: ValidationThresholdsV1) -> str: + payload = { + "metric_policy_digest": metric_policy_digest(), + "thresholds": thresholds.model_dump(mode="json"), + } + return hashlib.sha256(canonical_json(payload).encode()).hexdigest() + + +def _state_with_disposition( + default: FindingState, finding_id: str, dispositions: Mapping[str, str] +) -> tuple[FindingState, Literal["accepted", "rejected"] | None]: + disposition = dispositions.get(finding_id) + if disposition not in (None, "accepted", "rejected"): + raise ValueError("human disposition must be accepted or rejected") + if disposition is not None and default != "needs_review": + raise ValueError("human dispositions apply only to review findings") + return (disposition if disposition is not None else default), disposition # type: ignore[return-value] + + +def validate_curated_dataset( + manifest: CuratedDatasetManifestV1, + decoded_frames: Mapping[str, DecodedFrame], + thresholds: ValidationThresholdsV1 | None = None, + *, + human_dispositions: Mapping[str, str] | None = None, + cancelled: Callable[[], bool] = lambda: False, +) -> CuratedValidationReport: + configured = thresholds or ValidationThresholdsV1() + dispositions = human_dispositions or {} + dataset_digest = curated_dataset_digest(manifest) + policy_digest = _validation_policy_digest(configured) + sequence_count = len(manifest.sequences) + frame_count = sum(len(sequence.frames) for sequence in manifest.sequences) + comparison_count = frame_count * (frame_count - 1) // 2 + budget_code = ( + "validation_sequence_budget_exceeded" + if sequence_count > configured.max_sequences + else "validation_frame_budget_exceeded" + if frame_count > configured.max_frames + else "validation_comparison_budget_exceeded" + if comparison_count > configured.max_comparisons + else None + ) + if budget_code is not None: + if dispositions: + raise ValueError("human disposition does not bind current immutable finding evidence") + finding = _sequence_finding( + budget_code, + "rejected", + manifest.sequences[0].sequence_id, + (), + (), + quantize_metric( + sequence_count + if "sequence" in budget_code + else frame_count + if "frame" in budget_code + else comparison_count + ), + quantize_metric( + configured.max_sequences + if "sequence" in budget_code + else configured.max_frames + if "frame" in budget_code + else configured.max_comparisons + ), + dispositions, + dataset_digest, + policy_digest, + ) + return CuratedValidationReport( + dataset_digest=dataset_digest, + thresholds=configured, + metric_policy_digest=metric_policy_digest(), + state="rejected", + frame_findings=(), + sequence_findings=(finding,), + ) + expected_ids = {frame.frame_id for sequence in manifest.sequences for frame in sequence.frames} + if set(decoded_frames) != expected_ids: + raise ValueError("decoded frame identities must exactly match the manifest") + try: + for frame_id in sorted(decoded_frames): + validate_decoded_frame(decoded_frames[frame_id]) + except CuratedSourceError: + raise ValueError("decoded frame evidence is not canonical") from None + frame_findings: list[FrameFinding] = [] + sequence_findings: list[SequenceFinding] = [] + seen_exact: dict[str, tuple[str, str]] = {} + seen_hash: list[tuple[int, str, str, str]] = [] + frame_models = { + frame.frame_id: frame for sequence in manifest.sequences for frame in sequence.frames + } + + for sequence in manifest.sequences: + if cancelled(): + raise ValueError("curated_validation_cancelled") + previous: tuple[str, DecodedFrame] | None = None + expected_shape: tuple[int, int, int] | None = None + for frame in sequence.frames: + if cancelled(): + raise ValueError("curated_validation_cancelled") + decoded = decoded_frames[frame.frame_id] + shape = (decoded.width, decoded.height, decoded.channels) + if expected_shape is None: + expected_shape = shape + elif shape != expected_shape: + frame_findings.append( + _frame_finding( + "dimension_inconsistency", + "rejected", + sequence.sequence_id, + frame, + "1.0000", + "0.0000", + frame_models[previous[0]] if previous is not None else None, + dataset_digest, + policy_digest, + ) + ) + if decoded.pixel_digest != frame.pixel_digest: + frame_findings.append( + _frame_finding( + "decoded_digest_mismatch", + "rejected", + sequence.sequence_id, + frame, + "1.0000", + "0.0000", + None, + dataset_digest, + policy_digest, + observed_pixel_digest=decoded.pixel_digest, + ) + ) + exact = seen_exact.get(decoded.pixel_digest) + if exact is not None: + frame_findings.append( + _frame_finding( + "exact_duplicate", + "rejected", + sequence.sequence_id, + frame, + "0.0000", + "0.0000", + frame_models[exact[1]], + dataset_digest, + policy_digest, + ) + ) + else: + seen_exact[decoded.pixel_digest] = (sequence.sequence_id, frame.frame_id) + phash = perceptual_hash64(decoded) + for other_hash, other_sequence, other_frame, other_digest in seen_hash: + if cancelled(): + raise ValueError("curated_validation_cancelled") + distance = hamming_distance64(phash, other_hash) + if ( + distance <= configured.perceptual_duplicate_hamming_max + and other_digest != decoded.pixel_digest + ): + frame_findings.append( + _frame_finding( + "perceptual_duplicate", + "rejected", + sequence.sequence_id, + frame, + quantize_metric(distance), + quantize_metric(configured.perceptual_duplicate_hamming_max), + frame_models[other_frame], + dataset_digest, + policy_digest, + ) + ) + if ( + distance <= configured.perceptual_duplicate_hamming_max + and other_sequence != sequence.sequence_id + ): + left_split = next( + item.split + for item in manifest.sequences + if item.sequence_id == other_sequence + ) + if left_split != sequence.split: + related = tuple(sorted((other_sequence, sequence.sequence_id))) + sequence_findings.append( + _sequence_finding( + "near_duplicate_split_leakage", + "rejected", + sequence.sequence_id, + related, + tuple( + sorted( + ( + _implicated(frame), + _implicated(frame_models[other_frame]), + ), + key=lambda item: item.frame_id, + ) + ), + quantize_metric(distance), + quantize_metric(configured.perceptual_duplicate_hamming_max), + dispositions, + dataset_digest, + policy_digest, + ) + ) + seen_hash.append((phash, sequence.sequence_id, frame.frame_id, decoded.pixel_digest)) + blur = blur_variance(decoded) + if Decimal(str(blur)) < Decimal(configured.blur_variance_min): + frame_findings.append( + _frame_finding( + "blur_below_minimum", + "rejected", + sequence.sequence_id, + frame, + quantize_metric(blur), + configured.blur_variance_min, + None, + dataset_digest, + policy_digest, + ) + ) + if previous is not None: + previous_id, previous_decoded = previous + scene = histogram_discontinuity(previous_decoded, decoded) + if Decimal(str(scene)) > Decimal(configured.scene_cut_histogram_max): + frame_findings.append( + _frame_finding( + "scene_cut", + "rejected", + sequence.sequence_id, + frame, + quantize_metric(scene), + configured.scene_cut_histogram_max, + frame_models[previous_id], + dataset_digest, + policy_digest, + ) + ) + motion = temporal_motion(previous_decoded, decoded) + if Decimal(str(motion)) > Decimal(configured.temporal_motion_max): + frame_findings.append( + _frame_finding( + "temporal_discontinuity", + "rejected", + sequence.sequence_id, + frame, + quantize_metric(motion), + configured.temporal_motion_max, + frame_models[previous_id], + dataset_digest, + policy_digest, + ) + ) + previous = (frame.frame_id, decoded) + + identities: dict[str, tuple[tuple[str, ...], str]] = {} + sequences_by_id = {sequence.sequence_id: sequence for sequence in manifest.sequences} + for sequence in manifest.sequences: + if cancelled(): + raise ValueError("curated_validation_cancelled") + prior = identities.setdefault( + sequence.identity_id, (sequence.caption.immutable_attributes, sequence.sequence_id) + ) + if prior[0] != sequence.caption.immutable_attributes: + implicated = tuple( + sorted( + ( + _implicated(frame) + for implicated_sequence in ( + sequences_by_id[prior[1]], + sequence, + ) + for frame in implicated_sequence.frames + ), + key=lambda item: item.frame_id, + ) + ) + sequence_findings.append( + _sequence_finding( + "identity_inconsistency", + "needs_review", + sequence.sequence_id, + tuple(sorted((prior[1], sequence.sequence_id))), + implicated, + "1.0000", + "0.0000", + dispositions, + dataset_digest, + policy_digest, + ) + ) + sequence_findings.append( + _sequence_finding( + "caption_identity_mismatch", + "needs_review", + sequence.sequence_id, + tuple(sorted((prior[1], sequence.sequence_id))), + implicated, + "1.0000", + "0.0000", + dispositions, + dataset_digest, + policy_digest, + ) + ) + + frame_findings.sort( + key=lambda item: (item.sequence_id, item.frame_id, item.reason_code, item.finding_id) + ) + unique_sequence = {item.finding_id: item for item in sequence_findings} + ordered_sequence = tuple( + sorted( + unique_sequence.values(), + key=lambda item: (item.sequence_id, item.reason_code, item.finding_id), + ) + ) + states = [item.state for item in frame_findings] + [item.state for item in ordered_sequence] + review_ids = { + item.finding_id + for item in frame_findings + if item.human_disposition is not None or item.state == "needs_review" + } | { + item.finding_id + for item in ordered_sequence + if item.human_disposition is not None or item.state == "needs_review" + } + unknown_dispositions = set(dispositions) - review_ids + if unknown_dispositions: + raise ValueError("human disposition does not bind current immutable finding evidence") + state: FindingState = ( + "rejected" + if "rejected" in states + else "needs_review" + if "needs_review" in states + else "accepted" + ) + if cancelled(): + raise ValueError("curated_validation_cancelled") + split_evidence = ( + None + if any(item.reason_code == "decoded_digest_mismatch" for item in frame_findings) + else produce_split_assignment_evidence( + manifest, + decoded_frames, + perceptual_hamming_threshold=configured.perceptual_duplicate_hamming_max, + max_sequences=configured.max_sequences, + max_frames=configured.max_frames, + max_comparisons=configured.max_comparisons, + ) + ) + return CuratedValidationReport( + dataset_digest=dataset_digest, + thresholds=configured, + metric_policy_digest=metric_policy_digest(), + split_assignment_evidence=split_evidence, + state=state, + frame_findings=tuple(frame_findings), + sequence_findings=ordered_sequence, + ) + + +def _frame_finding( + code: str, + state: FindingState, + sequence_id: str, + frame: CuratedFrameV1, + observed: str, + threshold: str, + related_frame: CuratedFrameV1 | None, + dataset_digest: str, + policy_digest: str, + observed_pixel_digest: str | None = None, +) -> FrameFinding: + payload = { + "finding_schema": "frame-finding.v1", + "dataset_digest": dataset_digest, + "validation_policy_digest": policy_digest, + "reason_code": code, + "sequence_id": sequence_id, + "frame_id": frame.frame_id, + "source_digest": frame.source_digest, + "pixel_digest": frame.pixel_digest, + "observed_pixel_digest": observed_pixel_digest, + "observed_value": observed, + "threshold": threshold, + "related_frame_id": related_frame.frame_id if related_frame else None, + "related_source_digest": related_frame.source_digest if related_frame else None, + "related_pixel_digest": related_frame.pixel_digest if related_frame else None, + } + return FrameFinding( + finding_id=_finding_id(payload), + state=state, + reason_code=code, + frame_id=frame.frame_id, + sequence_id=sequence_id, + source_digest=frame.source_digest, + pixel_digest=frame.pixel_digest, + observed_pixel_digest=observed_pixel_digest, + observed_value=observed, + threshold=threshold, + related_frame_id=related_frame.frame_id if related_frame else None, + related_source_digest=related_frame.source_digest if related_frame else None, + related_pixel_digest=related_frame.pixel_digest if related_frame else None, + ) + + +def _sequence_finding( + code: str, + state: FindingState, + sequence_id: str, + related: tuple[str, ...], + implicated_frames: tuple[ImplicatedFrameEvidence, ...], + observed: str, + threshold: str, + dispositions: Mapping[str, str], + dataset_digest: str, + policy_digest: str, +) -> SequenceFinding: + payload = { + "finding_schema": "sequence-finding.v1", + "dataset_digest": dataset_digest, + "validation_policy_digest": policy_digest, + "reason_code": code, + "sequence_id": sequence_id, + "related_sequence_ids": related, + "implicated_frames": tuple(frame.model_dump(mode="json") for frame in implicated_frames), + "observed_value": observed, + "threshold": threshold, + } + finding_id = _finding_id(payload) + final_state, disposition = _state_with_disposition(state, finding_id, dispositions) + return SequenceFinding( + finding_id=finding_id, + state=final_state, + reason_code=code, + sequence_id=sequence_id, + related_sequence_ids=related, + implicated_frames=implicated_frames, + observed_value=observed, + threshold=threshold, + human_disposition=disposition, + ) + + +def _implicated(frame: CuratedFrameV1) -> ImplicatedFrameEvidence: + return ImplicatedFrameEvidence( + frame_id=frame.frame_id, + source_digest=frame.source_digest, + pixel_digest=frame.pixel_digest, + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/database.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/database.py new file mode 100644 index 0000000..dce1b89 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/database.py @@ -0,0 +1,6351 @@ +from __future__ import annotations + +# ruff: noqa: E501 +import hashlib +import json +import sqlite3 +from datetime import datetime +from pathlib import Path +from types import TracebackType +from typing import Any, Literal, cast + +from video_lab.adaptive_study_models import adaptive_observation_provenance_digest +from video_lab.character_models import CharacterPackDraft +from video_lab.character_pack import character_pack_id +from video_lab.domain import Manifest, canonical_json, utc_now +from video_lab.preference_optimizer import CandidatePair, OptimizerConfig, PreferenceOutcome +from video_lab.training_dataset import validate_training_recipe_dataset +from video_lab.training_models import ( + CalibrationOperationRequest, + CancelOperationReceipt, + CancelOperationRequest, + EvidenceOperationReceipt, + FrozenTrainingDataset, + ImportedTrainingSample, + ProbeOperationRequest, + ResumeOperationReceipt, + ResumeOperationRequest, + StartOperationReceipt, + StartOperationRequest, + TrainingCheckpointEvidence, + TrainingCurationConfirmation, + TrainingDatasetDraft, + TrainingRecipe, + TrainingSample, + checkpoint_steps, + imported_training_sample_id, + training_confirmation_id, + training_dataset_id, + training_recipe_id, +) + +SCHEMA_VERSION = 49 + +_CHECKPOINT_MANIFEST_KEYS = { + "schema_version", + "run_id", + "reviewer_id", + "recipe_id", + "dataset_digest", + "character_id", + "step", + "fraction", + "parent_checkpoint_id", + "artifact_digest", + "safetensors_size_bytes", + "backend_digest", + "runtime_digest", + "material_digest", + "source_digest", + "optimizer_state_digest", + "trainer_state_digest", + "telemetry_digest", + "telemetry_sequence_start", + "telemetry_sequence_end", + "trainable_targets_digest", + "frozen_parameters_digest", + "finite_gradient_proof_digest", + "cancellation_boundary", + "inventory", +} +_CHECKPOINT_EVIDENCE_KEYS = { + "schema_version", + "run_id", + "recipe_digest", + "dataset_digest", + "backend_digest", + "runtime_digest", + "material_digest", + "source_digest", + "step", + "fraction", + "parent_checkpoint_digest", + "safetensors_digest", + "safetensors_size_bytes", + "optimizer_state_digest", + "trainer_state_digest", + "telemetry_digest", + "telemetry_sequence_start", + "telemetry_sequence_end", + "trainable_targets_digest", + "frozen_parameters_digest", + "finite_gradient_proof_digest", + "cancellation_boundary", + "inventory", +} +_CHECKPOINT_INVENTORY_KEYS = {"relative_path", "digest", "size_bytes", "role"} + + +def _sha256_digest_is_valid(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 71 + and value.startswith("sha256:") + and all(character in "0123456789abcdef" for character in value[7:]) + ) + + +def _training_checkpoint_json_is_exact(manifest_json: object, evidence_json: object) -> int: + try: + manifest = json.loads(str(manifest_json)) + evidence = json.loads(str(evidence_json)) + if not isinstance(manifest, dict) or not isinstance(evidence, dict): + return 0 + if set(manifest) != _CHECKPOINT_MANIFEST_KEYS or set(evidence) != _CHECKPOINT_EVIDENCE_KEYS: + return 0 + if manifest["schema_version"] != "character-training-checkpoint.v1": + return 0 + manifest_inventory = manifest["inventory"] + evidence_inventory = evidence["inventory"] + if manifest_inventory != evidence_inventory or not isinstance(evidence_inventory, list): + return 0 + if len(evidence_inventory) != 7: + return 0 + if any( + not isinstance(item, dict) or set(item) != _CHECKPOINT_INVENTORY_KEYS + for item in evidence_inventory + ): + return 0 + TrainingCheckpointEvidence.model_validate(evidence) + digest_values = [ + value + for document in (manifest, evidence) + for key, value in document.items() + if key.endswith("digest") and value is not None + ] + [item["digest"] for item in evidence_inventory] + return int(all(_sha256_digest_is_valid(value) for value in digest_values)) + except (TypeError, ValueError, json.JSONDecodeError): + return 0 + + +def validation_experiment_id(campaign_id: str, trial_id: str) -> str: + if not campaign_id or not trial_id: + raise ValueError("validation campaign and trial ids must not be empty") + payload = canonical_json( + { + "schema_version": "validation-experiment-identity.v1", + "domain": "video-lab.validation.exp", + "campaign_id": campaign_id, + "trial_id": trial_id, + "role": "experiment", + } + ) + return f"validation_exp_{hashlib.sha256(payload.encode()).hexdigest()}" + + +def character_benchmark_reviewer_id(campaign_id: str, trial_id: str, reviewer_id: str) -> str: + payload = f"character-benchmark-review.v1\x00{campaign_id}\x00{trial_id}\x00{reviewer_id}" + return "character_reviewer_" + hashlib.sha256(payload.encode()).hexdigest() + + +def character_benchmark_experiment_id(campaign_id: str, trial_id: str) -> str: + payload = canonical_json( + { + "domain": "video-lab.character-benchmark.exp", + "campaign_id": campaign_id, + "identity": trial_id, + } + ) + return "character_exp_" + hashlib.sha256(payload.encode()).hexdigest() + + +def _training_dataset_identity_is_valid( + connection: sqlite3.Connection, storage_id: object, digest: object, manifest_json: object +) -> int: + try: + frozen = FrozenTrainingDataset.model_validate_json(str(manifest_json)) + row = connection.execute( + "SELECT character_id,character_pack_digest,canonical_outfit_id,reserved_token," + "source_tier_weights_json FROM training_datasets WHERE storage_id=?", + (storage_id,), + ).fetchone() + sample_rows = connection.execute( + "SELECT sample.*,annotation.annotation_json FROM training_samples sample " + "JOIN training_sample_annotations annotation USING(dataset_storage_id,sample_id) " + "WHERE sample.dataset_storage_id=? ORDER BY sample.ordinal", + (storage_id,), + ).fetchall() + samples = tuple( + TrainingSample.model_validate( + { + "sample_id": item["sample_id"], + "source_tier": item["source_tier"], + "sample_digest": item["sample_digest"], + "source_lineage": item["source_lineage"], + "caption": item["caption"], + "split": item["split"], + "media": json.loads(str(item["media_json"])), + "annotation": json.loads(str(item["annotation_json"])), + "accepted": bool(item["accepted"]), + } + ) + for item in sample_rows + ) + if row is None: + return 0 + draft = TrainingDatasetDraft( + character_id=str(row["character_id"]), + character_pack_digest=str(row["character_pack_digest"]), + canonical_outfit_id=str(row["canonical_outfit_id"]), + reserved_token=str(row["reserved_token"]), + source_tier_weights=json.loads(str(row["source_tier_weights_json"])), + samples=samples, + imported_sample_ids=frozen.imported_sample_ids, + curation_confirmation_ids=frozen.curation_confirmation_ids, + ) + expected = training_dataset_id(draft) + return int( + str(digest) == expected + and frozen.dataset_digest == expected + and TrainingDatasetDraft.model_validate( + frozen.model_dump(mode="python", exclude={"coverage", "dataset_digest"}) + ) + == draft + ) + except (sqlite3.DatabaseError, TypeError, ValueError): + return 0 + + +def _training_recipe_identity_is_valid( + connection: sqlite3.Connection, + dataset_storage_id: object, + recipe_id: object, + recipe_json: object, +) -> int: + try: + recipe = TrainingRecipe.model_validate_json(str(recipe_json)) + row = connection.execute( + "SELECT state,dataset_digest,manifest_json FROM training_datasets WHERE storage_id=?", + (dataset_storage_id,), + ).fetchone() + if row is None or str(row["state"]) != "sealed": + return 0 + dataset = FrozenTrainingDataset.model_validate_json(str(row["manifest_json"])) + validate_training_recipe_dataset(recipe, dataset) + return int( + str(recipe_id) == training_recipe_id(recipe) + and recipe.dataset_digest == str(row["dataset_digest"]) + ) + except (sqlite3.DatabaseError, TypeError, ValueError): + return 0 + + +def _canonical_json_digest(value: object) -> str | None: + try: + parsed = json.loads(str(value)) + return "sha256:" + hashlib.sha256(canonical_json(parsed).encode()).hexdigest() + except (TypeError, ValueError): + return None + + +def _canonical_json_is_exact(value: object) -> int: + if not isinstance(value, str): + return 0 + try: + return int(canonical_json(json.loads(value)) == value) + except (TypeError, ValueError, json.JSONDecodeError): + return 0 + + +def _training_operation_json_is_exact(kind: object, phase: object, value: object) -> int: + request_models = { + "probe": ProbeOperationRequest, + "calibration": CalibrationOperationRequest, + "start": StartOperationRequest, + "resume": ResumeOperationRequest, + "cancel": CancelOperationRequest, + } + receipt_models = { + "probe": EvidenceOperationReceipt, + "calibration": EvidenceOperationReceipt, + "start": StartOperationReceipt, + "resume": ResumeOperationReceipt, + "cancel": CancelOperationReceipt, + } + try: + models = ( + request_models if phase == "request" else receipt_models if phase == "receipt" else {} + ) + model = cast(Any, models[str(kind)]) + parsed = model.model_validate_json(str(value)) + return int(canonical_json(parsed.model_dump(mode="json")) == value) + except (KeyError, TypeError, ValueError): + return 0 + + +def _training_operation_lifecycle_is_valid(kind: object, state: object) -> int: + if str(kind) == "resume": + return int(str(state) in {"training", "cancelled"}) + return int( + str(state) + == { + "probe": "draft", + "calibration": "probed", + "start": "preregistered", + "cancel": "training", + }.get(str(kind)) + ) + + +def _training_operation_completion_target_is_valid(kind: object, state: object) -> int: + return int( + str(state) + == { + "probe": "probed", + "calibration": "calibrated", + "start": "evaluating", + "resume": "evaluating", + "cancel": "cancelled", + }.get(str(kind)) + ) + + +def _training_operation_receipt_links_are_exact( + connection: sqlite3.Connection, operation_id: object, receipt_json: object +) -> int: + try: + operation = connection.execute( + "SELECT operation_kind FROM training_orchestration_operations WHERE id=?", + (str(operation_id),), + ).fetchone() + if operation is None: + return 0 + kind = str(operation[0]) + payload = json.loads(str(receipt_json)) + if kind in {"start", "resume"}: + expected: set[tuple[str, str, str]] = set() + for field, link_kind in ( + ("events", "event"), + ("telemetry", "telemetry"), + ("checkpoints", "checkpoint"), + ("artifacts", "artifact"), + ("inventory", "inventory"), + ): + expected.update( + (link_kind, entry["evidence_id"], entry["digest"]) for entry in payload[field] + ) + if kind == "resume": + expected.add( + ("resume", payload["resume"]["evidence_id"], payload["resume"]["digest"]) + ) + if payload["evaluation"] is not None: + expected.add( + ( + "evaluation", + payload["evaluation"]["evidence_id"], + payload["evaluation"]["digest"], + ) + ) + elif kind == "cancel": + expected = { + ( + "cancel", + payload["cancellation"]["evidence_id"], + payload["cancellation"]["digest"], + ), + ( + "event", + payload["terminal_event"]["evidence_id"], + payload["terminal_event"]["digest"], + ), + ( + "checkpoint", + payload["boundary_checkpoint"]["evidence_id"], + payload["boundary_checkpoint"]["digest"], + ), + } + else: + expected = {(kind, payload["evidence_id"], payload["evidence_digest"])} + actual = { + (str(row[0]), str(row[1]), str(row[2])) + for row in connection.execute( + "SELECT evidence_kind,evidence_id,evidence_digest " + "FROM training_operation_evidence_links WHERE operation_id=?", + (str(operation_id),), + ) + } + return int(actual == expected) + except (KeyError, TypeError, ValueError, sqlite3.DatabaseError): + return 0 + + +def _training_operation_request_identity_is_valid( + connection: sqlite3.Connection, kind: object, run_id: object, payload_json: object +) -> int: + try: + payload = json.loads(str(payload_json)) + row = connection.execute( + "SELECT recipe.id recipe_digest,recipe.dataset_digest,recipe.recipe_json," + "material.runtime_digest,material.stage_3_campaign_id,material.stage_3_result_id," + "material.base_model_artifact_id,material.lightning_adapter_artifact_ids_json," + "material.source_snapshot_artifact_id,source.sha256 source_sha256 " + "FROM training_runs run JOIN training_recipes recipe " + "ON recipe.storage_id=run.recipe_storage_id JOIN training_recipe_materials material " + "ON material.recipe_storage_id=recipe.storage_id JOIN artifacts source " + "ON source.id=material.source_snapshot_artifact_id WHERE run.id=?", + (str(run_id),), + ).fetchone() + if row is None: + return 0 + material_payload = { + "stage_3_campaign_id": row["stage_3_campaign_id"], + "stage_3_result_id": row["stage_3_result_id"], + "base_model_artifact_id": row["base_model_artifact_id"], + "lightning_adapter_artifact_ids": json.loads( + row["lightning_adapter_artifact_ids_json"] + ), + "source_snapshot_artifact_id": row["source_snapshot_artifact_id"], + "runtime_digest": row["runtime_digest"], + } + common = ( + payload["recipe_digest"] == row["recipe_digest"] + and payload["dataset_digest"] == row["dataset_digest"] + and payload["backend_digest"] + == _canonical_json_digest(json.dumps(json.loads(row["recipe_json"])["backend_config"])) + and payload["runtime_digest"] == row["runtime_digest"] + and payload["source_digest"] == "sha256:" + row["source_sha256"] + and payload["material_digest"] == _canonical_json_digest(json.dumps(material_payload)) + ) + if not common: + return 0 + if kind == "calibration": + recipe_payload = json.loads(row["recipe_json"]) + legacy_keys = { + "request_id", + "run_id", + "recipe_digest", + "dataset_digest", + "backend_digest", + "runtime_digest", + "material_digest", + "source_digest", + "probe_operation_id", + "capability_digest", + "steps", + "trainable_targets", + "seed", + } + current_keys = legacy_keys | {"lora"} + if set(payload) not in (legacy_keys, current_keys): + return 0 + expected_lora = { + "rank": recipe_payload["lora_rank"], + "alpha": recipe_payload["lora_alpha"], + "dropout": recipe_payload["lora_dropout"], + } + lora_is_valid = "lora" not in payload or payload["lora"] == expected_lora + return int( + payload["steps"] == recipe_payload["calibration_steps"] + and payload["trainable_targets"] == recipe_payload["trainable_targets"] + and lora_is_valid + and payload["seed"] == recipe_payload["seed"] + and connection.execute( + "SELECT 1 FROM training_orchestration_operations operation " + "JOIN training_probe_evidence evidence " + "ON evidence.id=json_extract(operation.receipt_payload_json,'$.evidence_id') " + "WHERE operation.id=? AND operation.run_id=? " + "AND operation.operation_kind='probe' AND operation.status='completed' " + "AND evidence.capability_digest=? AND evidence.succeeded=1", + (payload["probe_operation_id"], str(run_id), payload["capability_digest"]), + ).fetchone() + is not None + ) + if kind == "start": + preregistered = connection.execute( + "SELECT 1 FROM training_run_idempotency WHERE run_id=? " + "AND idempotency_key=? AND request_digest=?", + ( + str(run_id), + payload["preregistration_id"], + payload["preregistration_request_digest"], + ), + ).fetchone() + recipe_payload = json.loads(row["recipe_json"]) + fractions = recipe_payload["checkpoint_plan"]["fractions"] + steps = [ + _training_checkpoint_step(recipe_payload["total_steps"], fraction) + for fraction in fractions + ] + exact_materials = { + "base_model": row["base_model_artifact_id"], + "source_snapshot": row["source_snapshot_artifact_id"], + **{ + f"lightning:{name}": artifact_id + for name, artifact_id in sorted( + json.loads(row["lightning_adapter_artifact_ids_json"]).items() + ) + }, + } + return int( + preregistered is not None + and payload["checkpoint_fractions"] == fractions + and payload["checkpoint_steps"] == steps + and payload["material_artifact_ids"] == exact_materials + ) + if kind == "resume": + checkpoint = connection.execute( + "SELECT evidence_digest FROM training_checkpoints WHERE id=? AND run_id=? " + "AND publication_state='finalized'", + (payload["parent_checkpoint_id"], str(run_id)), + ).fetchone() + if checkpoint is None or checkpoint[0] != payload["parent_checkpoint_digest"]: + return 0 + persisted = { + ( + str(item["role"]), + str(item["artifact_id"]), + str(item["artifact_digest"]), + int(item["size_bytes"]), + ) + for item in connection.execute( + "SELECT role,artifact_id,artifact_digest,size_bytes " + "FROM training_checkpoint_inventory_artifacts WHERE checkpoint_id=?", + (payload["parent_checkpoint_id"],), + ) + } + requested = { + ( + item["role"], + item["artifact_id"], + item["digest"], + int(item["size_bytes"]), + ) + for item in payload["parent_inventory"] + } + return int(persisted == requested and len(persisted) == 7) + if kind == "cancel": + request = connection.execute( + "SELECT requested_step,request_digest FROM training_cancellation_requests " + "WHERE id=? AND run_id=? AND reviewer_id=(SELECT reviewer_id FROM training_runs WHERE id=?)", + (payload["cancellation_request_id"], str(run_id), str(run_id)), + ).fetchone() + return int( + request is not None + and int(request[0]) == payload["requested_step"] + and str(request[1]) == payload["request_digest"] + ) + return 1 + except (KeyError, TypeError, ValueError, sqlite3.DatabaseError): + return 0 + + +def _training_operation_completion_is_derived( + connection: sqlite3.Connection, operation_id: object +) -> int: + try: + operation = connection.execute( + "SELECT operation_kind,run_id,updated_at FROM training_orchestration_operations " + "WHERE id=? AND status='running'", + (str(operation_id),), + ).fetchone() + if operation is None: + return 0 + kind, run_id, started_at = map(str, operation) + state = connection.execute( + "SELECT lifecycle_state FROM training_runs WHERE id=?", (run_id,) + ).fetchone()[0] + if _training_operation_completion_target_is_valid(kind, state) != 1: + return 0 + links: dict[str, set[str]] = {} + for row in connection.execute( + "SELECT evidence_kind,evidence_id FROM training_operation_evidence_links " + "WHERE operation_id=?", + (str(operation_id),), + ): + links.setdefault(str(row[0]), set()).add(str(row[1])) + if kind in {"probe", "calibration"}: + return int(len(links.get(kind, set())) == 1 and sum(map(len, links.values())) == 1) + events = { + str(row[0]) + for row in connection.execute( + "SELECT sequence FROM training_run_events WHERE run_id=? " + "AND julianday(created_at)>=julianday(?)", + (run_id, started_at), + ) + } + if links.get("event", set()) != events or not events: + return 0 + if kind == "cancel": + if set(links) != {"cancel", "event", "checkpoint"} or any( + len(links[item]) != 1 for item in ("cancel", "event", "checkpoint") + ): + return 0 + cancellation_id = next(iter(links["cancel"])) + event_id = next(iter(links["event"])) + checkpoint_id = next(iter(links["checkpoint"])) + exact = connection.execute( + "SELECT 1 FROM training_cancellation_evidence cancellation " + "JOIN training_checkpoints checkpoint ON checkpoint.id=? " + "JOIN training_run_events event ON event.run_id=cancellation.run_id " + "AND CAST(event.sequence AS TEXT)=? " + "WHERE cancellation.id=? AND cancellation.run_id=? " + "AND checkpoint.run_id=cancellation.run_id " + "AND checkpoint.step=cancellation.boundary_step " + "AND checkpoint.artifact_digest=cancellation.checkpoint_digest " + "AND event.previous_state='training' AND event.new_state='cancelled' " + "AND json_extract(event.payload_json,'$.cancellation_id')=cancellation.id " + "AND json_extract(event.payload_json,'$.cancellation_request_id')=" + "cancellation.cancellation_request_id " + "AND json_extract(event.payload_json,'$.request_digest')=cancellation.request_digest " + "AND json_extract(event.payload_json,'$.checkpoint_digest')=checkpoint.artifact_digest " + "AND json_extract(event.payload_json,'$.boundary_step')=checkpoint.step", + (checkpoint_id, event_id, cancellation_id, run_id), + ).fetchone() + return int(exact is not None) + telemetry = { + str(row[0]) + for row in connection.execute( + "SELECT sequence FROM training_telemetry WHERE run_id=? " + "AND julianday(created_at)>=julianday(?)", + (run_id, started_at), + ) + } + checkpoints = { + str(row[0]) + for row in connection.execute( + "SELECT id FROM training_checkpoints WHERE run_id=? " + "AND julianday(created_at)>=julianday(?) AND publication_state='finalized'", + (run_id, started_at), + ) + } + inventory = { + f"{row[0]}:{row[1]}" + for row in connection.execute( + "SELECT binding.checkpoint_id,binding.role " + "FROM training_checkpoint_inventory_artifacts binding " + "JOIN training_checkpoints checkpoint ON checkpoint.id=binding.checkpoint_id " + "WHERE checkpoint.run_id=? AND julianday(checkpoint.created_at)>=julianday(?)", + (run_id, started_at), + ) + } + artifacts = { + str(row[0]) + for row in connection.execute( + "SELECT binding.artifact_id FROM training_checkpoint_inventory_artifacts binding " + "JOIN training_checkpoints checkpoint ON checkpoint.id=binding.checkpoint_id " + "WHERE checkpoint.run_id=? AND julianday(checkpoint.created_at)>=julianday(?)", + (run_id, started_at), + ) + } + required_kinds = {"event", "telemetry", "checkpoint", "inventory", "artifact"} + if kind == "resume": + required_kinds.add("resume") + if "evaluation" in links: + required_kinds.add("evaluation") + return int( + bool(telemetry and checkpoints and inventory and artifacts) + and set(links) == required_kinds + and (kind != "resume" or len(links["resume"]) == 1) + and links.get("telemetry", set()) == telemetry + and links.get("checkpoint", set()) == checkpoints + and links.get("inventory", set()) == inventory + and links.get("artifact", set()) == artifacts + ) + except (KeyError, TypeError, sqlite3.DatabaseError): + return 0 + + +def _training_operation_evidence_digest( + connection: sqlite3.Connection, operation_id: object, kind: object, evidence_id: object +) -> str | None: + try: + operation = connection.execute( + "SELECT run_id FROM training_orchestration_operations WHERE id=?", + (str(operation_id),), + ).fetchone() + if operation is None: + return None + run_id = str(operation[0]) + table_specs = { + "probe": ("training_probe_evidence", "id"), + "calibration": ("training_calibration_evidence", "id"), + "resume": ("training_resume_evidence", "id"), + "cancel": ("training_cancellation_evidence", "id"), + "evaluation": ("training_evaluation_evidence", "id"), + } + if str(kind) in table_specs: + table, key = table_specs[str(kind)] + row = connection.execute( + f"SELECT * FROM {table} WHERE {key}=? AND run_id=?", # noqa: S608 + (str(evidence_id), run_id), + ).fetchone() + if row is None: + return None + payload = {name: row[name] for name in row.keys() if name != "created_at"} + return "sha256:" + hashlib.sha256(canonical_json(payload).encode()).hexdigest() + if str(kind) in {"event", "telemetry"}: + table = "training_run_events" if str(kind) == "event" else "training_telemetry" + row = connection.execute( + f"SELECT * FROM {table} WHERE run_id=? AND sequence=?", # noqa: S608 + (run_id, int(str(evidence_id))), + ).fetchone() + if row is None: + return None + payload = { + name: (json.loads(str(row[name])) if name == "payload_json" else row[name]) + for name in row.keys() + if name != "created_at" + } + return "sha256:" + hashlib.sha256(canonical_json(payload).encode()).hexdigest() + if str(kind) == "checkpoint": + row = connection.execute( + "SELECT evidence_digest FROM training_checkpoints WHERE id=? AND run_id=?", + (str(evidence_id), run_id), + ).fetchone() + return None if row is None else str(row[0]) + if str(kind) == "artifact": + row = connection.execute( + "SELECT sha256 FROM artifacts WHERE id=?", (str(evidence_id),) + ).fetchone() + return None if row is None else "sha256:" + str(row[0]) + if str(kind) == "inventory": + checkpoint_id, role = str(evidence_id).rsplit(":", 1) + row = connection.execute( + "SELECT * FROM training_checkpoint_inventory_artifacts " + "WHERE checkpoint_id=? AND role=?", + (checkpoint_id, role), + ).fetchone() + if row is None: + return None + payload = {name: row[name] for name in row.keys()} + return "sha256:" + hashlib.sha256(canonical_json(payload).encode()).hexdigest() + return None + except (TypeError, ValueError, sqlite3.DatabaseError): + return None + + +def _v43_training_operation_results( + connection: sqlite3.Connection, +) -> dict[str, dict[str, object]]: + """Derive public results or reject before any v44 schema mutation.""" + derived: dict[str, dict[str, object]] = {} + for row in connection.execute( + "SELECT id,operation_kind,run_id,request_payload_json,receipt_payload_json " + "FROM training_orchestration_operations WHERE status='completed'" + ): + kind = str(row["operation_kind"]) + if kind in {"probe", "calibration"}: + raise sqlite3.IntegrityError( + "training operation typed result is not derivable from v43 receipt" + ) + receipt = json.loads(str(row["receipt_payload_json"])) + if kind == "cancel": + derived[str(row["id"])] = receipt + continue + if kind not in {"start", "resume"}: + raise sqlite3.IntegrityError( + "training operation typed result is not derivable from v43 receipt" + ) + request = json.loads(str(row["request_payload_json"])) + checkpoints = receipt.get("checkpoints") + if not isinstance(checkpoints, list) or not checkpoints: + raise sqlite3.IntegrityError( + "training operation typed result is not derivable from v43 receipt" + ) + checkpoint_ids = [item.get("evidence_id") for item in checkpoints] + if any(not isinstance(item, str) for item in checkpoint_ids): + raise sqlite3.IntegrityError( + "training operation typed result is not derivable from v43 receipt" + ) + final = connection.execute( + "SELECT json_extract(evidence_json,'$.telemetry_digest') " + "FROM training_checkpoints WHERE id=? AND run_id=? AND publication_state='finalized'", + (checkpoint_ids[-1], str(row["run_id"])), + ).fetchone() + if final is None or not isinstance(final[0], str): + raise sqlite3.IntegrityError( + "training operation typed result is not derivable from v43 receipt" + ) + derived[str(row["id"])] = { + "schema_version": "character-training-publication-receipt.v1", + "request_id": request["request_id"], + "run_id": str(row["run_id"]), + "lifecycle_state": receipt["lifecycle_state"], + "checkpoint_ids": checkpoint_ids, + "telemetry_digest": str(final[0]), + } + return derived + + +def _apply_migration_44(connection: sqlite3.Connection) -> None: + results = _v43_training_operation_results(connection) + statements = _migration_statements(_MIGRATION_44) + connection.execute(statements[0]) + connection.execute(statements[1]) + trigger = connection.execute( + "SELECT sql FROM sqlite_master WHERE type='trigger' " + "AND name='validate_training_operation_update_v42'" + ).fetchone() + if trigger is None or not isinstance(trigger[0], str): + raise sqlite3.IntegrityError("v43 training operation update trigger is missing") + connection.execute("DROP TRIGGER validate_training_operation_update_v42") + try: + for operation_id, payload in results.items(): + payload_json = canonical_json(payload) + connection.execute( + "UPDATE training_orchestration_operations SET result_digest=?," + "result_payload_json=? WHERE id=? AND status='completed'", + ( + "sha256:" + hashlib.sha256(payload_json.encode()).hexdigest(), + payload_json, + operation_id, + ), + ) + finally: + connection.execute(str(trigger[0])) + for statement in statements[3:]: + connection.execute(statement) + + +def _training_resume_evidence_matches_operation_request( + connection: sqlite3.Connection, operation_id: object, evidence_id: object +) -> int: + """Bind resume evidence to the exact requested parent and seven artifact handles.""" + try: + operation = connection.execute( + "SELECT run_id,request_payload_json FROM training_orchestration_operations " + "WHERE id=? AND operation_kind='resume'", + (str(operation_id),), + ).fetchone() + evidence = connection.execute( + "SELECT * FROM training_resume_evidence WHERE id=?", + (str(evidence_id),), + ).fetchone() + if operation is None or evidence is None: + return 0 + request = json.loads(str(operation["request_payload_json"])) + if not ( + evidence["run_id"] == operation["run_id"] == request["run_id"] + and evidence["parent_checkpoint_id"] == request["parent_checkpoint_id"] + and evidence["parent_checkpoint_digest"] == request["parent_checkpoint_digest"] + and all( + evidence[field] == request[field] + for field in ( + "recipe_digest", + "dataset_digest", + "backend_digest", + "runtime_digest", + "material_digest", + "source_digest", + ) + ) + ): + return 0 + requested = { + ( + item["role"], + item["artifact_id"], + item["digest"], + int(item["size_bytes"]), + ) + for item in request["parent_inventory"] + } + persisted = { + ( + str(row["role"]), + str(row["artifact_id"]), + str(row["artifact_digest"]), + int(row["size_bytes"]), + ) + for row in connection.execute( + "SELECT role,artifact_id,artifact_digest,size_bytes " + "FROM training_checkpoint_inventory_artifacts WHERE checkpoint_id=?", + (evidence["parent_checkpoint_id"],), + ) + } + if len(requested) != 7 or persisted != requested: + return 0 + by_role = {role: digest for role, _, digest, _ in persisted} + checkpoint = connection.execute( + "SELECT artifact_digest,evidence_json,evidence_digest FROM training_checkpoints " + "WHERE id=? AND run_id=? AND publication_state='finalized'", + (evidence["parent_checkpoint_id"], operation["run_id"]), + ).fetchone() + if checkpoint is None: + return 0 + checkpoint_evidence = json.loads(str(checkpoint["evidence_json"])) + role_fields = { + "weights": "safetensors_digest", + "optimizer": "optimizer_state_digest", + "trainer_state": "trainer_state_digest", + "telemetry": "telemetry_digest", + "trainable_targets": "trainable_targets_digest", + "frozen_parameters": "frozen_parameters_digest", + "finite_gradient_proof": "finite_gradient_proof_digest", + } + return int( + checkpoint["evidence_digest"] == request["parent_checkpoint_digest"] + and checkpoint["artifact_digest"] == by_role["weights"] + and evidence["parent_artifact_digest"] == by_role["weights"] + and evidence["optimizer_state_digest"] == by_role["optimizer"] + and all( + checkpoint_evidence[field] == by_role[role] for role, field in role_fields.items() + ) + and all( + checkpoint_evidence[field] == request[field] + for field in ( + "recipe_digest", + "dataset_digest", + "backend_digest", + "runtime_digest", + "material_digest", + "source_digest", + ) + ) + ) + except (KeyError, TypeError, ValueError, sqlite3.DatabaseError): + return 0 + + +def _training_operation_evidence_link_is_valid( + connection: sqlite3.Connection, + operation_id: object, + attempt: object, + claim_token: object, + kind: object, + evidence_id: object, + digest: object, +) -> int: + """Validate one link against the exact live attempt and operation-specific vocabulary.""" + try: + operation = connection.execute( + "SELECT operation_kind,run_id,updated_at FROM training_orchestration_operations " + "WHERE id=? AND status='running' AND attempt=? AND claim_token=?", + (str(operation_id), int(str(attempt)), str(claim_token)), + ).fetchone() + if operation is None: + return 0 + permitted = { + "probe": {"probe"}, + "calibration": {"calibration"}, + "start": {"event", "telemetry", "checkpoint", "inventory", "artifact", "evaluation"}, + "resume": { + "resume", + "event", + "telemetry", + "checkpoint", + "inventory", + "artifact", + "evaluation", + }, + "cancel": {"cancel", "event", "checkpoint", "inventory", "artifact"}, + } + evidence_kind = str(kind) + if evidence_kind not in permitted[str(operation[0])]: + return 0 + canonical_digest = _training_operation_evidence_digest( + connection, operation_id, evidence_kind, evidence_id + ) + if canonical_digest != str(digest): + return 0 + if ( + evidence_kind == "resume" + and _training_resume_evidence_matches_operation_request( + connection, operation_id, evidence_id + ) + != 1 + ): + return 0 + if ( + evidence_kind == "calibration" + and connection.execute( + "SELECT 1 FROM training_calibration_operation_bindings WHERE evidence_id=? " + "AND operation_id=? AND attempt=? AND claim_token=?", + (str(evidence_id), str(operation_id), int(str(attempt)), str(claim_token)), + ).fetchone() + is None + ): + return 0 + if ( + evidence_kind == "cancel" + and connection.execute( + "SELECT 1 FROM training_cancellation_operation_claims claim " + "JOIN training_cancellation_evidence evidence " + "ON evidence.cancellation_request_id=claim.cancellation_request_id " + "WHERE claim.operation_id=? AND claim.attempt=? AND claim.claim_token=? " + "AND evidence.id=? AND evidence.request_digest=claim.request_digest " + "AND evidence.requested_step=claim.requested_step", + (str(operation_id), int(str(attempt)), str(claim_token), str(evidence_id)), + ).fetchone() + is None + ): + return 0 + run_id, started_at = str(operation[1]), str(operation[2]) + if evidence_kind in {"probe", "calibration", "resume", "cancel", "evaluation"}: + table = { + "probe": "training_probe_evidence", + "calibration": "training_calibration_evidence", + "resume": "training_resume_evidence", + "cancel": "training_cancellation_evidence", + "evaluation": "training_evaluation_evidence", + }[evidence_kind] + row = connection.execute( + f"SELECT created_at FROM {table} WHERE id=? AND run_id=?", # noqa: S608 + (str(evidence_id), run_id), + ).fetchone() + elif evidence_kind in {"event", "telemetry"}: + table = "training_run_events" if evidence_kind == "event" else "training_telemetry" + row = connection.execute( + f"SELECT created_at FROM {table} WHERE run_id=? AND sequence=?", # noqa: S608 + (run_id, int(str(evidence_id))), + ).fetchone() + else: + checkpoint_id = ( + str(evidence_id).rsplit(":", 1)[0] + if evidence_kind == "inventory" + else str(evidence_id) + ) + if evidence_kind == "artifact": + row = connection.execute( + "SELECT checkpoint.created_at FROM training_checkpoint_inventory_artifacts binding " + "JOIN training_checkpoints checkpoint ON checkpoint.id=binding.checkpoint_id " + "WHERE binding.artifact_id=? AND checkpoint.run_id=? " + "ORDER BY julianday(checkpoint.created_at) DESC LIMIT 1", + (checkpoint_id, run_id), + ).fetchone() + else: + row = connection.execute( + "SELECT created_at FROM training_checkpoints WHERE id=? AND run_id=?", + (checkpoint_id, run_id), + ).fetchone() + return int( + row is not None + and connection.execute( + "SELECT julianday(?)>=julianday(?)", (row[0], started_at) + ).fetchone()[0] + == 1 + ) + except (KeyError, TypeError, ValueError, sqlite3.DatabaseError): + return 0 + + +def _training_import_identity_is_valid(identity: object, payload: object) -> int: + try: + model = ImportedTrainingSample.model_validate_json(str(payload)) + return int(str(identity) == model.import_id == imported_training_sample_id(model)) + except (TypeError, ValueError): + return 0 + + +def _training_confirmation_identity_is_valid(identity: object, payload: object) -> int: + try: + model = TrainingCurationConfirmation.model_validate_json(str(payload)) + return int(str(identity) == model.confirmation_id == training_confirmation_id(model)) + except (TypeError, ValueError): + return 0 + + +def _training_ledger_snapshot_is_complete( + connection: sqlite3.Connection, + manifest_json: object, + lineage_id: object, + reviewer_id: object, + character_id: object, +) -> int: + try: + payload = json.loads(str(manifest_json)) + draft = TrainingDatasetDraft.model_validate( + { + key: value + for key, value in payload.items() + if key not in {"coverage", "dataset_digest"} + } + ) + imports = tuple( + str(row[0]) + for row in connection.execute( + "SELECT id FROM training_imported_samples WHERE lineage_id=? " + "AND reviewer_id=? AND character_id=? ORDER BY id", + (lineage_id, reviewer_id, character_id), + ) + ) + confirmations = tuple( + str(row[0]) + for row in connection.execute( + "SELECT confirmation.id FROM training_curation_confirmations confirmation " + "JOIN training_imported_samples imported ON imported.id=confirmation.import_id " + "WHERE imported.lineage_id=? AND confirmation.reviewer_id=? " + "AND imported.character_id=? ORDER BY confirmation.id", + (lineage_id, reviewer_id, character_id), + ) + ) + if draft.imported_sample_ids != imports or draft.curation_confirmation_ids != confirmations: + return 0 + samples = [canonical_json(sample.model_dump(mode="json")) for sample in draft.samples] + for confirmation_id in confirmations: + row = connection.execute( + "SELECT confirmation.payload_json confirmation_json,imported.payload_json import_json " + "FROM training_curation_confirmations confirmation " + "JOIN training_imported_samples imported ON imported.id=confirmation.import_id " + "WHERE confirmation.id=?", + (confirmation_id,), + ).fetchone() + if row is None: + return 0 + confirmation = TrainingCurationConfirmation.model_validate_json( + str(row["confirmation_json"]) + ) + imported = ImportedTrainingSample.model_validate_json(str(row["import_json"])) + if confirmation.decision.value == "reject": + if any( + sample.sample_digest == imported.artifact_digest for sample in draft.samples + ): + return 0 + elif ( + canonical_json(confirmation.accepted_sample(imported).model_dump(mode="json")) + not in samples + ): + return 0 + return 1 + except (sqlite3.DatabaseError, TypeError, ValueError): + return 0 + + +def _training_ledger_transition_is_valid( + connection: sqlite3.Connection, + old_storage_id: object, + new_storage_id: object, + new_state: object, +) -> int: + try: + rows = connection.execute( + "SELECT storage_id,manifest_json FROM training_datasets WHERE storage_id IN (?,?)", + (old_storage_id, new_storage_id), + ).fetchall() + manifests = {str(row["storage_id"]): str(row["manifest_json"]) for row in rows} + old_payload = json.loads(manifests[str(old_storage_id)]) + old = TrainingDatasetDraft.model_validate( + { + key: value + for key, value in old_payload.items() + if key not in {"coverage", "dataset_digest"} + } + ) + new_payload = json.loads(manifests[str(new_storage_id)]) + new = TrainingDatasetDraft.model_validate( + { + key: value + for key, value in new_payload.items() + if key not in {"coverage", "dataset_digest"} + } + ) + old_samples = sorted(canonical_json(item.model_dump(mode="json")) for item in old.samples) + new_samples = sorted(canonical_json(item.model_dump(mode="json")) for item in new.samples) + old_imports, new_imports = set(old.imported_sample_ids), set(new.imported_sample_ids) + old_confirmations = set(old.curation_confirmation_ids) + new_confirmations = set(new.curation_confirmation_ids) + if not old_imports and not new_imports and not old_confirmations and not new_confirmations: + return 1 + if str(new_state) == "sealed": + return int( + str(old_storage_id) == str(new_storage_id) + and old_imports == new_imports + and old_confirmations == new_confirmations + and old_samples == new_samples + ) + added_imports = new_imports - old_imports + added_confirmations = new_confirmations - old_confirmations + if added_imports and not added_confirmations: + return int( + old_imports < new_imports + and old_confirmations == new_confirmations + and old_samples == new_samples + ) + if len(added_confirmations) != 1 or old_imports != new_imports: + return 0 + confirmation_row = connection.execute( + "SELECT payload_json FROM training_curation_confirmations WHERE id=?", + (next(iter(added_confirmations)),), + ).fetchone() + if confirmation_row is None: + return 0 + confirmation = TrainingCurationConfirmation.model_validate_json( + str(confirmation_row["payload_json"]) + ) + imported_row = connection.execute( + "SELECT payload_json FROM training_imported_samples WHERE id=?", + (confirmation.import_id,), + ).fetchone() + if imported_row is None: + return 0 + if confirmation.decision.value == "reject": + return int(old_samples == new_samples) + imported = ImportedTrainingSample.model_validate_json(str(imported_row["payload_json"])) + expected = sorted( + old_samples + + [canonical_json(confirmation.accepted_sample(imported).model_dump(mode="json"))] + ) + return int(new_samples == expected) + except (KeyError, sqlite3.DatabaseError, TypeError, ValueError): + return 0 + + +def _training_draft_identity_is_valid( + connection: sqlite3.Connection, storage_id: object, identity: object, manifest_json: object +) -> int: + try: + draft = TrainingDatasetDraft.model_validate_json(str(manifest_json)) + rows = connection.execute( + "SELECT sample.*,annotation.annotation_json FROM training_samples sample " + "JOIN training_sample_annotations annotation USING(dataset_storage_id,sample_id) " + "WHERE sample.dataset_storage_id=? ORDER BY sample.ordinal", + (storage_id,), + ).fetchall() + persisted = tuple( + TrainingSample.model_validate( + { + "sample_id": row["sample_id"], + "source_tier": row["source_tier"], + "sample_digest": row["sample_digest"], + "source_lineage": row["source_lineage"], + "caption": row["caption"], + "split": row["split"], + "media": json.loads(str(row["media_json"])), + "annotation": json.loads(str(row["annotation_json"])), + "accepted": bool(row["accepted"]), + } + ) + for row in rows + ) + return int(draft.samples == persisted and training_dataset_id(draft) == str(identity)) + except (sqlite3.DatabaseError, TypeError, ValueError): + return 0 + + +def _training_draft_manifest_is_valid( + identity: object, + character_id: object, + character_pack_digest: object, + canonical_outfit_id: object, + reserved_token: object, + source_tier_weights_json: object, + manifest_json: object, +) -> int: + """Validate every duplicated dataset column against one canonical draft manifest.""" + try: + draft = TrainingDatasetDraft.model_validate_json(str(manifest_json)) + weights = json.loads(str(source_tier_weights_json)) + expected_weights = { + tier.value: weight for tier, weight in draft.source_tier_weights.items() + } + return int( + str(identity) == training_dataset_id(draft) + and str(character_id) == draft.character_id + and str(character_pack_digest) == draft.character_pack_digest + and str(canonical_outfit_id) == draft.canonical_outfit_id + and str(reserved_token) == draft.reserved_token + and weights == expected_weights + ) + except (TypeError, ValueError): + return 0 + + +def _training_checkpoint_step(total_steps: object, fraction: object) -> int | None: + try: + fractions = (0.25, 0.5, 0.75, 1.0) + return checkpoint_steps(int(str(total_steps)))[fractions.index(float(str(fraction)))] + except (TypeError, ValueError): + return None + + +def _character_benchmark_provenance_is_valid( + connection: sqlite3.Connection, campaign_id: object +) -> int: + try: + row = connection.execute( + "SELECT campaign.*,result.result_json,result.content_digest AS result_digest " + "FROM character_benchmark_campaigns campaign " + "JOIN character_benchmark_results result ON result.campaign_id=campaign.id " + "WHERE campaign.id=?", + (campaign_id,), + ).fetchone() + if row is None: + return 0 + campaign_payload = { + "definition": json.loads(str(row["definition_json"])), + "character_pack_id": str(row["character_pack_id"]), + "character_pack_storage_id": str(row["character_pack_storage_id"]), + "stage_2_campaign_id": str(row["stage_2_campaign_id"]), + "stage_2_manifest": json.loads(str(row["stage_2_manifest_json"])), + "identity_prompt": str(row["identity_prompt"]), + } + campaign_digest = ( + "sha256:" + hashlib.sha256(canonical_json(campaign_payload).encode()).hexdigest() + ) + result_digest = ( + "sha256:" + + hashlib.sha256( + canonical_json(json.loads(str(row["result_json"]))).encode() + ).hexdigest() + ) + stage_2_manifest_digest = _canonical_json_digest(row["stage_2_manifest_json"]) + stage_2_evidence_digest = _validation_evidence_digest( + connection, str(row["stage_2_campaign_id"]), str(row["reviewer_id"]) + ) + definition = json.loads(str(row["definition_json"])) + return int( + campaign_digest == str(row["content_digest"]) + and result_digest == str(row["result_digest"]) + and definition.get("stage_2_manifest_digest") == stage_2_manifest_digest + and definition.get("stage_2_evidence_digest") == stage_2_evidence_digest + ) + except (sqlite3.DatabaseError, TypeError, ValueError): + return 0 + + +def _validation_evidence_digest( + connection: sqlite3.Connection, campaign_id: str, reviewer_id: str +) -> str | None: + """Rebuild the canonical Stage-2 export from its owned persisted graph.""" + try: + campaign = connection.execute( + "SELECT * FROM validation_campaigns WHERE id=? AND reviewer_id=?", + (campaign_id, reviewer_id), + ).fetchone() + result = connection.execute( + "SELECT * FROM validation_results WHERE campaign_id=?", (campaign_id,) + ).fetchone() + state = connection.execute( + "SELECT new_state FROM validation_events WHERE campaign_id=? " + "ORDER BY sequence DESC LIMIT 1", + (campaign_id,), + ).fetchone() + if campaign is None or result is None or state is None or state[0] != "qualified": + return None + trials = connection.execute( + "SELECT * FROM validation_trials WHERE campaign_id=? ORDER BY ordinal", (campaign_id,) + ).fetchall() + events = connection.execute( + "SELECT * FROM validation_events WHERE campaign_id=? ORDER BY sequence", (campaign_id,) + ).fetchall() + judgments = connection.execute( + "SELECT * FROM validation_judgments WHERE campaign_id=? ORDER BY ordinal", + (campaign_id,), + ).fetchall() + experiment_bindings = [ + dict(item) + for item in connection.execute( + "SELECT binding.campaign_id,binding.trial_id,binding.experiment_id " + "FROM validation_experiment_bindings binding JOIN validation_trials trial " + "ON trial.campaign_id=binding.campaign_id AND trial.trial_id=binding.trial_id " + "WHERE binding.campaign_id=? ORDER BY trial.ordinal", + (campaign_id,), + ) + ] + review_rows = connection.execute( + "SELECT binding.campaign_id,binding.trial_id,binding.experiment_id,binding.review_id," + "binding.reused_from_trial_id,trial.ordinal,trial.winner_display_position," + "judgment.choice,review.session_id,review.revision,review.display_preference," + "review.preferred_variant_id,session.left_variant_id,session.left_job_id," + "session.left_artifact_id,session.left_artifact_sha256,session.right_variant_id," + "session.right_job_id,session.right_artifact_id,session.right_artifact_sha256," + "baseline.id AS baseline_id FROM validation_trial_bindings binding " + "JOIN validation_trials trial ON trial.campaign_id=binding.campaign_id " + "AND trial.trial_id=binding.trial_id JOIN validation_judgments judgment " + "ON judgment.campaign_id=binding.campaign_id AND judgment.trial_id=binding.trial_id " + "JOIN review_revisions review ON review.id=binding.review_id " + "JOIN review_sessions session ON session.id=review.session_id " + "JOIN variants baseline ON baseline.experiment_id=binding.experiment_id " + "AND baseline.label='A' WHERE binding.campaign_id=? ORDER BY trial.ordinal", + (campaign_id,), + ).fetchall() + if ( + len(trials) != 40 + or sum(not bool(item["hidden_repeat"]) for item in trials) != 36 + or len(events) != 6 + or len(judgments) != 40 + or len(experiment_bindings) != 36 + or len(review_rows) != 40 + ): + return None + trial_payloads = [ + { + "id": item["trial_id"], + "ordinal": item["ordinal"], + "archetype": item["archetype"], + "prompt": item["prompt"], + "negative_prompt": item["negative_prompt"] or None, + "seed": item["seed"], + "challenger": item["challenger"], + "hidden_repeat": bool(item["hidden_repeat"]), + "reuses_trial_id": item["reuses_trial_id"], + "winner_display_position": item["winner_display_position"], + } + for item in trials + ] + binding_payloads: list[dict[str, object]] = [] + review_payloads: list[dict[str, object]] = [] + for item in review_rows: + binding_payloads.append( + { + "campaign_id": item["campaign_id"], + "trial_id": item["trial_id"], + "experiment_id": item["experiment_id"], + "review_id": item["review_id"], + "reused_from_trial_id": item["reused_from_trial_id"], + } + ) + winner = "left" if item["left_variant_id"] == item["baseline_id"] else "right" + swap = winner != item["winner_display_position"] + mapping: dict[str, dict[str, str]] = {} + for position in ("left", "right"): + underlying = ("right" if position == "left" else "left") if swap else position + mapping[position] = { + "variant_id": str(item[f"{underlying}_variant_id"]), + "job_id": str(item[f"{underlying}_job_id"]), + "artifact_id": str(item[f"{underlying}_artifact_id"]), + "artifact_sha256": str(item[f"{underlying}_artifact_sha256"]), + } + review_payloads.append( + { + "trial_id": item["trial_id"], + "review_id": item["review_id"], + "session_id": item["session_id"], + "revision": item["revision"], + "choice": item["choice"], + "persisted_display_preference": item["display_preference"], + "preferred_variant_id": item["preferred_variant_id"], + "display_mapping": mapping, + } + ) + provenance = { + "experiment_bindings": experiment_bindings, + "trial_bindings": binding_payloads, + "reviews": review_payloads, + } + campaign_payload = { + "id": campaign["id"], + "schema_version": campaign["schema_version"], + "stage_1_study_id": campaign["stage_1_study_id"], + "stage_1_evidence_digest": campaign["stage_1_evidence_digest"], + "base_manifest": json.loads(str(campaign["base_manifest_json"])), + "candidates": json.loads(str(campaign["candidates_json"])), + "panel_input": json.loads(str(campaign["panel_input_json"])), + "protocol_version": campaign["protocol_version"], + "analysis_version": campaign["analysis_version"], + "created_at": _pydantic_json_datetime(campaign["created_at"]), + } + evidence = { + "schema_version": "balanced-validation-evidence.v2", + "campaign": campaign_payload, + "panel": { + "campaign_id": campaign_id, + "schema_version": "validation-panel.v1", + "trials": trial_payloads, + "content_digest": trials[0]["panel_digest"], + }, + "events": [ + { + "campaign_id": item["campaign_id"], + "sequence": item["sequence"], + "previous_state": item["previous_state"], + "new_state": item["new_state"], + "created_at": _pydantic_json_datetime(item["created_at"]), + } + for item in events + ], + "judgments": [ + { + "campaign_id": item["campaign_id"], + "trial_id": item["trial_id"], + "ordinal": item["ordinal"], + "archetype": item["archetype"], + "challenger": item["challenger"], + "seed": item["seed"], + "hidden_repeat": bool(item["hidden_repeat"]), + "winner_display_position": item["winner_display_position"], + "choice": item["choice"], + "created_at": _pydantic_json_datetime(item["created_at"]), + } + for item in judgments + ], + **provenance, + "provenance_digest": "sha256:" + + hashlib.sha256(canonical_json(provenance).encode()).hexdigest(), + "result": { + "campaign_id": result["campaign_id"], + "disposition": result["disposition"], + "reason": result["reason"], + "challengers": json.loads(str(result["challengers_json"])), + "archetype_reversals": json.loads(str(result["archetype_reversals_json"])), + "repeat_agreements": result["repeat_agreements"], + }, + } + return "sha256:" + hashlib.sha256(canonical_json(evidence).encode()).hexdigest() + except (sqlite3.DatabaseError, TypeError, ValueError, KeyError): + return None + + +def _pydantic_json_datetime(value: object) -> str: + text = str(value) + return text[:-6] + "Z" if text.endswith("+00:00") else text + + +_MIGRATION_1 = """ +CREATE TABLE schema_migrations(version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL); +CREATE TABLE experiments( + id TEXT PRIMARY KEY, + schema_version TEXT NOT NULL, + variable_path TEXT NOT NULL, + changed_paths_json TEXT NOT NULL, + created_at TEXT NOT NULL +); +CREATE TABLE variants( + id TEXT PRIMARY KEY, + experiment_id TEXT NOT NULL REFERENCES experiments(id), + label TEXT NOT NULL CHECK(label IN ('A','B')), + manifest_json TEXT NOT NULL, + UNIQUE(experiment_id, label) +); +CREATE TABLE jobs( + id TEXT PRIMARY KEY, + variant_id TEXT NOT NULL UNIQUE REFERENCES variants(id), + state TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + error_code TEXT, + error_message TEXT +); +""" + +_MIGRATION_2 = """ +CREATE TABLE artifacts( + id TEXT PRIMARY KEY, + job_id TEXT REFERENCES jobs(id), + sha256 TEXT NOT NULL, + byte_size INTEGER NOT NULL, + media_type TEXT NOT NULL, + relative_path TEXT NOT NULL, + created_at TEXT NOT NULL, + UNIQUE(sha256, media_type) +); +""" + +_MIGRATION_3 = """ +CREATE TABLE job_artifacts( + job_id TEXT NOT NULL REFERENCES jobs(id), + artifact_id TEXT NOT NULL REFERENCES artifacts(id), + created_at TEXT NOT NULL, + PRIMARY KEY(job_id, artifact_id) +); +INSERT INTO job_artifacts(job_id, artifact_id, created_at) +SELECT job_id, id, created_at FROM artifacts WHERE job_id IS NOT NULL; +""" + +_MIGRATION_4 = """ +CREATE TABLE run_attempts( + id TEXT PRIMARY KEY, + job_id TEXT NOT NULL REFERENCES jobs(id), + number INTEGER NOT NULL CHECK(number > 0), + worker_id TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('active','completed','failed','cancelled','interrupted')), + started_at TEXT NOT NULL, + finished_at TEXT, + error_code TEXT, + error_message TEXT, + artifact_id TEXT REFERENCES artifacts(id), + UNIQUE(job_id, number) +); +CREATE TABLE run_events( + id INTEGER PRIMARY KEY AUTOINCREMENT, + job_id TEXT NOT NULL REFERENCES jobs(id), + attempt_id TEXT NOT NULL REFERENCES run_attempts(id), + kind TEXT NOT NULL, + payload_json TEXT NOT NULL, + created_at TEXT NOT NULL +); +CREATE TABLE telemetry_samples( + id INTEGER PRIMARY KEY AUTOINCREMENT, + attempt_id TEXT NOT NULL REFERENCES run_attempts(id), + recorded_at TEXT NOT NULL, + elapsed_seconds REAL NOT NULL CHECK(elapsed_seconds >= 0), + process_rss_bytes INTEGER NOT NULL CHECK(process_rss_bytes >= 0), + system_used_bytes INTEGER NOT NULL CHECK(system_used_bytes >= 0), + system_available_bytes INTEGER NOT NULL CHECK(system_available_bytes >= 0), + swap_used_bytes INTEGER NOT NULL CHECK(swap_used_bytes >= 0), + thermal_pressure TEXT +); +""" + +_MIGRATION_5 = """ +ALTER TABLE jobs ADD COLUMN claimed_by TEXT; +ALTER TABLE jobs ADD COLUMN claimed_at TEXT; +""" + +_MIGRATION_6 = """ +CREATE TABLE review_sessions( + id TEXT PRIMARY KEY, + experiment_id TEXT NOT NULL REFERENCES experiments(id), + reviewer_id TEXT NOT NULL, + left_variant_id TEXT NOT NULL REFERENCES variants(id), + right_variant_id TEXT NOT NULL REFERENCES variants(id), + left_job_id TEXT NOT NULL REFERENCES jobs(id), + left_artifact_id TEXT NOT NULL REFERENCES artifacts(id), + left_artifact_sha256 TEXT NOT NULL, + right_job_id TEXT NOT NULL REFERENCES jobs(id), + right_artifact_id TEXT NOT NULL REFERENCES artifacts(id), + right_artifact_sha256 TEXT NOT NULL, + created_at TEXT NOT NULL, + CHECK(left_variant_id <> right_variant_id), + CHECK(left_job_id <> right_job_id), + CHECK(left_artifact_id <> right_artifact_id), + CHECK(left_artifact_sha256 <> right_artifact_sha256), + UNIQUE(experiment_id, reviewer_id) +); +CREATE TABLE review_revisions( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL REFERENCES review_sessions(id), + preferred_variant_id TEXT REFERENCES variants(id), + display_preference TEXT NOT NULL CHECK(display_preference IN ('left','right','tie')), + notes TEXT, + created_at TEXT NOT NULL, + revision INTEGER NOT NULL CHECK(revision > 0), + supersedes_review_id TEXT, + UNIQUE(session_id, revision), + UNIQUE(id, session_id), + FOREIGN KEY(supersedes_review_id, session_id) + REFERENCES review_revisions(id, session_id) +); +CREATE UNIQUE INDEX one_initial_review_per_session + ON review_revisions(session_id) WHERE supersedes_review_id IS NULL; +CREATE UNIQUE INDEX one_successor_per_review + ON review_revisions(session_id, supersedes_review_id) + WHERE supersedes_review_id IS NOT NULL; +CREATE TABLE rubric_scores( + review_id TEXT NOT NULL REFERENCES review_revisions(id), + display_position TEXT NOT NULL CHECK(display_position IN ('left','right')), + dimension TEXT NOT NULL CHECK(dimension IN ( + 'prompt_adherence','visual_quality','motion_quality', + 'temporal_consistency','character_consistency' + )), + score INTEGER NOT NULL CHECK(score BETWEEN 1 AND 5), + UNIQUE(review_id, display_position, dimension) +); +""" + +_MIGRATION_7 = """ +CREATE TABLE feasibility_runs( + id TEXT PRIMARY KEY, + adapter_id TEXT NOT NULL, + protocol_version TEXT NOT NULL, + input_fingerprint TEXT NOT NULL CHECK( + length(input_fingerprint) = 64 AND input_fingerprint = lower(input_fingerprint) + AND input_fingerprint NOT GLOB '*[^0-9a-f]*' + ), + configuration_json TEXT NOT NULL CHECK( + json_valid(configuration_json) AND json_type(configuration_json) = 'object' + ), + verdict TEXT NOT NULL CHECK(verdict IN ( + 'pending','incompatible','resource_limited', + 'experimental_local_hybrid','standard_lab_qualified' + )), + created_at TEXT NOT NULL CHECK(substr(created_at, -6) = '+00:00'), + updated_at TEXT NOT NULL CHECK(substr(updated_at, -6) = '+00:00') +); +CREATE TABLE feasibility_stage_attempts( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL REFERENCES feasibility_runs(id), + stage TEXT NOT NULL CHECK(stage IN ( + 'resolve_sources','inventory_sources','storage_preflight','download_sources', + 'load_text_stack','load_vae','load_high_transformer','load_low_transformer', + 'assemble_pipeline','generate_minimum_1','generate_minimum_2','qualify_standard', + 'validate_controlled_pair','publish_verdict' + )), + ordinal INTEGER NOT NULL CHECK(ordinal > 0), + input_fingerprint TEXT NOT NULL CHECK( + length(input_fingerprint) = 64 AND input_fingerprint = lower(input_fingerprint) + AND input_fingerprint NOT GLOB '*[^0-9a-f]*' + ), + outcome TEXT CHECK(outcome IN ( + 'passed','incompatible','resource_exhausted','interrupted', + 'invalid_evidence','unexpected_failure' + )), + started_at TEXT NOT NULL CHECK(substr(started_at, -6) = '+00:00'), + finished_at TEXT CHECK(finished_at IS NULL OR substr(finished_at, -6) = '+00:00'), + evidence_json TEXT CHECK( + evidence_json IS NULL OR (json_valid(evidence_json) AND json_type(evidence_json) = 'object') + ), + diagnostic_code TEXT CHECK( + diagnostic_code IS NULL OR length(diagnostic_code) BETWEEN 1 AND 64 + ), + diagnostic_message TEXT CHECK( + diagnostic_message IS NULL OR length(diagnostic_message) BETWEEN 1 AND 1000 + ), + UNIQUE(run_id, stage, ordinal), + CHECK( + (outcome IS NULL AND finished_at IS NULL AND evidence_json IS NULL + AND diagnostic_code IS NULL AND diagnostic_message IS NULL) + OR (outcome IS NOT NULL AND finished_at IS NOT NULL) + ) +); +CREATE UNIQUE INDEX one_active_feasibility_stage_attempt + ON feasibility_stage_attempts(run_id, stage) WHERE outcome IS NULL; +CREATE INDEX feasibility_stage_attempt_order + ON feasibility_stage_attempts(run_id, started_at, stage, ordinal, id); +""" + +_MIGRATION_8 = """ +CREATE TRIGGER guard_feasibility_stage_attempt_updates +BEFORE UPDATE ON feasibility_stage_attempts +BEGIN + SELECT CASE + WHEN OLD.outcome IS NOT NULL THEN + RAISE(ABORT, 'finished feasibility stage attempts are immutable') + WHEN NOT ( + NEW.id IS OLD.id + AND NEW.run_id IS OLD.run_id + AND NEW.stage IS OLD.stage + AND NEW.ordinal IS OLD.ordinal + AND NEW.input_fingerprint IS OLD.input_fingerprint + AND NEW.started_at IS OLD.started_at + AND NEW.outcome IS NOT NULL + AND NEW.finished_at IS NOT NULL + AND NEW.evidence_json IS NOT NULL + ) THEN + RAISE(ABORT, 'invalid feasibility stage terminal transition') + END; +END; +""" + +_MIGRATION_9 = """ +CREATE TABLE adaptive_studies( + id TEXT PRIMARY KEY CHECK(length(id) > 0), + schema_version TEXT NOT NULL CHECK(schema_version = 'adaptive-study.v1'), + prompt TEXT NOT NULL CHECK(length(prompt) > 0), + negative_prompt TEXT, + seed INTEGER NOT NULL CHECK(typeof(seed) = 'integer'), + base_manifest_json TEXT NOT NULL CHECK( + COALESCE( + json_valid(base_manifest_json) + AND json_type(base_manifest_json) = 'object' + AND json_extract(base_manifest_json, '$.schema_version') = 'manifest.v1' + AND json_extract(base_manifest_json, '$.prompt') = prompt + AND json_extract(base_manifest_json, '$.negative_prompt') = COALESCE(negative_prompt, '') + AND json_extract(base_manifest_json, '$.seed') = seed, + 0 + ) + ), + optimizer_config_json TEXT NOT NULL CHECK( + COALESCE( + json_valid(optimizer_config_json) + AND json_type(optimizer_config_json) = 'object' + AND json_extract(optimizer_config_json, '$.schema_version') + = 'preference-optimizer-config.v1' + AND json_extract(optimizer_config_json, '$.optimizer_version') + = 'preference-gp-laplace.v1', + 0 + ) + ), + input_fingerprint TEXT NOT NULL CHECK( + length(input_fingerprint) = 64 + AND input_fingerprint = lower(input_fingerprint) + AND input_fingerprint NOT GLOB '*[^0-9a-f]*' + ), + created_at TEXT NOT NULL CHECK( + julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00' + ) +); +CREATE TRIGGER validate_adaptive_study_definition_insert +BEFORE INSERT ON adaptive_studies BEGIN + SELECT CASE WHEN adaptive_manifest_json_is_valid(NEW.base_manifest_json) <> 1 + THEN RAISE(ABORT, 'invalid adaptive study manifest') END; + SELECT CASE WHEN adaptive_optimizer_config_json_is_valid(NEW.optimizer_config_json) <> 1 + THEN RAISE(ABORT, 'invalid adaptive optimizer config') END; +END; +CREATE TABLE adaptive_study_events( + study_id TEXT NOT NULL REFERENCES adaptive_studies(id), + schema_version TEXT NOT NULL CHECK(schema_version = 'adaptive-study-event.v1'), + sequence INTEGER NOT NULL CHECK(typeof(sequence) = 'integer' AND sequence > 0), + previous_state TEXT NOT NULL CHECK(previous_state IN ( + 'draft','search_ready','generating_pair','awaiting_preference','updating_posterior', + 'search_converged','search_budget_exhausted','validating','completed','paused', + 'optimizer_failed' + )), + new_state TEXT NOT NULL CHECK(new_state IN ( + 'draft','search_ready','generating_pair','awaiting_preference','updating_posterior', + 'search_converged','search_budget_exhausted','validating','completed','paused', + 'optimizer_failed' + )), + payload_json TEXT NOT NULL CHECK( + json_valid(payload_json) AND json_type(payload_json) = 'object' + ), + created_at TEXT NOT NULL CHECK( + julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00' + ), + PRIMARY KEY(study_id, sequence), + CHECK(previous_state <> new_state) +); +CREATE TABLE adaptive_optimizer_steps( + study_id TEXT NOT NULL REFERENCES adaptive_studies(id), + schema_version TEXT NOT NULL CHECK(schema_version = 'adaptive-optimizer-step.v1'), + ordinal INTEGER NOT NULL CHECK(typeof(ordinal) = 'integer' AND ordinal > 0), + input_digest TEXT NOT NULL CHECK( + length(input_digest) = 64 + AND input_digest = lower(input_digest) + AND input_digest NOT GLOB '*[^0-9a-f]*' + ), + output_digest TEXT NOT NULL CHECK( + length(output_digest) = 64 + AND output_digest = lower(output_digest) + AND output_digest NOT GLOB '*[^0-9a-f]*' + ), + posterior_json TEXT NOT NULL CHECK( + COALESCE( + json_valid(posterior_json) + AND json_type(posterior_json) = 'object' + AND json_type(posterior_json, '$.grid') = 'array' + AND json_array_length(posterior_json, '$.grid') > 0 + AND json_type(posterior_json, '$.mean') = 'array' + AND json_array_length(posterior_json, '$.mean') + = json_array_length(posterior_json, '$.grid') + AND json_type(posterior_json, '$.covariance') = 'array' + AND json_array_length(posterior_json, '$.covariance') + = json_array_length(posterior_json, '$.grid') + AND json_type(posterior_json, '$.argmax_probability') = 'array' + AND json_array_length(posterior_json, '$.argmax_probability') + = json_array_length(posterior_json, '$.grid') + AND json_type(posterior_json, '$.recommendation') IN ('integer', 'real') + AND json_type(posterior_json, '$.concentration_interval') = 'array' + AND json_array_length(posterior_json, '$.concentration_interval') = 2 + AND json_type(posterior_json, '$.concentration_probability') IN ('integer', 'real') + AND json_type(posterior_json, '$.content_digest') = 'text' + AND length(json_extract(posterior_json, '$.content_digest')) = 71 + AND substr(json_extract(posterior_json, '$.content_digest'), 1, 7) = 'sha256:' + AND substr(json_extract(posterior_json, '$.content_digest'), 8) + = lower(substr(json_extract(posterior_json, '$.content_digest'), 8)) + AND substr(json_extract(posterior_json, '$.content_digest'), 8) + NOT GLOB '*[^0-9a-f]*', + 0 + ) + ), + acquisition_ledger_json TEXT NOT NULL CHECK( + json_valid(acquisition_ledger_json) AND json_type(acquisition_ledger_json) = 'array' + ), + selected_pair_json TEXT CHECK( + selected_pair_json IS NULL + OR (json_valid(selected_pair_json) AND json_type(selected_pair_json) = 'object') + ), + stop_reason TEXT NOT NULL CHECK(stop_reason IN ( + 'continue','search_converged','search_budget_exhausted','candidate_space_exhausted' + )), + disposition TEXT CHECK(disposition IS NULL OR disposition = 'inconclusive_within_budget'), + comparison_count INTEGER NOT NULL CHECK( + typeof(comparison_count) = 'integer' AND comparison_count >= 0 + ), + maximum_information_bits REAL NOT NULL CHECK( + typeof(maximum_information_bits) = 'real' + AND maximum_information_bits >= 0 + AND maximum_information_bits <= 1.0e308 + ), + created_at TEXT NOT NULL CHECK( + julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00' + ), + PRIMARY KEY(study_id, ordinal), + CHECK(ordinal = comparison_count + 1), + CHECK(COALESCE( + (stop_reason = 'search_budget_exhausted' AND disposition = 'inconclusive_within_budget') + OR (stop_reason <> 'search_budget_exhausted' AND disposition IS NULL), + 0 + )), + CHECK( + (stop_reason = 'continue' AND selected_pair_json IS NOT NULL) + OR (stop_reason <> 'continue' AND selected_pair_json IS NULL) + ) +); +CREATE TABLE adaptive_observations( + study_id TEXT NOT NULL REFERENCES adaptive_studies(id), + schema_version TEXT NOT NULL CHECK(schema_version = 'adaptive-observation.v1'), + ordinal INTEGER NOT NULL CHECK(typeof(ordinal) = 'integer' AND ordinal > 0), + optimizer_step_ordinal INTEGER NOT NULL CHECK( + typeof(optimizer_step_ordinal) = 'integer' + AND optimizer_step_ordinal = ordinal + ), + value_a REAL NOT NULL CHECK(typeof(value_a) = 'real' AND abs(value_a) <= 1.0e308), + value_b REAL NOT NULL CHECK(typeof(value_b) = 'real' AND abs(value_b) <= 1.0e308), + experiment_id TEXT NOT NULL UNIQUE REFERENCES experiments(id), + review_id TEXT NOT NULL UNIQUE REFERENCES review_revisions(id), + display_mapping_digest TEXT NOT NULL CHECK( + length(display_mapping_digest) = 64 + AND display_mapping_digest = lower(display_mapping_digest) + AND display_mapping_digest NOT GLOB '*[^0-9a-f]*' + ), + outcome TEXT NOT NULL CHECK(outcome IN ('a','b','no_perceptible_difference')), + created_at TEXT NOT NULL CHECK( + julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00' + ), + PRIMARY KEY(study_id, ordinal), + UNIQUE(study_id, optimizer_step_ordinal), + FOREIGN KEY(study_id, optimizer_step_ordinal) + REFERENCES adaptive_optimizer_steps(study_id, ordinal), + CHECK(value_a < value_b) +); +CREATE TRIGGER validate_adaptive_study_event_insert +BEFORE INSERT ON adaptive_study_events BEGIN + SELECT CASE WHEN NOT ( + (NEW.previous_state = 'draft' AND NEW.new_state = 'search_ready') + OR (NEW.previous_state = 'search_ready' AND NEW.new_state IN ( + 'generating_pair','paused','optimizer_failed' + )) + OR (NEW.previous_state = 'generating_pair' AND NEW.new_state IN ( + 'awaiting_preference','paused' + )) + OR (NEW.previous_state = 'awaiting_preference' AND NEW.new_state IN ( + 'updating_posterior','paused' + )) + OR (NEW.previous_state = 'updating_posterior' AND NEW.new_state IN ( + 'generating_pair','search_converged','search_budget_exhausted','paused', + 'optimizer_failed' + )) + OR (NEW.previous_state IN ('search_converged','search_budget_exhausted') + AND NEW.new_state = 'validating') + OR (NEW.previous_state = 'validating' AND NEW.new_state IN ('completed','paused')) + OR (NEW.previous_state = 'paused' AND NEW.new_state IN ( + 'search_ready','generating_pair','awaiting_preference','updating_posterior','validating' + )) + OR (NEW.previous_state = 'optimizer_failed' AND NEW.new_state IN ( + 'search_ready','updating_posterior','paused' + )) + ) THEN RAISE(ABORT, 'invalid adaptive study transition') END; + SELECT CASE WHEN NEW.sequence <> COALESCE(( + SELECT MAX(sequence) + 1 FROM adaptive_study_events WHERE study_id = NEW.study_id + ), 1) THEN RAISE(ABORT, 'invalid adaptive study event sequence') END; + SELECT CASE WHEN NEW.previous_state <> COALESCE(( + SELECT new_state FROM adaptive_study_events WHERE study_id = NEW.study_id + ORDER BY sequence DESC LIMIT 1 + ), 'draft') THEN RAISE(ABORT, 'invalid adaptive study prior state') END; +END; +CREATE TRIGGER validate_adaptive_optimizer_step_insert +BEFORE INSERT ON adaptive_optimizer_steps BEGIN + SELECT CASE WHEN NEW.ordinal <> COALESCE(( + SELECT MAX(ordinal) + 1 FROM adaptive_optimizer_steps WHERE study_id = NEW.study_id + ), 1) THEN RAISE(ABORT, 'invalid adaptive optimizer step ordinal') END; + SELECT CASE WHEN NEW.comparison_count <> ( + SELECT COUNT(*) FROM adaptive_observations WHERE study_id = NEW.study_id + ) THEN RAISE(ABORT, 'invalid adaptive optimizer comparison count') END; + SELECT CASE WHEN NOT ( + (NEW.ordinal = 1 AND COALESCE(( + SELECT new_state FROM adaptive_study_events WHERE study_id = NEW.study_id + ORDER BY sequence DESC LIMIT 1 + ), 'draft') = 'search_ready') + OR (NEW.ordinal > 1 AND COALESCE(( + SELECT new_state FROM adaptive_study_events WHERE study_id = NEW.study_id + ORDER BY sequence DESC LIMIT 1 + ), 'draft') = 'updating_posterior') + ) THEN RAISE(ABORT, 'invalid adaptive optimizer state') END; + SELECT CASE WHEN (SELECT COUNT(*) FROM json_each(NEW.posterior_json)) <> 8 + THEN RAISE(ABORT, 'invalid adaptive optimizer posterior object') END; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM json_each(NEW.posterior_json, '$.grid') + WHERE type NOT IN ('integer','real') OR abs(CAST(value AS REAL)) > 1.0e308 + ) THEN RAISE(ABORT, 'invalid adaptive optimizer posterior grid') END; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM json_each(NEW.posterior_json, '$.mean') + WHERE type NOT IN ('integer','real') OR abs(CAST(value AS REAL)) > 1.0e308 + ) THEN RAISE(ABORT, 'invalid adaptive optimizer posterior mean') END; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM json_each(NEW.posterior_json, '$.argmax_probability') + WHERE type NOT IN ('integer','real') OR abs(CAST(value AS REAL)) > 1.0e308 + ) THEN RAISE(ABORT, 'invalid adaptive optimizer argmax probability') END; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM json_each(NEW.posterior_json, '$.concentration_interval') + WHERE type NOT IN ('integer','real') OR abs(CAST(value AS REAL)) > 1.0e308 + ) THEN RAISE(ABORT, 'invalid adaptive optimizer concentration interval') END; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM json_each(NEW.posterior_json, '$.covariance') AS covariance_row + WHERE covariance_row.type <> 'array' + OR json_array_length(covariance_row.value) + <> json_array_length(NEW.posterior_json, '$.grid') + ) THEN RAISE(ABORT, 'invalid adaptive optimizer covariance shape') END; + SELECT CASE WHEN EXISTS ( + SELECT 1 + FROM json_each(NEW.posterior_json, '$.covariance') AS covariance_row, + json_each(covariance_row.value) AS covariance_value + WHERE covariance_value.type NOT IN ('integer','real') + OR abs(CAST(covariance_value.value AS REAL)) > 1.0e308 + ) THEN RAISE(ABORT, 'invalid adaptive optimizer covariance value') END; + SELECT CASE WHEN NOT ( + json_type(NEW.posterior_json, '$.recommendation') IN ('integer','real') + AND abs(CAST(json_extract(NEW.posterior_json, '$.recommendation') AS REAL)) <= 1.0e308 + AND json_type(NEW.posterior_json, '$.concentration_probability') IN ('integer','real') + AND abs(CAST( + json_extract(NEW.posterior_json, '$.concentration_probability') AS REAL + )) <= 1.0e308 + ) THEN RAISE(ABORT, 'invalid adaptive optimizer posterior scalar') END; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM json_each(NEW.acquisition_ledger_json) AS score + WHERE score.type <> 'object' + OR (SELECT COUNT(*) FROM json_each(score.value)) <> 2 + OR json_type(score.value, '$.pair') <> 'object' + OR (SELECT COUNT(*) FROM json_each(json_extract(score.value, '$.pair'))) <> 2 + OR json_type(score.value, '$.pair.value_a') NOT IN ('integer','real') + OR json_type(score.value, '$.pair.value_b') NOT IN ('integer','real') + OR abs(CAST(json_extract(score.value, '$.pair.value_a') AS REAL)) > 1.0e308 + OR abs(CAST(json_extract(score.value, '$.pair.value_b') AS REAL)) > 1.0e308 + OR CAST(json_extract(score.value, '$.pair.value_a') AS REAL) + >= CAST(json_extract(score.value, '$.pair.value_b') AS REAL) + OR json_type(score.value, '$.expected_information_bits') NOT IN ('integer','real') + OR CAST(json_extract(score.value, '$.expected_information_bits') AS REAL) < 0 + OR abs(CAST( + json_extract(score.value, '$.expected_information_bits') AS REAL + )) > 1.0e308 + ) THEN RAISE(ABORT, 'invalid adaptive optimizer acquisition ledger') END; + SELECT CASE WHEN EXISTS ( + SELECT 1 + FROM json_each(NEW.acquisition_ledger_json) AS score + GROUP BY + json_extract(score.value, '$.pair.value_a'), + json_extract(score.value, '$.pair.value_b') + HAVING COUNT(*) > 1 + ) THEN RAISE(ABORT, 'invalid adaptive optimizer duplicate acquisition pair') END; + SELECT CASE WHEN NEW.selected_pair_json IS NOT NULL AND NOT ( + (SELECT COUNT(*) FROM json_each(NEW.selected_pair_json)) = 2 + AND json_type(NEW.selected_pair_json, '$.value_a') IN ('integer','real') + AND json_type(NEW.selected_pair_json, '$.value_b') IN ('integer','real') + AND abs(CAST(json_extract(NEW.selected_pair_json, '$.value_a') AS REAL)) <= 1.0e308 + AND abs(CAST(json_extract(NEW.selected_pair_json, '$.value_b') AS REAL)) <= 1.0e308 + AND CAST(json_extract(NEW.selected_pair_json, '$.value_a') AS REAL) + < CAST(json_extract(NEW.selected_pair_json, '$.value_b') AS REAL) + ) THEN RAISE(ABORT, 'invalid adaptive optimizer selected pair') END; +END; +CREATE TRIGGER validate_adaptive_observation_insert +BEFORE INSERT ON adaptive_observations BEGIN + SELECT CASE WHEN COALESCE(( + SELECT new_state FROM adaptive_study_events WHERE study_id = NEW.study_id + ORDER BY sequence DESC LIMIT 1 + ), 'draft') <> 'awaiting_preference' + THEN RAISE(ABORT, 'invalid adaptive observation state') END; + SELECT CASE WHEN NEW.ordinal <> COALESCE(( + SELECT MAX(ordinal) + 1 FROM adaptive_observations WHERE study_id = NEW.study_id + ), 1) THEN RAISE(ABORT, 'invalid adaptive observation ordinal') END; + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM adaptive_optimizer_steps AS optimizer + WHERE optimizer.study_id = NEW.study_id + AND optimizer.ordinal = NEW.optimizer_step_ordinal + AND optimizer.comparison_count = NEW.ordinal - 1 + AND optimizer.stop_reason = 'continue' + AND CAST(json_extract(optimizer.selected_pair_json, '$.value_a') AS REAL) = NEW.value_a + AND CAST(json_extract(optimizer.selected_pair_json, '$.value_b') AS REAL) = NEW.value_b + ) THEN RAISE(ABORT, 'invalid adaptive observation optimizer binding') END; + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 + FROM review_revisions AS review + JOIN review_sessions AS session ON session.id = review.session_id + JOIN experiments AS experiment ON experiment.id = session.experiment_id + JOIN variants AS left_variant ON left_variant.id = session.left_variant_id + JOIN variants AS right_variant ON right_variant.id = session.right_variant_id + JOIN jobs AS left_job ON left_job.id = session.left_job_id + JOIN jobs AS right_job ON right_job.id = session.right_job_id + JOIN artifacts AS left_artifact ON left_artifact.id = session.left_artifact_id + JOIN artifacts AS right_artifact ON right_artifact.id = session.right_artifact_id + JOIN job_artifacts AS left_link ON left_link.job_id = session.left_job_id + AND left_link.artifact_id = session.left_artifact_id + JOIN job_artifacts AS right_link ON right_link.job_id = session.right_job_id + AND right_link.artifact_id = session.right_artifact_id + JOIN adaptive_studies AS study ON study.id = NEW.study_id + WHERE review.id = NEW.review_id + AND NEW.experiment_id = session.experiment_id + AND review.id = ( + SELECT latest.id FROM review_revisions AS latest + WHERE latest.session_id = session.id ORDER BY latest.revision DESC LIMIT 1 + ) + AND left_variant.experiment_id = session.experiment_id + AND right_variant.experiment_id = session.experiment_id + AND left_job.variant_id = session.left_variant_id + AND right_job.variant_id = session.right_variant_id + AND left_job.state = 'completed' + AND right_job.state = 'completed' + AND left_artifact.sha256 = session.left_artifact_sha256 + AND right_artifact.sha256 = session.right_artifact_sha256 + AND experiment.schema_version = 'manifest.v1' + AND experiment.variable_path = 'guidance_scale' + AND experiment.changed_paths_json = '["guidance_scale"]' + AND json_remove(left_variant.manifest_json, '$.guidance_scale') + = json_remove(study.base_manifest_json, '$.guidance_scale') + AND json_remove(right_variant.manifest_json, '$.guidance_scale') + = json_remove(study.base_manifest_json, '$.guidance_scale') + AND min( + CAST(json_extract(left_variant.manifest_json, '$.guidance_scale') AS REAL), + CAST(json_extract(right_variant.manifest_json, '$.guidance_scale') AS REAL) + ) = NEW.value_a + AND max( + CAST(json_extract(left_variant.manifest_json, '$.guidance_scale') AS REAL), + CAST(json_extract(right_variant.manifest_json, '$.guidance_scale') AS REAL) + ) = NEW.value_b + AND ( + (review.display_preference = 'tie' + AND review.preferred_variant_id IS NULL + AND NEW.outcome = 'no_perceptible_difference') + OR (review.display_preference = 'left' + AND review.preferred_variant_id = session.left_variant_id + AND NEW.outcome = CASE + WHEN CAST(json_extract( + left_variant.manifest_json, '$.guidance_scale' + ) AS REAL) = NEW.value_a THEN 'a' ELSE 'b' END) + OR (review.display_preference = 'right' + AND review.preferred_variant_id = session.right_variant_id + AND NEW.outcome = CASE + WHEN CAST(json_extract( + right_variant.manifest_json, '$.guidance_scale' + ) AS REAL) = NEW.value_a THEN 'a' ELSE 'b' END) + ) + ) THEN RAISE(ABORT, 'invalid adaptive observation review binding') END; + SELECT CASE WHEN NEW.display_mapping_digest IS NOT adaptive_observation_provenance_digest( + NEW.study_id, + NEW.schema_version, + NEW.ordinal, + NEW.optimizer_step_ordinal, + NEW.value_a, + NEW.value_b, + NEW.experiment_id, + NEW.review_id, + NEW.outcome, + NEW.created_at + ) THEN RAISE(ABORT, 'invalid adaptive observation provenance digest') END; +END; +CREATE TRIGGER guard_adaptive_studies_update +BEFORE UPDATE ON adaptive_studies BEGIN + SELECT RAISE(ABORT, 'adaptive study evidence is append-only'); +END; +CREATE TRIGGER guard_adaptive_studies_delete +BEFORE DELETE ON adaptive_studies BEGIN + SELECT RAISE(ABORT, 'adaptive study evidence is append-only'); +END; +CREATE TRIGGER guard_adaptive_study_events_update +BEFORE UPDATE ON adaptive_study_events BEGIN + SELECT RAISE(ABORT, 'adaptive study evidence is append-only'); +END; +CREATE TRIGGER guard_adaptive_study_events_delete +BEFORE DELETE ON adaptive_study_events BEGIN + SELECT RAISE(ABORT, 'adaptive study evidence is append-only'); +END; +CREATE TRIGGER guard_adaptive_observations_update +BEFORE UPDATE ON adaptive_observations BEGIN + SELECT RAISE(ABORT, 'adaptive study evidence is append-only'); +END; +CREATE TRIGGER guard_adaptive_observations_delete +BEFORE DELETE ON adaptive_observations BEGIN + SELECT RAISE(ABORT, 'adaptive study evidence is append-only'); +END; +CREATE TRIGGER guard_adaptive_optimizer_steps_update +BEFORE UPDATE ON adaptive_optimizer_steps BEGIN + SELECT RAISE(ABORT, 'adaptive study evidence is append-only'); +END; +CREATE TRIGGER guard_adaptive_optimizer_steps_delete +BEFORE DELETE ON adaptive_optimizer_steps BEGIN + SELECT RAISE(ABORT, 'adaptive study evidence is append-only'); +END; +""" + +_MIGRATION_10 = """ +CREATE TABLE adaptive_web_creation_grants( + token_digest TEXT NOT NULL CHECK( + length(token_digest) = 64 + AND token_digest = lower(token_digest) + AND token_digest NOT GLOB '*[^0-9a-f]*' + ), + reviewer_id TEXT NOT NULL CHECK(length(reviewer_id) > 0 AND reviewer_id = trim(reviewer_id)), + created_at TEXT NOT NULL CHECK( + julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00' + ), + expires_at TEXT NOT NULL CHECK( + julianday(expires_at) IS NOT NULL + AND substr(expires_at, -6) = '+00:00' + AND julianday(expires_at) > julianday(created_at) + ), + input_fingerprint TEXT CHECK( + input_fingerprint IS NULL OR ( + length(input_fingerprint) = 64 + AND input_fingerprint = lower(input_fingerprint) + AND input_fingerprint NOT GLOB '*[^0-9a-f]*' + ) + ), + study_id TEXT CHECK(study_id IS NULL OR length(study_id) > 0), + bound_at TEXT CHECK( + bound_at IS NULL OR ( + julianday(bound_at) IS NOT NULL + AND substr(bound_at, -6) = '+00:00' + AND julianday(bound_at) >= julianday(created_at) + AND julianday(bound_at) < julianday(expires_at) + ) + ), + PRIMARY KEY(token_digest, reviewer_id), + CHECK( + (input_fingerprint IS NULL AND study_id IS NULL AND bound_at IS NULL) + OR (input_fingerprint IS NOT NULL AND study_id IS NOT NULL AND bound_at IS NOT NULL) + ) +); +CREATE INDEX adaptive_web_creation_grants_owner_expiry +ON adaptive_web_creation_grants(reviewer_id, expires_at); +CREATE TRIGGER validate_adaptive_web_creation_grant_update +BEFORE UPDATE ON adaptive_web_creation_grants BEGIN + SELECT CASE WHEN NOT ( + OLD.input_fingerprint IS NULL + AND OLD.study_id IS NULL + AND OLD.bound_at IS NULL + AND NEW.token_digest IS OLD.token_digest + AND NEW.reviewer_id IS OLD.reviewer_id + AND NEW.created_at IS OLD.created_at + AND NEW.expires_at IS OLD.expires_at + AND NEW.input_fingerprint IS NOT NULL + AND NEW.study_id IS NOT NULL + AND NEW.bound_at IS NOT NULL + ) THEN RAISE(ABORT, 'invalid adaptive web creation grant transition') END; +END; +""" + +_MIGRATION_11 = """ +CREATE TABLE validation_campaigns( + id TEXT PRIMARY KEY CHECK(length(id) > 0), + schema_version TEXT NOT NULL CHECK(schema_version = 'validation-campaign.v1'), + reviewer_id TEXT NOT NULL CHECK(length(reviewer_id) > 0 AND reviewer_id = trim(reviewer_id)), + stage_1_study_id TEXT NOT NULL REFERENCES adaptive_studies(id) CHECK(length(stage_1_study_id) > 0), + stage_1_evidence_digest TEXT NOT NULL CHECK( + length(stage_1_evidence_digest) = 64 + AND stage_1_evidence_digest = lower(stage_1_evidence_digest) + AND stage_1_evidence_digest NOT GLOB '*[^0-9a-f]*' + ), + base_manifest_json TEXT NOT NULL CHECK(json_valid(base_manifest_json) AND json_type(base_manifest_json) = 'object'), + candidates_json TEXT NOT NULL CHECK(json_valid(candidates_json) AND json_type(candidates_json) = 'object'), + panel_input_json TEXT NOT NULL CHECK(json_valid(panel_input_json) AND json_type(panel_input_json) = 'object'), + protocol_version TEXT NOT NULL CHECK(protocol_version = 'balanced-validation.v1'), + analysis_version TEXT NOT NULL CHECK(analysis_version = 'dirichlet-validation.v1'), + created_at TEXT NOT NULL CHECK(julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00') +); +CREATE TABLE validation_events( + campaign_id TEXT NOT NULL REFERENCES validation_campaigns(id), + sequence INTEGER NOT NULL CHECK(typeof(sequence) = 'integer' AND sequence > 0), + previous_state TEXT NOT NULL CHECK(previous_state IN ('draft','preregistered','generating','ready','judging','analyzing','qualified','not_qualified','inconclusive')), + new_state TEXT NOT NULL CHECK(new_state IN ('draft','preregistered','generating','ready','judging','analyzing','qualified','not_qualified','inconclusive')), + created_at TEXT NOT NULL CHECK(julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00'), + PRIMARY KEY(campaign_id, sequence), + CHECK(previous_state <> new_state) +); +CREATE TABLE validation_trials( + campaign_id TEXT NOT NULL REFERENCES validation_campaigns(id), + trial_id TEXT NOT NULL CHECK(length(trial_id) > 0), + ordinal INTEGER NOT NULL CHECK(typeof(ordinal) = 'integer' AND ordinal > 0), + archetype TEXT NOT NULL CHECK(archetype IN ('portrait_motion','full_body_action','camera_motion','environment_interaction')), + prompt TEXT NOT NULL CHECK(length(prompt) > 0), + negative_prompt TEXT NOT NULL, + seed INTEGER NOT NULL CHECK(typeof(seed) = 'integer'), + challenger REAL NOT NULL CHECK(typeof(challenger) = 'real' AND abs(challenger) <= 1.0e308), + hidden_repeat INTEGER NOT NULL CHECK(hidden_repeat IN (0, 1)), + reuses_trial_id TEXT, + winner_display_position TEXT NOT NULL CHECK(winner_display_position IN ('left', 'right')), + panel_digest TEXT NOT NULL CHECK( + length(panel_digest) = 71 AND substr(panel_digest, 1, 7) = 'sha256:' + AND substr(panel_digest, 8) = lower(substr(panel_digest, 8)) + AND substr(panel_digest, 8) NOT GLOB '*[^0-9a-f]*' + ), + PRIMARY KEY(campaign_id, trial_id), + UNIQUE(campaign_id, ordinal), + FOREIGN KEY(campaign_id, reuses_trial_id) REFERENCES validation_trials(campaign_id, trial_id), + CHECK((hidden_repeat = 1) = (reuses_trial_id IS NOT NULL)) +); +CREATE UNIQUE INDEX validation_primary_context_challenger +ON validation_trials(campaign_id, archetype, prompt, negative_prompt, seed, challenger) +WHERE hidden_repeat = 0; +CREATE TABLE validation_trial_bindings( + campaign_id TEXT NOT NULL, + trial_id TEXT NOT NULL, + experiment_id TEXT, + review_id TEXT, + reused_from_trial_id TEXT, + PRIMARY KEY(campaign_id, trial_id), + FOREIGN KEY(campaign_id, trial_id) REFERENCES validation_trials(campaign_id, trial_id), + FOREIGN KEY(campaign_id, reused_from_trial_id) REFERENCES validation_trials(campaign_id, trial_id), + FOREIGN KEY(experiment_id) REFERENCES experiments(id), + FOREIGN KEY(review_id) REFERENCES review_revisions(id), + CHECK((experiment_id IS NULL) = (review_id IS NULL)) +); +CREATE TABLE validation_judgments( + campaign_id TEXT NOT NULL, + trial_id TEXT NOT NULL, + ordinal INTEGER NOT NULL CHECK(typeof(ordinal) = 'integer' AND ordinal > 0), + archetype TEXT NOT NULL CHECK(archetype IN ('portrait_motion','full_body_action','camera_motion','environment_interaction')), + challenger REAL NOT NULL CHECK(typeof(challenger) = 'real' AND abs(challenger) <= 1.0e308), + seed INTEGER NOT NULL CHECK(typeof(seed) = 'integer'), + hidden_repeat INTEGER NOT NULL CHECK(hidden_repeat IN (0, 1)), + winner_display_position TEXT NOT NULL CHECK(winner_display_position IN ('left', 'right')), + choice TEXT NOT NULL CHECK(choice IN ('a', 'b', 'tie')), + created_at TEXT NOT NULL CHECK(julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00'), + PRIMARY KEY(campaign_id, trial_id), + UNIQUE(campaign_id, ordinal), + FOREIGN KEY(campaign_id, trial_id) REFERENCES validation_trials(campaign_id, trial_id) +); +CREATE TABLE validation_results( + campaign_id TEXT PRIMARY KEY REFERENCES validation_campaigns(id), + disposition TEXT NOT NULL CHECK(disposition IN ('qualified','not_qualified','inconclusive')), + reason TEXT NOT NULL CHECK(reason IN ('all_gates_passed','challenger_rejection','archetype_reversal','repeat_reliability_failed','incomplete_primary_evidence','insufficient_primary_evidence')), + challengers_json TEXT NOT NULL CHECK(json_valid(challengers_json) AND json_type(challengers_json) = 'array'), + archetype_reversals_json TEXT NOT NULL CHECK(json_valid(archetype_reversals_json) AND json_type(archetype_reversals_json) = 'array'), + repeat_agreements INTEGER NOT NULL CHECK(typeof(repeat_agreements) = 'integer' AND repeat_agreements BETWEEN 0 AND 4), + created_at TEXT NOT NULL CHECK(julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00') +); +CREATE TRIGGER validate_validation_event_insert +BEFORE INSERT ON validation_events BEGIN + SELECT CASE WHEN NEW.sequence <> COALESCE((SELECT MAX(sequence) + 1 FROM validation_events WHERE campaign_id = NEW.campaign_id), 1) + THEN RAISE(ABORT, 'invalid validation event sequence') END; + SELECT CASE WHEN NEW.previous_state <> COALESCE((SELECT new_state FROM validation_events WHERE campaign_id = NEW.campaign_id ORDER BY sequence DESC LIMIT 1), 'draft') + THEN RAISE(ABORT, 'invalid validation event prior state') END; + SELECT CASE WHEN NOT ( + (NEW.previous_state = 'draft' AND NEW.new_state = 'preregistered') + OR (NEW.previous_state = 'preregistered' AND NEW.new_state = 'generating') + OR (NEW.previous_state = 'generating' AND NEW.new_state = 'ready') + OR (NEW.previous_state = 'ready' AND NEW.new_state = 'judging') + OR (NEW.previous_state = 'judging' AND NEW.new_state = 'analyzing') + OR (NEW.previous_state = 'analyzing' AND NEW.new_state IN ('qualified','not_qualified','inconclusive')) + ) THEN RAISE(ABORT, 'invalid validation lifecycle transition') END; +END; +CREATE TRIGGER validate_validation_trial_insert +BEFORE INSERT ON validation_trials BEGIN + SELECT CASE WHEN COALESCE((SELECT new_state FROM validation_events WHERE campaign_id = NEW.campaign_id ORDER BY sequence DESC LIMIT 1), 'draft') <> 'preregistered' + THEN RAISE(ABORT, 'validation trials require preregistered campaign') END; + SELECT CASE WHEN NEW.hidden_repeat = 1 AND NOT EXISTS(SELECT 1 FROM validation_trials WHERE campaign_id = NEW.campaign_id AND trial_id = NEW.reuses_trial_id) + THEN RAISE(ABORT, 'validation repeat source is missing') END; +END; +CREATE TRIGGER validate_validation_trial_binding_insert +BEFORE INSERT ON validation_trial_bindings BEGIN + SELECT CASE WHEN COALESCE((SELECT new_state FROM validation_events WHERE campaign_id = NEW.campaign_id ORDER BY sequence DESC LIMIT 1), 'draft') NOT IN ('generating','ready','judging') + THEN RAISE(ABORT, 'validation trial binding has invalid lifecycle state') END; + SELECT CASE WHEN NEW.review_id IS NOT NULL AND NOT EXISTS( + SELECT 1 FROM review_revisions AS review JOIN review_sessions AS session ON session.id = review.session_id + JOIN validation_campaigns AS campaign ON campaign.id = NEW.campaign_id + WHERE review.id = NEW.review_id AND session.experiment_id = NEW.experiment_id AND session.reviewer_id = campaign.reviewer_id + ) THEN RAISE(ABORT, 'validation trial review owner mismatch') END; +END; +CREATE TRIGGER validate_validation_judgment_insert +BEFORE INSERT ON validation_judgments BEGIN + SELECT CASE WHEN COALESCE((SELECT new_state FROM validation_events WHERE campaign_id = NEW.campaign_id ORDER BY sequence DESC LIMIT 1), 'draft') <> 'judging' + THEN RAISE(ABORT, 'validation judgment has invalid lifecycle state') END; + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM validation_trials AS trial + WHERE trial.campaign_id = NEW.campaign_id AND trial.trial_id = NEW.trial_id + AND trial.ordinal = NEW.ordinal AND trial.archetype = NEW.archetype + AND trial.challenger = NEW.challenger AND trial.seed = NEW.seed + AND trial.hidden_repeat = NEW.hidden_repeat AND trial.winner_display_position = NEW.winner_display_position + ) THEN RAISE(ABORT, 'validation judgment does not match trial') END; +END; +CREATE TRIGGER validate_validation_result_insert +BEFORE INSERT ON validation_results BEGIN + SELECT CASE WHEN COALESCE((SELECT new_state FROM validation_events WHERE campaign_id = NEW.campaign_id ORDER BY sequence DESC LIMIT 1), 'draft') <> 'analyzing' + THEN RAISE(ABORT, 'validation result has invalid lifecycle state') END; +END; +CREATE TRIGGER guard_validation_campaigns_update BEFORE UPDATE ON validation_campaigns BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +CREATE TRIGGER guard_validation_campaigns_delete BEFORE DELETE ON validation_campaigns BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +CREATE TRIGGER guard_validation_events_update BEFORE UPDATE ON validation_events BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +CREATE TRIGGER guard_validation_events_delete BEFORE DELETE ON validation_events BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +CREATE TRIGGER guard_validation_trials_update BEFORE UPDATE ON validation_trials BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +CREATE TRIGGER guard_validation_trials_delete BEFORE DELETE ON validation_trials BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +CREATE TRIGGER guard_validation_trial_bindings_update BEFORE UPDATE ON validation_trial_bindings BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +CREATE TRIGGER guard_validation_trial_bindings_delete BEFORE DELETE ON validation_trial_bindings BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +CREATE TRIGGER guard_validation_judgments_update BEFORE UPDATE ON validation_judgments BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +CREATE TRIGGER guard_validation_judgments_delete BEFORE DELETE ON validation_judgments BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +CREATE TRIGGER guard_validation_results_update BEFORE UPDATE ON validation_results BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +CREATE TRIGGER guard_validation_results_delete BEFORE DELETE ON validation_results BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +""" + +_MIGRATION_12 = """ +CREATE TABLE validation_experiment_bindings( + campaign_id TEXT NOT NULL, + trial_id TEXT NOT NULL, + experiment_id TEXT NOT NULL REFERENCES experiments(id), + PRIMARY KEY(campaign_id, trial_id), + FOREIGN KEY(campaign_id, trial_id) REFERENCES validation_trials(campaign_id, trial_id) +); +CREATE UNIQUE INDEX validation_experiment_binding_experiment +ON validation_experiment_bindings(experiment_id); +CREATE TRIGGER validate_validation_experiment_binding_insert +BEFORE INSERT ON validation_experiment_bindings BEGIN + SELECT CASE WHEN COALESCE((SELECT new_state FROM validation_events WHERE campaign_id = NEW.campaign_id ORDER BY sequence DESC LIMIT 1), 'draft') NOT IN ('generating','ready','judging') + AND EXISTS(SELECT 1 FROM schema_migrations WHERE version = 12) + THEN RAISE(ABORT, 'validation experiment binding has invalid lifecycle state') END; + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM validation_trials AS trial + WHERE trial.campaign_id = NEW.campaign_id AND trial.trial_id = NEW.trial_id + AND trial.hidden_repeat = 0 + ) THEN RAISE(ABORT, 'validation experiment binding requires a primary trial') END; + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM experiments AS experiment + WHERE experiment.id = NEW.experiment_id + AND experiment.schema_version = 'validation-controlled.v1' + ) THEN RAISE(ABORT, 'validation experiment binding requires a controlled experiment') END; +END; +CREATE TRIGGER guard_validation_experiment_bindings_update BEFORE UPDATE ON validation_experiment_bindings BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +CREATE TRIGGER guard_validation_experiment_bindings_delete BEFORE DELETE ON validation_experiment_bindings BEGIN SELECT RAISE(ABORT, 'validation evidence is append-only'); END; +""" + +_MIGRATION_13 = """ +DROP TRIGGER validate_validation_trial_binding_insert; +CREATE TRIGGER validate_validation_trial_binding_insert +BEFORE INSERT ON validation_trial_bindings BEGIN + SELECT CASE WHEN COALESCE((SELECT new_state FROM validation_events WHERE campaign_id = NEW.campaign_id ORDER BY sequence DESC LIMIT 1), 'draft') NOT IN ('generating','ready','judging') + THEN RAISE(ABORT, 'validation trial binding has invalid lifecycle state') END; + SELECT CASE WHEN NEW.review_id IS NOT NULL AND NOT EXISTS( + SELECT 1 FROM review_revisions AS review JOIN review_sessions AS session ON session.id = review.session_id + JOIN validation_campaigns AS campaign ON campaign.id = NEW.campaign_id + JOIN validation_trials AS trial ON trial.campaign_id = NEW.campaign_id AND trial.trial_id = NEW.trial_id + WHERE review.id = NEW.review_id AND session.experiment_id = NEW.experiment_id + AND session.reviewer_id = validation_session_reviewer( + NEW.campaign_id, NEW.trial_id, campaign.reviewer_id, trial.hidden_repeat + ) + ) THEN RAISE(ABORT, 'validation trial review owner mismatch') END; +END; +""" + +_MIGRATION_14 = """ +CREATE TABLE characters( + id TEXT PRIMARY KEY, + reviewer_id TEXT NOT NULL, + name TEXT NOT NULL CHECK(length(name) > 0 AND name = trim(name)), + created_at TEXT NOT NULL, + UNIQUE(id, reviewer_id), + UNIQUE(reviewer_id, name) +); +CREATE TABLE character_packs( + id TEXT PRIMARY KEY, + character_id TEXT NOT NULL, + reviewer_id TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('draft','preregistered')), + supersedes_pack_id TEXT, + manifest_json TEXT, + created_at TEXT NOT NULL, + preregistered_at TEXT, + UNIQUE(character_id, id), + FOREIGN KEY(character_id, reviewer_id) REFERENCES characters(id, reviewer_id), + FOREIGN KEY(character_id, supersedes_pack_id) REFERENCES character_packs(character_id, id), + CHECK((state = 'draft' AND manifest_json IS NULL AND preregistered_at IS NULL) + OR (state = 'preregistered' AND manifest_json IS NOT NULL AND preregistered_at IS NOT NULL)) +); +CREATE TABLE character_pack_aliases( + alias_id TEXT PRIMARY KEY, + pack_id TEXT NOT NULL REFERENCES character_packs(id) ON UPDATE CASCADE +); +CREATE TABLE character_references( + pack_id TEXT NOT NULL REFERENCES character_packs(id) ON UPDATE CASCADE, + reference_id TEXT NOT NULL, + artifact_id TEXT NOT NULL REFERENCES artifacts(id), + source_name TEXT NOT NULL CHECK(length(source_name) > 0 AND substr(source_name, 1, 1) != '/'), + image_sha256 TEXT NOT NULL CHECK(length(image_sha256) = 64 AND image_sha256 = lower(image_sha256)), + media_type TEXT NOT NULL CHECK(media_type IN ('image/jpeg','image/png','image/webp')), + dhash_hex TEXT NOT NULL CHECK(length(dhash_hex) = 16 AND dhash_hex = lower(dhash_hex)), + disposition TEXT NOT NULL CHECK(disposition IN ('active','rejected')), + ordinal INTEGER NOT NULL CHECK(ordinal >= 0), + PRIMARY KEY(pack_id, reference_id), + UNIQUE(pack_id, ordinal), + UNIQUE(pack_id, artifact_id) +); +CREATE TABLE character_annotations( + pack_id TEXT NOT NULL, + reference_id TEXT NOT NULL, + framing TEXT NOT NULL CHECK(framing IN ('portrait','full_body')), + head_angle TEXT NOT NULL CHECK(head_angle IN ('front','three_quarter','profile')), + body_visibility TEXT NOT NULL CHECK(body_visibility IN ('clear_face','partial_body','full_body')), + expression TEXT NOT NULL CHECK(expression IN ('neutral','smile','serious','angry','surprised')), + lighting TEXT NOT NULL CHECK(lighting IN ('ordinary','directional')), + occlusion TEXT NOT NULL CHECK(occlusion IN ('none','partial','heavy')), + identity_confirmed INTEGER NOT NULL CHECK(identity_confirmed IN (0,1)), + PRIMARY KEY(pack_id, reference_id), + FOREIGN KEY(pack_id, reference_id) REFERENCES character_references(pack_id, reference_id) ON UPDATE CASCADE +); +CREATE TABLE character_pack_warnings( + pack_id TEXT NOT NULL REFERENCES character_packs(id) ON UPDATE CASCADE, + left_reference_id TEXT NOT NULL, + right_reference_id TEXT NOT NULL, + distance INTEGER NOT NULL CHECK(distance BETWEEN 0 AND 64), + resolution TEXT NOT NULL CHECK(resolution IN ('unresolved','accepted','dismissed')), + PRIMARY KEY(pack_id, left_reference_id, right_reference_id), + FOREIGN KEY(pack_id, left_reference_id) REFERENCES character_references(pack_id, reference_id) ON UPDATE CASCADE, + FOREIGN KEY(pack_id, right_reference_id) REFERENCES character_references(pack_id, reference_id) ON UPDATE CASCADE, + CHECK(left_reference_id < right_reference_id) +); +CREATE TABLE character_pack_events( + pack_id TEXT NOT NULL REFERENCES character_packs(id) ON UPDATE CASCADE, + sequence INTEGER NOT NULL CHECK(sequence > 0), + reviewer_id TEXT NOT NULL, + event_type TEXT NOT NULL CHECK(event_type IN ('draft_created','preregistered')), + payload_json TEXT NOT NULL CHECK(json_valid(payload_json) AND json_type(payload_json) = 'object'), + created_at TEXT NOT NULL, + PRIMARY KEY(pack_id, sequence) +); +CREATE TRIGGER validate_character_pack_event_insert BEFORE INSERT ON character_pack_events BEGIN + SELECT CASE WHEN NEW.sequence != COALESCE((SELECT MAX(sequence) + 1 FROM character_pack_events WHERE pack_id = NEW.pack_id), 1) + THEN RAISE(ABORT, 'character pack event sequence is not append-only') END; + SELECT CASE WHEN NEW.reviewer_id != (SELECT reviewer_id FROM character_packs WHERE id = NEW.pack_id) + THEN RAISE(ABORT, 'character pack event reviewer mismatch') END; + SELECT CASE WHEN NOT ( + (NEW.sequence = 1 AND NEW.event_type = 'draft_created' AND (SELECT state FROM character_packs WHERE id = NEW.pack_id) = 'draft') + OR (NEW.sequence = 2 AND NEW.event_type = 'preregistered' AND (SELECT state FROM character_packs WHERE id = NEW.pack_id) = 'preregistered') + ) THEN RAISE(ABORT, 'character pack event does not match lifecycle') END; +END; +CREATE TRIGGER guard_character_frozen_reference_insert BEFORE INSERT ON character_references WHEN (SELECT state FROM character_packs WHERE id = NEW.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_frozen_annotation_insert BEFORE INSERT ON character_annotations WHEN (SELECT state FROM character_packs WHERE id = NEW.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_frozen_warning_insert BEFORE INSERT ON character_pack_warnings WHEN (SELECT state FROM character_packs WHERE id = NEW.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_characters_update BEFORE UPDATE ON characters BEGIN SELECT RAISE(ABORT, 'character identity is append-only'); END; +CREATE TRIGGER guard_characters_delete BEFORE DELETE ON characters BEGIN SELECT RAISE(ABORT, 'character identity is append-only'); END; +CREATE TRIGGER guard_character_pack_alias_update BEFORE UPDATE ON character_pack_aliases WHEN EXISTS(SELECT 1 FROM character_packs WHERE id = OLD.pack_id) BEGIN SELECT RAISE(ABORT, 'character pack alias is append-only'); END; +CREATE TRIGGER guard_character_pack_alias_delete BEFORE DELETE ON character_pack_aliases BEGIN SELECT RAISE(ABORT, 'character pack alias is append-only'); END; +CREATE TRIGGER guard_character_pack_frozen_update BEFORE UPDATE ON character_packs WHEN OLD.state = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER validate_character_pack_preregister_update BEFORE UPDATE ON character_packs WHEN OLD.state = 'draft' BEGIN + SELECT CASE WHEN NEW.character_id != OLD.character_id OR NEW.reviewer_id != OLD.reviewer_id OR NEW.supersedes_pack_id IS NOT OLD.supersedes_pack_id OR NEW.created_at != OLD.created_at + OR NOT ((NEW.state = 'preregistered' AND NEW.id = OLD.id) OR (NEW.state = 'draft' AND NEW.manifest_json IS NULL AND NEW.preregistered_at IS NULL)) + THEN RAISE(ABORT, 'invalid character pack transition') END; +END; +CREATE TRIGGER guard_character_pack_frozen_delete BEFORE DELETE ON character_packs WHEN OLD.state = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_reference_update BEFORE UPDATE ON character_references WHEN (SELECT state FROM character_packs WHERE id = OLD.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_reference_delete BEFORE DELETE ON character_references WHEN (SELECT state FROM character_packs WHERE id = OLD.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_annotation_update BEFORE UPDATE ON character_annotations WHEN (SELECT state FROM character_packs WHERE id = OLD.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_annotation_delete BEFORE DELETE ON character_annotations WHEN (SELECT state FROM character_packs WHERE id = OLD.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_warning_update BEFORE UPDATE ON character_pack_warnings WHEN + (OLD.resolution != 'unresolved' AND (NEW.pack_id = OLD.pack_id OR NEW.left_reference_id != OLD.left_reference_id OR NEW.right_reference_id != OLD.right_reference_id OR NEW.distance != OLD.distance OR NEW.resolution != OLD.resolution)) + OR (SELECT state FROM character_packs WHERE id = OLD.pack_id) = 'preregistered' + BEGIN SELECT RAISE(ABORT, 'character warning is append-only'); END; +CREATE TRIGGER guard_character_warning_delete BEFORE DELETE ON character_pack_warnings WHEN OLD.resolution != 'unresolved' OR (SELECT state FROM character_packs WHERE id = OLD.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character warning is append-only'); END; +CREATE TRIGGER guard_character_pack_events_update BEFORE UPDATE ON character_pack_events WHEN + NEW.pack_id = OLD.pack_id OR NEW.sequence != OLD.sequence OR NEW.reviewer_id != OLD.reviewer_id OR NEW.event_type != OLD.event_type OR NEW.payload_json != OLD.payload_json OR NEW.created_at != OLD.created_at + BEGIN SELECT RAISE(ABORT, 'character pack events are append-only'); END; +CREATE TRIGGER guard_character_pack_events_delete BEFORE DELETE ON character_pack_events BEGIN SELECT RAISE(ABORT, 'character pack events are append-only'); END; +""" + +_MIGRATION_15 = """ +PRAGMA defer_foreign_keys=ON; +DROP TRIGGER validate_character_pack_event_insert; +DROP TRIGGER guard_character_frozen_reference_insert; +DROP TRIGGER guard_character_frozen_annotation_insert; +DROP TRIGGER guard_character_frozen_warning_insert; +DROP TRIGGER guard_characters_update; +DROP TRIGGER guard_characters_delete; +DROP TRIGGER guard_character_pack_alias_update; +DROP TRIGGER guard_character_pack_alias_delete; +DROP TRIGGER guard_character_pack_frozen_update; +DROP TRIGGER validate_character_pack_preregister_update; +DROP TRIGGER guard_character_pack_frozen_delete; +DROP TRIGGER guard_character_reference_update; +DROP TRIGGER guard_character_reference_delete; +DROP TRIGGER guard_character_annotation_update; +DROP TRIGGER guard_character_annotation_delete; +DROP TRIGGER guard_character_warning_update; +DROP TRIGGER guard_character_warning_delete; +DROP TRIGGER guard_character_pack_events_update; +DROP TRIGGER guard_character_pack_events_delete; +ALTER TABLE character_pack_events RENAME TO character_pack_events_v14; +ALTER TABLE character_pack_warnings RENAME TO character_pack_warnings_v14; +ALTER TABLE character_annotations RENAME TO character_annotations_v14; +ALTER TABLE character_references RENAME TO character_references_v14; +ALTER TABLE character_pack_aliases RENAME TO character_pack_aliases_v14; +ALTER TABLE character_packs RENAME TO character_packs_v14; +CREATE TABLE character_packs( + storage_id TEXT PRIMARY KEY, + id TEXT NOT NULL UNIQUE, + character_id TEXT NOT NULL, + reviewer_id TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('draft','preregistered')), + supersedes_storage_id TEXT, + manifest_json TEXT, + created_at TEXT NOT NULL, + preregistered_at TEXT, + UNIQUE(character_id, id), + UNIQUE(character_id, storage_id), + FOREIGN KEY(character_id, reviewer_id) REFERENCES characters(id, reviewer_id), + FOREIGN KEY(character_id, supersedes_storage_id) REFERENCES character_packs(character_id, storage_id), + CHECK((state = 'draft' AND manifest_json IS NULL AND preregistered_at IS NULL) + OR (state = 'preregistered' AND manifest_json IS NOT NULL AND preregistered_at IS NOT NULL)) +); +CREATE TABLE character_pack_aliases( + alias_id TEXT PRIMARY KEY, + pack_storage_id TEXT NOT NULL REFERENCES character_packs(storage_id) +); +CREATE TABLE character_references( + pack_id TEXT NOT NULL REFERENCES character_packs(storage_id), + reference_id TEXT NOT NULL, + artifact_id TEXT NOT NULL REFERENCES artifacts(id), + source_name TEXT NOT NULL CHECK(length(source_name) > 0 AND substr(source_name, 1, 1) != '/'), + image_sha256 TEXT NOT NULL CHECK(length(image_sha256) = 64 AND image_sha256 = lower(image_sha256)), + media_type TEXT NOT NULL CHECK(media_type IN ('image/jpeg','image/png','image/webp')), + dhash_hex TEXT NOT NULL CHECK(length(dhash_hex) = 16 AND dhash_hex = lower(dhash_hex)), + disposition TEXT NOT NULL CHECK(disposition IN ('active','rejected')), + ordinal INTEGER NOT NULL CHECK(ordinal >= 0), + PRIMARY KEY(pack_id, reference_id), + UNIQUE(pack_id, ordinal), + UNIQUE(pack_id, artifact_id) +); +CREATE TABLE character_annotations( + pack_id TEXT NOT NULL, + reference_id TEXT NOT NULL, + framing TEXT NOT NULL CHECK(framing IN ('portrait','full_body')), + head_angle TEXT NOT NULL CHECK(head_angle IN ('front','three_quarter','profile')), + body_visibility TEXT NOT NULL CHECK(body_visibility IN ('clear_face','partial_body','full_body')), + expression TEXT NOT NULL CHECK(expression IN ('neutral','smile','serious','angry','surprised')), + lighting TEXT NOT NULL CHECK(lighting IN ('ordinary','directional')), + occlusion TEXT NOT NULL CHECK(occlusion IN ('none','partial','heavy')), + identity_confirmed INTEGER NOT NULL CHECK(identity_confirmed IN (0,1)), + PRIMARY KEY(pack_id, reference_id), + FOREIGN KEY(pack_id, reference_id) REFERENCES character_references(pack_id, reference_id) +); +CREATE TABLE character_pack_warnings( + pack_id TEXT NOT NULL REFERENCES character_packs(storage_id), + left_reference_id TEXT NOT NULL, + right_reference_id TEXT NOT NULL, + distance INTEGER NOT NULL CHECK(distance BETWEEN 0 AND 64), + resolution TEXT NOT NULL CHECK(resolution IN ('unresolved','accepted','dismissed')), + PRIMARY KEY(pack_id, left_reference_id, right_reference_id), + FOREIGN KEY(pack_id, left_reference_id) REFERENCES character_references(pack_id, reference_id), + FOREIGN KEY(pack_id, right_reference_id) REFERENCES character_references(pack_id, reference_id), + CHECK(left_reference_id < right_reference_id) +); +CREATE TABLE character_pack_events( + pack_id TEXT NOT NULL REFERENCES character_packs(storage_id), + sequence INTEGER NOT NULL CHECK(sequence > 0), + reviewer_id TEXT NOT NULL, + event_type TEXT NOT NULL CHECK(event_type IN ('draft_created','preregistered')), + payload_json TEXT NOT NULL CHECK(json_valid(payload_json) AND json_type(payload_json) = 'object'), + created_at TEXT NOT NULL, + PRIMARY KEY(pack_id, sequence) +); +INSERT INTO character_packs( + storage_id, id, character_id, reviewer_id, state, supersedes_storage_id, + manifest_json, created_at, preregistered_at +) SELECT id, id, character_id, reviewer_id, state, supersedes_pack_id, + manifest_json, created_at, preregistered_at FROM character_packs_v14; +INSERT INTO character_pack_aliases(alias_id, pack_storage_id) + SELECT alias_id, pack_id FROM character_pack_aliases_v14; +INSERT INTO character_references SELECT * FROM character_references_v14; +INSERT INTO character_annotations SELECT * FROM character_annotations_v14; +INSERT INTO character_pack_warnings SELECT * FROM character_pack_warnings_v14; +INSERT INTO character_pack_events SELECT * FROM character_pack_events_v14; +DROP TABLE character_pack_events_v14; +DROP TABLE character_pack_warnings_v14; +DROP TABLE character_annotations_v14; +DROP TABLE character_references_v14; +DROP TABLE character_pack_aliases_v14; +DROP TABLE character_packs_v14; +CREATE TRIGGER validate_character_pack_event_insert BEFORE INSERT ON character_pack_events BEGIN + SELECT CASE WHEN NEW.sequence != COALESCE((SELECT MAX(sequence) + 1 FROM character_pack_events WHERE pack_id = NEW.pack_id), 1) + THEN RAISE(ABORT, 'character pack event sequence is not append-only') END; + SELECT CASE WHEN NEW.reviewer_id != (SELECT reviewer_id FROM character_packs WHERE storage_id = NEW.pack_id) + THEN RAISE(ABORT, 'character pack event reviewer mismatch') END; + SELECT CASE WHEN NOT ( + (NEW.sequence = 1 AND NEW.event_type = 'draft_created' AND (SELECT state FROM character_packs WHERE storage_id = NEW.pack_id) = 'draft') + OR (NEW.sequence = 2 AND NEW.event_type = 'preregistered' AND (SELECT state FROM character_packs WHERE storage_id = NEW.pack_id) = 'preregistered') + ) THEN RAISE(ABORT, 'character pack event does not match lifecycle') END; +END; +CREATE TRIGGER guard_character_frozen_reference_insert BEFORE INSERT ON character_references WHEN (SELECT state FROM character_packs WHERE storage_id = NEW.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_frozen_annotation_insert BEFORE INSERT ON character_annotations WHEN (SELECT state FROM character_packs WHERE storage_id = NEW.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_frozen_warning_insert BEFORE INSERT ON character_pack_warnings WHEN (SELECT state FROM character_packs WHERE storage_id = NEW.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_characters_update BEFORE UPDATE ON characters BEGIN SELECT RAISE(ABORT, 'character identity is append-only'); END; +CREATE TRIGGER guard_characters_delete BEFORE DELETE ON characters BEGIN SELECT RAISE(ABORT, 'character identity is append-only'); END; +CREATE TRIGGER guard_character_pack_alias_update BEFORE UPDATE ON character_pack_aliases BEGIN SELECT RAISE(ABORT, 'character pack alias is append-only'); END; +CREATE TRIGGER guard_character_pack_alias_delete BEFORE DELETE ON character_pack_aliases BEGIN SELECT RAISE(ABORT, 'character pack alias is append-only'); END; +CREATE TRIGGER guard_character_pack_frozen_update BEFORE UPDATE ON character_packs WHEN OLD.state = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER validate_character_pack_preregister_update BEFORE UPDATE ON character_packs WHEN OLD.state = 'draft' BEGIN + SELECT CASE WHEN NEW.storage_id != OLD.storage_id OR NEW.character_id != OLD.character_id OR NEW.reviewer_id != OLD.reviewer_id OR NEW.supersedes_storage_id IS NOT OLD.supersedes_storage_id OR NEW.created_at != OLD.created_at + OR NOT ((NEW.state = 'preregistered' AND NEW.id = OLD.id) OR (NEW.state = 'draft' AND NEW.manifest_json IS NULL AND NEW.preregistered_at IS NULL)) + THEN RAISE(ABORT, 'invalid character pack transition') END; +END; +CREATE TRIGGER guard_character_pack_frozen_delete BEFORE DELETE ON character_packs WHEN OLD.state = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_reference_update BEFORE UPDATE ON character_references WHEN (SELECT state FROM character_packs WHERE storage_id = OLD.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_reference_delete BEFORE DELETE ON character_references WHEN (SELECT state FROM character_packs WHERE storage_id = OLD.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_annotation_update BEFORE UPDATE ON character_annotations WHEN (SELECT state FROM character_packs WHERE storage_id = OLD.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_annotation_delete BEFORE DELETE ON character_annotations WHEN (SELECT state FROM character_packs WHERE storage_id = OLD.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character pack is append-only'); END; +CREATE TRIGGER guard_character_warning_update BEFORE UPDATE ON character_pack_warnings WHEN + OLD.resolution != 'unresolved' OR (SELECT state FROM character_packs WHERE storage_id = OLD.pack_id) = 'preregistered' + BEGIN SELECT RAISE(ABORT, 'character warning is append-only'); END; +CREATE TRIGGER guard_character_warning_delete BEFORE DELETE ON character_pack_warnings WHEN OLD.resolution != 'unresolved' OR (SELECT state FROM character_packs WHERE storage_id = OLD.pack_id) = 'preregistered' BEGIN SELECT RAISE(ABORT, 'character warning is append-only'); END; +CREATE TRIGGER guard_character_pack_events_update BEFORE UPDATE ON character_pack_events BEGIN SELECT RAISE(ABORT, 'character pack events are append-only'); END; +CREATE TRIGGER guard_character_pack_events_delete BEFORE DELETE ON character_pack_events BEGIN SELECT RAISE(ABORT, 'character pack events are append-only'); END; +""" + +_MIGRATION_16 = """ +CREATE UNIQUE INDEX character_pack_storage_reviewer +ON character_packs(storage_id, reviewer_id); +CREATE TABLE character_curation_grants( + token_digest TEXT PRIMARY KEY + CHECK(length(token_digest) = 64 AND token_digest = lower(token_digest)), + reviewer_id TEXT NOT NULL CHECK(length(reviewer_id) > 0 AND reviewer_id = trim(reviewer_id)), + character_id TEXT NOT NULL, + pack_storage_id TEXT NOT NULL, + created_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + FOREIGN KEY(character_id, reviewer_id) REFERENCES characters(id, reviewer_id), + FOREIGN KEY(character_id, pack_storage_id) + REFERENCES character_packs(character_id, storage_id), + FOREIGN KEY(pack_storage_id, reviewer_id) + REFERENCES character_packs(storage_id, reviewer_id), + CHECK(julianday(expires_at) > julianday(created_at)) +); +CREATE INDEX character_curation_grants_expiry +ON character_curation_grants(expires_at); +CREATE INDEX character_curation_grants_owner +ON character_curation_grants(reviewer_id); +CREATE TRIGGER guard_character_curation_grants_update +BEFORE UPDATE ON character_curation_grants BEGIN + SELECT RAISE(ABORT, 'character curation grants cannot be changed'); +END; +""" + +_MIGRATION_17 = """ +CREATE TABLE character_benchmark_campaigns( + id TEXT PRIMARY KEY CHECK(substr(id, 1, 20) = 'character_benchmark_'), + reviewer_id TEXT NOT NULL CHECK(length(reviewer_id) > 0 AND reviewer_id = trim(reviewer_id)), + content_digest TEXT NOT NULL UNIQUE CHECK(length(content_digest) = 71 AND substr(content_digest, 1, 7) = 'sha256:'), + definition_json TEXT NOT NULL CHECK(json_valid(definition_json) AND json_type(definition_json) = 'object'), + character_pack_id TEXT NOT NULL, + character_pack_storage_id TEXT NOT NULL REFERENCES character_packs(storage_id), + stage_2_campaign_id TEXT NOT NULL REFERENCES validation_campaigns(id), + stage_2_manifest_json TEXT NOT NULL CHECK(json_valid(stage_2_manifest_json) AND json_type(stage_2_manifest_json) = 'object'), + identity_prompt TEXT NOT NULL CHECK(length(identity_prompt) > 0 AND identity_prompt = trim(identity_prompt)), + created_at TEXT NOT NULL CHECK(julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00'), + UNIQUE(id, reviewer_id), + FOREIGN KEY(character_pack_storage_id, reviewer_id) REFERENCES character_packs(storage_id, reviewer_id) +); +CREATE TABLE character_benchmark_events( + campaign_id TEXT NOT NULL REFERENCES character_benchmark_campaigns(id), + sequence INTEGER NOT NULL CHECK(sequence BETWEEN 1 AND 6), + previous_state TEXT NOT NULL CHECK(previous_state IN ('draft','preregistered','generating','ready','judging','analyzing','completed')), + new_state TEXT NOT NULL CHECK(new_state IN ('draft','preregistered','generating','ready','judging','analyzing','completed')), + payload_digest TEXT NOT NULL CHECK(length(payload_digest) = 71 AND substr(payload_digest, 1, 7) = 'sha256:'), + created_at TEXT NOT NULL CHECK(julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00'), + PRIMARY KEY(campaign_id, sequence), + CHECK(previous_state <> new_state) +); +CREATE TABLE character_benchmark_trials( + campaign_id TEXT NOT NULL REFERENCES character_benchmark_campaigns(id), + trial_id TEXT NOT NULL, + ordinal INTEGER NOT NULL CHECK(ordinal BETWEEN 1 AND 40), + trial_json TEXT NOT NULL CHECK(json_valid(trial_json) AND json_type(trial_json) = 'object'), + hidden_repeat INTEGER NOT NULL CHECK(hidden_repeat IN (0,1)), + reuses_trial_id TEXT, + candidate_a_position TEXT NOT NULL CHECK(candidate_a_position IN ('left','right')), + PRIMARY KEY(campaign_id, trial_id), + UNIQUE(campaign_id, ordinal), + FOREIGN KEY(campaign_id, reuses_trial_id) REFERENCES character_benchmark_trials(campaign_id, trial_id) DEFERRABLE INITIALLY DEFERRED, + CHECK((hidden_repeat = 1) = (reuses_trial_id IS NOT NULL)) +); +CREATE TABLE character_benchmark_experiment_bindings( + campaign_id TEXT NOT NULL, + trial_id TEXT NOT NULL, + experiment_id TEXT NOT NULL UNIQUE REFERENCES experiments(id), + PRIMARY KEY(campaign_id, trial_id), + FOREIGN KEY(campaign_id, trial_id) REFERENCES character_benchmark_trials(campaign_id, trial_id) +); +CREATE TABLE character_benchmark_review_bindings( + campaign_id TEXT NOT NULL, + trial_id TEXT NOT NULL, + experiment_id TEXT NOT NULL REFERENCES experiments(id), + review_id TEXT NOT NULL UNIQUE REFERENCES review_revisions(id), + session_id TEXT NOT NULL UNIQUE REFERENCES review_sessions(id), + reviewer_alias TEXT NOT NULL, + reused_from_trial_id TEXT, + display_mapping_json TEXT NOT NULL CHECK(json_valid(display_mapping_json) AND json_type(display_mapping_json) = 'object'), + PRIMARY KEY(campaign_id, trial_id), + FOREIGN KEY(campaign_id, trial_id) REFERENCES character_benchmark_trials(campaign_id, trial_id), + FOREIGN KEY(campaign_id, reused_from_trial_id) REFERENCES character_benchmark_trials(campaign_id, trial_id) +); +CREATE TABLE character_benchmark_identity_judgments( + campaign_id TEXT NOT NULL, + trial_id TEXT NOT NULL, + review_id TEXT NOT NULL, + choice TEXT NOT NULL CHECK(choice IN ('a','b','both','neither')), + display_choice TEXT NOT NULL CHECK(display_choice IN ('left','right','both','neither')), + observation_json TEXT NOT NULL CHECK(json_valid(observation_json) AND json_type(observation_json) = 'object'), + content_digest TEXT NOT NULL CHECK(length(content_digest) = 71 AND substr(content_digest, 1, 7) = 'sha256:'), + created_at TEXT NOT NULL CHECK(julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00'), + PRIMARY KEY(campaign_id, trial_id), + UNIQUE(review_id), + FOREIGN KEY(campaign_id, trial_id) REFERENCES character_benchmark_trials(campaign_id, trial_id) +); +CREATE TABLE character_benchmark_secondary_judgments( + campaign_id TEXT NOT NULL, + trial_id TEXT NOT NULL, + dimension TEXT NOT NULL CHECK(dimension IN ('prompt_adherence','motion_quality','temporal_consistency','visual_quality')), + choice TEXT NOT NULL CHECK(choice IN ('a','b','tie')), + observation_json TEXT NOT NULL CHECK(json_valid(observation_json) AND json_type(observation_json) = 'object'), + content_digest TEXT NOT NULL CHECK(length(content_digest) = 71 AND substr(content_digest, 1, 7) = 'sha256:'), + created_at TEXT NOT NULL CHECK(julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00'), + PRIMARY KEY(campaign_id, trial_id, dimension), + FOREIGN KEY(campaign_id, trial_id) REFERENCES character_benchmark_identity_judgments(campaign_id, trial_id) +); +CREATE TABLE character_benchmark_results( + campaign_id TEXT PRIMARY KEY REFERENCES character_benchmark_campaigns(id), + result_json TEXT NOT NULL CHECK(json_valid(result_json) AND json_type(result_json) = 'object'), + content_digest TEXT NOT NULL CHECK(length(content_digest) = 71 AND substr(content_digest, 1, 7) = 'sha256:'), + created_at TEXT NOT NULL CHECK(julianday(created_at) IS NOT NULL AND substr(created_at, -6) = '+00:00') +); +CREATE TRIGGER validate_character_benchmark_event_insert BEFORE INSERT ON character_benchmark_events BEGIN + SELECT CASE WHEN NEW.sequence <> COALESCE((SELECT MAX(sequence)+1 FROM character_benchmark_events WHERE campaign_id=NEW.campaign_id),1) THEN RAISE(ABORT,'invalid character benchmark event sequence') END; + SELECT CASE WHEN NEW.previous_state <> COALESCE((SELECT new_state FROM character_benchmark_events WHERE campaign_id=NEW.campaign_id ORDER BY sequence DESC LIMIT 1),'draft') THEN RAISE(ABORT,'invalid character benchmark prior state') END; + SELECT CASE WHEN NOT ((NEW.previous_state='draft' AND NEW.new_state='preregistered') OR (NEW.previous_state='preregistered' AND NEW.new_state='generating') OR (NEW.previous_state='generating' AND NEW.new_state='ready') OR (NEW.previous_state='ready' AND NEW.new_state='judging') OR (NEW.previous_state='judging' AND NEW.new_state='analyzing') OR (NEW.previous_state='analyzing' AND NEW.new_state='completed')) THEN RAISE(ABORT,'invalid character benchmark lifecycle transition') END; +END; +CREATE TRIGGER validate_character_benchmark_trial_insert BEFORE INSERT ON character_benchmark_trials BEGIN + SELECT CASE WHEN COALESCE((SELECT new_state FROM character_benchmark_events WHERE campaign_id=NEW.campaign_id ORDER BY sequence DESC LIMIT 1),'draft') <> 'preregistered' THEN RAISE(ABORT,'character benchmark trials require preregistration') END; +END; +CREATE TRIGGER guard_character_benchmark_campaign_update BEFORE UPDATE ON character_benchmark_campaigns BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_campaign_delete BEFORE DELETE ON character_benchmark_campaigns BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_trial_update BEFORE UPDATE ON character_benchmark_trials BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_trial_delete BEFORE DELETE ON character_benchmark_trials BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_experiment_binding_update BEFORE UPDATE ON character_benchmark_experiment_bindings BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_experiment_binding_delete BEFORE DELETE ON character_benchmark_experiment_bindings BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_review_binding_update BEFORE UPDATE ON character_benchmark_review_bindings BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_review_binding_delete BEFORE DELETE ON character_benchmark_review_bindings BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_identity_update BEFORE UPDATE ON character_benchmark_identity_judgments BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_identity_delete BEFORE DELETE ON character_benchmark_identity_judgments BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_secondary_update BEFORE UPDATE ON character_benchmark_secondary_judgments BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_secondary_delete BEFORE DELETE ON character_benchmark_secondary_judgments BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_event_update BEFORE UPDATE ON character_benchmark_events BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_event_delete BEFORE DELETE ON character_benchmark_events BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_result_update BEFORE UPDATE ON character_benchmark_results BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_result_delete BEFORE DELETE ON character_benchmark_results BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +""" + +_MIGRATION_18 = """ +DROP TRIGGER guard_character_benchmark_trial_update; +DROP TRIGGER guard_character_benchmark_secondary_update; +ALTER TABLE character_benchmark_trials ADD COLUMN expected_reviewer_id TEXT; +UPDATE character_benchmark_trials +SET expected_reviewer_id = character_benchmark_reviewer_id( + campaign_id, + trial_id, + (SELECT campaign.reviewer_id FROM character_benchmark_campaigns AS campaign + WHERE campaign.id = character_benchmark_trials.campaign_id) +); +ALTER TABLE character_benchmark_secondary_judgments ADD COLUMN review_id TEXT; +UPDATE character_benchmark_secondary_judgments +SET review_id = ( + SELECT identity.review_id FROM character_benchmark_identity_judgments AS identity + WHERE identity.campaign_id = character_benchmark_secondary_judgments.campaign_id + AND identity.trial_id = character_benchmark_secondary_judgments.trial_id +); +CREATE TRIGGER guard_character_benchmark_trial_update BEFORE UPDATE ON character_benchmark_trials BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER guard_character_benchmark_secondary_update BEFORE UPDATE ON character_benchmark_secondary_judgments BEGIN SELECT RAISE(ABORT,'character benchmark evidence is append-only'); END; +CREATE TRIGGER validate_character_benchmark_campaign_insert_v18 +BEFORE INSERT ON character_benchmark_campaigns BEGIN + SELECT CASE WHEN json_extract(NEW.definition_json, '$.id') IS NOT NEW.id + THEN RAISE(ABORT, 'character benchmark definition campaign mismatch') END; + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM character_packs AS pack + WHERE pack.storage_id = NEW.character_pack_storage_id + AND pack.id = NEW.character_pack_id + AND pack.reviewer_id = NEW.reviewer_id + AND pack.state = 'preregistered' + ) THEN RAISE(ABORT, 'character benchmark pack is not owned and preregistered') END; + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM validation_campaigns AS campaign + JOIN validation_results AS result ON result.campaign_id = campaign.id + WHERE campaign.id = NEW.stage_2_campaign_id + AND campaign.reviewer_id = NEW.reviewer_id + AND result.disposition = 'qualified' + AND (SELECT event.new_state FROM validation_events AS event + WHERE event.campaign_id = campaign.id + ORDER BY event.sequence DESC LIMIT 1) = 'qualified' + ) THEN RAISE(ABORT, 'character benchmark Stage 2 campaign is not owned and qualified') END; +END; +CREATE TRIGGER validate_character_benchmark_trial_insert_v18 +BEFORE INSERT ON character_benchmark_trials BEGIN + SELECT CASE WHEN NEW.expected_reviewer_id IS NULL OR NEW.expected_reviewer_id != + character_benchmark_reviewer_id( + NEW.campaign_id, + NEW.trial_id, + (SELECT campaign.reviewer_id FROM character_benchmark_campaigns AS campaign + WHERE campaign.id = NEW.campaign_id) + ) THEN RAISE(ABORT, 'character benchmark trial reviewer mismatch') END; + SELECT CASE WHEN json_extract(NEW.trial_json, '$.benchmark_id') IS NOT NEW.campaign_id + OR json_extract(NEW.trial_json, '$.id') IS NOT NEW.trial_id + OR json_extract(NEW.trial_json, '$.ordinal') IS NOT NEW.ordinal + OR json_extract(NEW.trial_json, '$.hidden_repeat') IS NOT NEW.hidden_repeat + OR json_extract(NEW.trial_json, '$.reuses_trial_id') IS NOT NEW.reuses_trial_id + OR json_extract(NEW.trial_json, '$.candidate_a_position') IS NOT NEW.candidate_a_position + THEN RAISE(ABORT, 'character benchmark trial columns do not match trial evidence') END; +END; +CREATE TRIGGER validate_character_benchmark_experiment_binding_insert_v18 +BEFORE INSERT ON character_benchmark_experiment_bindings BEGIN + SELECT CASE WHEN COALESCE( + (SELECT event.new_state FROM character_benchmark_events AS event + WHERE event.campaign_id = NEW.campaign_id ORDER BY event.sequence DESC LIMIT 1), + 'draft' + ) != 'generating' THEN RAISE(ABORT, 'character benchmark experiment has invalid lifecycle') END; + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM character_benchmark_trials AS trial + WHERE trial.campaign_id = NEW.campaign_id + AND trial.trial_id = NEW.trial_id + AND trial.hidden_repeat = 0 + AND NEW.experiment_id = character_benchmark_experiment_id( + NEW.campaign_id, NEW.trial_id + ) + ) THEN RAISE(ABORT, 'character benchmark experiment does not match trial') END; +END; +CREATE TRIGGER validate_character_benchmark_review_binding_insert_v18 +BEFORE INSERT ON character_benchmark_review_bindings BEGIN + SELECT CASE WHEN COALESCE( + (SELECT event.new_state FROM character_benchmark_events AS event + WHERE event.campaign_id = NEW.campaign_id ORDER BY event.sequence DESC LIMIT 1), + 'draft' + ) != 'judging' THEN RAISE(ABORT, 'character benchmark review has invalid lifecycle') END; + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM character_benchmark_trials AS trial + JOIN character_benchmark_experiment_bindings AS source + ON source.campaign_id = trial.campaign_id + AND source.trial_id = COALESCE(trial.reuses_trial_id, trial.trial_id) + AND source.experiment_id = NEW.experiment_id + JOIN review_sessions AS session + ON session.id = NEW.session_id + AND session.experiment_id = NEW.experiment_id + AND session.reviewer_id = trial.expected_reviewer_id + JOIN review_revisions AS revision + ON revision.id = NEW.review_id + AND revision.session_id = NEW.session_id + AND revision.revision = 1 + AND revision.supersedes_review_id IS NULL + WHERE trial.campaign_id = NEW.campaign_id + AND trial.trial_id = NEW.trial_id + AND NEW.reviewer_alias = trial.expected_reviewer_id + AND NEW.reused_from_trial_id IS trial.reuses_trial_id + ) THEN RAISE(ABORT, 'character benchmark review binding is inconsistent') END; +END; +CREATE TRIGGER validate_character_benchmark_identity_insert_v18 +BEFORE INSERT ON character_benchmark_identity_judgments BEGIN + SELECT CASE WHEN COALESCE( + (SELECT event.new_state FROM character_benchmark_events AS event + WHERE event.campaign_id = NEW.campaign_id ORDER BY event.sequence DESC LIMIT 1), + 'draft' + ) != 'judging' THEN RAISE(ABORT, 'character benchmark identity has invalid lifecycle') END; + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM character_benchmark_review_bindings AS binding + WHERE binding.campaign_id = NEW.campaign_id + AND binding.trial_id = NEW.trial_id + AND binding.review_id = NEW.review_id + ) THEN RAISE(ABORT, 'character benchmark identity review binding is inconsistent') END; + SELECT CASE WHEN json_extract(NEW.observation_json, '$.benchmark_id') IS NOT NEW.campaign_id + OR json_extract(NEW.observation_json, '$.trial_id') IS NOT NEW.trial_id + OR json_extract(NEW.observation_json, '$.choice') IS NOT NEW.choice + THEN RAISE(ABORT, 'character benchmark identity columns do not match observation') END; +END; +CREATE TRIGGER validate_character_benchmark_secondary_insert_v18 +BEFORE INSERT ON character_benchmark_secondary_judgments BEGIN + SELECT CASE WHEN COALESCE( + (SELECT event.new_state FROM character_benchmark_events AS event + WHERE event.campaign_id = NEW.campaign_id ORDER BY event.sequence DESC LIMIT 1), + 'draft' + ) != 'judging' THEN RAISE(ABORT, 'character benchmark secondary has invalid lifecycle') END; + SELECT CASE WHEN NEW.review_id IS NULL OR NOT EXISTS( + SELECT 1 FROM character_benchmark_identity_judgments AS identity + WHERE identity.campaign_id = NEW.campaign_id + AND identity.trial_id = NEW.trial_id + AND identity.review_id = NEW.review_id + AND identity.choice != 'neither' + ) THEN RAISE(ABORT, 'character benchmark secondary identity binding is inconsistent') END; + SELECT CASE WHEN json_extract(NEW.observation_json, '$.benchmark_id') IS NOT NEW.campaign_id + OR json_extract(NEW.observation_json, '$.trial_id') IS NOT NEW.trial_id + OR json_extract(NEW.observation_json, '$.dimension') IS NOT NEW.dimension + OR json_extract(NEW.observation_json, '$.choice') IS NOT NEW.choice + THEN RAISE(ABORT, 'character benchmark secondary columns do not match observation') END; +END; +CREATE TRIGGER validate_character_benchmark_result_insert_v18 +BEFORE INSERT ON character_benchmark_results BEGIN + SELECT CASE WHEN COALESCE( + (SELECT event.new_state FROM character_benchmark_events AS event + WHERE event.campaign_id = NEW.campaign_id ORDER BY event.sequence DESC LIMIT 1), + 'draft' + ) NOT IN ('analyzing','completed') + THEN RAISE(ABORT, 'character benchmark result has invalid lifecycle') END; + SELECT CASE WHEN json_extract(NEW.result_json, '$.benchmark_id') IS NOT NEW.campaign_id + THEN RAISE(ABORT, 'character benchmark result campaign mismatch') END; +END; +""" + +_MIGRATION_19 = """ +DROP TRIGGER validate_character_benchmark_secondary_insert_v18; +CREATE TRIGGER validate_character_benchmark_secondary_insert_v19 +BEFORE INSERT ON character_benchmark_secondary_judgments BEGIN + SELECT CASE WHEN COALESCE( + (SELECT event.new_state FROM character_benchmark_events AS event + WHERE event.campaign_id = NEW.campaign_id ORDER BY event.sequence DESC LIMIT 1), + 'draft' + ) != 'judging' THEN RAISE(ABORT, 'character benchmark secondary has invalid lifecycle') END; + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM character_benchmark_trials AS trial + WHERE trial.campaign_id = NEW.campaign_id + AND trial.trial_id = NEW.trial_id + AND trial.hidden_repeat = 0 + ) THEN RAISE(ABORT, 'character benchmark secondary requires a primary trial') END; + SELECT CASE WHEN NEW.review_id IS NULL OR NOT EXISTS( + SELECT 1 FROM character_benchmark_identity_judgments AS identity + WHERE identity.campaign_id = NEW.campaign_id + AND identity.trial_id = NEW.trial_id + AND identity.review_id = NEW.review_id + AND identity.choice != 'neither' + ) THEN RAISE(ABORT, 'character benchmark secondary identity binding is inconsistent') END; + SELECT CASE WHEN json_extract(NEW.observation_json, '$.benchmark_id') IS NOT NEW.campaign_id + OR json_extract(NEW.observation_json, '$.trial_id') IS NOT NEW.trial_id + OR json_extract(NEW.observation_json, '$.dimension') IS NOT NEW.dimension + OR json_extract(NEW.observation_json, '$.choice') IS NOT NEW.choice + THEN RAISE(ABORT, 'character benchmark secondary columns do not match observation') END; +END; +""" + +_MIGRATION_20 = """ +CREATE TRIGGER validate_character_pack_duplicate_provenance_insert +BEFORE INSERT ON character_packs WHEN NEW.state = 'preregistered' BEGIN + SELECT CASE WHEN + json_extract(NEW.manifest_json, '$.duplicate_hash_algorithm_version') + IS NOT 'dhash64-exif-transpose-lanczos.v1' + OR json_extract(NEW.manifest_json, '$.near_duplicate_threshold_inclusive') IS NOT 5 + THEN RAISE(ABORT, 'character pack duplicate provenance is incomplete') END; +END; +CREATE TRIGGER validate_character_pack_duplicate_provenance_update +BEFORE UPDATE ON character_packs +WHEN OLD.state = 'draft' AND NEW.state = 'preregistered' BEGIN + SELECT CASE WHEN + json_extract(NEW.manifest_json, '$.duplicate_hash_algorithm_version') + IS NOT 'dhash64-exif-transpose-lanczos.v1' + OR json_extract(NEW.manifest_json, '$.near_duplicate_threshold_inclusive') IS NOT 5 + THEN RAISE(ABORT, 'character pack duplicate provenance is incomplete') END; +END; +""" + +_MIGRATION_21 = "" + +_MIGRATION_22 = """ +CREATE TABLE training_datasets( + storage_id TEXT PRIMARY KEY, + id TEXT NOT NULL UNIQUE, + reviewer_id TEXT NOT NULL, + character_id TEXT NOT NULL, + character_pack_storage_id TEXT NOT NULL, + character_pack_digest TEXT NOT NULL CHECK(length(character_pack_digest)=71 AND substr(character_pack_digest,1,7)='sha256:'), + state TEXT NOT NULL CHECK(state IN ('draft','sealed')), + canonical_outfit_id TEXT NOT NULL, + reserved_token TEXT NOT NULL, + source_tier_weights_json TEXT NOT NULL CHECK(json_valid(source_tier_weights_json)), + manifest_json TEXT NOT NULL CHECK(json_valid(manifest_json) AND json_type(manifest_json)='object'), + dataset_digest TEXT UNIQUE, + created_at TEXT NOT NULL, + sealed_at TEXT, + UNIQUE(storage_id, reviewer_id), + UNIQUE(storage_id, character_id), + FOREIGN KEY(character_id, reviewer_id) REFERENCES characters(id, reviewer_id), + FOREIGN KEY(character_id,character_pack_storage_id) REFERENCES character_packs(character_id,storage_id), + FOREIGN KEY(character_pack_storage_id,reviewer_id) REFERENCES character_packs(storage_id,reviewer_id), + CHECK((state='draft' AND sealed_at IS NULL AND dataset_digest IS NULL) OR (state='sealed' AND sealed_at IS NOT NULL AND dataset_digest IS NOT NULL)) +); +CREATE TABLE training_dataset_aliases( + alias_id TEXT PRIMARY KEY, + dataset_storage_id TEXT NOT NULL REFERENCES training_datasets(storage_id) +); +CREATE TABLE training_samples( + dataset_storage_id TEXT NOT NULL REFERENCES training_datasets(storage_id), + sample_id TEXT NOT NULL, + source_tier TEXT NOT NULL CHECK(source_tier IN ('gold_still','gold_clip','synthetic_clip')), + sample_digest TEXT NOT NULL, + source_lineage TEXT NOT NULL, + caption TEXT NOT NULL, + split TEXT NOT NULL CHECK(split IN ('train','validation','sealed_holdout')), + media_json TEXT NOT NULL CHECK(json_valid(media_json)), + accepted INTEGER NOT NULL CHECK(accepted IN (0,1)), + ordinal INTEGER NOT NULL CHECK(ordinal>=0), + PRIMARY KEY(dataset_storage_id,sample_id), + UNIQUE(dataset_storage_id,ordinal), + UNIQUE(dataset_storage_id,sample_digest) +); +CREATE TABLE training_sample_annotations( + dataset_storage_id TEXT NOT NULL, + sample_id TEXT NOT NULL, + annotation_json TEXT NOT NULL CHECK(json_valid(annotation_json) AND json_type(annotation_json)='object'), + PRIMARY KEY(dataset_storage_id,sample_id), + FOREIGN KEY(dataset_storage_id,sample_id) REFERENCES training_samples(dataset_storage_id,sample_id) +); +CREATE INDEX training_datasets_owner ON training_datasets(reviewer_id,character_id,state); +CREATE INDEX training_samples_split ON training_samples(dataset_storage_id,split,source_tier); +CREATE TRIGGER validate_training_dataset_insert BEFORE INSERT ON training_datasets BEGIN SELECT CASE WHEN NEW.state!='draft' OR training_draft_manifest_is_valid(NEW.id,NEW.character_id,NEW.character_pack_digest,NEW.canonical_outfit_id,NEW.reserved_token,NEW.source_tier_weights_json,NEW.manifest_json)!=1 OR NOT EXISTS(SELECT 1 FROM character_packs pack WHERE pack.storage_id=NEW.character_pack_storage_id AND pack.character_id=NEW.character_id AND pack.reviewer_id=NEW.reviewer_id AND pack.state='preregistered' AND canonical_json_digest(pack.manifest_json)=NEW.character_pack_digest) THEN RAISE(ABORT,'training dataset identity or character pack provenance mismatch') END; END; +CREATE TRIGGER materialize_training_dataset AFTER INSERT ON training_datasets BEGIN + INSERT INTO training_dataset_aliases(alias_id,dataset_storage_id) VALUES(NEW.id,NEW.storage_id); + INSERT INTO training_samples(dataset_storage_id,sample_id,source_tier,sample_digest,source_lineage,caption,split,media_json,accepted,ordinal) SELECT NEW.storage_id,json_extract(item.value,'$.sample_id'),json_extract(item.value,'$.source_tier'),json_extract(item.value,'$.sample_digest'),json_extract(item.value,'$.source_lineage'),json_extract(item.value,'$.caption'),json_extract(item.value,'$.split'),json_extract(item.value,'$.media'),json_extract(item.value,'$.accepted'),CAST(item.key AS INTEGER) FROM json_each(NEW.manifest_json,'$.samples') item; + INSERT INTO training_sample_annotations(dataset_storage_id,sample_id,annotation_json) SELECT NEW.storage_id,json_extract(item.value,'$.sample_id'),json_extract(item.value,'$.annotation') FROM json_each(NEW.manifest_json,'$.samples') item; +END; +CREATE TRIGGER validate_training_dataset_seal BEFORE UPDATE ON training_datasets WHEN OLD.state='draft' AND NEW.state='sealed' BEGIN SELECT CASE WHEN NEW.storage_id!=OLD.storage_id OR NEW.reviewer_id!=OLD.reviewer_id OR NEW.character_id!=OLD.character_id OR NEW.character_pack_storage_id!=OLD.character_pack_storage_id OR NEW.character_pack_digest!=OLD.character_pack_digest OR NEW.canonical_outfit_id!=OLD.canonical_outfit_id OR NEW.reserved_token!=OLD.reserved_token OR NEW.source_tier_weights_json!=OLD.source_tier_weights_json OR NEW.created_at!=OLD.created_at OR NEW.dataset_digest IS NULL OR NEW.sealed_at IS NULL OR training_dataset_identity_is_valid(OLD.storage_id,NEW.dataset_digest,NEW.manifest_json)!=1 THEN RAISE(ABORT,'invalid training dataset seal') END; END; +CREATE TRIGGER guard_training_dataset_other_update BEFORE UPDATE ON training_datasets WHEN OLD.state IN ('draft','sealed') AND NOT (OLD.state='draft' AND NEW.state='sealed') BEGIN SELECT RAISE(ABORT,'training dataset snapshots are immutable'); END; +CREATE TRIGGER guard_training_dataset_delete BEFORE DELETE ON training_datasets BEGIN SELECT RAISE(ABORT,'training dataset snapshots are immutable'); END; +CREATE TRIGGER validate_training_alias_insert BEFORE INSERT ON training_dataset_aliases BEGIN SELECT CASE WHEN NEW.alias_id!=(SELECT id FROM training_datasets WHERE storage_id=NEW.dataset_storage_id) OR EXISTS(SELECT 1 FROM training_dataset_aliases WHERE dataset_storage_id=NEW.dataset_storage_id) THEN RAISE(ABORT,'training dataset alias is not canonical') END; END; +CREATE TRIGGER guard_training_alias_update BEFORE UPDATE ON training_dataset_aliases BEGIN SELECT RAISE(ABORT,'training dataset aliases are immutable'); END; +CREATE TRIGGER guard_training_alias_delete BEFORE DELETE ON training_dataset_aliases BEGIN SELECT RAISE(ABORT,'training dataset aliases are immutable'); END; +CREATE TRIGGER validate_training_sample_insert BEFORE INSERT ON training_samples BEGIN SELECT CASE WHEN NOT EXISTS(SELECT 1 FROM training_datasets parent,json_each(parent.manifest_json,'$.samples') item WHERE parent.storage_id=NEW.dataset_storage_id AND CAST(item.key AS INTEGER)=NEW.ordinal AND json_extract(item.value,'$.sample_id')=NEW.sample_id AND json_extract(item.value,'$.source_tier')=NEW.source_tier AND json_extract(item.value,'$.sample_digest')=NEW.sample_digest AND json_extract(item.value,'$.source_lineage')=NEW.source_lineage AND json_extract(item.value,'$.caption')=NEW.caption AND json_extract(item.value,'$.split')=NEW.split AND canonical_json_digest(json_extract(item.value,'$.media'))=canonical_json_digest(NEW.media_json) AND json_extract(item.value,'$.accepted')=NEW.accepted) THEN RAISE(ABORT,'training sample does not match canonical manifest') END; END; +CREATE TRIGGER guard_training_sample_update BEFORE UPDATE ON training_samples BEGIN SELECT RAISE(ABORT,'training samples are immutable'); END; +CREATE TRIGGER guard_training_sample_delete BEFORE DELETE ON training_samples BEGIN SELECT RAISE(ABORT,'training samples are immutable'); END; +CREATE TRIGGER validate_training_annotation_insert BEFORE INSERT ON training_sample_annotations BEGIN SELECT CASE WHEN NOT EXISTS(SELECT 1 FROM training_datasets parent,json_each(parent.manifest_json,'$.samples') item WHERE parent.storage_id=NEW.dataset_storage_id AND json_extract(item.value,'$.sample_id')=NEW.sample_id AND canonical_json_digest(json_extract(item.value,'$.annotation'))=canonical_json_digest(NEW.annotation_json)) THEN RAISE(ABORT,'training annotation does not match canonical manifest') END; END; +CREATE TRIGGER guard_training_annotation_update BEFORE UPDATE ON training_sample_annotations BEGIN SELECT RAISE(ABORT,'training annotations are immutable'); END; +CREATE TRIGGER guard_training_annotation_delete BEFORE DELETE ON training_sample_annotations BEGIN SELECT RAISE(ABORT,'training annotations are immutable'); END; +""" + +_MIGRATION_23 = """ +CREATE TABLE training_recipes( + storage_id TEXT PRIMARY KEY, + id TEXT NOT NULL UNIQUE, + reviewer_id TEXT NOT NULL, + dataset_storage_id TEXT NOT NULL, + dataset_digest TEXT NOT NULL, + character_id TEXT NOT NULL, + character_pack_digest TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('draft','preregistered')), + recipe_json TEXT NOT NULL CHECK(json_valid(recipe_json) AND json_type(recipe_json)='object'), + created_at TEXT NOT NULL, + preregistered_at TEXT, + UNIQUE(storage_id,reviewer_id), + UNIQUE(storage_id,dataset_storage_id), + FOREIGN KEY(dataset_storage_id,reviewer_id) REFERENCES training_datasets(storage_id,reviewer_id), + FOREIGN KEY(dataset_storage_id,character_id) REFERENCES training_datasets(storage_id,character_id), + CHECK((state='draft' AND preregistered_at IS NULL) OR (state='preregistered' AND preregistered_at IS NOT NULL)) +); +CREATE TABLE training_runs( + id TEXT PRIMARY KEY, + reviewer_id TEXT NOT NULL, + recipe_storage_id TEXT NOT NULL, + dataset_storage_id TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('created','running','completed','failed','cancelled')), + total_steps INTEGER NOT NULL CHECK(total_steps>0), + created_at TEXT NOT NULL, + terminal_at TEXT, + UNIQUE(id,reviewer_id), UNIQUE(id,recipe_storage_id), + FOREIGN KEY(recipe_storage_id,reviewer_id) REFERENCES training_recipes(storage_id,reviewer_id), + FOREIGN KEY(recipe_storage_id,dataset_storage_id) REFERENCES training_recipes(storage_id,dataset_storage_id), + CHECK((state IN ('created','running') AND terminal_at IS NULL) OR (state IN ('completed','failed','cancelled') AND terminal_at IS NOT NULL)) +); +CREATE TABLE training_run_events( + run_id TEXT NOT NULL, reviewer_id TEXT NOT NULL, sequence INTEGER NOT NULL CHECK(sequence>0), + previous_state TEXT NOT NULL, new_state TEXT NOT NULL, payload_json TEXT NOT NULL CHECK(json_valid(payload_json)), created_at TEXT NOT NULL, + PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,reviewer_id) REFERENCES training_runs(id,reviewer_id) +); +CREATE TABLE training_telemetry( + run_id TEXT NOT NULL, reviewer_id TEXT NOT NULL, sequence INTEGER NOT NULL CHECK(sequence>0), step INTEGER NOT NULL CHECK(step>=0), payload_json TEXT NOT NULL CHECK(json_valid(payload_json)), created_at TEXT NOT NULL, + PRIMARY KEY(run_id,sequence), UNIQUE(run_id,step), FOREIGN KEY(run_id,reviewer_id) REFERENCES training_runs(id,reviewer_id) +); +CREATE TABLE training_failures( + id TEXT PRIMARY KEY, run_id TEXT NOT NULL, reviewer_id TEXT NOT NULL, code TEXT NOT NULL, message TEXT NOT NULL, created_at TEXT NOT NULL, + FOREIGN KEY(run_id,reviewer_id) REFERENCES training_runs(id,reviewer_id) +); +CREATE INDEX training_recipes_dataset ON training_recipes(dataset_storage_id,state); +CREATE INDEX training_runs_recipe ON training_runs(recipe_storage_id,state); +CREATE TRIGGER validate_training_recipe_insert BEFORE INSERT ON training_recipes BEGIN SELECT CASE WHEN (SELECT state FROM training_datasets WHERE storage_id=NEW.dataset_storage_id)!='sealed' OR (SELECT dataset_digest FROM training_datasets WHERE storage_id=NEW.dataset_storage_id)!=NEW.dataset_digest OR json_extract(NEW.recipe_json,'$.dataset_digest')!=NEW.dataset_digest OR json_extract(NEW.recipe_json,'$.character_pack_digest')!=NEW.character_pack_digest OR (SELECT character_pack_digest FROM training_datasets WHERE storage_id=NEW.dataset_storage_id)!=NEW.character_pack_digest OR training_recipe_identity_is_valid(NEW.dataset_storage_id,NEW.id,NEW.recipe_json)!=1 OR NOT EXISTS(SELECT 1 FROM training_datasets dataset JOIN character_benchmark_campaigns benchmark ON benchmark.character_pack_storage_id=dataset.character_pack_storage_id AND benchmark.reviewer_id=NEW.reviewer_id JOIN character_benchmark_results result ON result.campaign_id=benchmark.id WHERE dataset.storage_id=NEW.dataset_storage_id AND character_benchmark_provenance_is_valid(benchmark.id)=1 AND result.content_digest=json_extract(NEW.recipe_json,'$.stage_3_evidence_digest') AND json_extract(benchmark.definition_json,'$.character_pack_digest')=NEW.character_pack_digest AND json_extract(benchmark.definition_json,'$.stage_2_manifest_digest')=json_extract(NEW.recipe_json,'$.stage_2_manifest_digest') AND json_extract(benchmark.definition_json,'$.stage_2_evidence_digest')=json_extract(NEW.recipe_json,'$.stage_2_evidence_digest') AND (SELECT new_state FROM character_benchmark_events WHERE campaign_id=benchmark.id ORDER BY sequence DESC LIMIT 1)='completed') THEN RAISE(ABORT,'training recipe dataset identity mismatch') END; END; +CREATE TRIGGER guard_training_recipe_update BEFORE UPDATE ON training_recipes WHEN OLD.state='preregistered' BEGIN SELECT RAISE(ABORT,'preregistered training recipe is immutable'); END; +CREATE TRIGGER validate_training_recipe_preregister BEFORE UPDATE ON training_recipes WHEN OLD.state='draft' BEGIN SELECT CASE WHEN NEW.storage_id!=OLD.storage_id OR NEW.id!=OLD.id OR NEW.reviewer_id!=OLD.reviewer_id OR NEW.dataset_storage_id!=OLD.dataset_storage_id OR NEW.dataset_digest!=OLD.dataset_digest OR NEW.character_id!=OLD.character_id OR NEW.character_pack_digest!=OLD.character_pack_digest OR NEW.recipe_json!=OLD.recipe_json OR NEW.created_at!=OLD.created_at OR NEW.state!='preregistered' OR NEW.preregistered_at IS NULL OR NOT EXISTS(SELECT 1 FROM training_datasets dataset JOIN character_benchmark_campaigns benchmark ON benchmark.character_pack_storage_id=dataset.character_pack_storage_id AND benchmark.reviewer_id=NEW.reviewer_id JOIN character_benchmark_results result ON result.campaign_id=benchmark.id WHERE dataset.storage_id=NEW.dataset_storage_id AND character_benchmark_provenance_is_valid(benchmark.id)=1 AND result.content_digest=json_extract(NEW.recipe_json,'$.stage_3_evidence_digest') AND json_extract(benchmark.definition_json,'$.character_pack_digest')=NEW.character_pack_digest AND json_extract(benchmark.definition_json,'$.stage_2_manifest_digest')=json_extract(NEW.recipe_json,'$.stage_2_manifest_digest') AND json_extract(benchmark.definition_json,'$.stage_2_evidence_digest')=json_extract(NEW.recipe_json,'$.stage_2_evidence_digest') AND (SELECT new_state FROM character_benchmark_events WHERE campaign_id=benchmark.id ORDER BY sequence DESC LIMIT 1)='completed') THEN RAISE(ABORT,'invalid training recipe preregistration') END; END; +CREATE TRIGGER guard_training_recipe_delete BEFORE DELETE ON training_recipes WHEN OLD.state='preregistered' BEGIN SELECT RAISE(ABORT,'preregistered training recipe is immutable'); END; +CREATE TRIGGER validate_training_run_insert BEFORE INSERT ON training_runs BEGIN SELECT CASE WHEN (SELECT state FROM training_recipes WHERE storage_id=NEW.recipe_storage_id)!='preregistered' OR (SELECT dataset_storage_id FROM training_recipes WHERE storage_id=NEW.recipe_storage_id)!=NEW.dataset_storage_id OR NEW.total_steps!=(SELECT json_extract(recipe_json,'$.total_steps') FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) THEN RAISE(ABORT,'training run identity mismatch') END; END; +CREATE TRIGGER validate_training_event_insert BEFORE INSERT ON training_run_events BEGIN + SELECT CASE WHEN NEW.sequence!=COALESCE((SELECT max(sequence)+1 FROM training_run_events WHERE run_id=NEW.run_id),1) THEN RAISE(ABORT,'training event sequence is not append-only') END; + SELECT CASE WHEN (SELECT state FROM training_runs WHERE id=NEW.run_id)!=NEW.previous_state OR NOT ((NEW.previous_state='created' AND NEW.new_state='running') OR (NEW.previous_state='running' AND NEW.new_state IN ('completed','failed','cancelled'))) THEN RAISE(ABORT,'invalid training lifecycle transition') END; + SELECT CASE WHEN NEW.new_state='completed' AND (COALESCE((SELECT max(step) FROM training_telemetry WHERE run_id=NEW.run_id),-1)!=(SELECT total_steps FROM training_runs WHERE id=NEW.run_id) OR NOT EXISTS(SELECT 1 FROM training_checkpoints WHERE run_id=NEW.run_id AND fraction=1.0)) THEN RAISE(ABORT,'training completion evidence is incomplete') END; +END; +CREATE TRIGGER apply_training_event AFTER INSERT ON training_run_events BEGIN UPDATE training_runs SET state=NEW.new_state,terminal_at=CASE WHEN NEW.new_state IN ('completed','failed','cancelled') THEN NEW.created_at ELSE NULL END WHERE id=NEW.run_id; END; +CREATE TRIGGER guard_training_event_update BEFORE UPDATE ON training_run_events BEGIN SELECT RAISE(ABORT,'training events are immutable'); END; +CREATE TRIGGER guard_training_event_delete BEFORE DELETE ON training_run_events BEGIN SELECT RAISE(ABORT,'training events are immutable'); END; +CREATE TRIGGER validate_training_telemetry_insert BEFORE INSERT ON training_telemetry BEGIN SELECT CASE WHEN (SELECT state FROM training_runs WHERE id=NEW.run_id)!='running' OR NEW.sequence!=COALESCE((SELECT max(sequence)+1 FROM training_telemetry WHERE run_id=NEW.run_id),1) OR NEW.step>(SELECT total_steps FROM training_runs WHERE id=NEW.run_id) OR NEW.step<=COALESCE((SELECT max(step) FROM training_telemetry WHERE run_id=NEW.run_id),-1) THEN RAISE(ABORT,'invalid training telemetry sequence') END; END; +CREATE TRIGGER guard_training_telemetry_update BEFORE UPDATE ON training_telemetry BEGIN SELECT RAISE(ABORT,'training telemetry is immutable'); END; +CREATE TRIGGER guard_training_telemetry_delete BEFORE DELETE ON training_telemetry BEGIN SELECT RAISE(ABORT,'training telemetry is immutable'); END; +CREATE TRIGGER guard_training_failure_update BEFORE UPDATE ON training_failures BEGIN SELECT RAISE(ABORT,'training failures are immutable'); END; +CREATE TRIGGER guard_training_failure_delete BEFORE DELETE ON training_failures BEGIN SELECT RAISE(ABORT,'training failures are immutable'); END; +""" + +_MIGRATION_24 = """ +CREATE TABLE training_checkpoints( + id TEXT PRIMARY KEY, run_id TEXT NOT NULL, reviewer_id TEXT NOT NULL, recipe_storage_id TEXT NOT NULL, dataset_storage_id TEXT NOT NULL, + step INTEGER NOT NULL CHECK(step>0), fraction REAL NOT NULL CHECK(fraction IN (0.25,0.5,0.75,1.0)), parent_checkpoint_id TEXT, + artifact_digest TEXT NOT NULL CHECK(length(artifact_digest)=71 AND substr(artifact_digest,1,7)='sha256:'), + manifest_digest TEXT NOT NULL CHECK(length(manifest_digest)=71 AND substr(manifest_digest,1,7)='sha256:'), manifest_json TEXT NOT NULL CHECK(json_valid(manifest_json) AND json_type(manifest_json)='object'), created_at TEXT NOT NULL, + UNIQUE(run_id,step), UNIQUE(run_id,fraction), UNIQUE(run_id,artifact_digest), UNIQUE(run_id,manifest_digest), + FOREIGN KEY(run_id,recipe_storage_id) REFERENCES training_runs(id,recipe_storage_id), + FOREIGN KEY(run_id,reviewer_id) REFERENCES training_runs(id,reviewer_id), + FOREIGN KEY(recipe_storage_id,dataset_storage_id) REFERENCES training_recipes(storage_id,dataset_storage_id), + FOREIGN KEY(parent_checkpoint_id) REFERENCES training_checkpoints(id) +); +CREATE INDEX training_checkpoints_run ON training_checkpoints(run_id,step); +CREATE TRIGGER validate_training_checkpoint_insert BEFORE INSERT ON training_checkpoints BEGIN + SELECT CASE WHEN NEW.dataset_storage_id!=(SELECT dataset_storage_id FROM training_runs WHERE id=NEW.run_id) OR NEW.step!=training_checkpoint_step((SELECT total_steps FROM training_runs WHERE id=NEW.run_id),NEW.fraction) OR NEW.step>(SELECT total_steps FROM training_runs WHERE id=NEW.run_id) OR (SELECT state FROM training_runs WHERE id=NEW.run_id)!='running' OR NEW.step>COALESCE((SELECT max(step) FROM training_telemetry WHERE run_id=NEW.run_id),-1) OR NOT EXISTS(SELECT 1 FROM training_recipes recipe,json_each(recipe.recipe_json,'$.checkpoint_plan.fractions') declared WHERE recipe.storage_id=NEW.recipe_storage_id AND declared.value=NEW.fraction) OR json_extract(NEW.manifest_json,'$.schema_version')!='character-training-checkpoint.v1' OR json_extract(NEW.manifest_json,'$.run_id')!=NEW.run_id OR json_extract(NEW.manifest_json,'$.reviewer_id')!=NEW.reviewer_id OR json_extract(NEW.manifest_json,'$.recipe_id')!=(SELECT id FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) OR json_extract(NEW.manifest_json,'$.dataset_digest')!=(SELECT dataset_digest FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) OR json_extract(NEW.manifest_json,'$.character_id')!=(SELECT character_id FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) OR json_extract(NEW.manifest_json,'$.step')!=NEW.step OR json_extract(NEW.manifest_json,'$.fraction')!=NEW.fraction OR json_extract(NEW.manifest_json,'$.parent_checkpoint_id') IS NOT NEW.parent_checkpoint_id OR json_extract(NEW.manifest_json,'$.artifact_digest')!=NEW.artifact_digest OR canonical_json_digest(NEW.manifest_json)!=NEW.manifest_digest THEN RAISE(ABORT,'training checkpoint identity mismatch') END; + SELECT CASE WHEN (NEW.fraction=0.25 AND NEW.parent_checkpoint_id IS NOT NULL) OR (NEW.fraction>0.25 AND (SELECT run_id FROM training_checkpoints WHERE id=NEW.parent_checkpoint_id) IS NOT NEW.run_id) OR (NEW.fraction>0.25 AND (SELECT step FROM training_checkpoints WHERE id=NEW.parent_checkpoint_id)!=(SELECT max(step) FROM training_checkpoints WHERE run_id=NEW.run_id)) OR (NEW.fraction>0.25 AND (SELECT fraction FROM training_checkpoints WHERE id=NEW.parent_checkpoint_id)!=NEW.fraction-0.25) THEN RAISE(ABORT,'training checkpoint parent mismatch') END; +END; +CREATE TRIGGER guard_training_checkpoint_update BEFORE UPDATE ON training_checkpoints BEGIN SELECT RAISE(ABORT,'training checkpoints are immutable'); END; +CREATE TRIGGER guard_training_checkpoint_delete BEFORE DELETE ON training_checkpoints BEGIN SELECT RAISE(ABORT,'training checkpoints are immutable'); END; +""" + +_MIGRATION_25 = """ +CREATE TABLE training_dataset_lineages( + id TEXT PRIMARY KEY, + reviewer_id TEXT NOT NULL, + character_id TEXT NOT NULL, + current_dataset_storage_id TEXT NOT NULL UNIQUE, + head_version INTEGER NOT NULL CHECK(head_version>0), + state TEXT NOT NULL CHECK(state IN ('draft','sealed')), + created_at TEXT NOT NULL, + UNIQUE(id,reviewer_id), UNIQUE(id,character_id), + FOREIGN KEY(character_id,reviewer_id) REFERENCES characters(id,reviewer_id), + FOREIGN KEY(current_dataset_storage_id,reviewer_id) REFERENCES training_datasets(storage_id,reviewer_id), + FOREIGN KEY(current_dataset_storage_id,character_id) REFERENCES training_datasets(storage_id,character_id) +); +CREATE TABLE training_dataset_head_events( + lineage_id TEXT NOT NULL, + version INTEGER NOT NULL CHECK(version>0), + reviewer_id TEXT NOT NULL, + character_id TEXT NOT NULL, + previous_dataset_storage_id TEXT, + dataset_storage_id TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('draft','sealed')), + created_at TEXT NOT NULL, + PRIMARY KEY(lineage_id,version), + FOREIGN KEY(lineage_id,reviewer_id) REFERENCES training_dataset_lineages(id,reviewer_id), + FOREIGN KEY(lineage_id,character_id) REFERENCES training_dataset_lineages(id,character_id), + FOREIGN KEY(dataset_storage_id,reviewer_id) REFERENCES training_datasets(storage_id,reviewer_id), + FOREIGN KEY(dataset_storage_id,character_id) REFERENCES training_datasets(storage_id,character_id), + FOREIGN KEY(previous_dataset_storage_id) REFERENCES training_datasets(storage_id) +); +CREATE TABLE training_imported_samples( + id TEXT PRIMARY KEY, + reviewer_id TEXT NOT NULL, + character_id TEXT NOT NULL, + lineage_id TEXT NOT NULL, + artifact_id TEXT NOT NULL, + artifact_digest TEXT NOT NULL CHECK(length(artifact_digest)=71 AND substr(artifact_digest,1,7)='sha256:'), + payload_json TEXT NOT NULL CHECK(json_valid(payload_json) AND json_type(payload_json)='object'), + created_at TEXT NOT NULL, + UNIQUE(lineage_id,artifact_digest), + FOREIGN KEY(lineage_id,reviewer_id) REFERENCES training_dataset_lineages(id,reviewer_id), + FOREIGN KEY(lineage_id,character_id) REFERENCES training_dataset_lineages(id,character_id), + FOREIGN KEY(artifact_id) REFERENCES artifacts(id) +); +CREATE TABLE training_curation_confirmations( + id TEXT PRIMARY KEY, + import_id TEXT NOT NULL, + reviewer_id TEXT NOT NULL, + version INTEGER NOT NULL CHECK(version>0), + decision TEXT NOT NULL CHECK(decision IN ('accept','reject')), + payload_json TEXT NOT NULL CHECK(json_valid(payload_json) AND json_type(payload_json)='object'), + created_at TEXT NOT NULL, + UNIQUE(import_id,reviewer_id,version), + FOREIGN KEY(import_id) REFERENCES training_imported_samples(id) +); +CREATE INDEX training_imports_lineage ON training_imported_samples(lineage_id,created_at,id); +CREATE INDEX training_confirmations_import ON training_curation_confirmations(import_id,version); +CREATE TRIGGER validate_training_lineage_insert BEFORE INSERT ON training_dataset_lineages BEGIN + SELECT CASE WHEN NEW.head_version!=1 OR NEW.state!='draft' OR NOT EXISTS( + SELECT 1 FROM training_datasets d WHERE d.storage_id=NEW.current_dataset_storage_id + AND d.reviewer_id=NEW.reviewer_id AND d.character_id=NEW.character_id AND d.state='draft' + ) THEN RAISE(ABORT,'invalid initial training dataset head') END; +END; +CREATE TRIGGER validate_training_head_event_insert BEFORE INSERT ON training_dataset_head_events WHEN NEW.version>1 BEGIN + SELECT CASE WHEN NEW.reviewer_id!=(SELECT reviewer_id FROM training_dataset_lineages WHERE id=NEW.lineage_id) + OR NEW.character_id!=(SELECT character_id FROM training_dataset_lineages WHERE id=NEW.lineage_id) + OR NEW.version!=(SELECT head_version+1 FROM training_dataset_lineages WHERE id=NEW.lineage_id) + OR NEW.previous_dataset_storage_id!=(SELECT current_dataset_storage_id FROM training_dataset_lineages WHERE id=NEW.lineage_id) + OR (SELECT state FROM training_dataset_lineages WHERE id=NEW.lineage_id)!='draft' + OR NOT EXISTS(SELECT 1 FROM training_datasets d WHERE d.storage_id=NEW.dataset_storage_id + AND d.reviewer_id=NEW.reviewer_id AND d.character_id=NEW.character_id AND d.state=NEW.state) + OR (NEW.state='draft' AND EXISTS(SELECT 1 FROM training_dataset_head_events old + WHERE old.lineage_id=NEW.lineage_id AND old.dataset_storage_id=NEW.dataset_storage_id)) + OR (NEW.state='sealed' AND NEW.dataset_storage_id!=NEW.previous_dataset_storage_id) + THEN RAISE(ABORT,'invalid training dataset head event') END; +END; +CREATE TRIGGER validate_initial_training_head_event BEFORE INSERT ON training_dataset_head_events WHEN NEW.version=1 BEGIN + SELECT CASE WHEN NEW.previous_dataset_storage_id IS NOT NULL + OR NEW.dataset_storage_id!=(SELECT current_dataset_storage_id FROM training_dataset_lineages WHERE id=NEW.lineage_id) + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_dataset_lineages WHERE id=NEW.lineage_id) + OR NEW.character_id!=(SELECT character_id FROM training_dataset_lineages WHERE id=NEW.lineage_id) + OR NEW.state!='draft' THEN RAISE(ABORT,'invalid initial training dataset head event') END; +END; +CREATE TRIGGER validate_training_lineage_update BEFORE UPDATE ON training_dataset_lineages BEGIN + SELECT CASE WHEN NEW.id!=OLD.id OR NEW.reviewer_id!=OLD.reviewer_id OR NEW.character_id!=OLD.character_id + OR NEW.created_at!=OLD.created_at OR OLD.state!='draft' OR NEW.head_version!=OLD.head_version+1 + OR NOT EXISTS(SELECT 1 FROM training_dataset_head_events e WHERE e.lineage_id=OLD.id + AND e.version=NEW.head_version AND e.previous_dataset_storage_id=OLD.current_dataset_storage_id + AND e.dataset_storage_id=NEW.current_dataset_storage_id AND e.state=NEW.state) + THEN RAISE(ABORT,'invalid training dataset head CAS') END; +END; +CREATE TRIGGER guard_training_lineage_delete BEFORE DELETE ON training_dataset_lineages BEGIN SELECT RAISE(ABORT,'training dataset lineages are immutable'); END; +CREATE TRIGGER guard_training_head_event_update BEFORE UPDATE ON training_dataset_head_events BEGIN SELECT RAISE(ABORT,'training dataset head history is immutable'); END; +CREATE TRIGGER guard_training_head_event_delete BEFORE DELETE ON training_dataset_head_events BEGIN SELECT RAISE(ABORT,'training dataset head history is immutable'); END; +CREATE TRIGGER validate_training_import_insert BEFORE INSERT ON training_imported_samples BEGIN + SELECT CASE WHEN json_extract(NEW.payload_json,'$.import_id')!=NEW.id + OR json_extract(NEW.payload_json,'$.reviewer_id')!=NEW.reviewer_id + OR json_extract(NEW.payload_json,'$.character_id')!=NEW.character_id + OR json_extract(NEW.payload_json,'$.lineage_id')!=NEW.lineage_id + OR json_extract(NEW.payload_json,'$.artifact_id')!=NEW.artifact_id + OR json_extract(NEW.payload_json,'$.artifact_digest')!=NEW.artifact_digest + OR NOT EXISTS(SELECT 1 FROM artifacts a WHERE a.id=NEW.artifact_id AND 'sha256:'||a.sha256=NEW.artifact_digest) + THEN RAISE(ABORT,'invalid imported training sample') END; +END; +CREATE TRIGGER guard_training_import_update BEFORE UPDATE ON training_imported_samples BEGIN SELECT RAISE(ABORT,'imported training samples are immutable'); END; +CREATE TRIGGER guard_training_import_delete BEFORE DELETE ON training_imported_samples BEGIN SELECT RAISE(ABORT,'imported training samples are immutable'); END; +CREATE TRIGGER validate_training_confirmation_insert BEFORE INSERT ON training_curation_confirmations BEGIN + SELECT CASE WHEN json_extract(NEW.payload_json,'$.confirmation_id')!=NEW.id + OR json_extract(NEW.payload_json,'$.import_id')!=NEW.import_id + OR json_extract(NEW.payload_json,'$.reviewer_id')!=NEW.reviewer_id + OR json_extract(NEW.payload_json,'$.version')!=NEW.version + OR json_extract(NEW.payload_json,'$.decision')!=NEW.decision + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_imported_samples WHERE id=NEW.import_id) + OR NEW.version!=COALESCE((SELECT max(version)+1 FROM training_curation_confirmations WHERE import_id=NEW.import_id AND reviewer_id=NEW.reviewer_id),1) + THEN RAISE(ABORT,'invalid training curation confirmation') END; +END; +CREATE TRIGGER guard_training_confirmation_update BEFORE UPDATE ON training_curation_confirmations BEGIN SELECT RAISE(ABORT,'training confirmations are immutable'); END; +CREATE TRIGGER guard_training_confirmation_delete BEFORE DELETE ON training_curation_confirmations BEGIN SELECT RAISE(ABORT,'training confirmations are immutable'); END; +""" + +_MIGRATION_26 = """ +DROP TRIGGER validate_training_lineage_insert; +DROP TRIGGER validate_training_head_event_insert; +DROP TRIGGER validate_initial_training_head_event; +DROP TRIGGER validate_training_lineage_update; +DROP TRIGGER guard_training_head_event_update; +DROP TRIGGER guard_training_head_event_delete; +DROP TRIGGER validate_training_import_insert; +ALTER TABLE training_dataset_head_events RENAME TO training_dataset_head_events_v25; +CREATE TABLE training_dataset_snapshot_lineages( + dataset_storage_id TEXT PRIMARY KEY, + lineage_id TEXT NOT NULL, + reviewer_id TEXT NOT NULL, + character_id TEXT NOT NULL, + first_version INTEGER NOT NULL CHECK(first_version>0), + FOREIGN KEY(lineage_id,reviewer_id) REFERENCES training_dataset_lineages(id,reviewer_id), + FOREIGN KEY(lineage_id,character_id) REFERENCES training_dataset_lineages(id,character_id), + FOREIGN KEY(dataset_storage_id,reviewer_id) REFERENCES training_datasets(storage_id,reviewer_id), + FOREIGN KEY(dataset_storage_id,character_id) REFERENCES training_datasets(storage_id,character_id) +); +CREATE TABLE training_dataset_snapshots( + lineage_id TEXT NOT NULL, + version INTEGER NOT NULL CHECK(version>0), + reviewer_id TEXT NOT NULL, + character_id TEXT NOT NULL, + previous_dataset_storage_id TEXT, + dataset_storage_id TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('draft','sealed')), + created_at TEXT NOT NULL, + PRIMARY KEY(lineage_id,version), + FOREIGN KEY(lineage_id,reviewer_id) REFERENCES training_dataset_lineages(id,reviewer_id), + FOREIGN KEY(lineage_id,character_id) REFERENCES training_dataset_lineages(id,character_id), + FOREIGN KEY(dataset_storage_id,reviewer_id) REFERENCES training_datasets(storage_id,reviewer_id), + FOREIGN KEY(dataset_storage_id,character_id) REFERENCES training_datasets(storage_id,character_id), + FOREIGN KEY(previous_dataset_storage_id) REFERENCES training_datasets(storage_id) +); +INSERT INTO training_dataset_snapshot_lineages + SELECT dataset_storage_id,lineage_id,reviewer_id,character_id,min(version) + FROM training_dataset_head_events_v25 GROUP BY dataset_storage_id; +INSERT INTO training_dataset_snapshots + SELECT lineage_id,version,reviewer_id,character_id,previous_dataset_storage_id, + dataset_storage_id,state,created_at FROM training_dataset_head_events_v25; +DROP TABLE training_dataset_head_events_v25; +CREATE VIEW training_dataset_head_events AS SELECT * FROM training_dataset_snapshots; +CREATE TRIGGER reject_training_head_event_insert INSTEAD OF INSERT ON training_dataset_head_events + BEGIN SELECT RAISE(ABORT,'training dataset head history is derived'); END; +CREATE TRIGGER reject_training_head_event_update INSTEAD OF UPDATE ON training_dataset_head_events + BEGIN SELECT RAISE(ABORT,'training dataset head history is derived'); END; +CREATE TRIGGER reject_training_head_event_delete INSTEAD OF DELETE ON training_dataset_head_events + BEGIN SELECT RAISE(ABORT,'training dataset head history is derived'); END; +CREATE TRIGGER validate_training_lineage_insert_v26 BEFORE INSERT ON training_dataset_lineages BEGIN + SELECT CASE WHEN NEW.head_version!=1 OR NEW.state!='draft' OR EXISTS( + SELECT 1 FROM training_dataset_snapshot_lineages + WHERE dataset_storage_id=NEW.current_dataset_storage_id + ) OR NOT EXISTS(SELECT 1 FROM training_datasets d WHERE d.storage_id=NEW.current_dataset_storage_id + AND d.reviewer_id=NEW.reviewer_id AND d.character_id=NEW.character_id AND d.state='draft') + THEN RAISE(ABORT,'invalid initial training dataset head') END; +END; +CREATE TRIGGER derive_initial_training_head_v26 AFTER INSERT ON training_dataset_lineages BEGIN + INSERT INTO training_dataset_snapshot_lineages VALUES( + NEW.current_dataset_storage_id,NEW.id,NEW.reviewer_id,NEW.character_id,1 + ); + INSERT INTO training_dataset_snapshots VALUES( + NEW.id,1,NEW.reviewer_id,NEW.character_id,NULL,NEW.current_dataset_storage_id,'draft',NEW.created_at + ); +END; +CREATE TRIGGER validate_training_lineage_update_v26 BEFORE UPDATE ON training_dataset_lineages BEGIN + SELECT CASE WHEN NEW.id!=OLD.id OR NEW.reviewer_id!=OLD.reviewer_id + OR NEW.character_id!=OLD.character_id OR NEW.created_at!=OLD.created_at OR OLD.state!='draft' + OR NEW.head_version!=OLD.head_version+1 + OR NOT EXISTS(SELECT 1 FROM training_dataset_snapshots old_head + WHERE old_head.lineage_id=OLD.id AND old_head.version=OLD.head_version + AND old_head.dataset_storage_id=OLD.current_dataset_storage_id) + OR NOT EXISTS(SELECT 1 FROM training_datasets d WHERE d.storage_id=NEW.current_dataset_storage_id + AND d.reviewer_id=NEW.reviewer_id AND d.character_id=NEW.character_id AND d.state=NEW.state) + OR (NEW.state='draft' AND NEW.current_dataset_storage_id=OLD.current_dataset_storage_id) + OR (NEW.state='sealed' AND NEW.current_dataset_storage_id!=OLD.current_dataset_storage_id) + OR (NEW.state='draft' AND EXISTS(SELECT 1 FROM training_dataset_snapshot_lineages used + WHERE used.dataset_storage_id=NEW.current_dataset_storage_id)) + THEN RAISE(ABORT,'invalid training dataset head CAS') END; +END; +CREATE TRIGGER derive_training_head_v26 AFTER UPDATE ON training_dataset_lineages BEGIN + INSERT INTO training_dataset_snapshot_lineages + SELECT NEW.current_dataset_storage_id,NEW.id,NEW.reviewer_id,NEW.character_id,NEW.head_version + WHERE NEW.state='draft'; + INSERT INTO training_dataset_snapshots VALUES( + NEW.id,NEW.head_version,NEW.reviewer_id,NEW.character_id,OLD.current_dataset_storage_id, + NEW.current_dataset_storage_id,NEW.state,strftime('%Y-%m-%dT%H:%M:%fZ','now') + ); +END; +CREATE TRIGGER validate_training_snapshot_insert_v26 BEFORE INSERT ON training_dataset_snapshots BEGIN + SELECT CASE WHEN NEW.lineage_id!=(SELECT id FROM training_dataset_lineages + WHERE id=NEW.lineage_id AND reviewer_id=NEW.reviewer_id AND character_id=NEW.character_id) + OR NEW.version!=(SELECT head_version FROM training_dataset_lineages WHERE id=NEW.lineage_id) + OR NEW.dataset_storage_id!=(SELECT current_dataset_storage_id FROM training_dataset_lineages WHERE id=NEW.lineage_id) + OR NEW.state!=(SELECT state FROM training_dataset_lineages WHERE id=NEW.lineage_id) + OR (NEW.version=1 AND NEW.previous_dataset_storage_id IS NOT NULL) + OR (NEW.version>1 AND NEW.previous_dataset_storage_id!=(SELECT dataset_storage_id + FROM training_dataset_snapshots WHERE lineage_id=NEW.lineage_id AND version=NEW.version-1)) + THEN RAISE(ABORT,'invalid derived training dataset history') END; +END; +CREATE TRIGGER guard_training_snapshot_update_v26 BEFORE UPDATE ON training_dataset_snapshots + BEGIN SELECT RAISE(ABORT,'training dataset history is immutable'); END; +CREATE TRIGGER guard_training_snapshot_delete_v26 BEFORE DELETE ON training_dataset_snapshots + BEGIN SELECT RAISE(ABORT,'training dataset history is immutable'); END; +CREATE TRIGGER guard_training_snapshot_lineage_update_v26 + BEFORE UPDATE ON training_dataset_snapshot_lineages + BEGIN SELECT RAISE(ABORT,'training dataset lineage binding is immutable'); END; +CREATE TRIGGER guard_training_snapshot_lineage_delete_v26 + BEFORE DELETE ON training_dataset_snapshot_lineages + BEGIN SELECT RAISE(ABORT,'training dataset lineage binding is immutable'); END; +CREATE TRIGGER validate_training_import_insert_v26 BEFORE INSERT ON training_imported_samples BEGIN + SELECT CASE WHEN json_extract(NEW.payload_json,'$.import_id')!=NEW.id + OR json_extract(NEW.payload_json,'$.reviewer_id')!=NEW.reviewer_id + OR json_extract(NEW.payload_json,'$.character_id')!=NEW.character_id + OR json_extract(NEW.payload_json,'$.lineage_id')!=NEW.lineage_id + OR json_extract(NEW.payload_json,'$.artifact_id')!=NEW.artifact_id + OR json_extract(NEW.payload_json,'$.artifact_digest')!=NEW.artifact_digest + OR json_extract(NEW.payload_json,'$.media.sha256')!=NEW.artifact_digest + OR json_extract(NEW.payload_json,'$.artifact_byte_size')!=json_extract(NEW.payload_json,'$.media.byte_size') + OR NOT EXISTS(SELECT 1 FROM artifacts a WHERE a.id=NEW.artifact_id + AND 'sha256:'||a.sha256=NEW.artifact_digest + AND a.byte_size=json_extract(NEW.payload_json,'$.artifact_byte_size') + AND a.media_type=json_extract(NEW.payload_json,'$.media.media_type')) + THEN RAISE(ABORT,'invalid imported training sample') END; +END; +""" + +_MIGRATION_27 = """ +CREATE TRIGGER authorize_training_snapshot_lineage_insert_v27 + BEFORE INSERT ON training_dataset_snapshot_lineages BEGIN + SELECT CASE WHEN training_transition_authorized( + NEW.lineage_id,NEW.dataset_storage_id,NEW.reviewer_id,NEW.character_id,NEW.first_version + )!=1 THEN RAISE(ABORT,'training snapshot lineage transition is not authorized') END; +END; +DROP TRIGGER validate_training_import_insert_v26; +CREATE TRIGGER validate_training_import_insert_v27 BEFORE INSERT ON training_imported_samples BEGIN + SELECT CASE WHEN json_extract(NEW.payload_json,'$.import_id')!=NEW.id + OR json_extract(NEW.payload_json,'$.reviewer_id')!=NEW.reviewer_id + OR json_extract(NEW.payload_json,'$.character_id')!=NEW.character_id + OR json_extract(NEW.payload_json,'$.lineage_id')!=NEW.lineage_id + OR json_extract(NEW.payload_json,'$.artifact_id')!=NEW.artifact_id + OR json_extract(NEW.payload_json,'$.artifact_digest')!=NEW.artifact_digest + OR json_extract(NEW.payload_json,'$.media.sha256')!=NEW.artifact_digest + OR json_extract(NEW.payload_json,'$.imported_at')!=NEW.created_at + OR json_extract(NEW.payload_json,'$.artifact_byte_size')!=json_extract(NEW.payload_json,'$.media.byte_size') + OR NOT EXISTS(SELECT 1 FROM artifacts a WHERE a.id=NEW.artifact_id + AND 'sha256:'||a.sha256=NEW.artifact_digest + AND a.byte_size=json_extract(NEW.payload_json,'$.artifact_byte_size') + AND a.media_type=json_extract(NEW.payload_json,'$.media.media_type')) + THEN RAISE(ABORT,'invalid imported training sample') END; +END; +DROP TRIGGER validate_training_confirmation_insert; +CREATE TRIGGER validate_training_confirmation_insert_v27 + BEFORE INSERT ON training_curation_confirmations BEGIN + SELECT CASE WHEN json_extract(NEW.payload_json,'$.confirmation_id')!=NEW.id + OR json_extract(NEW.payload_json,'$.import_id')!=NEW.import_id + OR json_extract(NEW.payload_json,'$.reviewer_id')!=NEW.reviewer_id + OR json_extract(NEW.payload_json,'$.version')!=NEW.version + OR json_extract(NEW.payload_json,'$.decision')!=NEW.decision + OR json_extract(NEW.payload_json,'$.confirmed_at')!=NEW.created_at + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_imported_samples WHERE id=NEW.import_id) + OR NEW.version!=COALESCE((SELECT max(version)+1 FROM training_curation_confirmations + WHERE import_id=NEW.import_id AND reviewer_id=NEW.reviewer_id),1) + THEN RAISE(ABORT,'invalid training curation confirmation') END; +END; +""" + +_MIGRATION_28 = """ +ALTER TABLE training_datasets ADD COLUMN lineage_id TEXT; +DROP TRIGGER guard_training_dataset_other_update; +UPDATE training_datasets SET lineage_id=( + SELECT binding.lineage_id FROM training_dataset_snapshot_lineages binding + WHERE binding.dataset_storage_id=training_datasets.storage_id +) WHERE EXISTS(SELECT 1 FROM training_dataset_snapshot_lineages binding + WHERE binding.dataset_storage_id=training_datasets.storage_id); +CREATE TRIGGER guard_training_dataset_other_update BEFORE UPDATE ON training_datasets + WHEN OLD.state IN ('draft','sealed') AND NOT (OLD.state='draft' AND NEW.state='sealed') + BEGIN SELECT RAISE(ABORT,'training dataset snapshots are immutable'); END; +DROP TRIGGER authorize_training_snapshot_lineage_insert_v27; +DROP TRIGGER guard_training_snapshot_lineage_update_v26; +DROP TRIGGER guard_training_snapshot_lineage_delete_v26; +ALTER TABLE training_dataset_snapshot_lineages RENAME TO training_dataset_snapshot_lineages_v27; +CREATE VIEW training_dataset_snapshot_lineages AS + SELECT dataset.storage_id AS dataset_storage_id,dataset.lineage_id, + dataset.reviewer_id,dataset.character_id,min(history.version) AS first_version + FROM training_datasets dataset JOIN training_dataset_snapshots history + ON history.dataset_storage_id=dataset.storage_id AND history.lineage_id=dataset.lineage_id + WHERE dataset.lineage_id IS NOT NULL GROUP BY dataset.storage_id; +DROP TABLE training_dataset_snapshot_lineages_v27; +CREATE TRIGGER reject_training_snapshot_lineage_insert_v28 + INSTEAD OF INSERT ON training_dataset_snapshot_lineages + BEGIN SELECT RAISE(ABORT,'training snapshot lineage is derived'); END; +CREATE TRIGGER reject_training_snapshot_lineage_update_v28 + INSTEAD OF UPDATE ON training_dataset_snapshot_lineages + BEGIN SELECT RAISE(ABORT,'training snapshot lineage is derived'); END; +CREATE TRIGGER reject_training_snapshot_lineage_delete_v28 + INSTEAD OF DELETE ON training_dataset_snapshot_lineages + BEGIN SELECT RAISE(ABORT,'training snapshot lineage is derived'); END; +DROP TRIGGER validate_training_lineage_insert_v26; +DROP TRIGGER derive_initial_training_head_v26; +DROP TRIGGER validate_training_lineage_update_v26; +DROP TRIGGER derive_training_head_v26; +CREATE TRIGGER guard_training_dataset_lineage_update_v28 + BEFORE UPDATE OF lineage_id ON training_datasets + WHEN NEW.lineage_id IS NOT OLD.lineage_id + BEGIN SELECT RAISE(ABORT,'training dataset declared lineage is immutable'); END; +CREATE TRIGGER validate_training_lineage_insert_v28 BEFORE INSERT ON training_dataset_lineages BEGIN + SELECT CASE WHEN NEW.head_version!=1 OR NEW.state!='draft' OR NOT EXISTS( + SELECT 1 FROM training_datasets d WHERE d.storage_id=NEW.current_dataset_storage_id + AND d.reviewer_id=NEW.reviewer_id AND d.character_id=NEW.character_id + AND d.state='draft' AND d.lineage_id=NEW.id + ) THEN RAISE(ABORT,'invalid initial training dataset head') END; +END; +CREATE TRIGGER derive_initial_training_head_v28 AFTER INSERT ON training_dataset_lineages BEGIN + INSERT INTO training_dataset_snapshots VALUES( + NEW.id,1,NEW.reviewer_id,NEW.character_id,NULL,NEW.current_dataset_storage_id,'draft',NEW.created_at + ); +END; +CREATE TRIGGER validate_training_lineage_update_v28 BEFORE UPDATE ON training_dataset_lineages BEGIN + SELECT CASE WHEN NEW.id!=OLD.id OR NEW.reviewer_id!=OLD.reviewer_id + OR NEW.character_id!=OLD.character_id OR NEW.created_at!=OLD.created_at OR OLD.state!='draft' + OR NEW.head_version!=OLD.head_version+1 + OR NOT EXISTS(SELECT 1 FROM training_dataset_snapshots old_head + WHERE old_head.lineage_id=OLD.id AND old_head.version=OLD.head_version + AND old_head.dataset_storage_id=OLD.current_dataset_storage_id) + OR NOT EXISTS(SELECT 1 FROM training_datasets d WHERE d.storage_id=NEW.current_dataset_storage_id + AND d.reviewer_id=NEW.reviewer_id AND d.character_id=NEW.character_id + AND d.state=NEW.state AND d.lineage_id=NEW.id) + OR (NEW.state='draft' AND NEW.current_dataset_storage_id=OLD.current_dataset_storage_id) + OR (NEW.state='sealed' AND NEW.current_dataset_storage_id!=OLD.current_dataset_storage_id) + OR (NEW.state='draft' AND EXISTS(SELECT 1 FROM training_dataset_snapshots used + WHERE used.dataset_storage_id=NEW.current_dataset_storage_id)) + THEN RAISE(ABORT,'invalid training dataset head CAS') END; +END; +CREATE TRIGGER derive_training_head_v28 AFTER UPDATE ON training_dataset_lineages BEGIN + INSERT INTO training_dataset_snapshots VALUES( + NEW.id,NEW.head_version,NEW.reviewer_id,NEW.character_id,OLD.current_dataset_storage_id, + NEW.current_dataset_storage_id,NEW.state,strftime('%Y-%m-%dT%H:%M:%fZ','now') + ); +END; +""" + +_MIGRATION_29 = """ +CREATE TABLE training_dataset_snapshot_imports( + dataset_storage_id TEXT NOT NULL, + import_id TEXT NOT NULL, + PRIMARY KEY(dataset_storage_id,import_id), + FOREIGN KEY(dataset_storage_id) REFERENCES training_datasets(storage_id), + FOREIGN KEY(import_id) REFERENCES training_imported_samples(id) +); +CREATE TABLE training_dataset_snapshot_confirmations( + dataset_storage_id TEXT NOT NULL, + confirmation_id TEXT NOT NULL, + PRIMARY KEY(dataset_storage_id,confirmation_id), + FOREIGN KEY(dataset_storage_id) REFERENCES training_datasets(storage_id), + FOREIGN KEY(confirmation_id) REFERENCES training_curation_confirmations(id) +); +CREATE TRIGGER validate_training_ledger_snapshot_v29 BEFORE INSERT ON training_datasets +WHEN NEW.lineage_id IS NOT NULL BEGIN + SELECT CASE WHEN EXISTS( + SELECT 1 FROM json_each(NEW.manifest_json,'$.imported_sample_ids') ledger + LEFT JOIN training_imported_samples imported ON imported.id=ledger.value + WHERE imported.id IS NULL OR imported.reviewer_id!=NEW.reviewer_id + OR imported.character_id!=NEW.character_id OR imported.lineage_id!=NEW.lineage_id + ) OR EXISTS( + SELECT 1 FROM json_each(NEW.manifest_json,'$.curation_confirmation_ids') ledger + LEFT JOIN training_curation_confirmations confirmation ON confirmation.id=ledger.value + LEFT JOIN training_imported_samples imported ON imported.id=confirmation.import_id + WHERE confirmation.id IS NULL OR confirmation.reviewer_id!=NEW.reviewer_id + OR imported.character_id!=NEW.character_id OR imported.lineage_id!=NEW.lineage_id + OR NOT EXISTS(SELECT 1 FROM json_each(NEW.manifest_json,'$.imported_sample_ids') ids + WHERE ids.value=confirmation.import_id) + ) OR EXISTS( + SELECT confirmation.import_id + FROM json_each(NEW.manifest_json,'$.curation_confirmation_ids') ledger + JOIN training_curation_confirmations confirmation ON confirmation.id=ledger.value + GROUP BY confirmation.import_id HAVING count(*)>1 + ) THEN RAISE(ABORT,'training dataset ledger record mismatch') END; + SELECT CASE WHEN EXISTS( + SELECT 1 FROM json_each(NEW.manifest_json,'$.curation_confirmation_ids') ledger + JOIN training_curation_confirmations confirmation ON confirmation.id=ledger.value + JOIN training_imported_samples imported ON imported.id=confirmation.import_id + WHERE (confirmation.decision='reject' AND EXISTS( + SELECT 1 FROM json_each(NEW.manifest_json,'$.samples') sample + WHERE json_extract(sample.value,'$.sample_digest')=imported.artifact_digest)) + OR (confirmation.decision='accept' AND NOT EXISTS( + SELECT 1 FROM json_each(NEW.manifest_json,'$.samples') sample + WHERE json_extract(sample.value,'$.sample_id')=json_extract(confirmation.payload_json,'$.sample_id') + AND json_extract(sample.value,'$.sample_digest')=imported.artifact_digest + AND json_extract(sample.value,'$.source_tier')=json_extract(confirmation.payload_json,'$.source_tier') + AND json_extract(sample.value,'$.source_lineage')=json_extract(confirmation.payload_json,'$.source_lineage') + AND json_extract(sample.value,'$.caption')=json_extract(confirmation.payload_json,'$.caption') + AND json_extract(sample.value,'$.split')=json_extract(confirmation.payload_json,'$.split') + AND canonical_json_digest(json_extract(sample.value,'$.media'))=canonical_json_digest(json_extract(imported.payload_json,'$.media')) + AND canonical_json_digest(json_extract(sample.value,'$.annotation'))=canonical_json_digest(json_extract(confirmation.payload_json,'$.annotation')) + AND json_extract(sample.value,'$.accepted')=1)) + ) THEN RAISE(ABORT,'training dataset confirmation/sample mismatch') END; +END; +CREATE TRIGGER materialize_training_snapshot_ledgers_v29 AFTER INSERT ON training_datasets +WHEN NEW.lineage_id IS NOT NULL BEGIN + INSERT INTO training_dataset_snapshot_imports SELECT NEW.storage_id,value + FROM json_each(NEW.manifest_json,'$.imported_sample_ids'); + INSERT INTO training_dataset_snapshot_confirmations SELECT NEW.storage_id,value + FROM json_each(NEW.manifest_json,'$.curation_confirmation_ids'); +END; +CREATE TRIGGER validate_training_snapshot_import_insert_v29 BEFORE INSERT ON training_dataset_snapshot_imports BEGIN + SELECT CASE WHEN NOT EXISTS(SELECT 1 FROM training_datasets dataset, + json_each(dataset.manifest_json,'$.imported_sample_ids') ledger + WHERE dataset.storage_id=NEW.dataset_storage_id AND ledger.value=NEW.import_id) + THEN RAISE(ABORT,'training snapshot import binding is not canonical') END; +END; +CREATE TRIGGER validate_training_snapshot_confirmation_insert_v29 BEFORE INSERT ON training_dataset_snapshot_confirmations BEGIN + SELECT CASE WHEN NOT EXISTS(SELECT 1 FROM training_datasets dataset, + json_each(dataset.manifest_json,'$.curation_confirmation_ids') ledger + WHERE dataset.storage_id=NEW.dataset_storage_id AND ledger.value=NEW.confirmation_id) + THEN RAISE(ABORT,'training snapshot confirmation binding is not canonical') END; +END; +CREATE TRIGGER guard_training_snapshot_import_update_v29 BEFORE UPDATE ON training_dataset_snapshot_imports BEGIN SELECT RAISE(ABORT,'training snapshot import bindings are immutable'); END; +CREATE TRIGGER guard_training_snapshot_import_delete_v29 BEFORE DELETE ON training_dataset_snapshot_imports BEGIN SELECT RAISE(ABORT,'training snapshot import bindings are immutable'); END; +CREATE TRIGGER guard_training_snapshot_confirmation_update_v29 BEFORE UPDATE ON training_dataset_snapshot_confirmations BEGIN SELECT RAISE(ABORT,'training snapshot confirmation bindings are immutable'); END; +CREATE TRIGGER guard_training_snapshot_confirmation_delete_v29 BEFORE DELETE ON training_dataset_snapshot_confirmations BEGIN SELECT RAISE(ABORT,'training snapshot confirmation bindings are immutable'); END; +CREATE TRIGGER validate_training_ledger_head_advance_v29 BEFORE UPDATE ON training_dataset_lineages +WHEN NEW.state='draft' AND ( + EXISTS(SELECT 1 FROM training_dataset_snapshot_imports WHERE dataset_storage_id=OLD.current_dataset_storage_id) + OR EXISTS(SELECT 1 FROM training_dataset_snapshot_confirmations WHERE dataset_storage_id=OLD.current_dataset_storage_id) + OR EXISTS(SELECT 1 FROM training_dataset_snapshot_imports WHERE dataset_storage_id=NEW.current_dataset_storage_id) + OR EXISTS(SELECT 1 FROM training_dataset_snapshot_confirmations WHERE dataset_storage_id=NEW.current_dataset_storage_id) +) BEGIN + SELECT CASE WHEN EXISTS( + SELECT import_id FROM training_dataset_snapshot_imports WHERE dataset_storage_id=OLD.current_dataset_storage_id + EXCEPT SELECT import_id FROM training_dataset_snapshot_imports WHERE dataset_storage_id=NEW.current_dataset_storage_id + ) OR EXISTS( + SELECT confirmation_id FROM training_dataset_snapshot_confirmations WHERE dataset_storage_id=OLD.current_dataset_storage_id + EXCEPT SELECT confirmation_id FROM training_dataset_snapshot_confirmations WHERE dataset_storage_id=NEW.current_dataset_storage_id + ) THEN RAISE(ABORT,'training dataset ledger rollback') END; + SELECT CASE WHEN + (SELECT count(*) FROM training_dataset_snapshot_confirmations WHERE dataset_storage_id=NEW.current_dataset_storage_id)> + (SELECT count(*) FROM training_dataset_snapshot_confirmations WHERE dataset_storage_id=OLD.current_dataset_storage_id) + AND ((SELECT count(*) FROM training_dataset_snapshot_confirmations WHERE dataset_storage_id=NEW.current_dataset_storage_id)!= + (SELECT count(*)+1 FROM training_dataset_snapshot_confirmations WHERE dataset_storage_id=OLD.current_dataset_storage_id) + OR (SELECT count(*) FROM training_dataset_snapshot_imports WHERE dataset_storage_id=NEW.current_dataset_storage_id)!= + (SELECT count(*) FROM training_dataset_snapshot_imports WHERE dataset_storage_id=OLD.current_dataset_storage_id)) + THEN RAISE(ABORT,'invalid training confirmation action') END; + SELECT CASE WHEN + (SELECT count(*) FROM training_dataset_snapshot_confirmations WHERE dataset_storage_id=NEW.current_dataset_storage_id)= + (SELECT count(*) FROM training_dataset_snapshot_confirmations WHERE dataset_storage_id=OLD.current_dataset_storage_id) + AND (SELECT count(*) FROM training_dataset_snapshot_imports WHERE dataset_storage_id=NEW.current_dataset_storage_id)<= + (SELECT count(*) FROM training_dataset_snapshot_imports WHERE dataset_storage_id=OLD.current_dataset_storage_id) + THEN RAISE(ABORT,'training dataset ledger action is empty') END; +END; +DROP TRIGGER validate_training_import_insert_v27; +CREATE TRIGGER validate_training_import_insert_v29 BEFORE INSERT ON training_imported_samples BEGIN + SELECT CASE WHEN training_import_identity_is_valid(NEW.id,NEW.payload_json)!=1 + OR json_extract(NEW.payload_json,'$.reviewer_id')!=NEW.reviewer_id + OR json_extract(NEW.payload_json,'$.character_id')!=NEW.character_id + OR json_extract(NEW.payload_json,'$.lineage_id')!=NEW.lineage_id + OR json_extract(NEW.payload_json,'$.artifact_id')!=NEW.artifact_id + OR json_extract(NEW.payload_json,'$.artifact_digest')!=NEW.artifact_digest + OR json_extract(NEW.payload_json,'$.imported_at')!=NEW.created_at + OR NOT EXISTS(SELECT 1 FROM artifacts a WHERE a.id=NEW.artifact_id + AND 'sha256:'||a.sha256=NEW.artifact_digest + AND a.byte_size=json_extract(NEW.payload_json,'$.artifact_byte_size') + AND a.media_type=json_extract(NEW.payload_json,'$.media.media_type')) + THEN RAISE(ABORT,'invalid imported training sample') END; +END; +DROP TRIGGER validate_training_confirmation_insert_v27; +CREATE TRIGGER validate_training_confirmation_insert_v29 BEFORE INSERT ON training_curation_confirmations BEGIN + SELECT CASE WHEN training_confirmation_identity_is_valid(NEW.id,NEW.payload_json)!=1 + OR json_extract(NEW.payload_json,'$.import_id')!=NEW.import_id + OR json_extract(NEW.payload_json,'$.reviewer_id')!=NEW.reviewer_id + OR json_extract(NEW.payload_json,'$.version')!=NEW.version + OR json_extract(NEW.payload_json,'$.decision')!=NEW.decision + OR json_extract(NEW.payload_json,'$.confirmed_at')!=NEW.created_at + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_imported_samples WHERE id=NEW.import_id) + OR NEW.version!=COALESCE((SELECT max(version)+1 FROM training_curation_confirmations WHERE import_id=NEW.import_id AND reviewer_id=NEW.reviewer_id),1) + THEN RAISE(ABORT,'invalid training curation confirmation') END; +END; +""" + +_MIGRATION_30 = """ +CREATE TRIGGER validate_complete_training_ledger_snapshot_v30 + BEFORE INSERT ON training_datasets WHEN NEW.lineage_id IS NOT NULL BEGIN + SELECT CASE WHEN training_ledger_snapshot_is_complete( + NEW.manifest_json,NEW.lineage_id,NEW.reviewer_id,NEW.character_id + )!=1 THEN RAISE(ABORT,'training dataset ledger is incomplete') END; +END; +DROP TRIGGER validate_training_ledger_head_advance_v29; +CREATE TRIGGER validate_training_ledger_head_advance_v30 + BEFORE UPDATE ON training_dataset_lineages BEGIN + SELECT CASE WHEN training_ledger_transition_is_valid( + OLD.current_dataset_storage_id,NEW.current_dataset_storage_id,NEW.state + )!=1 THEN RAISE(ABORT,'invalid training dataset ledger transition') END; +END; +DROP TRIGGER validate_training_confirmation_insert_v29; +CREATE TRIGGER validate_training_confirmation_insert_v30 + BEFORE INSERT ON training_curation_confirmations BEGIN + SELECT CASE WHEN training_confirmation_identity_is_valid(NEW.id,NEW.payload_json)!=1 + OR json_extract(NEW.payload_json,'$.import_id')!=NEW.import_id + OR json_extract(NEW.payload_json,'$.reviewer_id')!=NEW.reviewer_id + OR json_extract(NEW.payload_json,'$.version')!=NEW.version + OR json_extract(NEW.payload_json,'$.decision')!=NEW.decision + OR json_extract(NEW.payload_json,'$.confirmed_at')!=NEW.created_at + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_imported_samples WHERE id=NEW.import_id) + OR NEW.version!=1 + OR EXISTS(SELECT 1 FROM training_curation_confirmations WHERE import_id=NEW.import_id) + THEN RAISE(ABORT,'invalid training curation confirmation') END; +END; +""" + +_MIGRATION_31 = """ +CREATE TRIGGER validate_training_ledger_seal_v31 + BEFORE UPDATE ON training_dataset_lineages WHEN NEW.state='sealed' BEGIN + SELECT CASE WHEN training_ledger_snapshot_is_complete( + (SELECT manifest_json FROM training_datasets WHERE storage_id=NEW.current_dataset_storage_id), + NEW.id,NEW.reviewer_id,NEW.character_id + )!=1 THEN RAISE(ABORT,'sealed training dataset ledger is incomplete') END; +END; +CREATE TRIGGER reject_training_import_for_sealed_lineage_v31 + BEFORE INSERT ON training_imported_samples + WHEN (SELECT state FROM training_dataset_lineages WHERE id=NEW.lineage_id)='sealed' + BEGIN SELECT RAISE(ABORT,'sealed training lineage rejects imports'); END; +CREATE TRIGGER reject_training_confirmation_for_sealed_lineage_v31 + BEFORE INSERT ON training_curation_confirmations + WHEN (SELECT lineage.state FROM training_imported_samples imported + JOIN training_dataset_lineages lineage ON lineage.id=imported.lineage_id + WHERE imported.id=NEW.import_id)='sealed' + BEGIN SELECT RAISE(ABORT,'sealed training lineage rejects confirmations'); END; +""" + +_MIGRATION_32 = """ +CREATE TABLE training_recipe_materials( + recipe_storage_id TEXT PRIMARY KEY, + reviewer_id TEXT NOT NULL, + base_model_artifact_id TEXT NOT NULL, + lightning_adapter_artifact_ids_json TEXT NOT NULL CHECK(json_valid(lightning_adapter_artifact_ids_json) AND json_type(lightning_adapter_artifact_ids_json)='object'), + source_snapshot_artifact_id TEXT NOT NULL, + stage_3_campaign_id TEXT NOT NULL, + stage_3_result_id TEXT NOT NULL, + runtime_digest TEXT NOT NULL CHECK(length(runtime_digest)=71 AND substr(runtime_digest,1,7)='sha256:'), + FOREIGN KEY(recipe_storage_id,reviewer_id) REFERENCES training_recipes(storage_id,reviewer_id), + FOREIGN KEY(base_model_artifact_id) REFERENCES artifacts(id), + FOREIGN KEY(source_snapshot_artifact_id) REFERENCES artifacts(id), + FOREIGN KEY(stage_3_campaign_id) REFERENCES character_benchmark_campaigns(id) +); +CREATE TRIGGER guard_training_recipe_material_update_v32 BEFORE UPDATE ON training_recipe_materials BEGIN SELECT RAISE(ABORT,'training recipe materials are immutable'); END; +CREATE TRIGGER guard_training_recipe_material_delete_v32 BEFORE DELETE ON training_recipe_materials BEGIN SELECT RAISE(ABORT,'training recipe materials are immutable'); END; +""" + +_MIGRATION_33 = """ +CREATE UNIQUE INDEX training_recipe_materials_owner_v33 +ON training_recipe_materials(recipe_storage_id,reviewer_id); +CREATE TABLE training_recipe_lightning_materials( + recipe_storage_id TEXT NOT NULL, + reviewer_id TEXT NOT NULL, + material_name TEXT NOT NULL CHECK(length(material_name)>0), + artifact_id TEXT NOT NULL, + PRIMARY KEY(recipe_storage_id,material_name), + FOREIGN KEY(recipe_storage_id,reviewer_id) + REFERENCES training_recipe_materials(recipe_storage_id,reviewer_id), + FOREIGN KEY(artifact_id) REFERENCES artifacts(id) +); +INSERT INTO training_recipe_lightning_materials +SELECT material.recipe_storage_id,material.reviewer_id,item.key,item.value +FROM training_recipe_materials material,json_each(material.lightning_adapter_artifact_ids_json) item; +CREATE TRIGGER validate_training_recipe_material_insert_v33 +BEFORE INSERT ON training_recipe_materials BEGIN + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM training_recipes recipe + JOIN artifacts base ON base.id=NEW.base_model_artifact_id + JOIN artifacts source ON source.id=NEW.source_snapshot_artifact_id + JOIN character_benchmark_campaigns campaign ON campaign.id=NEW.stage_3_campaign_id + AND campaign.reviewer_id=NEW.reviewer_id + JOIN character_benchmark_results result ON result.campaign_id=campaign.id + WHERE recipe.storage_id=NEW.recipe_storage_id AND recipe.reviewer_id=NEW.reviewer_id + AND 'sha256:'||base.sha256=json_extract(recipe.recipe_json,'$.base_model_digest') + AND 'sha256:'||source.sha256=json_extract(recipe.recipe_json,'$.source_code_digest') + AND NEW.runtime_digest=canonical_json_digest(json_extract(recipe.recipe_json,'$.runtime_identity')) + AND result.content_digest=NEW.stage_3_result_id + AND result.content_digest=json_extract(recipe.recipe_json,'$.stage_3_evidence_digest') + ) THEN RAISE(ABORT,'training recipe material provenance mismatch') END; +END; +CREATE TRIGGER validate_training_recipe_lightning_insert_v33 +BEFORE INSERT ON training_recipe_lightning_materials BEGIN + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM training_recipes recipe JOIN artifacts artifact ON artifact.id=NEW.artifact_id + WHERE recipe.storage_id=NEW.recipe_storage_id AND recipe.reviewer_id=NEW.reviewer_id + AND 'sha256:'||artifact.sha256=json_extract( + recipe.recipe_json,'$.lightning_adapter_digests.'||NEW.material_name) + ) THEN RAISE(ABORT,'training recipe lightning provenance mismatch') END; +END; +CREATE TRIGGER guard_training_recipe_lightning_update_v33 BEFORE UPDATE ON training_recipe_lightning_materials BEGIN SELECT RAISE(ABORT,'training recipe lightning materials are immutable'); END; +CREATE TRIGGER guard_training_recipe_lightning_delete_v33 BEFORE DELETE ON training_recipe_lightning_materials BEGIN SELECT RAISE(ABORT,'training recipe lightning materials are immutable'); END; +CREATE TRIGGER validate_training_recipe_materials_preregister_v33 +BEFORE UPDATE OF state ON training_recipes WHEN NEW.state='preregistered' BEGIN + SELECT CASE WHEN + (SELECT count(*) FROM training_recipe_lightning_materials WHERE recipe_storage_id=NEW.storage_id) + !=(SELECT count(*) FROM json_each(json_extract(NEW.recipe_json,'$.lightning_adapter_digests'))) + THEN RAISE(ABORT,'training recipe material set mismatch') END; +END; +""" + +_MIGRATION_34 = """ +DROP TRIGGER validate_training_recipe_material_insert_v33; +DROP TRIGGER validate_training_recipe_materials_preregister_v33; +CREATE TRIGGER validate_training_recipe_material_insert_v34 +BEFORE INSERT ON training_recipe_materials BEGIN + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM training_recipes recipe + JOIN training_datasets dataset ON dataset.storage_id=recipe.dataset_storage_id + AND dataset.reviewer_id=recipe.reviewer_id AND dataset.state='sealed' + JOIN artifacts base ON base.id=NEW.base_model_artifact_id + JOIN artifacts source ON source.id=NEW.source_snapshot_artifact_id + JOIN character_benchmark_campaigns campaign ON campaign.id=NEW.stage_3_campaign_id + AND campaign.reviewer_id=NEW.reviewer_id + AND campaign.character_pack_storage_id=dataset.character_pack_storage_id + JOIN character_benchmark_results result ON result.campaign_id=campaign.id + WHERE recipe.storage_id=NEW.recipe_storage_id AND recipe.reviewer_id=NEW.reviewer_id + AND recipe.dataset_digest=dataset.dataset_digest + AND recipe.character_pack_digest=dataset.character_pack_digest + AND 'sha256:'||base.sha256=json_extract(recipe.recipe_json,'$.base_model_digest') + AND 'sha256:'||source.sha256=json_extract(recipe.recipe_json,'$.source_code_digest') + AND NEW.runtime_digest=canonical_json_digest(json_extract(recipe.recipe_json,'$.runtime_identity')) + AND result.content_digest=NEW.stage_3_result_id + AND result.content_digest=json_extract(recipe.recipe_json,'$.stage_3_evidence_digest') + AND json_extract(result.result_json,'$.baseline_disposition')='baseline_established' + AND character_benchmark_provenance_is_valid(campaign.id)=1 + AND json_extract(campaign.definition_json,'$.character_pack_digest')=recipe.character_pack_digest + AND json_extract(campaign.definition_json,'$.stage_2_manifest_digest')=json_extract(recipe.recipe_json,'$.stage_2_manifest_digest') + AND json_extract(campaign.definition_json,'$.stage_2_evidence_digest')=json_extract(recipe.recipe_json,'$.stage_2_evidence_digest') + AND (SELECT new_state FROM character_benchmark_events WHERE campaign_id=campaign.id ORDER BY sequence DESC LIMIT 1)='completed' + ) THEN RAISE(ABORT,'training recipe material provenance mismatch') END; +END; +CREATE TRIGGER validate_training_recipe_materials_preregister_v34 +BEFORE UPDATE OF state ON training_recipes WHEN NEW.state='preregistered' BEGIN + SELECT CASE WHEN EXISTS( + SELECT 1 FROM json_each(json_extract(NEW.recipe_json,'$.lightning_adapter_digests')) expected + LEFT JOIN training_recipe_lightning_materials child + ON child.recipe_storage_id=NEW.storage_id AND child.material_name=expected.key + LEFT JOIN training_recipe_materials parent ON parent.recipe_storage_id=NEW.storage_id + LEFT JOIN json_each(parent.lightning_adapter_artifact_ids_json) binding ON binding.key=expected.key + LEFT JOIN artifacts artifact ON artifact.id=child.artifact_id + WHERE child.artifact_id IS NULL OR child.artifact_id IS NOT binding.value + OR 'sha256:'||artifact.sha256 IS NOT expected.value + ) OR EXISTS( + SELECT 1 FROM training_recipe_lightning_materials child + WHERE child.recipe_storage_id=NEW.storage_id AND NOT EXISTS( + SELECT 1 FROM json_each(json_extract(NEW.recipe_json,'$.lightning_adapter_digests')) expected + WHERE expected.key=child.material_name + ) + ) THEN RAISE(ABORT,'training recipe material set mismatch') END; +END; +""" + +_MIGRATION_35 = """ +ALTER TABLE training_runs ADD COLUMN lifecycle_state TEXT NOT NULL DEFAULT 'draft' + CHECK(lifecycle_state IN ('draft','probed','calibrated','preregistered','training','evaluating','qualified','rejected','failed','cancelled')); +ALTER TABLE training_runs ADD COLUMN lifecycle_version INTEGER NOT NULL DEFAULT 0 CHECK(lifecycle_version>=0); +ALTER TABLE training_runs ADD COLUMN idempotency_key TEXT; +ALTER TABLE training_runs ADD COLUMN request_digest TEXT CHECK(request_digest IS NULL OR (length(request_digest)=71 AND substr(request_digest,1,7)='sha256:')); +UPDATE training_runs SET lifecycle_state=CASE state WHEN 'created' THEN 'draft' ELSE 'failed' END, + lifecycle_version=0; +CREATE UNIQUE INDEX training_runs_idempotency ON training_runs(reviewer_id,idempotency_key) WHERE idempotency_key IS NOT NULL; +CREATE TABLE training_probe_evidence( + id TEXT PRIMARY KEY,run_id TEXT NOT NULL,reviewer_id TEXT NOT NULL,recipe_digest TEXT NOT NULL,dataset_digest TEXT NOT NULL, + backend_digest TEXT NOT NULL,runtime_digest TEXT NOT NULL,material_digest TEXT NOT NULL,source_digest TEXT NOT NULL, + capability_digest TEXT NOT NULL,succeeded INTEGER NOT NULL CHECK(succeeded IN (0,1)),created_at TEXT NOT NULL, + UNIQUE(run_id,id),FOREIGN KEY(run_id,reviewer_id) REFERENCES training_runs(id,reviewer_id) +); +CREATE TABLE training_calibration_evidence( + id TEXT PRIMARY KEY,run_id TEXT NOT NULL,reviewer_id TEXT NOT NULL,probe_evidence_id TEXT NOT NULL,calibration_digest TEXT NOT NULL, + recipe_digest TEXT NOT NULL,dataset_digest TEXT NOT NULL,backend_digest TEXT NOT NULL,runtime_digest TEXT NOT NULL, + material_digest TEXT NOT NULL,source_digest TEXT NOT NULL, + succeeded INTEGER NOT NULL CHECK(succeeded IN (0,1)),created_at TEXT NOT NULL, + UNIQUE(run_id,id),FOREIGN KEY(run_id,reviewer_id) REFERENCES training_runs(id,reviewer_id), + FOREIGN KEY(probe_evidence_id) REFERENCES training_probe_evidence(id) +); +CREATE TABLE training_cancellation_evidence( + id TEXT PRIMARY KEY,run_id TEXT NOT NULL,reviewer_id TEXT NOT NULL,requested_step INTEGER NOT NULL CHECK(requested_step>=0), + boundary_step INTEGER NOT NULL CHECK(boundary_step>=requested_step),checkpoint_digest TEXT NOT NULL,created_at TEXT NOT NULL, + UNIQUE(run_id),FOREIGN KEY(run_id,reviewer_id) REFERENCES training_runs(id,reviewer_id) +); +CREATE TABLE training_resume_evidence( + id TEXT PRIMARY KEY,run_id TEXT NOT NULL,reviewer_id TEXT NOT NULL,parent_checkpoint_id TEXT NOT NULL,parent_checkpoint_digest TEXT NOT NULL,parent_artifact_digest TEXT NOT NULL, + recipe_digest TEXT NOT NULL,dataset_digest TEXT NOT NULL,backend_digest TEXT NOT NULL,runtime_digest TEXT NOT NULL, + material_digest TEXT NOT NULL,source_digest TEXT NOT NULL,optimizer_state_digest TEXT NOT NULL,created_at TEXT NOT NULL, + UNIQUE(run_id,id),FOREIGN KEY(run_id,reviewer_id) REFERENCES training_runs(id,reviewer_id), + FOREIGN KEY(parent_checkpoint_id) REFERENCES training_checkpoints(id) +); +CREATE TABLE training_evaluation_evidence( + id TEXT PRIMARY KEY,run_id TEXT NOT NULL,reviewer_id TEXT NOT NULL,disposition TEXT NOT NULL CHECK(disposition IN ('qualified','rejected')), + evidence_digest TEXT NOT NULL CHECK(length(evidence_digest)=71 AND substr(evidence_digest,1,7)='sha256:'),created_at TEXT NOT NULL, + UNIQUE(run_id),FOREIGN KEY(run_id,reviewer_id) REFERENCES training_runs(id,reviewer_id) +); +CREATE UNIQUE INDEX training_probe_one_per_run ON training_probe_evidence(run_id); +CREATE UNIQUE INDEX training_calibration_one_per_run ON training_calibration_evidence(run_id); +ALTER TABLE training_checkpoints ADD COLUMN evidence_json TEXT CHECK(evidence_json IS NULL OR (json_valid(evidence_json) AND json_type(evidence_json)='object')); +ALTER TABLE training_checkpoints ADD COLUMN evidence_digest TEXT CHECK(evidence_digest IS NULL OR (length(evidence_digest)=71 AND substr(evidence_digest,1,7)='sha256:')); +CREATE TABLE training_checkpoint_inventory( + checkpoint_id TEXT NOT NULL,ordinal INTEGER NOT NULL CHECK(ordinal>=0),relative_path TEXT NOT NULL, + artifact_digest TEXT NOT NULL CHECK(length(artifact_digest)=71 AND substr(artifact_digest,1,7)='sha256:'), + size_bytes INTEGER NOT NULL CHECK(size_bytes>0),role TEXT NOT NULL CHECK(role IN ('weights','optimizer','trainer_state','telemetry','evidence')), + PRIMARY KEY(checkpoint_id,ordinal),UNIQUE(checkpoint_id,relative_path), + FOREIGN KEY(checkpoint_id) REFERENCES training_checkpoints(id) +); +DROP TRIGGER validate_training_event_insert; +DROP TRIGGER apply_training_event; +DROP TRIGGER validate_training_telemetry_insert; +DROP TRIGGER validate_training_checkpoint_insert; +CREATE TRIGGER validate_training_event_insert BEFORE INSERT ON training_run_events BEGIN + SELECT CASE WHEN NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.sequence!=COALESCE((SELECT max(sequence)+1 FROM training_run_events WHERE run_id=NEW.run_id),1) + OR NEW.previous_state!=(SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id) + OR NOT ((NEW.previous_state='draft' AND NEW.new_state='probed' AND EXISTS(SELECT 1 FROM training_probe_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND succeeded=1)) + OR (NEW.previous_state='probed' AND NEW.new_state='calibrated' AND EXISTS(SELECT 1 FROM training_calibration_evidence c JOIN training_probe_evidence p ON p.id=c.probe_evidence_id WHERE c.run_id=NEW.run_id AND c.reviewer_id=NEW.reviewer_id AND c.succeeded=1 AND p.succeeded=1 AND c.recipe_digest=p.recipe_digest AND c.dataset_digest=p.dataset_digest AND c.backend_digest=p.backend_digest AND c.runtime_digest=p.runtime_digest AND c.material_digest=p.material_digest AND c.source_digest=p.source_digest)) + OR (NEW.previous_state='calibrated' AND NEW.new_state='preregistered' AND length(json_extract(NEW.payload_json,'$.idempotency_key'))>0 AND length(json_extract(NEW.payload_json,'$.request_digest'))=71 AND substr(json_extract(NEW.payload_json,'$.request_digest'),1,7)='sha256:' AND json_extract(NEW.payload_json,'$.probe_evidence_id')=(SELECT id FROM training_probe_evidence WHERE run_id=NEW.run_id AND succeeded=1 ORDER BY created_at DESC LIMIT 1) AND json_extract(NEW.payload_json,'$.calibration_evidence_id')=(SELECT id FROM training_calibration_evidence WHERE run_id=NEW.run_id AND succeeded=1 ORDER BY created_at DESC LIMIT 1)) + OR (NEW.previous_state='preregistered' AND NEW.new_state='training') + OR (NEW.previous_state='training' AND NEW.new_state='evaluating' AND EXISTS(SELECT 1 FROM training_checkpoints c WHERE c.run_id=NEW.run_id AND c.fraction=1.0 AND (SELECT count(*) FROM training_checkpoint_inventory i WHERE i.checkpoint_id=c.id)=json_array_length(json_extract(c.evidence_json,'$.inventory')) AND NOT EXISTS(SELECT 1 FROM json_each(c.evidence_json,'$.inventory') item WHERE NOT EXISTS(SELECT 1 FROM training_checkpoint_inventory i WHERE i.checkpoint_id=c.id AND i.ordinal=CAST(item.key AS INTEGER) AND i.relative_path=json_extract(item.value,'$.relative_path') AND i.artifact_digest=json_extract(item.value,'$.digest') AND i.size_bytes=json_extract(item.value,'$.size_bytes') AND i.role=json_extract(item.value,'$.role'))) AND (SELECT max(step) FROM training_telemetry WHERE run_id=NEW.run_id)=(SELECT total_steps FROM training_runs WHERE id=NEW.run_id))) + OR (NEW.previous_state='evaluating' AND NEW.new_state IN ('qualified','rejected') AND EXISTS(SELECT 1 FROM training_evaluation_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND disposition=NEW.new_state AND evidence_digest=json_extract(NEW.payload_json,'$.evaluation_evidence_digest'))) + OR (NEW.previous_state IN ('draft','probed','calibrated','preregistered','training','evaluating') AND NEW.new_state='failed' AND EXISTS(SELECT 1 FROM training_failures WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id)) + OR (NEW.previous_state IN ('preregistered','training') AND NEW.new_state='cancelled' AND EXISTS(SELECT 1 FROM training_cancellation_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id))) + THEN RAISE(ABORT,'invalid scientific training lifecycle transition') END; +END; +CREATE TRIGGER apply_training_event AFTER INSERT ON training_run_events BEGIN + UPDATE training_runs SET lifecycle_state=NEW.new_state,lifecycle_version=lifecycle_version+1, + state=CASE WHEN NEW.new_state IN ('qualified','rejected') THEN 'completed' + WHEN NEW.new_state IN ('failed','cancelled') THEN NEW.new_state + WHEN NEW.new_state IN ('training','evaluating') THEN 'running' ELSE 'created' END, + terminal_at=CASE WHEN NEW.new_state IN ('qualified','rejected','failed','cancelled') THEN NEW.created_at ELSE NULL END, + idempotency_key=CASE WHEN NEW.new_state='preregistered' THEN json_extract(NEW.payload_json,'$.idempotency_key') ELSE idempotency_key END, + request_digest=CASE WHEN NEW.new_state='preregistered' THEN json_extract(NEW.payload_json,'$.request_digest') ELSE request_digest END + WHERE id=NEW.run_id; +END; +CREATE TRIGGER guard_scientific_training_run_update BEFORE UPDATE ON training_runs BEGIN + SELECT CASE WHEN NEW.id!=OLD.id OR NEW.reviewer_id!=OLD.reviewer_id + OR NEW.recipe_storage_id!=OLD.recipe_storage_id OR NEW.dataset_storage_id!=OLD.dataset_storage_id + OR NEW.total_steps!=OLD.total_steps OR NEW.created_at!=OLD.created_at + OR NEW.state!=CASE WHEN NEW.lifecycle_state IN ('qualified','rejected') THEN 'completed' + WHEN NEW.lifecycle_state IN ('failed','cancelled') THEN NEW.lifecycle_state + WHEN NEW.lifecycle_state IN ('training','evaluating') THEN 'running' ELSE 'created' END + OR NEW.lifecycle_version!=OLD.lifecycle_version+1 + OR NOT EXISTS(SELECT 1 FROM training_run_events e WHERE e.run_id=OLD.id + AND e.sequence=(SELECT max(sequence) FROM training_run_events WHERE run_id=OLD.id) + AND e.previous_state=OLD.lifecycle_state AND e.new_state=NEW.lifecycle_state) + THEN RAISE(ABORT,'scientific training runs change only through evidence events') END; +END; +CREATE TRIGGER validate_training_telemetry_insert BEFORE INSERT ON training_telemetry BEGIN + SELECT CASE WHEN (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id)!='training' + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.sequence!=COALESCE((SELECT max(sequence)+1 FROM training_telemetry WHERE run_id=NEW.run_id),1) + OR NEW.step>(SELECT total_steps FROM training_runs WHERE id=NEW.run_id) + OR NEW.step<=COALESCE((SELECT max(step) FROM training_telemetry WHERE run_id=NEW.run_id),-1) + THEN RAISE(ABORT,'invalid training telemetry sequence') END; +END; +CREATE TRIGGER validate_training_checkpoint_insert BEFORE INSERT ON training_checkpoints BEGIN + SELECT CASE WHEN NEW.dataset_storage_id!=(SELECT dataset_storage_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.step!=training_checkpoint_step((SELECT total_steps FROM training_runs WHERE id=NEW.run_id),NEW.fraction) + OR (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id)!='training' + OR NEW.step>COALESCE((SELECT max(step) FROM training_telemetry WHERE run_id=NEW.run_id),-1) + OR NOT EXISTS(SELECT 1 FROM training_recipes recipe,json_each(recipe.recipe_json,'$.checkpoint_plan.fractions') declared WHERE recipe.storage_id=NEW.recipe_storage_id AND declared.value=NEW.fraction) + OR NEW.evidence_json IS NULL OR canonical_json_digest(NEW.evidence_json)!=NEW.evidence_digest + OR json_extract(NEW.evidence_json,'$.schema_version')!='character-training-checkpoint-evidence.v1' + OR json_extract(NEW.evidence_json,'$.run_id')!=NEW.run_id + OR json_extract(NEW.evidence_json,'$.recipe_digest')!=(SELECT id FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) + OR json_extract(NEW.evidence_json,'$.dataset_digest')!=(SELECT dataset_digest FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) + OR NOT EXISTS(SELECT 1 FROM training_calibration_evidence c WHERE c.run_id=NEW.run_id AND c.succeeded=1 AND c.backend_digest=json_extract(NEW.evidence_json,'$.backend_digest') AND c.runtime_digest=json_extract(NEW.evidence_json,'$.runtime_digest') AND c.material_digest=json_extract(NEW.evidence_json,'$.material_digest') AND c.source_digest=json_extract(NEW.evidence_json,'$.source_digest')) + OR json_extract(NEW.evidence_json,'$.step')!=NEW.step OR json_extract(NEW.evidence_json,'$.fraction')!=NEW.fraction + OR json_extract(NEW.evidence_json,'$.parent_checkpoint_digest') IS NOT (SELECT evidence_digest FROM training_checkpoints WHERE id=NEW.parent_checkpoint_id) + OR json_extract(NEW.evidence_json,'$.safetensors_digest')!=NEW.artifact_digest + OR json_array_length(json_extract(NEW.evidence_json,'$.inventory'))<4 + OR json_extract(NEW.evidence_json,'$.telemetry_sequence_start')<1 + OR json_extract(NEW.evidence_json,'$.telemetry_sequence_end')<(SELECT min(sequence) FROM training_telemetry WHERE run_id=NEW.run_id) + OR json_extract(NEW.evidence_json,'$.telemetry_sequence_end')>(SELECT max(sequence) FROM training_telemetry WHERE run_id=NEW.run_id) + THEN RAISE(ABORT,'training checkpoint evidence mismatch') END; + SELECT CASE WHEN (NEW.fraction=0.25 AND NEW.parent_checkpoint_id IS NOT NULL) + OR (NEW.fraction>0.25 AND (SELECT run_id FROM training_checkpoints WHERE id=NEW.parent_checkpoint_id) IS NOT NEW.run_id) + OR (NEW.fraction>0.25 AND (SELECT step FROM training_checkpoints WHERE id=NEW.parent_checkpoint_id)!=(SELECT max(step) FROM training_checkpoints WHERE run_id=NEW.run_id)) + OR (NEW.fraction>0.25 AND (SELECT fraction FROM training_checkpoints WHERE id=NEW.parent_checkpoint_id)!=NEW.fraction-0.25) + THEN RAISE(ABORT,'training checkpoint parent mismatch') END; +END; +CREATE TRIGGER validate_training_probe_insert BEFORE INSERT ON training_probe_evidence BEGIN + SELECT CASE WHEN (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id)!='draft' + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.recipe_digest!=(SELECT recipe.id FROM training_runs run JOIN training_recipes recipe ON recipe.storage_id=run.recipe_storage_id WHERE run.id=NEW.run_id) + OR NEW.dataset_digest!=(SELECT recipe.dataset_digest FROM training_runs run JOIN training_recipes recipe ON recipe.storage_id=run.recipe_storage_id WHERE run.id=NEW.run_id) + OR NEW.backend_digest!=(SELECT canonical_json_digest(json_extract(recipe.recipe_json,'$.backend_config')) FROM training_runs run JOIN training_recipes recipe ON recipe.storage_id=run.recipe_storage_id WHERE run.id=NEW.run_id) + OR NEW.runtime_digest!=(SELECT material.runtime_digest FROM training_runs run JOIN training_recipe_materials material ON material.recipe_storage_id=run.recipe_storage_id WHERE run.id=NEW.run_id) + OR NEW.source_digest!=(SELECT 'sha256:'||artifact.sha256 FROM training_runs run JOIN training_recipe_materials material ON material.recipe_storage_id=run.recipe_storage_id JOIN artifacts artifact ON artifact.id=material.source_snapshot_artifact_id WHERE run.id=NEW.run_id) + THEN RAISE(ABORT,'training probe identity mismatch') END; +END; +CREATE TRIGGER validate_training_calibration_insert BEFORE INSERT ON training_calibration_evidence BEGIN + SELECT CASE WHEN (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id)!='probed' + OR NOT EXISTS(SELECT 1 FROM training_probe_evidence p WHERE p.id=NEW.probe_evidence_id AND p.run_id=NEW.run_id AND p.reviewer_id=NEW.reviewer_id AND p.succeeded=1 AND p.recipe_digest=NEW.recipe_digest AND p.dataset_digest=NEW.dataset_digest AND p.backend_digest=NEW.backend_digest AND p.runtime_digest=NEW.runtime_digest AND p.material_digest=NEW.material_digest AND p.source_digest=NEW.source_digest) + THEN RAISE(ABORT,'training calibration identity mismatch') END; +END; +CREATE TRIGGER validate_training_cancellation_insert BEFORE INSERT ON training_cancellation_evidence BEGIN + SELECT CASE WHEN (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id) NOT IN ('preregistered','training') + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NOT EXISTS(SELECT 1 FROM training_checkpoints c WHERE c.run_id=NEW.run_id AND c.step=NEW.boundary_step AND c.artifact_digest=NEW.checkpoint_digest AND json_extract(c.evidence_json,'$.cancellation_boundary')=1) + THEN RAISE(ABORT,'training cancellation boundary mismatch') END; +END; +CREATE TRIGGER validate_training_resume_insert BEFORE INSERT ON training_resume_evidence BEGIN + SELECT CASE WHEN (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id)!='training' + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NOT EXISTS(SELECT 1 FROM training_checkpoints c JOIN training_runs parent_run ON parent_run.id=c.run_id JOIN training_recipes recipe ON recipe.storage_id=parent_run.recipe_storage_id WHERE c.id=NEW.parent_checkpoint_id AND c.run_id=NEW.run_id AND c.evidence_digest=NEW.parent_checkpoint_digest AND c.artifact_digest=NEW.parent_artifact_digest AND recipe.id=NEW.recipe_digest AND recipe.dataset_digest=NEW.dataset_digest AND json_extract(c.evidence_json,'$.backend_digest')=NEW.backend_digest AND json_extract(c.evidence_json,'$.runtime_digest')=NEW.runtime_digest AND json_extract(c.evidence_json,'$.material_digest')=NEW.material_digest AND json_extract(c.evidence_json,'$.source_digest')=NEW.source_digest AND json_extract(c.evidence_json,'$.optimizer_state_digest')=NEW.optimizer_state_digest) + THEN RAISE(ABORT,'training resume identity mismatch') END; +END; +CREATE TRIGGER validate_training_checkpoint_inventory_insert BEFORE INSERT ON training_checkpoint_inventory BEGIN + SELECT CASE WHEN (SELECT lifecycle_state FROM training_runs WHERE id=(SELECT run_id FROM training_checkpoints WHERE id=NEW.checkpoint_id))!='training' + OR NOT EXISTS(SELECT 1 FROM training_checkpoints c,json_each(c.evidence_json,'$.inventory') item WHERE c.id=NEW.checkpoint_id AND CAST(item.key AS INTEGER)=NEW.ordinal AND json_extract(item.value,'$.relative_path')=NEW.relative_path AND json_extract(item.value,'$.digest')=NEW.artifact_digest AND json_extract(item.value,'$.size_bytes')=NEW.size_bytes AND json_extract(item.value,'$.role')=NEW.role) + THEN RAISE(ABORT,'training checkpoint inventory mismatch') END; +END; +CREATE TRIGGER validate_training_failure_insert_v35 BEFORE INSERT ON training_failures BEGIN + SELECT CASE WHEN NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id) NOT IN ('draft','probed','calibrated','preregistered','training','evaluating') + THEN RAISE(ABORT,'training failure identity or lifecycle mismatch') END; +END; +CREATE TRIGGER validate_training_evaluation_insert BEFORE INSERT ON training_evaluation_evidence BEGIN + SELECT CASE WHEN NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id)!='evaluating' + THEN RAISE(ABORT,'training evaluation identity or lifecycle mismatch') END; +END; +CREATE TRIGGER guard_training_probe_update BEFORE UPDATE ON training_probe_evidence BEGIN SELECT RAISE(ABORT,'training probe evidence is immutable'); END; +CREATE TRIGGER guard_training_probe_delete BEFORE DELETE ON training_probe_evidence BEGIN SELECT RAISE(ABORT,'training probe evidence is immutable'); END; +CREATE TRIGGER guard_training_calibration_update BEFORE UPDATE ON training_calibration_evidence BEGIN SELECT RAISE(ABORT,'training calibration evidence is immutable'); END; +CREATE TRIGGER guard_training_calibration_delete BEFORE DELETE ON training_calibration_evidence BEGIN SELECT RAISE(ABORT,'training calibration evidence is immutable'); END; +CREATE TRIGGER guard_training_cancellation_update BEFORE UPDATE ON training_cancellation_evidence BEGIN SELECT RAISE(ABORT,'training cancellation evidence is immutable'); END; +CREATE TRIGGER guard_training_cancellation_delete BEFORE DELETE ON training_cancellation_evidence BEGIN SELECT RAISE(ABORT,'training cancellation evidence is immutable'); END; +CREATE TRIGGER guard_training_resume_update BEFORE UPDATE ON training_resume_evidence BEGIN SELECT RAISE(ABORT,'training resume evidence is immutable'); END; +CREATE TRIGGER guard_training_resume_delete BEFORE DELETE ON training_resume_evidence BEGIN SELECT RAISE(ABORT,'training resume evidence is immutable'); END; +CREATE TRIGGER guard_training_evaluation_update BEFORE UPDATE ON training_evaluation_evidence BEGIN SELECT RAISE(ABORT,'training evaluation evidence is immutable'); END; +CREATE TRIGGER guard_training_evaluation_delete BEFORE DELETE ON training_evaluation_evidence BEGIN SELECT RAISE(ABORT,'training evaluation evidence is immutable'); END; +CREATE TRIGGER guard_training_checkpoint_inventory_update BEFORE UPDATE ON training_checkpoint_inventory BEGIN SELECT RAISE(ABORT,'training checkpoint inventory is immutable'); END; +CREATE TRIGGER guard_training_checkpoint_inventory_delete BEFORE DELETE ON training_checkpoint_inventory BEGIN SELECT RAISE(ABORT,'training checkpoint inventory is immutable'); END; +""" + +_MIGRATION_36 = """ +ALTER TABLE training_checkpoints ADD COLUMN publication_state TEXT NOT NULL DEFAULT 'building' + CHECK(publication_state IN ('building','finalized')); +DROP TRIGGER validate_training_event_insert; +DROP TRIGGER guard_training_checkpoint_inventory_update; +DROP TRIGGER guard_training_checkpoint_inventory_delete; +ALTER TABLE training_checkpoint_inventory RENAME TO training_checkpoint_inventory_v35; +CREATE TABLE training_checkpoint_inventory( + checkpoint_id TEXT NOT NULL,ordinal INTEGER NOT NULL CHECK(ordinal>=0),relative_path TEXT NOT NULL, + artifact_digest TEXT NOT NULL CHECK(length(artifact_digest)=71 AND substr(artifact_digest,1,7)='sha256:'), + size_bytes INTEGER NOT NULL CHECK(size_bytes>0),role TEXT NOT NULL CHECK(role IN + ('weights','optimizer','trainer_state','telemetry','trainable_targets','frozen_parameters','finite_gradient_proof')), + PRIMARY KEY(checkpoint_id,ordinal),UNIQUE(checkpoint_id,relative_path), + FOREIGN KEY(checkpoint_id) REFERENCES training_checkpoints(id)); +INSERT INTO training_checkpoint_inventory SELECT * FROM training_checkpoint_inventory_v35; +DROP TABLE training_checkpoint_inventory_v35; +CREATE TABLE training_run_idempotency( + reviewer_id TEXT NOT NULL,recipe_storage_id TEXT NOT NULL,idempotency_key TEXT NOT NULL, + request_digest TEXT NOT NULL CHECK(length(request_digest)=71 AND substr(request_digest,1,7)='sha256:'), + run_id TEXT NOT NULL,probe_evidence_id TEXT NOT NULL,calibration_evidence_id TEXT NOT NULL, + created_at TEXT NOT NULL,PRIMARY KEY(reviewer_id,recipe_storage_id,idempotency_key), + FOREIGN KEY(run_id,reviewer_id) REFERENCES training_runs(id,reviewer_id), + FOREIGN KEY(probe_evidence_id) REFERENCES training_probe_evidence(id), + FOREIGN KEY(calibration_evidence_id) REFERENCES training_calibration_evidence(id) +); +CREATE TABLE training_cancellation_requests( + id TEXT PRIMARY KEY,run_id TEXT NOT NULL UNIQUE,reviewer_id TEXT NOT NULL, + requested_step INTEGER NOT NULL CHECK(requested_step>=0),request_digest TEXT NOT NULL + CHECK(length(request_digest)=71 AND substr(request_digest,1,7)='sha256:'),created_at TEXT NOT NULL, + FOREIGN KEY(run_id,reviewer_id) REFERENCES training_runs(id,reviewer_id) +); +DROP TRIGGER validate_training_checkpoint_insert; +DROP TRIGGER guard_training_checkpoint_update; +DROP TRIGGER validate_training_cancellation_insert; +DROP TRIGGER validate_training_probe_insert; +CREATE TRIGGER validate_training_event_insert_v36 BEFORE INSERT ON training_run_events BEGIN + SELECT CASE WHEN NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.sequence!=COALESCE((SELECT max(sequence)+1 FROM training_run_events WHERE run_id=NEW.run_id),1) + OR NEW.previous_state!=(SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id) + OR NOT ((NEW.previous_state='draft' AND NEW.new_state='probed' AND EXISTS(SELECT 1 FROM training_probe_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND succeeded=1)) + OR (NEW.previous_state='probed' AND NEW.new_state='calibrated' AND EXISTS(SELECT 1 FROM training_calibration_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND succeeded=1)) + OR (NEW.previous_state='calibrated' AND NEW.new_state='preregistered' AND EXISTS(SELECT 1 FROM training_run_idempotency b WHERE b.run_id=NEW.run_id AND b.reviewer_id=NEW.reviewer_id AND b.idempotency_key=json_extract(NEW.payload_json,'$.idempotency_key') AND b.request_digest=json_extract(NEW.payload_json,'$.request_digest'))) + OR (NEW.previous_state='preregistered' AND NEW.new_state='training') + OR (NEW.previous_state='training' AND NEW.new_state='evaluating' AND EXISTS(SELECT 1 FROM training_checkpoints c WHERE c.run_id=NEW.run_id AND c.fraction=1.0 AND c.publication_state='finalized') AND (SELECT max(step) FROM training_telemetry WHERE run_id=NEW.run_id)=(SELECT total_steps FROM training_runs WHERE id=NEW.run_id)) + OR (NEW.previous_state='evaluating' AND NEW.new_state IN ('qualified','rejected') AND EXISTS(SELECT 1 FROM training_evaluation_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND disposition=NEW.new_state AND evidence_digest=json_extract(NEW.payload_json,'$.evaluation_evidence_digest'))) + OR (NEW.previous_state IN ('draft','probed','calibrated','preregistered','training','evaluating') AND NEW.new_state='failed' AND EXISTS(SELECT 1 FROM training_failures WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id)) + OR (NEW.previous_state='training' AND NEW.new_state='cancelled' AND EXISTS(SELECT 1 FROM training_cancellation_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id))) + THEN RAISE(ABORT,'invalid scientific training lifecycle transition') END; +END; +CREATE TRIGGER validate_training_checkpoint_insert_v36 BEFORE INSERT ON training_checkpoints BEGIN + SELECT CASE WHEN NEW.publication_state!='building' OR NEW.evidence_json IS NULL + OR canonical_json_digest(NEW.manifest_json)!=NEW.manifest_digest + OR canonical_json_digest(NEW.evidence_json)!=NEW.evidence_digest + OR json_extract(NEW.manifest_json,'$.schema_version')!='character-training-checkpoint.v1' + OR json_extract(NEW.evidence_json,'$.schema_version')!='character-training-checkpoint-evidence.v1' + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id)!='training' + OR NEW.recipe_storage_id!=(SELECT recipe_storage_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.dataset_storage_id!=(SELECT dataset_storage_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.step!=training_checkpoint_step((SELECT total_steps FROM training_runs WHERE id=NEW.run_id),NEW.fraction) + OR json_extract(NEW.manifest_json,'$.run_id')!=NEW.run_id + OR json_extract(NEW.manifest_json,'$.reviewer_id')!=NEW.reviewer_id + OR json_extract(NEW.manifest_json,'$.recipe_id')!=(SELECT id FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) + OR json_extract(NEW.manifest_json,'$.dataset_digest')!=(SELECT dataset_digest FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) + OR json_extract(NEW.manifest_json,'$.character_id')!=(SELECT character_id FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) + OR json_extract(NEW.manifest_json,'$.step')!=NEW.step OR json_extract(NEW.manifest_json,'$.fraction')!=NEW.fraction + OR json_extract(NEW.manifest_json,'$.parent_checkpoint_id') IS NOT NEW.parent_checkpoint_id + OR json_extract(NEW.manifest_json,'$.artifact_digest')!=NEW.artifact_digest + OR json_extract(NEW.manifest_json,'$.safetensors_size_bytes')!=json_extract(NEW.evidence_json,'$.safetensors_size_bytes') + OR json_extract(NEW.manifest_json,'$.backend_digest')!=json_extract(NEW.evidence_json,'$.backend_digest') + OR json_extract(NEW.manifest_json,'$.runtime_digest')!=json_extract(NEW.evidence_json,'$.runtime_digest') + OR json_extract(NEW.manifest_json,'$.material_digest')!=json_extract(NEW.evidence_json,'$.material_digest') + OR json_extract(NEW.manifest_json,'$.source_digest')!=json_extract(NEW.evidence_json,'$.source_digest') + OR json_extract(NEW.manifest_json,'$.optimizer_state_digest')!=json_extract(NEW.evidence_json,'$.optimizer_state_digest') + OR json_extract(NEW.manifest_json,'$.trainer_state_digest')!=json_extract(NEW.evidence_json,'$.trainer_state_digest') + OR json_extract(NEW.manifest_json,'$.telemetry_digest')!=json_extract(NEW.evidence_json,'$.telemetry_digest') + OR json_extract(NEW.manifest_json,'$.telemetry_sequence_start')!=json_extract(NEW.evidence_json,'$.telemetry_sequence_start') + OR json_extract(NEW.manifest_json,'$.telemetry_sequence_end')!=json_extract(NEW.evidence_json,'$.telemetry_sequence_end') + OR json_extract(NEW.manifest_json,'$.trainable_targets_digest')!=json_extract(NEW.evidence_json,'$.trainable_targets_digest') + OR json_extract(NEW.manifest_json,'$.frozen_parameters_digest')!=json_extract(NEW.evidence_json,'$.frozen_parameters_digest') + OR json_extract(NEW.manifest_json,'$.finite_gradient_proof_digest')!=json_extract(NEW.evidence_json,'$.finite_gradient_proof_digest') + OR json_extract(NEW.manifest_json,'$.cancellation_boundary')!=json_extract(NEW.evidence_json,'$.cancellation_boundary') + OR json_extract(NEW.manifest_json,'$.inventory')!=json_extract(NEW.evidence_json,'$.inventory') + OR json_extract(NEW.evidence_json,'$.run_id')!=NEW.run_id + OR json_extract(NEW.evidence_json,'$.recipe_digest')!=(SELECT id FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) + OR json_extract(NEW.evidence_json,'$.dataset_digest')!=(SELECT dataset_digest FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) + OR NOT EXISTS(SELECT 1 FROM training_calibration_evidence c WHERE c.run_id=NEW.run_id AND c.succeeded=1 + AND c.backend_digest=json_extract(NEW.evidence_json,'$.backend_digest') + AND c.runtime_digest=json_extract(NEW.evidence_json,'$.runtime_digest') + AND c.material_digest=json_extract(NEW.evidence_json,'$.material_digest') + AND c.source_digest=json_extract(NEW.evidence_json,'$.source_digest')) + OR json_extract(NEW.evidence_json,'$.step')!=NEW.step OR json_extract(NEW.evidence_json,'$.fraction')!=NEW.fraction + OR json_extract(NEW.evidence_json,'$.parent_checkpoint_digest') IS NOT + (SELECT evidence_digest FROM training_checkpoints WHERE id=NEW.parent_checkpoint_id AND publication_state='finalized') + OR json_extract(NEW.evidence_json,'$.safetensors_digest')!=NEW.artifact_digest + OR json_extract(NEW.evidence_json,'$.safetensors_size_bytes')<=0 + OR json_extract(NEW.evidence_json,'$.telemetry_sequence_start')<1 + OR json_extract(NEW.evidence_json,'$.telemetry_sequence_end')COALESCE((SELECT max(step) FROM training_telemetry WHERE run_id=NEW.run_id),0) + THEN RAISE(ABORT,'training cancellation request lifecycle mismatch') END; +END; +""" + +_MIGRATION_37 = """ +DROP TRIGGER validate_training_event_insert_v36; +DROP TRIGGER validate_training_checkpoint_insert_v36; +DROP TRIGGER finalize_training_checkpoint_v36; +DROP TRIGGER guard_finalized_training_checkpoint_v36; +DROP TRIGGER validate_training_checkpoint_inventory_insert_v36; +DROP TRIGGER validate_training_cancellation_insert_v36; +ALTER TABLE training_cancellation_evidence ADD COLUMN cancellation_request_id TEXT; +ALTER TABLE training_cancellation_evidence ADD COLUMN request_digest TEXT; +UPDATE training_cancellation_evidence SET + cancellation_request_id=(SELECT id FROM training_cancellation_requests request + WHERE request.run_id=training_cancellation_evidence.run_id + AND request.reviewer_id=training_cancellation_evidence.reviewer_id + AND request.requested_step=training_cancellation_evidence.requested_step), + request_digest=(SELECT request_digest FROM training_cancellation_requests request + WHERE request.run_id=training_cancellation_evidence.run_id + AND request.reviewer_id=training_cancellation_evidence.reviewer_id + AND request.requested_step=training_cancellation_evidence.requested_step); +CREATE TRIGGER validate_training_checkpoint_insert_v37 BEFORE INSERT ON training_checkpoints BEGIN + SELECT CASE WHEN NEW.publication_state!='finalized' + OR training_checkpoint_json_is_exact(NEW.manifest_json,NEW.evidence_json)!=1 + OR canonical_json_digest(NEW.manifest_json)!=NEW.manifest_digest + OR canonical_json_digest(NEW.evidence_json)!=NEW.evidence_digest + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id)!='training' + OR NEW.recipe_storage_id!=(SELECT recipe_storage_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.dataset_storage_id!=(SELECT dataset_storage_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.step!=training_checkpoint_step((SELECT total_steps FROM training_runs WHERE id=NEW.run_id),NEW.fraction) + OR json_extract(NEW.manifest_json,'$.run_id')!=NEW.run_id + OR json_extract(NEW.manifest_json,'$.reviewer_id')!=NEW.reviewer_id + OR json_extract(NEW.manifest_json,'$.recipe_id')!=(SELECT id FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) + OR json_extract(NEW.manifest_json,'$.dataset_digest')!=(SELECT dataset_digest FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) + OR json_extract(NEW.manifest_json,'$.character_id')!=(SELECT character_id FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) + OR json_extract(NEW.manifest_json,'$.step')!=NEW.step OR json_extract(NEW.manifest_json,'$.fraction')!=NEW.fraction + OR json_extract(NEW.manifest_json,'$.parent_checkpoint_id') IS NOT NEW.parent_checkpoint_id + OR json_extract(NEW.manifest_json,'$.artifact_digest')!=NEW.artifact_digest + OR json_extract(NEW.manifest_json,'$.safetensors_size_bytes')!=json_extract(NEW.evidence_json,'$.safetensors_size_bytes') + OR json_extract(NEW.manifest_json,'$.backend_digest')!=json_extract(NEW.evidence_json,'$.backend_digest') + OR json_extract(NEW.manifest_json,'$.runtime_digest')!=json_extract(NEW.evidence_json,'$.runtime_digest') + OR json_extract(NEW.manifest_json,'$.material_digest')!=json_extract(NEW.evidence_json,'$.material_digest') + OR json_extract(NEW.manifest_json,'$.source_digest')!=json_extract(NEW.evidence_json,'$.source_digest') + OR json_extract(NEW.manifest_json,'$.optimizer_state_digest')!=json_extract(NEW.evidence_json,'$.optimizer_state_digest') + OR json_extract(NEW.manifest_json,'$.trainer_state_digest')!=json_extract(NEW.evidence_json,'$.trainer_state_digest') + OR json_extract(NEW.manifest_json,'$.telemetry_digest')!=json_extract(NEW.evidence_json,'$.telemetry_digest') + OR json_extract(NEW.manifest_json,'$.telemetry_sequence_start')!=json_extract(NEW.evidence_json,'$.telemetry_sequence_start') + OR json_extract(NEW.manifest_json,'$.telemetry_sequence_end')!=json_extract(NEW.evidence_json,'$.telemetry_sequence_end') + OR json_extract(NEW.manifest_json,'$.trainable_targets_digest')!=json_extract(NEW.evidence_json,'$.trainable_targets_digest') + OR json_extract(NEW.manifest_json,'$.frozen_parameters_digest')!=json_extract(NEW.evidence_json,'$.frozen_parameters_digest') + OR json_extract(NEW.manifest_json,'$.finite_gradient_proof_digest')!=json_extract(NEW.evidence_json,'$.finite_gradient_proof_digest') + OR json_extract(NEW.manifest_json,'$.cancellation_boundary')!=json_extract(NEW.evidence_json,'$.cancellation_boundary') + OR json_extract(NEW.evidence_json,'$.run_id')!=NEW.run_id + OR json_extract(NEW.evidence_json,'$.recipe_digest')!=(SELECT id FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) + OR json_extract(NEW.evidence_json,'$.dataset_digest')!=(SELECT dataset_digest FROM training_recipes WHERE storage_id=NEW.recipe_storage_id) + OR NOT EXISTS(SELECT 1 FROM training_calibration_evidence c WHERE c.run_id=NEW.run_id AND c.succeeded=1 + AND c.backend_digest=json_extract(NEW.evidence_json,'$.backend_digest') + AND c.runtime_digest=json_extract(NEW.evidence_json,'$.runtime_digest') + AND c.material_digest=json_extract(NEW.evidence_json,'$.material_digest') + AND c.source_digest=json_extract(NEW.evidence_json,'$.source_digest')) + OR json_extract(NEW.evidence_json,'$.step')!=NEW.step OR json_extract(NEW.evidence_json,'$.fraction')!=NEW.fraction + OR json_extract(NEW.evidence_json,'$.parent_checkpoint_digest') IS NOT + (SELECT evidence_digest FROM training_checkpoints WHERE id=NEW.parent_checkpoint_id AND publication_state='finalized') + OR json_extract(NEW.evidence_json,'$.safetensors_digest')!=NEW.artifact_digest + OR (SELECT count(*) FROM training_telemetry t WHERE t.run_id=NEW.run_id AND t.sequence BETWEEN json_extract(NEW.evidence_json,'$.telemetry_sequence_start') AND json_extract(NEW.evidence_json,'$.telemetry_sequence_end'))!=json_extract(NEW.evidence_json,'$.telemetry_sequence_end')-json_extract(NEW.evidence_json,'$.telemetry_sequence_start')+1 + OR json_extract(NEW.evidence_json,'$.telemetry_digest')!=(SELECT canonical_json_digest(json_group_array(json_object('sequence',sequence,'step',step,'payload',json(payload_json)))) FROM (SELECT sequence,step,payload_json FROM training_telemetry WHERE run_id=NEW.run_id AND sequence BETWEEN json_extract(NEW.evidence_json,'$.telemetry_sequence_start') AND json_extract(NEW.evidence_json,'$.telemetry_sequence_end') ORDER BY sequence)) + THEN RAISE(ABORT,'training checkpoint canonical publication mismatch') END; +END; +CREATE TRIGGER materialize_training_checkpoint_inventory_v37 AFTER INSERT ON training_checkpoints BEGIN + INSERT INTO training_checkpoint_inventory(checkpoint_id,ordinal,relative_path,artifact_digest,size_bytes,role) + SELECT NEW.id,CAST(item.key AS INTEGER),json_extract(item.value,'$.relative_path'), + json_extract(item.value,'$.digest'),json_extract(item.value,'$.size_bytes'),json_extract(item.value,'$.role') + FROM json_each(NEW.evidence_json,'$.inventory') item ORDER BY CAST(item.key AS INTEGER); +END; +CREATE TRIGGER validate_training_checkpoint_inventory_insert_v37 BEFORE INSERT ON training_checkpoint_inventory BEGIN + SELECT CASE WHEN (SELECT publication_state FROM training_checkpoints WHERE id=NEW.checkpoint_id)!='finalized' + OR NEW.ordinal!=(SELECT count(*) FROM training_checkpoint_inventory WHERE checkpoint_id=NEW.checkpoint_id) + OR NEW.ordinal>=7 + OR NOT EXISTS(SELECT 1 FROM training_checkpoints c,json_each(c.evidence_json,'$.inventory') item + WHERE c.id=NEW.checkpoint_id AND CAST(item.key AS INTEGER)=NEW.ordinal + AND json_extract(item.value,'$.relative_path')=NEW.relative_path + AND json_extract(item.value,'$.digest')=NEW.artifact_digest + AND json_extract(item.value,'$.size_bytes')=NEW.size_bytes + AND json_extract(item.value,'$.role')=NEW.role) + THEN RAISE(ABORT,'training checkpoint inventory is derived') END; +END; +CREATE TRIGGER guard_training_checkpoint_inventory_update_v37 BEFORE UPDATE ON training_checkpoint_inventory BEGIN SELECT RAISE(ABORT,'training checkpoint inventory is immutable'); END; +CREATE TRIGGER guard_training_checkpoint_inventory_delete_v37 BEFORE DELETE ON training_checkpoint_inventory BEGIN SELECT RAISE(ABORT,'training checkpoint inventory is immutable'); END; +CREATE TRIGGER guard_training_checkpoint_update_v37 BEFORE UPDATE ON training_checkpoints BEGIN SELECT RAISE(ABORT,'training checkpoints are immutable'); END; +CREATE TRIGGER validate_training_cancellation_insert_v37 BEFORE INSERT ON training_cancellation_evidence BEGIN + SELECT CASE WHEN (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id)!='training' + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.cancellation_request_id IS NULL OR NEW.request_digest IS NULL + OR NOT EXISTS(SELECT 1 FROM training_cancellation_requests request + WHERE request.id=NEW.cancellation_request_id AND request.run_id=NEW.run_id + AND request.reviewer_id=NEW.reviewer_id AND request.requested_step=NEW.requested_step + AND request.request_digest=NEW.request_digest) + OR NOT EXISTS(SELECT 1 FROM training_checkpoints checkpoint WHERE checkpoint.run_id=NEW.run_id + AND checkpoint.publication_state='finalized' AND checkpoint.step=NEW.boundary_step + AND checkpoint.artifact_digest=NEW.checkpoint_digest + AND json_extract(checkpoint.evidence_json,'$.cancellation_boundary')=1) + THEN RAISE(ABORT,'training cancellation request or boundary mismatch') END; +END; +CREATE TRIGGER validate_training_event_insert_v37 BEFORE INSERT ON training_run_events BEGIN + SELECT CASE WHEN NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.sequence!=COALESCE((SELECT max(sequence)+1 FROM training_run_events WHERE run_id=NEW.run_id),1) + OR NEW.previous_state!=(SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id) + OR NOT ((NEW.previous_state='draft' AND NEW.new_state='probed' AND EXISTS(SELECT 1 FROM training_probe_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND succeeded=1)) + OR (NEW.previous_state='probed' AND NEW.new_state='calibrated' AND EXISTS(SELECT 1 FROM training_calibration_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND succeeded=1)) + OR (NEW.previous_state='calibrated' AND NEW.new_state='preregistered' AND EXISTS(SELECT 1 FROM training_run_idempotency binding WHERE binding.run_id=NEW.run_id AND binding.reviewer_id=NEW.reviewer_id AND binding.idempotency_key=json_extract(NEW.payload_json,'$.idempotency_key') AND binding.request_digest=json_extract(NEW.payload_json,'$.request_digest'))) + OR (NEW.previous_state='preregistered' AND NEW.new_state='training') + OR (NEW.previous_state='training' AND NEW.new_state='evaluating' AND EXISTS(SELECT 1 FROM training_checkpoints checkpoint WHERE checkpoint.run_id=NEW.run_id AND checkpoint.fraction=1.0 AND checkpoint.publication_state='finalized') AND (SELECT max(step) FROM training_telemetry WHERE run_id=NEW.run_id)=(SELECT total_steps FROM training_runs WHERE id=NEW.run_id)) + OR (NEW.previous_state='evaluating' AND NEW.new_state IN ('qualified','rejected') AND EXISTS(SELECT 1 FROM training_evaluation_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND disposition=NEW.new_state AND evidence_digest=json_extract(NEW.payload_json,'$.evaluation_evidence_digest'))) + OR (NEW.previous_state IN ('draft','probed','calibrated','preregistered','training','evaluating') AND NEW.new_state='failed' AND EXISTS(SELECT 1 FROM training_failures WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id)) + OR (NEW.previous_state='training' AND NEW.new_state='cancelled' AND EXISTS( + SELECT 1 FROM training_cancellation_evidence evidence WHERE evidence.run_id=NEW.run_id + AND evidence.reviewer_id=NEW.reviewer_id + AND evidence.id=json_extract(NEW.payload_json,'$.cancellation_id') + AND evidence.cancellation_request_id=json_extract(NEW.payload_json,'$.cancellation_request_id') + AND evidence.request_digest=json_extract(NEW.payload_json,'$.request_digest')))) + THEN RAISE(ABORT,'invalid scientific training lifecycle transition') END; +END; +DROP TRIGGER validate_training_probe_insert_v36; +CREATE TRIGGER validate_training_probe_insert_v37 BEFORE INSERT ON training_probe_evidence BEGIN + SELECT CASE WHEN length(NEW.capability_digest)!=71 OR substr(NEW.capability_digest,1,7)!='sha256:' + OR substr(NEW.capability_digest,8) GLOB '*[^0-9a-f]*' + OR (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id)!='draft' + OR NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.recipe_digest!=(SELECT recipe.id FROM training_runs run JOIN training_recipes recipe ON recipe.storage_id=run.recipe_storage_id WHERE run.id=NEW.run_id) + OR NEW.dataset_digest!=(SELECT recipe.dataset_digest FROM training_runs run JOIN training_recipes recipe ON recipe.storage_id=run.recipe_storage_id WHERE run.id=NEW.run_id) + OR NEW.backend_digest!=(SELECT canonical_json_digest(json_extract(recipe.recipe_json,'$.backend_config')) FROM training_runs run JOIN training_recipes recipe ON recipe.storage_id=run.recipe_storage_id WHERE run.id=NEW.run_id) + OR NEW.runtime_digest!=(SELECT material.runtime_digest FROM training_runs run JOIN training_recipe_materials material ON material.recipe_storage_id=run.recipe_storage_id WHERE run.id=NEW.run_id) + OR NEW.source_digest!=(SELECT 'sha256:'||artifact.sha256 FROM training_runs run JOIN training_recipe_materials material ON material.recipe_storage_id=run.recipe_storage_id JOIN artifacts artifact ON artifact.id=material.source_snapshot_artifact_id WHERE run.id=NEW.run_id) + OR NEW.material_digest!=(SELECT canonical_json_digest(json_object('stage_3_campaign_id',material.stage_3_campaign_id,'stage_3_result_id',material.stage_3_result_id,'base_model_artifact_id',material.base_model_artifact_id,'lightning_adapter_artifact_ids',json(material.lightning_adapter_artifact_ids_json),'source_snapshot_artifact_id',material.source_snapshot_artifact_id,'runtime_digest',material.runtime_digest)) FROM training_runs run JOIN training_recipe_materials material ON material.recipe_storage_id=run.recipe_storage_id WHERE run.id=NEW.run_id) + THEN RAISE(ABORT,'training probe persisted identity mismatch') END; +END; +DROP TRIGGER validate_training_calibration_insert; +CREATE TRIGGER validate_training_calibration_insert_v37 BEFORE INSERT ON training_calibration_evidence BEGIN + SELECT CASE WHEN length(NEW.calibration_digest)!=71 OR substr(NEW.calibration_digest,1,7)!='sha256:' + OR substr(NEW.calibration_digest,8) GLOB '*[^0-9a-f]*' + OR (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id)!='probed' + OR NOT EXISTS(SELECT 1 FROM training_probe_evidence probe WHERE probe.id=NEW.probe_evidence_id + AND probe.run_id=NEW.run_id AND probe.reviewer_id=NEW.reviewer_id AND probe.succeeded=1 + AND probe.recipe_digest=NEW.recipe_digest AND probe.dataset_digest=NEW.dataset_digest + AND probe.backend_digest=NEW.backend_digest AND probe.runtime_digest=NEW.runtime_digest + AND probe.material_digest=NEW.material_digest AND probe.source_digest=NEW.source_digest) + THEN RAISE(ABORT,'training calibration identity mismatch') END; +END; +CREATE TRIGGER validate_training_resume_digest_shape_v37 BEFORE INSERT ON training_resume_evidence BEGIN + SELECT CASE WHEN sha256_digest_is_valid(NEW.parent_checkpoint_digest)!=1 + OR sha256_digest_is_valid(NEW.parent_artifact_digest)!=1 + OR sha256_digest_is_valid(NEW.recipe_digest)!=1 OR sha256_digest_is_valid(NEW.dataset_digest)!=1 + OR sha256_digest_is_valid(NEW.backend_digest)!=1 OR sha256_digest_is_valid(NEW.runtime_digest)!=1 + OR sha256_digest_is_valid(NEW.material_digest)!=1 OR sha256_digest_is_valid(NEW.source_digest)!=1 + OR sha256_digest_is_valid(NEW.optimizer_state_digest)!=1 + THEN RAISE(ABORT,'training resume digest shape mismatch') END; +END; +CREATE TRIGGER validate_training_evaluation_digest_shape_v37 BEFORE INSERT ON training_evaluation_evidence BEGIN + SELECT CASE WHEN sha256_digest_is_valid(NEW.evidence_digest)!=1 + THEN RAISE(ABORT,'training evaluation digest shape mismatch') END; +END; +CREATE TRIGGER validate_training_idempotency_digest_shape_v37 BEFORE INSERT ON training_run_idempotency BEGIN + SELECT CASE WHEN sha256_digest_is_valid(NEW.request_digest)!=1 + THEN RAISE(ABORT,'training idempotency digest shape mismatch') END; +END; +CREATE TRIGGER validate_training_cancel_request_digest_shape_v37 BEFORE INSERT ON training_cancellation_requests BEGIN + SELECT CASE WHEN sha256_digest_is_valid(NEW.request_digest)!=1 + THEN RAISE(ABORT,'training cancellation request digest shape mismatch') END; +END; +""" + +_MIGRATION_38 = """ +CREATE TRIGGER validate_training_checkpoint_parent_chain_v38 +BEFORE INSERT ON training_checkpoints BEGIN + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM training_recipes recipe, + json_each(recipe.recipe_json,'$.checkpoint_plan.fractions') declared + WHERE recipe.storage_id=NEW.recipe_storage_id AND declared.value=NEW.fraction + AND ((CAST(declared.key AS INTEGER)=0 + AND NEW.parent_checkpoint_id IS NULL + AND NOT EXISTS(SELECT 1 FROM training_checkpoints prior + WHERE prior.run_id=NEW.run_id AND prior.publication_state='finalized')) + OR (CAST(declared.key AS INTEGER)>0 + AND (SELECT count(*) FROM training_checkpoints prior + WHERE prior.run_id=NEW.run_id AND prior.publication_state='finalized')=CAST(declared.key AS INTEGER) + AND EXISTS( + SELECT 1 FROM training_checkpoints parent + WHERE parent.id=NEW.parent_checkpoint_id AND parent.run_id=NEW.run_id + AND parent.reviewer_id=NEW.reviewer_id + AND parent.recipe_storage_id=NEW.recipe_storage_id + AND parent.dataset_storage_id=NEW.dataset_storage_id + AND parent.publication_state='finalized' + AND parent.fraction=json_extract( + recipe.recipe_json,'$.checkpoint_plan.fractions['||(CAST(declared.key AS INTEGER)-1)||']') + AND parent.step=training_checkpoint_step( + (SELECT total_steps FROM training_runs WHERE id=NEW.run_id),parent.fraction) + AND parent.step=(SELECT max(latest.step) FROM training_checkpoints latest + WHERE latest.run_id=NEW.run_id AND latest.publication_state='finalized') + AND parent.evidence_digest=json_extract(NEW.evidence_json,'$.parent_checkpoint_digest')))) + ) THEN RAISE(ABORT,'training checkpoint parent chain mismatch') END; +END; +""" + +_MIGRATION_39 = """ +CREATE TABLE training_orchestration_operations( + id TEXT PRIMARY KEY, + reviewer_id TEXT NOT NULL, + run_id TEXT NOT NULL, + recipe_storage_id TEXT NOT NULL, + operation_kind TEXT NOT NULL CHECK(operation_kind IN ('probe','calibration','start','resume','cancel')), + idempotency_key TEXT NOT NULL, + request_digest TEXT NOT NULL, + request_payload_json TEXT NOT NULL, + lifecycle_version INTEGER NOT NULL CHECK(lifecycle_version>=0), + lifecycle_state TEXT NOT NULL CHECK(lifecycle_state IN ('draft','probed','calibrated','preregistered','training','evaluating','qualified','rejected','failed','cancelled')), + status TEXT NOT NULL CHECK(status IN ('pending','running','completed','failed')), + attempt INTEGER NOT NULL DEFAULT 0 CHECK(attempt>=0), + claimant_id TEXT, + lease_expires_at TEXT, + receipt_digest TEXT, + receipt_payload_json TEXT, + failure_digest TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE(reviewer_id,operation_kind,idempotency_key), + FOREIGN KEY(run_id) REFERENCES training_runs(id), + FOREIGN KEY(recipe_storage_id) REFERENCES training_recipes(storage_id) +); +CREATE TRIGGER validate_training_operation_insert_v39 BEFORE INSERT ON training_orchestration_operations BEGIN + SELECT CASE WHEN NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.recipe_storage_id!=(SELECT recipe_storage_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.lifecycle_version!=(SELECT lifecycle_version FROM training_runs WHERE id=NEW.run_id) + OR NEW.lifecycle_state!=(SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id) + OR canonical_json_is_exact(NEW.request_payload_json)!=1 + OR canonical_json_digest(NEW.request_payload_json)!=NEW.request_digest + OR json_extract(NEW.request_payload_json,'$.run_id') IS NOT NEW.run_id + OR sha256_digest_is_valid(NEW.request_digest)!=1 + OR NEW.status!='pending' OR NEW.attempt!=0 OR NEW.claimant_id IS NOT NULL + OR NEW.lease_expires_at IS NOT NULL OR NEW.receipt_digest IS NOT NULL + OR NEW.receipt_payload_json IS NOT NULL OR NEW.failure_digest IS NOT NULL + THEN RAISE(ABORT,'training operation request mismatch') END; +END; +CREATE TRIGGER validate_training_operation_update_v39 BEFORE UPDATE ON training_orchestration_operations BEGIN + SELECT CASE WHEN NEW.id!=OLD.id OR NEW.reviewer_id!=OLD.reviewer_id OR NEW.run_id!=OLD.run_id + OR NEW.recipe_storage_id!=OLD.recipe_storage_id OR NEW.operation_kind!=OLD.operation_kind + OR NEW.idempotency_key!=OLD.idempotency_key OR NEW.request_digest!=OLD.request_digest + OR NEW.request_payload_json!=OLD.request_payload_json OR NEW.lifecycle_version!=OLD.lifecycle_version + OR NEW.lifecycle_state!=OLD.lifecycle_state OR NEW.created_at!=OLD.created_at + OR OLD.status IN ('completed','failed') + OR (NEW.status='completed' AND ( + json_extract(NEW.receipt_payload_json,'$.run_id') IS NOT NEW.run_id + OR json_extract(NEW.receipt_payload_json,'$.lifecycle_version') IS NOT (SELECT lifecycle_version FROM training_runs WHERE id=NEW.run_id) + OR (NEW.operation_kind='probe' AND NOT EXISTS(SELECT 1 FROM training_probe_evidence e + WHERE e.id=json_extract(NEW.receipt_payload_json,'$.evidence_id') AND e.run_id=NEW.run_id + AND e.reviewer_id=NEW.reviewer_id AND e.succeeded=1)) + OR (NEW.operation_kind='calibration' AND NOT EXISTS(SELECT 1 FROM training_calibration_evidence e + WHERE e.id=json_extract(NEW.receipt_payload_json,'$.evidence_id') AND e.run_id=NEW.run_id + AND e.reviewer_id=NEW.reviewer_id AND e.succeeded=1)) + OR (NEW.operation_kind='resume' AND NOT EXISTS(SELECT 1 FROM training_resume_evidence e + WHERE e.id=json_extract(NEW.receipt_payload_json,'$.resume_id') AND e.run_id=NEW.run_id + AND e.reviewer_id=NEW.reviewer_id)) + OR (NEW.operation_kind='cancel' AND NOT EXISTS(SELECT 1 FROM training_cancellation_evidence e + WHERE e.id=json_extract(NEW.receipt_payload_json,'$.cancellation_id') AND e.run_id=NEW.run_id + AND e.reviewer_id=NEW.reviewer_id)))) + OR NOT ((OLD.status='pending' AND NEW.status='running' AND NEW.attempt=OLD.attempt+1 + AND NEW.claimant_id IS NOT NULL AND NEW.lease_expires_at IS NOT NULL + AND NEW.receipt_digest IS NULL AND NEW.receipt_payload_json IS NULL AND NEW.failure_digest IS NULL) + OR (OLD.status='running' AND NEW.status='running' AND julianday(OLD.lease_expires_at)<=julianday('now') + AND NEW.attempt=OLD.attempt+1 AND NEW.claimant_id IS NOT NULL AND NEW.lease_expires_at IS NOT NULL + AND NEW.receipt_digest IS NULL AND NEW.receipt_payload_json IS NULL AND NEW.failure_digest IS NULL) + OR (OLD.status='running' AND NEW.status='completed' AND NEW.attempt=OLD.attempt + AND NEW.claimant_id IS NULL AND NEW.lease_expires_at IS NULL AND NEW.failure_digest IS NULL + AND NEW.receipt_payload_json IS NOT NULL AND canonical_json_is_exact(NEW.receipt_payload_json)=1 + AND canonical_json_digest(NEW.receipt_payload_json)=NEW.receipt_digest + AND sha256_digest_is_valid(NEW.receipt_digest)=1) + OR (OLD.status='running' AND NEW.status='failed' AND NEW.attempt=OLD.attempt + AND NEW.claimant_id IS NULL AND NEW.lease_expires_at IS NULL + AND NEW.receipt_digest IS NULL AND NEW.receipt_payload_json IS NULL + AND sha256_digest_is_valid(NEW.failure_digest)=1)) + THEN RAISE(ABORT,'training operation status transition mismatch') END; +END; +CREATE TRIGGER guard_training_operation_delete_v39 BEFORE DELETE ON training_orchestration_operations BEGIN + SELECT RAISE(ABORT,'training operation is immutable'); +END; +""" + +_MIGRATION_40 = """ +DROP TRIGGER validate_training_operation_insert_v39; +DROP TRIGGER validate_training_operation_update_v39; +ALTER TABLE training_orchestration_operations ADD COLUMN claim_token TEXT; +CREATE TABLE training_operation_evidence_links( + operation_id TEXT NOT NULL, + attempt INTEGER NOT NULL CHECK(attempt>0), + claim_token TEXT NOT NULL, + evidence_kind TEXT NOT NULL CHECK(evidence_kind IN ('probe','calibration','resume','cancel','event','telemetry','checkpoint','artifact','inventory','evaluation')), + evidence_id TEXT NOT NULL, + evidence_digest TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY(operation_id,evidence_kind,evidence_id), + UNIQUE(evidence_kind,evidence_id), + FOREIGN KEY(operation_id) REFERENCES training_orchestration_operations(id) +); +CREATE TRIGGER validate_training_operation_insert_v40 BEFORE INSERT ON training_orchestration_operations BEGIN + SELECT CASE WHEN NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.recipe_storage_id!=(SELECT recipe_storage_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.lifecycle_version!=(SELECT lifecycle_version FROM training_runs WHERE id=NEW.run_id) + OR NEW.lifecycle_state!=(SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id) + OR training_operation_json_is_exact(NEW.operation_kind,'request',NEW.request_payload_json)!=1 + OR training_operation_request_identity_is_valid(NEW.operation_kind,NEW.run_id,NEW.request_payload_json)!=1 + OR canonical_json_digest(NEW.request_payload_json)!=NEW.request_digest + OR json_extract(NEW.request_payload_json,'$.run_id') IS NOT NEW.run_id + OR sha256_digest_is_valid(NEW.request_digest)!=1 + OR NEW.status!='pending' OR NEW.attempt!=0 OR NEW.claimant_id IS NOT NULL + OR NEW.claim_token IS NOT NULL OR NEW.lease_expires_at IS NOT NULL + OR NEW.receipt_digest IS NOT NULL OR NEW.receipt_payload_json IS NOT NULL + OR NEW.failure_digest IS NOT NULL + THEN RAISE(ABORT,'training operation request mismatch') END; +END; +CREATE TRIGGER validate_training_operation_update_v40 BEFORE UPDATE ON training_orchestration_operations BEGIN + SELECT CASE WHEN NEW.id!=OLD.id OR NEW.reviewer_id!=OLD.reviewer_id OR NEW.run_id!=OLD.run_id + OR NEW.recipe_storage_id!=OLD.recipe_storage_id OR NEW.operation_kind!=OLD.operation_kind + OR NEW.idempotency_key!=OLD.idempotency_key OR NEW.request_digest!=OLD.request_digest + OR NEW.request_payload_json!=OLD.request_payload_json OR NEW.lifecycle_version!=OLD.lifecycle_version + OR NEW.lifecycle_state!=OLD.lifecycle_state OR NEW.created_at!=OLD.created_at + OR OLD.status IN ('completed','failed') + OR (NEW.status='completed' AND ( + training_operation_json_is_exact(NEW.operation_kind,'receipt',NEW.receipt_payload_json)!=1 + OR canonical_json_digest(NEW.receipt_payload_json)!=NEW.receipt_digest + OR json_extract(NEW.receipt_payload_json,'$.run_id') IS NOT NEW.run_id + OR json_extract(NEW.receipt_payload_json,'$.lifecycle_version') IS NOT (SELECT lifecycle_version FROM training_runs WHERE id=NEW.run_id) + OR json_extract(NEW.receipt_payload_json,'$.lifecycle_state') IS NOT (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id) + OR training_operation_receipt_links_are_exact(NEW.id,NEW.receipt_payload_json)!=1 + OR NOT EXISTS(SELECT 1 FROM training_operation_evidence_links link + WHERE link.operation_id=NEW.id AND link.attempt=OLD.attempt AND link.claim_token=OLD.claim_token))) + OR NOT ((OLD.status='pending' AND NEW.status='running' AND NEW.attempt=OLD.attempt+1 + AND NEW.claimant_id IS NOT NULL AND NEW.claim_token IS NOT NULL + AND NEW.lease_expires_at IS NOT NULL AND NEW.receipt_digest IS NULL + AND NEW.receipt_payload_json IS NULL AND NEW.failure_digest IS NULL) + OR (OLD.status='running' AND NEW.status='running' AND julianday(OLD.lease_expires_at)<=julianday('now') + AND NEW.attempt=OLD.attempt+1 AND NEW.claimant_id IS NOT NULL AND NEW.claim_token IS NOT NULL + AND NEW.claim_token!=OLD.claim_token AND NEW.lease_expires_at IS NOT NULL + AND NEW.receipt_digest IS NULL AND NEW.receipt_payload_json IS NULL AND NEW.failure_digest IS NULL) + OR (OLD.status='running' AND NEW.status='completed' AND NEW.attempt=OLD.attempt + AND NEW.claimant_id IS NULL AND NEW.claim_token IS NULL AND NEW.lease_expires_at IS NULL + AND NEW.failure_digest IS NULL AND NEW.receipt_payload_json IS NOT NULL + AND sha256_digest_is_valid(NEW.receipt_digest)=1) + OR (OLD.status='running' AND NEW.status='failed' AND NEW.attempt=OLD.attempt + AND NEW.claimant_id IS NULL AND NEW.claim_token IS NULL AND NEW.lease_expires_at IS NULL + AND NEW.receipt_digest IS NULL AND NEW.receipt_payload_json IS NULL + AND sha256_digest_is_valid(NEW.failure_digest)=1)) + THEN RAISE(ABORT,'training operation status transition mismatch') END; +END; +CREATE TRIGGER validate_training_operation_evidence_link_v40 BEFORE INSERT ON training_operation_evidence_links BEGIN + SELECT CASE WHEN NOT EXISTS(SELECT 1 FROM training_orchestration_operations operation + WHERE operation.id=NEW.operation_id AND operation.status='running' + AND operation.attempt=NEW.attempt AND operation.claim_token=NEW.claim_token) + OR sha256_digest_is_valid(NEW.evidence_digest)!=1 + OR training_operation_evidence_digest(NEW.operation_id,NEW.evidence_kind,NEW.evidence_id) + IS NOT NEW.evidence_digest + OR ((SELECT operation_kind FROM training_orchestration_operations WHERE id=NEW.operation_id) + IN ('probe','calibration','resume','cancel') + AND NEW.evidence_kind!=(SELECT operation_kind FROM training_orchestration_operations WHERE id=NEW.operation_id)) + OR NOT ( + (NEW.evidence_kind='probe' AND EXISTS(SELECT 1 FROM training_probe_evidence e + WHERE e.id=NEW.evidence_id AND e.run_id=(SELECT run_id FROM training_orchestration_operations WHERE id=NEW.operation_id) + AND julianday(e.created_at)>=julianday((SELECT updated_at FROM training_orchestration_operations WHERE id=NEW.operation_id)))) + OR (NEW.evidence_kind='calibration' AND EXISTS(SELECT 1 FROM training_calibration_evidence e + WHERE e.id=NEW.evidence_id AND e.run_id=(SELECT run_id FROM training_orchestration_operations WHERE id=NEW.operation_id) + AND julianday(e.created_at)>=julianday((SELECT updated_at FROM training_orchestration_operations WHERE id=NEW.operation_id)))) + OR (NEW.evidence_kind='resume' AND EXISTS(SELECT 1 FROM training_resume_evidence e + WHERE e.id=NEW.evidence_id AND e.run_id=(SELECT run_id FROM training_orchestration_operations WHERE id=NEW.operation_id) + AND julianday(e.created_at)>=julianday((SELECT updated_at FROM training_orchestration_operations WHERE id=NEW.operation_id)))) + OR (NEW.evidence_kind='cancel' AND EXISTS(SELECT 1 FROM training_cancellation_evidence e + WHERE e.id=NEW.evidence_id AND e.run_id=(SELECT run_id FROM training_orchestration_operations WHERE id=NEW.operation_id) + AND julianday(e.created_at)>=julianday((SELECT updated_at FROM training_orchestration_operations WHERE id=NEW.operation_id)))) + OR (NEW.evidence_kind='event' AND EXISTS(SELECT 1 FROM training_run_events e + WHERE e.run_id=(SELECT run_id FROM training_orchestration_operations WHERE id=NEW.operation_id) + AND CAST(e.sequence AS TEXT)=NEW.evidence_id + AND julianday(e.created_at)>=julianday((SELECT updated_at FROM training_orchestration_operations WHERE id=NEW.operation_id)))) + OR (NEW.evidence_kind='telemetry' AND EXISTS(SELECT 1 FROM training_telemetry e + WHERE e.run_id=(SELECT run_id FROM training_orchestration_operations WHERE id=NEW.operation_id) + AND CAST(e.sequence AS TEXT)=NEW.evidence_id + AND julianday(e.created_at)>=julianday((SELECT updated_at FROM training_orchestration_operations WHERE id=NEW.operation_id)))) + OR (NEW.evidence_kind='checkpoint' AND EXISTS(SELECT 1 FROM training_checkpoints e + WHERE e.run_id=(SELECT run_id FROM training_orchestration_operations WHERE id=NEW.operation_id) + AND e.id=NEW.evidence_id AND e.evidence_digest=NEW.evidence_digest + AND julianday(e.created_at)>=julianday((SELECT updated_at FROM training_orchestration_operations WHERE id=NEW.operation_id)))) + OR (NEW.evidence_kind='artifact' AND EXISTS(SELECT 1 FROM artifacts e + WHERE e.id=NEW.evidence_id AND 'sha256:'||e.sha256=NEW.evidence_digest + AND julianday(e.created_at)>=julianday((SELECT updated_at FROM training_orchestration_operations WHERE id=NEW.operation_id)))) + OR (NEW.evidence_kind='inventory' AND EXISTS(SELECT 1 FROM training_checkpoint_inventory e + JOIN training_checkpoints checkpoint ON checkpoint.id=e.checkpoint_id + WHERE checkpoint.run_id=(SELECT run_id FROM training_orchestration_operations WHERE id=NEW.operation_id) + AND e.checkpoint_id||':'||e.role=NEW.evidence_id + AND julianday(checkpoint.created_at)>=julianday((SELECT updated_at FROM training_orchestration_operations WHERE id=NEW.operation_id)))) + OR (NEW.evidence_kind='evaluation' AND EXISTS(SELECT 1 FROM training_evaluation_evidence e + WHERE e.run_id=(SELECT run_id FROM training_orchestration_operations WHERE id=NEW.operation_id) + AND e.id=NEW.evidence_id + AND julianday(e.created_at)>=julianday((SELECT updated_at FROM training_orchestration_operations WHERE id=NEW.operation_id))))) + THEN RAISE(ABORT,'training operation evidence fence mismatch') END; +END; +CREATE TRIGGER guard_training_operation_evidence_link_update_v40 BEFORE UPDATE ON training_operation_evidence_links BEGIN SELECT RAISE(ABORT,'training operation evidence link is immutable'); END; +CREATE TRIGGER guard_training_operation_evidence_link_delete_v40 BEFORE DELETE ON training_operation_evidence_links BEGIN SELECT RAISE(ABORT,'training operation evidence link is immutable'); END; +""" + +_MIGRATION_41 = """ +CREATE TABLE training_checkpoint_inventory_artifacts( + checkpoint_id TEXT NOT NULL, + ordinal INTEGER NOT NULL, + artifact_id TEXT NOT NULL, + role TEXT NOT NULL, + artifact_digest TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + PRIMARY KEY(checkpoint_id,ordinal), + UNIQUE(checkpoint_id,artifact_id), + FOREIGN KEY(checkpoint_id,ordinal) REFERENCES training_checkpoint_inventory(checkpoint_id,ordinal), + FOREIGN KEY(artifact_id) REFERENCES artifacts(id) +); +INSERT INTO training_checkpoint_inventory_artifacts +SELECT inventory.checkpoint_id,inventory.ordinal,artifact.id,inventory.role, + inventory.artifact_digest,inventory.size_bytes +FROM training_checkpoint_inventory inventory JOIN artifacts artifact + ON 'sha256:'||artifact.sha256=inventory.artifact_digest + AND artifact.byte_size=inventory.size_bytes; +CREATE TRIGGER validate_training_checkpoint_inventory_artifact_insert_v41 +BEFORE INSERT ON training_checkpoint_inventory_artifacts BEGIN + SELECT CASE WHEN NOT EXISTS(SELECT 1 FROM training_checkpoint_inventory inventory + JOIN training_checkpoints checkpoint ON checkpoint.id=inventory.checkpoint_id + JOIN training_runs run ON run.id=checkpoint.run_id + JOIN artifacts artifact ON artifact.id=NEW.artifact_id + WHERE inventory.checkpoint_id=NEW.checkpoint_id AND inventory.ordinal=NEW.ordinal + AND inventory.role=NEW.role AND inventory.artifact_digest=NEW.artifact_digest + AND inventory.size_bytes=NEW.size_bytes AND 'sha256:'||artifact.sha256=NEW.artifact_digest + AND artifact.byte_size=NEW.size_bytes) + THEN RAISE(ABORT,'training checkpoint artifact binding mismatch') END; +END; +CREATE TRIGGER guard_training_checkpoint_inventory_artifact_update_v41 +BEFORE UPDATE ON training_checkpoint_inventory_artifacts BEGIN SELECT RAISE(ABORT,'training checkpoint artifact binding is immutable'); END; +CREATE TRIGGER guard_training_checkpoint_inventory_artifact_delete_v41 +BEFORE DELETE ON training_checkpoint_inventory_artifacts BEGIN SELECT RAISE(ABORT,'training checkpoint artifact binding is immutable'); END; +CREATE TRIGGER validate_training_operation_lifecycle_insert_v41 BEFORE INSERT ON training_orchestration_operations BEGIN + SELECT CASE WHEN training_operation_lifecycle_is_valid( + NEW.operation_kind,NEW.lifecycle_state)!=1 + OR (NEW.operation_kind='resume' AND NOT EXISTS(SELECT 1 FROM training_checkpoints checkpoint + WHERE checkpoint.id=json_extract(NEW.request_payload_json,'$.parent_checkpoint_id') + AND checkpoint.run_id=NEW.run_id AND checkpoint.publication_state='finalized')) + THEN RAISE(ABORT,'training operation lifecycle mismatch') END; +END; +CREATE TRIGGER validate_training_operation_completion_v41 +BEFORE UPDATE OF status ON training_orchestration_operations WHEN NEW.status='completed' BEGIN + SELECT CASE WHEN training_operation_completion_is_derived(NEW.id)!=1 + THEN RAISE(ABORT,'training operation completion is not derived from complete attempt evidence') END; +END; +""" + +_MIGRATION_42 = """ +DROP TRIGGER validate_training_operation_evidence_link_v40; +DROP TRIGGER validate_training_operation_update_v40; +CREATE TRIGGER validate_training_operation_update_v42 +BEFORE UPDATE ON training_orchestration_operations BEGIN + SELECT CASE WHEN NEW.id!=OLD.id OR NEW.reviewer_id!=OLD.reviewer_id OR NEW.run_id!=OLD.run_id + OR NEW.recipe_storage_id!=OLD.recipe_storage_id OR NEW.operation_kind!=OLD.operation_kind + OR NEW.idempotency_key!=OLD.idempotency_key OR NEW.request_digest!=OLD.request_digest + OR NEW.request_payload_json!=OLD.request_payload_json OR NEW.lifecycle_version!=OLD.lifecycle_version + OR NEW.lifecycle_state!=OLD.lifecycle_state OR NEW.created_at!=OLD.created_at + OR OLD.status IN ('completed','failed') + OR (NEW.status='completed' AND ( + training_operation_json_is_exact(NEW.operation_kind,'receipt',NEW.receipt_payload_json)!=1 + OR canonical_json_digest(NEW.receipt_payload_json)!=NEW.receipt_digest + OR json_extract(NEW.receipt_payload_json,'$.run_id') IS NOT NEW.run_id + OR json_extract(NEW.receipt_payload_json,'$.lifecycle_version') IS NOT + (SELECT lifecycle_version FROM training_runs WHERE id=NEW.run_id) + OR json_extract(NEW.receipt_payload_json,'$.lifecycle_state') IS NOT + (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id) + OR training_operation_receipt_links_are_exact(NEW.id,NEW.receipt_payload_json)!=1 + OR NOT EXISTS(SELECT 1 FROM training_operation_evidence_links link + WHERE link.operation_id=NEW.id))) + OR NOT ((OLD.status='pending' AND NEW.status='running' AND NEW.attempt=OLD.attempt+1 + AND NEW.claimant_id IS NOT NULL AND NEW.claim_token IS NOT NULL + AND NEW.lease_expires_at IS NOT NULL AND NEW.receipt_digest IS NULL + AND NEW.receipt_payload_json IS NULL AND NEW.failure_digest IS NULL) + OR (OLD.status='running' AND NEW.status='running' + AND julianday(OLD.lease_expires_at)<=julianday('now') + AND NEW.attempt=OLD.attempt+1 AND NEW.claimant_id IS NOT NULL + AND NEW.claim_token IS NOT NULL AND NEW.claim_token!=OLD.claim_token + AND NEW.lease_expires_at IS NOT NULL AND NEW.receipt_digest IS NULL + AND NEW.receipt_payload_json IS NULL AND NEW.failure_digest IS NULL) + OR (OLD.status='running' AND NEW.status='completed' AND NEW.attempt=OLD.attempt + AND NEW.claimant_id IS NULL AND NEW.claim_token IS NULL AND NEW.lease_expires_at IS NULL + AND NEW.failure_digest IS NULL AND NEW.receipt_payload_json IS NOT NULL + AND sha256_digest_is_valid(NEW.receipt_digest)=1) + OR (OLD.status='running' AND NEW.status='failed' AND NEW.attempt=OLD.attempt + AND NEW.claimant_id IS NULL AND NEW.claim_token IS NULL AND NEW.lease_expires_at IS NULL + AND NEW.receipt_digest IS NULL AND NEW.receipt_payload_json IS NULL + AND sha256_digest_is_valid(NEW.failure_digest)=1)) + THEN RAISE(ABORT,'training operation status transition mismatch') END; +END; +CREATE TABLE training_calibration_operation_bindings( + evidence_id TEXT PRIMARY KEY, + operation_id TEXT NOT NULL UNIQUE, + request_digest TEXT NOT NULL, + attempt INTEGER NOT NULL CHECK(attempt>0), + claim_token TEXT NOT NULL, + steps INTEGER NOT NULL CHECK(steps>0), + trainable_targets_json TEXT NOT NULL, + seed INTEGER NOT NULL, + created_at TEXT NOT NULL, + FOREIGN KEY(evidence_id) REFERENCES training_calibration_evidence(id), + FOREIGN KEY(operation_id) REFERENCES training_orchestration_operations(id) +); +CREATE TRIGGER validate_training_calibration_operation_binding_v42 +BEFORE INSERT ON training_calibration_operation_bindings BEGIN + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM training_orchestration_operations operation + JOIN training_calibration_evidence evidence ON evidence.id=NEW.evidence_id + WHERE operation.id=NEW.operation_id AND operation.operation_kind='calibration' + AND operation.status='running' AND operation.attempt=NEW.attempt + AND operation.claim_token=NEW.claim_token AND operation.request_digest=NEW.request_digest + AND evidence.run_id=operation.run_id AND evidence.reviewer_id=operation.reviewer_id + AND evidence.recipe_digest=json_extract(operation.request_payload_json,'$.recipe_digest') + AND evidence.dataset_digest=json_extract(operation.request_payload_json,'$.dataset_digest') + AND evidence.backend_digest=json_extract(operation.request_payload_json,'$.backend_digest') + AND evidence.runtime_digest=json_extract(operation.request_payload_json,'$.runtime_digest') + AND evidence.material_digest=json_extract(operation.request_payload_json,'$.material_digest') + AND evidence.source_digest=json_extract(operation.request_payload_json,'$.source_digest') + AND evidence.probe_evidence_id=(SELECT json_extract(probe.receipt_payload_json,'$.evidence_id') + FROM training_orchestration_operations probe + WHERE probe.id=json_extract(operation.request_payload_json,'$.probe_operation_id')) + AND NEW.steps=json_extract(operation.request_payload_json,'$.steps') + AND NEW.trainable_targets_json=json_extract(operation.request_payload_json,'$.trainable_targets') + AND NEW.seed=json_extract(operation.request_payload_json,'$.seed') + ) THEN RAISE(ABORT,'training calibration operation binding mismatch') END; +END; +CREATE TRIGGER guard_training_calibration_operation_binding_update_v42 +BEFORE UPDATE ON training_calibration_operation_bindings BEGIN + SELECT RAISE(ABORT,'training calibration operation binding is immutable'); +END; +CREATE TRIGGER guard_training_calibration_operation_binding_delete_v42 +BEFORE DELETE ON training_calibration_operation_bindings BEGIN + SELECT RAISE(ABORT,'training calibration operation binding is immutable'); +END; +CREATE TABLE training_cancellation_operation_claims( + cancellation_request_id TEXT PRIMARY KEY, + operation_id TEXT NOT NULL UNIQUE, + request_digest TEXT NOT NULL, + requested_step INTEGER NOT NULL CHECK(requested_step>=0), + attempt INTEGER NOT NULL DEFAULT 0 CHECK(attempt>=0), + claim_token TEXT, + created_at TEXT NOT NULL, + FOREIGN KEY(cancellation_request_id) REFERENCES training_cancellation_requests(id), + FOREIGN KEY(operation_id) REFERENCES training_orchestration_operations(id) +); +CREATE TRIGGER consume_training_cancellation_request_v42 +AFTER INSERT ON training_orchestration_operations WHEN NEW.operation_kind='cancel' BEGIN + INSERT INTO training_cancellation_operation_claims( + cancellation_request_id,operation_id,request_digest,requested_step,attempt,claim_token,created_at) + SELECT request.id,NEW.id,request.request_digest,request.requested_step,0,NULL,NEW.created_at + FROM training_cancellation_requests request + WHERE request.id=json_extract(NEW.request_payload_json,'$.cancellation_request_id') + AND request.run_id=NEW.run_id AND request.reviewer_id=NEW.reviewer_id + AND request.request_digest=json_extract(NEW.request_payload_json,'$.request_digest') + AND request.requested_step=json_extract(NEW.request_payload_json,'$.requested_step'); + SELECT CASE WHEN changes()!=1 + THEN RAISE(ABORT,'training cancellation request consumption mismatch') END; +END; +CREATE TRIGGER claim_training_cancellation_request_v42 +AFTER UPDATE OF status,attempt,claim_token ON training_orchestration_operations +WHEN NEW.operation_kind='cancel' AND NEW.status='running' + AND (OLD.status!='running' OR NEW.attempt!=OLD.attempt) BEGIN + UPDATE training_cancellation_operation_claims SET attempt=NEW.attempt,claim_token=NEW.claim_token + WHERE operation_id=NEW.id AND attempt=OLD.attempt; + SELECT CASE WHEN changes()!=1 + THEN RAISE(ABORT,'training cancellation request claim mismatch') END; +END; +CREATE TRIGGER guard_training_cancellation_operation_claim_delete_v42 +BEFORE DELETE ON training_cancellation_operation_claims BEGIN + SELECT RAISE(ABORT,'training cancellation request consumption is immutable'); +END; +CREATE TRIGGER validate_training_operation_evidence_link_v42 +BEFORE INSERT ON training_operation_evidence_links BEGIN + SELECT CASE WHEN training_operation_evidence_link_is_valid( + NEW.operation_id,NEW.attempt,NEW.claim_token,NEW.evidence_kind,NEW.evidence_id,NEW.evidence_digest)!=1 + THEN RAISE(ABORT,'training operation evidence fence mismatch') END; +END; +""" + +_MIGRATION_43 = """ +DROP TRIGGER validate_training_operation_evidence_link_v42; +CREATE TRIGGER validate_training_operation_evidence_link_v43 +BEFORE INSERT ON training_operation_evidence_links BEGIN + SELECT CASE WHEN training_operation_evidence_link_is_valid( + NEW.operation_id,NEW.attempt,NEW.claim_token,NEW.evidence_kind,NEW.evidence_id,NEW.evidence_digest)!=1 + THEN RAISE(ABORT,'training operation evidence fence mismatch') END; +END; +CREATE TRIGGER guard_training_cancellation_operation_claim_update_v43 +BEFORE UPDATE ON training_cancellation_operation_claims BEGIN + SELECT CASE WHEN NEW.cancellation_request_id!=OLD.cancellation_request_id + OR NEW.operation_id!=OLD.operation_id OR NEW.request_digest!=OLD.request_digest + OR NEW.requested_step!=OLD.requested_step OR NEW.created_at!=OLD.created_at + OR NEW.attempt!=OLD.attempt+1 OR NEW.claim_token IS NULL + OR NEW.claim_token IS OLD.claim_token + OR NOT EXISTS( + SELECT 1 FROM training_orchestration_operations operation + WHERE operation.id=OLD.operation_id AND operation.operation_kind='cancel' + AND operation.status='running' AND operation.attempt=NEW.attempt + AND operation.claim_token=NEW.claim_token + AND operation.run_id=(SELECT run_id FROM training_cancellation_requests + WHERE id=OLD.cancellation_request_id) + AND json_extract(operation.request_payload_json,'$.cancellation_request_id')= + OLD.cancellation_request_id + AND json_extract(operation.request_payload_json,'$.request_digest')=OLD.request_digest + AND json_extract(operation.request_payload_json,'$.requested_step')=OLD.requested_step) + THEN RAISE(ABORT,'training cancellation claim changes only through exact live reclaim') END; +END; +""" + +_MIGRATION_44 = """ +ALTER TABLE training_orchestration_operations ADD COLUMN result_digest TEXT + CHECK(result_digest IS NULL OR sha256_digest_is_valid(result_digest)=1); +ALTER TABLE training_orchestration_operations ADD COLUMN result_payload_json TEXT + CHECK(result_payload_json IS NULL OR (json_valid(result_payload_json) + AND json_type(result_payload_json)='object' + AND canonical_json_is_exact(result_payload_json)=1)); +SELECT 1; +CREATE TRIGGER validate_training_operation_durable_result_v44 +BEFORE UPDATE OF status,result_digest,result_payload_json ON training_orchestration_operations BEGIN + SELECT CASE WHEN + (NEW.status='completed')!=(NEW.result_digest IS NOT NULL AND NEW.result_payload_json IS NOT NULL) + OR (NEW.result_payload_json IS NOT NULL + AND canonical_json_digest(NEW.result_payload_json)!=NEW.result_digest) + THEN RAISE(ABORT,'training operation durable result mismatch') END; +END; +CREATE TRIGGER guard_training_operation_durable_result_v44 +BEFORE UPDATE OF result_digest,result_payload_json ON training_orchestration_operations +WHEN OLD.status='completed' BEGIN + SELECT RAISE(ABORT,'training operation durable result is immutable'); +END; +""" + +_MIGRATION_45 = """ +DROP TRIGGER validate_training_calibration_operation_binding_v42; +DROP TRIGGER guard_training_calibration_operation_binding_update_v42; +DROP TRIGGER guard_training_calibration_operation_binding_delete_v42; +ALTER TABLE training_calibration_operation_bindings + RENAME TO training_calibration_operation_bindings_v44; +CREATE TABLE training_calibration_operation_bindings( + evidence_id TEXT PRIMARY KEY, + operation_id TEXT NOT NULL UNIQUE, + request_digest TEXT NOT NULL, + attempt INTEGER NOT NULL CHECK(attempt>0), + claim_token TEXT NOT NULL, + steps INTEGER NOT NULL CHECK(steps>=0), + trainable_targets_json TEXT NOT NULL, + seed INTEGER NOT NULL, + created_at TEXT NOT NULL, + FOREIGN KEY(evidence_id) REFERENCES training_calibration_evidence(id), + FOREIGN KEY(operation_id) REFERENCES training_orchestration_operations(id) +); +INSERT INTO training_calibration_operation_bindings( + evidence_id,operation_id,request_digest,attempt,claim_token,steps, + trainable_targets_json,seed,created_at +) SELECT evidence_id,operation_id,request_digest,attempt,claim_token,steps, + trainable_targets_json,seed,created_at + FROM training_calibration_operation_bindings_v44; +DROP TABLE training_calibration_operation_bindings_v44; +CREATE TRIGGER validate_training_calibration_operation_binding_v42 +BEFORE INSERT ON training_calibration_operation_bindings BEGIN + SELECT CASE WHEN NOT EXISTS( + SELECT 1 FROM training_orchestration_operations operation + JOIN training_calibration_evidence evidence ON evidence.id=NEW.evidence_id + WHERE operation.id=NEW.operation_id AND operation.operation_kind='calibration' + AND operation.status='running' AND operation.attempt=NEW.attempt + AND operation.claim_token=NEW.claim_token AND operation.request_digest=NEW.request_digest + AND evidence.run_id=operation.run_id AND evidence.reviewer_id=operation.reviewer_id + AND evidence.recipe_digest=json_extract(operation.request_payload_json,'$.recipe_digest') + AND evidence.dataset_digest=json_extract(operation.request_payload_json,'$.dataset_digest') + AND evidence.backend_digest=json_extract(operation.request_payload_json,'$.backend_digest') + AND evidence.runtime_digest=json_extract(operation.request_payload_json,'$.runtime_digest') + AND evidence.material_digest=json_extract(operation.request_payload_json,'$.material_digest') + AND evidence.source_digest=json_extract(operation.request_payload_json,'$.source_digest') + AND evidence.probe_evidence_id=(SELECT json_extract(probe.receipt_payload_json,'$.evidence_id') + FROM training_orchestration_operations probe + WHERE probe.id=json_extract(operation.request_payload_json,'$.probe_operation_id')) + AND NEW.steps=json_extract(operation.request_payload_json,'$.steps') + AND NEW.trainable_targets_json=json_extract(operation.request_payload_json,'$.trainable_targets') + AND NEW.seed=json_extract(operation.request_payload_json,'$.seed') + ) THEN RAISE(ABORT,'training calibration operation binding mismatch') END; +END; +CREATE TRIGGER guard_training_calibration_operation_binding_update_v42 +BEFORE UPDATE ON training_calibration_operation_bindings BEGIN + SELECT RAISE(ABORT,'training calibration operation binding is immutable'); +END; +CREATE TRIGGER guard_training_calibration_operation_binding_delete_v42 +BEFORE DELETE ON training_calibration_operation_bindings BEGIN + SELECT RAISE(ABORT,'training calibration operation binding is immutable'); +END; +""" + + +_MIGRATION_46 = """ +DROP TRIGGER validate_training_event_insert_v37; +CREATE TRIGGER validate_training_event_insert_v46 BEFORE INSERT ON training_run_events BEGIN + SELECT CASE WHEN NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.sequence!=COALESCE((SELECT max(sequence)+1 FROM training_run_events WHERE run_id=NEW.run_id),1) + OR NEW.previous_state!=(SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id) + OR NOT ((NEW.previous_state='draft' AND NEW.new_state='probed' AND EXISTS(SELECT 1 FROM training_probe_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND succeeded=1)) + OR (NEW.previous_state='probed' AND NEW.new_state='calibrated' AND EXISTS(SELECT 1 FROM training_calibration_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND succeeded=1)) + OR (NEW.previous_state='calibrated' AND NEW.new_state='preregistered' AND EXISTS(SELECT 1 FROM training_run_idempotency binding WHERE binding.run_id=NEW.run_id AND binding.reviewer_id=NEW.reviewer_id AND binding.idempotency_key=json_extract(NEW.payload_json,'$.idempotency_key') AND binding.request_digest=json_extract(NEW.payload_json,'$.request_digest'))) + OR (NEW.previous_state='preregistered' AND NEW.new_state='training') + OR (NEW.previous_state='cancelled' AND NEW.new_state='training' AND EXISTS(SELECT 1 FROM training_checkpoints checkpoint WHERE checkpoint.run_id=NEW.run_id AND checkpoint.id=json_extract(NEW.payload_json,'$.parent_checkpoint_id') AND checkpoint.evidence_digest=json_extract(NEW.payload_json,'$.parent_checkpoint_digest') AND checkpoint.publication_state='finalized')) + OR (NEW.previous_state='training' AND NEW.new_state='evaluating' AND EXISTS(SELECT 1 FROM training_checkpoints checkpoint WHERE checkpoint.run_id=NEW.run_id AND checkpoint.fraction=1.0 AND checkpoint.publication_state='finalized') AND (SELECT max(step) FROM training_telemetry WHERE run_id=NEW.run_id)=(SELECT total_steps FROM training_runs WHERE id=NEW.run_id)) + OR (NEW.previous_state='evaluating' AND NEW.new_state IN ('qualified','rejected') AND EXISTS(SELECT 1 FROM training_evaluation_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND disposition=NEW.new_state AND evidence_digest=json_extract(NEW.payload_json,'$.evaluation_evidence_digest'))) + OR (NEW.previous_state IN ('draft','probed','calibrated','preregistered','training','evaluating') AND NEW.new_state='failed' AND EXISTS(SELECT 1 FROM training_failures WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id)) + OR (NEW.previous_state='training' AND NEW.new_state='cancelled' AND EXISTS( + SELECT 1 FROM training_cancellation_evidence evidence WHERE evidence.run_id=NEW.run_id + AND evidence.reviewer_id=NEW.reviewer_id + AND evidence.id=json_extract(NEW.payload_json,'$.cancellation_id') + AND evidence.cancellation_request_id=json_extract(NEW.payload_json,'$.cancellation_request_id') + AND evidence.request_digest=json_extract(NEW.payload_json,'$.request_digest')))) + THEN RAISE(ABORT,'invalid scientific training lifecycle transition') END; +END; +""" + + +_MIGRATION_47 = """ +CREATE TABLE training_resume_attempt_failures( + operation_id TEXT NOT NULL,attempt INTEGER NOT NULL CHECK(attempt>0), + claim_token TEXT NOT NULL,failure_digest TEXT NOT NULL, + created_at TEXT NOT NULL,PRIMARY KEY(operation_id,attempt), + UNIQUE(operation_id,claim_token), + FOREIGN KEY(operation_id) REFERENCES training_orchestration_operations(id) +); +CREATE TRIGGER validate_training_resume_attempt_failure_insert_v47 +BEFORE INSERT ON training_resume_attempt_failures BEGIN + SELECT CASE WHEN sha256_digest_is_valid(NEW.failure_digest)!=1 OR NOT EXISTS( + SELECT 1 FROM training_orchestration_operations operation + WHERE operation.id=NEW.operation_id AND operation.operation_kind='resume' + AND operation.status='running' AND operation.attempt=NEW.attempt + AND operation.claim_token=NEW.claim_token) + THEN RAISE(ABORT,'training resume attempt failure mismatch') END; +END; +CREATE TRIGGER guard_training_resume_attempt_failure_update_v47 +BEFORE UPDATE ON training_resume_attempt_failures BEGIN + SELECT RAISE(ABORT,'training resume attempt failure is immutable'); +END; +CREATE TRIGGER guard_training_resume_attempt_failure_delete_v47 +BEFORE DELETE ON training_resume_attempt_failures BEGIN + SELECT RAISE(ABORT,'training resume attempt failure is immutable'); +END; +DROP TRIGGER validate_training_operation_update_v42; +CREATE TRIGGER validate_training_operation_update_v47 +BEFORE UPDATE ON training_orchestration_operations BEGIN + SELECT CASE WHEN NEW.id!=OLD.id OR NEW.reviewer_id!=OLD.reviewer_id OR NEW.run_id!=OLD.run_id + OR NEW.recipe_storage_id!=OLD.recipe_storage_id OR NEW.operation_kind!=OLD.operation_kind + OR NEW.idempotency_key!=OLD.idempotency_key OR NEW.request_digest!=OLD.request_digest + OR NEW.request_payload_json!=OLD.request_payload_json OR NEW.lifecycle_version!=OLD.lifecycle_version + OR NEW.lifecycle_state!=OLD.lifecycle_state OR NEW.created_at!=OLD.created_at + OR OLD.status IN ('completed','failed') + OR (NEW.status='completed' AND ( + training_operation_json_is_exact(NEW.operation_kind,'receipt',NEW.receipt_payload_json)!=1 + OR canonical_json_digest(NEW.receipt_payload_json)!=NEW.receipt_digest + OR json_extract(NEW.receipt_payload_json,'$.run_id') IS NOT NEW.run_id + OR json_extract(NEW.receipt_payload_json,'$.lifecycle_version') IS NOT + (SELECT lifecycle_version FROM training_runs WHERE id=NEW.run_id) + OR json_extract(NEW.receipt_payload_json,'$.lifecycle_state') IS NOT + (SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id) + OR training_operation_receipt_links_are_exact(NEW.id,NEW.receipt_payload_json)!=1 + OR NOT EXISTS(SELECT 1 FROM training_operation_evidence_links link + WHERE link.operation_id=NEW.id))) + OR NOT ((OLD.status='pending' AND NEW.status='running' AND NEW.attempt=OLD.attempt+1 + AND NEW.claimant_id IS NOT NULL AND NEW.claim_token IS NOT NULL + AND NEW.lease_expires_at IS NOT NULL AND NEW.receipt_digest IS NULL + AND NEW.receipt_payload_json IS NULL AND NEW.failure_digest IS NULL) + OR (OLD.status='running' AND NEW.status='running' + AND julianday(OLD.lease_expires_at)<=julianday('now') + AND NEW.attempt=OLD.attempt+1 AND NEW.claimant_id IS NOT NULL + AND NEW.claim_token IS NOT NULL AND NEW.claim_token!=OLD.claim_token + AND NEW.lease_expires_at IS NOT NULL AND NEW.receipt_digest IS NULL + AND NEW.receipt_payload_json IS NULL AND NEW.failure_digest IS NULL) + OR (OLD.status='running' AND NEW.status='running' AND NEW.attempt=OLD.attempt + AND NEW.claimant_id=OLD.claimant_id AND NEW.claim_token=OLD.claim_token + AND julianday(NEW.lease_expires_at)<=julianday('now') + AND EXISTS(SELECT 1 FROM training_resume_attempt_failures failure + WHERE failure.operation_id=OLD.id AND failure.attempt=OLD.attempt + AND failure.claim_token=OLD.claim_token) + AND NEW.receipt_digest IS NULL AND NEW.receipt_payload_json IS NULL + AND NEW.failure_digest IS NULL) + OR (OLD.status='running' AND NEW.status='completed' AND NEW.attempt=OLD.attempt + AND NEW.claimant_id IS NULL AND NEW.claim_token IS NULL AND NEW.lease_expires_at IS NULL + AND NEW.failure_digest IS NULL AND NEW.receipt_payload_json IS NOT NULL + AND sha256_digest_is_valid(NEW.receipt_digest)=1) + OR (OLD.status='running' AND NEW.status='failed' AND NEW.attempt=OLD.attempt + AND NEW.claimant_id IS NULL AND NEW.claim_token IS NULL AND NEW.lease_expires_at IS NULL + AND NEW.receipt_digest IS NULL AND NEW.receipt_payload_json IS NULL + AND sha256_digest_is_valid(NEW.failure_digest)=1)) + THEN RAISE(ABORT,'training operation status transition mismatch') END; +END; +DROP TRIGGER validate_training_event_insert_v46; +CREATE TRIGGER validate_training_event_insert_v47 BEFORE INSERT ON training_run_events BEGIN + SELECT CASE WHEN NEW.reviewer_id!=(SELECT reviewer_id FROM training_runs WHERE id=NEW.run_id) + OR NEW.sequence!=COALESCE((SELECT max(sequence)+1 FROM training_run_events WHERE run_id=NEW.run_id),1) + OR NEW.previous_state!=(SELECT lifecycle_state FROM training_runs WHERE id=NEW.run_id) + OR NOT ((NEW.previous_state='draft' AND NEW.new_state='probed' AND EXISTS(SELECT 1 FROM training_probe_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND succeeded=1)) + OR (NEW.previous_state='probed' AND NEW.new_state='calibrated' AND EXISTS(SELECT 1 FROM training_calibration_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND succeeded=1)) + OR (NEW.previous_state='calibrated' AND NEW.new_state='preregistered' AND EXISTS(SELECT 1 FROM training_run_idempotency binding WHERE binding.run_id=NEW.run_id AND binding.reviewer_id=NEW.reviewer_id AND binding.idempotency_key=json_extract(NEW.payload_json,'$.idempotency_key') AND binding.request_digest=json_extract(NEW.payload_json,'$.request_digest'))) + OR (NEW.previous_state='preregistered' AND NEW.new_state='training') + OR (NEW.previous_state='cancelled' AND NEW.new_state='training' AND EXISTS(SELECT 1 FROM training_checkpoints checkpoint WHERE checkpoint.run_id=NEW.run_id AND checkpoint.id=json_extract(NEW.payload_json,'$.parent_checkpoint_id') AND checkpoint.evidence_digest=json_extract(NEW.payload_json,'$.parent_checkpoint_digest') AND checkpoint.publication_state='finalized')) + OR (NEW.previous_state='training' AND NEW.new_state='evaluating' AND EXISTS(SELECT 1 FROM training_checkpoints checkpoint WHERE checkpoint.run_id=NEW.run_id AND checkpoint.fraction=1.0 AND checkpoint.publication_state='finalized') AND (SELECT max(step) FROM training_telemetry WHERE run_id=NEW.run_id)=(SELECT total_steps FROM training_runs WHERE id=NEW.run_id)) + OR (NEW.previous_state='evaluating' AND NEW.new_state IN ('qualified','rejected') AND EXISTS(SELECT 1 FROM training_evaluation_evidence WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id AND disposition=NEW.new_state AND evidence_digest=json_extract(NEW.payload_json,'$.evaluation_evidence_digest'))) + OR (NEW.previous_state IN ('draft','probed','calibrated','preregistered','training','evaluating') AND NEW.new_state='failed' AND EXISTS(SELECT 1 FROM training_failures WHERE run_id=NEW.run_id AND reviewer_id=NEW.reviewer_id)) + OR (NEW.previous_state='training' AND NEW.new_state='cancelled' AND EXISTS( + SELECT 1 FROM training_cancellation_evidence evidence WHERE evidence.run_id=NEW.run_id + AND evidence.reviewer_id=NEW.reviewer_id + AND evidence.id=json_extract(NEW.payload_json,'$.cancellation_id') + AND evidence.cancellation_request_id=json_extract(NEW.payload_json,'$.cancellation_request_id') + AND evidence.request_digest=json_extract(NEW.payload_json,'$.request_digest'))) + OR (NEW.previous_state='training' AND NEW.new_state='cancelled' AND EXISTS( + SELECT 1 FROM training_resume_attempt_failures failure + JOIN training_orchestration_operations operation ON operation.id=failure.operation_id + WHERE operation.run_id=NEW.run_id AND operation.reviewer_id=NEW.reviewer_id + AND failure.operation_id=json_extract(NEW.payload_json,'$.resume_operation_id') + AND failure.attempt=json_extract(NEW.payload_json,'$.resume_attempt') + AND failure.failure_digest=json_extract(NEW.payload_json,'$.resume_failure_digest')))) + THEN RAISE(ABORT,'invalid scientific training lifecycle transition') END; +END; +""" + + +_MIGRATION_48 = """ +CREATE TABLE IF NOT EXISTS dataset_compilation_operations( + id TEXT PRIMARY KEY, operation_key TEXT NOT NULL UNIQUE, compiler_id TEXT NOT NULL, + request_digest TEXT NOT NULL CHECK(length(request_digest)=64 AND request_digest NOT GLOB '*[^0-9a-f]*'), + request_json TEXT NOT NULL CHECK(json_valid(request_json) AND canonical_json_is_exact(request_json)=1), + status TEXT NOT NULL CHECK(status IN ('pending','running','retryable','completed','failed','cancelled')), + attempt INTEGER NOT NULL CHECK(attempt>=0), claimant_id TEXT, claim_token TEXT, + lease_expires_at TEXT, receipt_id TEXT UNIQUE, + receipt_digest TEXT CHECK(receipt_digest IS NULL OR (length(receipt_digest)=64 AND receipt_digest NOT GLOB '*[^0-9a-f]*')), + receipt_json TEXT CHECK(receipt_json IS NULL OR (json_valid(receipt_json) AND canonical_json_is_exact(receipt_json)=1)), + created_at TEXT NOT NULL, updated_at TEXT NOT NULL, + CHECK((status='running')=(claimant_id IS NOT NULL AND claim_token IS NOT NULL AND lease_expires_at IS NOT NULL)), + CHECK((status='completed')=(receipt_id IS NOT NULL AND receipt_digest IS NOT NULL AND receipt_json IS NOT NULL)) +); +CREATE TABLE IF NOT EXISTS dataset_compilation_inputs( + operation_id TEXT NOT NULL, role TEXT NOT NULL CHECK(role IN ('source_manifest','transform_recipe')), + artifact_id TEXT NOT NULL, sha256 TEXT NOT NULL CHECK(length(sha256)=64 AND sha256 NOT GLOB '*[^0-9a-f]*'), + byte_size INTEGER NOT NULL CHECK(byte_size>=0), media_type TEXT NOT NULL, created_at TEXT NOT NULL, + PRIMARY KEY(operation_id,role), FOREIGN KEY(operation_id) REFERENCES dataset_compilation_operations(id), + FOREIGN KEY(artifact_id) REFERENCES artifacts(id) +); +CREATE TABLE IF NOT EXISTS dataset_compilation_outputs( + receipt_id TEXT NOT NULL, operation_id TEXT NOT NULL, role TEXT NOT NULL, + artifact_id TEXT NOT NULL, sha256 TEXT NOT NULL CHECK(length(sha256)=64 AND sha256 NOT GLOB '*[^0-9a-f]*'), + byte_size INTEGER NOT NULL CHECK(byte_size>=0), media_type TEXT NOT NULL, + attempt INTEGER NOT NULL CHECK(attempt>0), created_at TEXT NOT NULL, + PRIMARY KEY(receipt_id,role), UNIQUE(operation_id,role), + FOREIGN KEY(operation_id) REFERENCES dataset_compilation_operations(id), + FOREIGN KEY(artifact_id) REFERENCES artifacts(id) +); +CREATE TABLE IF NOT EXISTS dataset_compilation_events( + operation_id TEXT NOT NULL, sequence INTEGER NOT NULL CHECK(sequence>0), + attempt INTEGER NOT NULL CHECK(attempt>=0), event_kind TEXT NOT NULL, + evidence_digest TEXT NOT NULL CHECK(length(evidence_digest)=64 AND evidence_digest NOT GLOB '*[^0-9a-f]*'), created_at TEXT NOT NULL, + PRIMARY KEY(operation_id,sequence), FOREIGN KEY(operation_id) REFERENCES dataset_compilation_operations(id) +); +CREATE TABLE IF NOT EXISTS dataset_compilation_failures( + operation_id TEXT NOT NULL, attempt INTEGER NOT NULL CHECK(attempt>0), claim_token TEXT NOT NULL, + code TEXT NOT NULL, failure_digest TEXT NOT NULL CHECK(length(failure_digest)=64 AND failure_digest NOT GLOB '*[^0-9a-f]*'), + created_at TEXT NOT NULL, PRIMARY KEY(operation_id,attempt), + FOREIGN KEY(operation_id) REFERENCES dataset_compilation_operations(id) +); +CREATE TABLE IF NOT EXISTS dataset_compilation_reproductions( + receipt_id TEXT PRIMARY KEY, original_receipt_id TEXT NOT NULL, + operation_id TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL, + FOREIGN KEY(operation_id) REFERENCES dataset_compilation_operations(id) +); +CREATE INDEX IF NOT EXISTS dataset_compilation_operation_status ON dataset_compilation_operations(status,lease_expires_at); +CREATE INDEX IF NOT EXISTS dataset_compilation_input_artifact ON dataset_compilation_inputs(artifact_id); +CREATE INDEX IF NOT EXISTS dataset_compilation_output_artifact ON dataset_compilation_outputs(artifact_id); +CREATE TRIGGER IF NOT EXISTS validate_dataset_compilation_input_insert_v48 + BEFORE INSERT ON dataset_compilation_inputs BEGIN + SELECT CASE WHEN NOT EXISTS(SELECT 1 FROM artifacts artifact WHERE artifact.id=NEW.artifact_id + AND artifact.sha256=NEW.sha256 AND artifact.byte_size=NEW.byte_size + AND artifact.media_type=NEW.media_type) + THEN RAISE(ABORT,'dataset compilation input artifact mismatch') END; +END; +CREATE TRIGGER IF NOT EXISTS validate_dataset_compilation_output_insert_v48 + BEFORE INSERT ON dataset_compilation_outputs BEGIN + SELECT CASE WHEN NOT EXISTS(SELECT 1 FROM dataset_compilation_operations operation + WHERE operation.id=NEW.operation_id AND operation.status='running' + AND operation.attempt=NEW.attempt) + OR NOT EXISTS(SELECT 1 FROM artifacts artifact WHERE artifact.id=NEW.artifact_id + AND artifact.sha256=NEW.sha256 AND artifact.byte_size=NEW.byte_size + AND artifact.media_type=NEW.media_type) + THEN RAISE(ABORT,'dataset compilation output claim or artifact mismatch') END; +END; +CREATE TRIGGER IF NOT EXISTS guard_dataset_compilation_operation_authority_v48 + BEFORE UPDATE ON dataset_compilation_operations BEGIN + SELECT CASE WHEN NEW.id!=OLD.id OR NEW.operation_key!=OLD.operation_key + OR NEW.compiler_id!=OLD.compiler_id OR NEW.request_digest!=OLD.request_digest + OR NEW.request_json!=OLD.request_json OR NEW.created_at!=OLD.created_at + OR OLD.status IN ('completed','failed','cancelled') + OR (NEW.status='completed' AND ( + NEW.receipt_digest!=substr(canonical_json_digest(NEW.receipt_json),8) + OR json_extract(NEW.receipt_json,'$.operation_id')!=NEW.id + OR json_extract(NEW.receipt_json,'$.operation_key')!=NEW.operation_key + OR json_extract(NEW.receipt_json,'$.compiler_id')!=NEW.compiler_id + OR json_extract(NEW.receipt_json,'$.request_digest')!=NEW.request_digest + OR json_extract(NEW.receipt_json,'$.attempt')!=NEW.attempt)) + THEN RAISE(ABORT,'dataset compilation operation authority mismatch') END; +END; +CREATE TRIGGER IF NOT EXISTS guard_dataset_compilation_input_update_v48 BEFORE UPDATE ON dataset_compilation_inputs + BEGIN SELECT RAISE(ABORT,'dataset compilation inputs are immutable'); END; +CREATE TRIGGER IF NOT EXISTS guard_dataset_compilation_input_delete_v48 BEFORE DELETE ON dataset_compilation_inputs + BEGIN SELECT RAISE(ABORT,'dataset compilation inputs are immutable'); END; +CREATE TRIGGER IF NOT EXISTS guard_dataset_compilation_output_update_v48 BEFORE UPDATE ON dataset_compilation_outputs + BEGIN SELECT RAISE(ABORT,'dataset compilation outputs are immutable'); END; +CREATE TRIGGER IF NOT EXISTS guard_dataset_compilation_output_delete_v48 BEFORE DELETE ON dataset_compilation_outputs + BEGIN SELECT RAISE(ABORT,'dataset compilation outputs are immutable'); END; +CREATE TRIGGER IF NOT EXISTS guard_dataset_compilation_event_update_v48 BEFORE UPDATE ON dataset_compilation_events + BEGIN SELECT RAISE(ABORT,'dataset compilation events are immutable'); END; +CREATE TRIGGER IF NOT EXISTS guard_dataset_compilation_event_delete_v48 BEFORE DELETE ON dataset_compilation_events + BEGIN SELECT RAISE(ABORT,'dataset compilation events are immutable'); END; +CREATE TRIGGER IF NOT EXISTS guard_dataset_compilation_failure_update_v48 BEFORE UPDATE ON dataset_compilation_failures + BEGIN SELECT RAISE(ABORT,'dataset compilation failures are immutable'); END; +CREATE TRIGGER IF NOT EXISTS guard_dataset_compilation_failure_delete_v48 BEFORE DELETE ON dataset_compilation_failures + BEGIN SELECT RAISE(ABORT,'dataset compilation failures are immutable'); END; +CREATE TRIGGER IF NOT EXISTS guard_dataset_compilation_reproduction_update_v48 BEFORE UPDATE ON dataset_compilation_reproductions + BEGIN SELECT RAISE(ABORT,'dataset compilation reproductions are immutable'); END; +CREATE TRIGGER IF NOT EXISTS guard_dataset_compilation_reproduction_delete_v48 BEFORE DELETE ON dataset_compilation_reproductions + BEGIN SELECT RAISE(ABORT,'dataset compilation reproductions are immutable'); END; +""" + + +_MIGRATION_49 = """ +CREATE TABLE IF NOT EXISTS curated_validation_operations( + operation_key TEXT PRIMARY KEY, + request_digest TEXT NOT NULL CHECK(length(request_digest)=64 AND request_digest NOT GLOB '*[^0-9a-f]*'), + request_json TEXT NOT NULL CHECK(json_valid(request_json) AND canonical_json_is_exact(request_json)=1), + state TEXT NOT NULL CHECK(state IN ('accepted','rejected','needs_review')), + report_artifact_id TEXT NOT NULL, + receipt_id TEXT NOT NULL UNIQUE, + receipt_digest TEXT NOT NULL CHECK(length(receipt_digest)=64 AND receipt_digest NOT GLOB '*[^0-9a-f]*'), + receipt_json TEXT NOT NULL CHECK(json_valid(receipt_json) AND canonical_json_is_exact(receipt_json)=1), + created_at TEXT NOT NULL, + FOREIGN KEY(report_artifact_id) REFERENCES artifacts(id) +); +CREATE UNIQUE INDEX IF NOT EXISTS curated_validation_request_digest + ON curated_validation_operations(request_digest); +CREATE TRIGGER IF NOT EXISTS guard_curated_validation_update_v49 + BEFORE UPDATE ON curated_validation_operations + BEGIN SELECT RAISE(ABORT,'curated validation operations are immutable'); END; +CREATE TRIGGER IF NOT EXISTS guard_curated_validation_delete_v49 + BEFORE DELETE ON curated_validation_operations + BEGIN SELECT RAISE(ABORT,'curated validation operations are immutable'); END; +""" + + +_MIGRATIONS = { + 1: _MIGRATION_1, + 2: _MIGRATION_2, + 3: _MIGRATION_3, + 4: _MIGRATION_4, + 5: _MIGRATION_5, + 6: _MIGRATION_6, + 7: _MIGRATION_7, + 8: _MIGRATION_8, + 9: _MIGRATION_9, + 10: _MIGRATION_10, + 11: _MIGRATION_11, + 12: _MIGRATION_12, + 13: _MIGRATION_13, + 14: _MIGRATION_14, + 15: _MIGRATION_15, + 16: _MIGRATION_16, + 17: _MIGRATION_17, + 18: _MIGRATION_18, + 19: _MIGRATION_19, + 20: _MIGRATION_20, + 21: _MIGRATION_21, + 22: _MIGRATION_22, + 23: _MIGRATION_23, + 24: _MIGRATION_24, + 25: _MIGRATION_25, + 26: _MIGRATION_26, + 27: _MIGRATION_27, + 28: _MIGRATION_28, + 29: _MIGRATION_29, + 30: _MIGRATION_30, + 31: _MIGRATION_31, + 32: _MIGRATION_32, + 33: _MIGRATION_33, + 34: _MIGRATION_34, + 35: _MIGRATION_35, + 36: _MIGRATION_36, + 37: _MIGRATION_37, + 38: _MIGRATION_38, + 39: _MIGRATION_39, + 40: _MIGRATION_40, + 41: _MIGRATION_41, + 42: _MIGRATION_42, + 43: _MIGRATION_43, + 44: _MIGRATION_44, + 45: _MIGRATION_45, + 46: _MIGRATION_46, + 47: _MIGRATION_47, + 48: _MIGRATION_48, + 49: _MIGRATION_49, +} + + +def _validate_v31_training_recipe_materials(connection: sqlite3.Connection) -> None: + """Version 31 has no canonical artifact handles from which exact bindings can be derived.""" + if connection.execute("SELECT 1 FROM training_recipes LIMIT 1").fetchone() is not None: + raise sqlite3.IntegrityError("v31 training recipes require explicit material migration") + + +def _validate_v32_training_recipe_materials(connection: sqlite3.Connection) -> None: + rows = connection.execute( + "SELECT recipe.recipe_json,recipe.reviewer_id AS recipe_reviewer," + "recipe.dataset_storage_id," + "material.* FROM training_recipes recipe JOIN training_recipe_materials material " + "ON material.recipe_storage_id=recipe.storage_id" + ).fetchall() + recipe_count = connection.execute("SELECT count(*) FROM training_recipes").fetchone()[0] + material_count = connection.execute( + "SELECT count(*) FROM training_recipe_materials" + ).fetchone()[0] + if len(rows) != recipe_count or material_count != recipe_count: + raise sqlite3.IntegrityError("v32 training recipe materials are incomplete") + for row in rows: + try: + recipe = TrainingRecipe.model_validate_json(str(row["recipe_json"])) + adapters = json.loads(str(row["lightning_adapter_artifact_ids_json"])) + if not isinstance(adapters, dict) or set(adapters) != set( + recipe.lightning_adapter_digests + ): + raise ValueError + dataset = connection.execute( + "SELECT * FROM training_datasets WHERE storage_id=? AND reviewer_id=? " + "AND state='sealed'", + (row["dataset_storage_id"], row["reviewer_id"]), + ).fetchone() + campaign = connection.execute( + "SELECT campaign.*,result.content_digest AS result_digest,result.result_json " + "FROM character_benchmark_campaigns campaign JOIN character_benchmark_results result " + "ON result.campaign_id=campaign.id WHERE campaign.id=? AND campaign.reviewer_id=?", + (row["stage_3_campaign_id"], row["reviewer_id"]), + ).fetchone() + base = connection.execute( + "SELECT sha256 FROM artifacts WHERE id=?", (row["base_model_artifact_id"],) + ).fetchone() + source = connection.execute( + "SELECT sha256 FROM artifacts WHERE id=?", (row["source_snapshot_artifact_id"],) + ).fetchone() + if dataset is None or campaign is None or base is None or source is None: + raise ValueError + definition = json.loads(str(campaign["definition_json"])) + valid = ( + str(row["recipe_reviewer"]) == str(row["reviewer_id"]) + and str(dataset["dataset_digest"]) == recipe.dataset_digest + and str(dataset["character_pack_digest"]) == recipe.character_pack_digest + and str(campaign["character_pack_storage_id"]) + == str(dataset["character_pack_storage_id"]) + and str(campaign["result_digest"]) + == str(row["stage_3_result_id"]) + == recipe.stage_3_evidence_digest + and json.loads(str(campaign["result_json"])).get("baseline_disposition") + == "baseline_established" + and definition.get("character_pack_digest") == recipe.character_pack_digest + and definition.get("stage_2_manifest_digest") == recipe.stage_2_manifest_digest + and definition.get("stage_2_evidence_digest") == recipe.stage_2_evidence_digest + and str(base["sha256"]) == recipe.base_model_digest[7:] + and str(source["sha256"]) == recipe.source_code_digest[7:] + and str(row["runtime_digest"]) + == _canonical_json_digest(canonical_json(dict(recipe.runtime_identity))) + and _character_benchmark_provenance_is_valid(connection, row["stage_3_campaign_id"]) + == 1 + and connection.execute( + "SELECT new_state FROM character_benchmark_events WHERE campaign_id=? " + "ORDER BY sequence DESC LIMIT 1", + (row["stage_3_campaign_id"],), + ).fetchone()[0] + == "completed" + ) + for name, artifact_id in adapters.items(): + artifact = connection.execute( + "SELECT sha256 FROM artifacts WHERE id=?", (artifact_id,) + ).fetchone() + valid = ( + valid + and artifact is not None + and ( + "sha256:" + str(artifact["sha256"]) + == recipe.lightning_adapter_digests[name] + ) + ) + if not valid: + raise ValueError + except (AttributeError, KeyError, TypeError, ValueError): + raise sqlite3.IntegrityError("v32 training recipe materials are invalid") from None + + +def _validate_v33_training_recipe_materials(connection: sqlite3.Connection) -> None: + """Audit every populated v33 material against the complete v34 contract.""" + try: + _validate_v32_training_recipe_materials(connection) + except sqlite3.IntegrityError: + raise sqlite3.IntegrityError("v33 training recipe materials are invalid") from None + rows = connection.execute( + "SELECT recipe.storage_id,recipe.reviewer_id AS recipe_reviewer,recipe.recipe_json," + "material.reviewer_id AS material_reviewer," + "material.lightning_adapter_artifact_ids_json " + "FROM training_recipes recipe JOIN training_recipe_materials material " + "ON material.recipe_storage_id=recipe.storage_id" + ).fetchall() + try: + expected_child_count = 0 + for row in rows: + recipe = TrainingRecipe.model_validate_json(str(row["recipe_json"])) + bindings = json.loads(str(row["lightning_adapter_artifact_ids_json"])) + if ( + row["recipe_reviewer"] != row["material_reviewer"] + or not isinstance(bindings, dict) + or canonical_json(bindings) != str(row["lightning_adapter_artifact_ids_json"]) + or set(bindings) != set(recipe.lightning_adapter_digests) + ): + raise ValueError + expected_child_count += len(bindings) + children = connection.execute( + "SELECT material_name,artifact_id,reviewer_id " + "FROM training_recipe_lightning_materials WHERE recipe_storage_id=?", + (row["storage_id"],), + ).fetchall() + child_bindings = { + str(child["material_name"]): str(child["artifact_id"]) for child in children + } + if ( + len(child_bindings) != len(children) + or child_bindings != bindings + or any(child["reviewer_id"] != row["recipe_reviewer"] for child in children) + ): + raise ValueError + for name, artifact_id in child_bindings.items(): + artifact = connection.execute( + "SELECT sha256 FROM artifacts WHERE id=?", (artifact_id,) + ).fetchone() + if ( + artifact is None + or "sha256:" + str(artifact["sha256"]) != recipe.lightning_adapter_digests[name] + ): + raise ValueError + actual_child_count = connection.execute( + "SELECT count(*) FROM training_recipe_lightning_materials" + ).fetchone()[0] + if actual_child_count != expected_child_count: + raise ValueError + except (AttributeError, KeyError, TypeError, ValueError): + raise sqlite3.IntegrityError("v33 training recipe materials are invalid") from None + + +def _validate_v34_training_runs(connection: sqlite3.Connection) -> None: + """Fail a v34 upgrade before DDL when persisted run identities are not exact.""" + invalid = connection.execute( + "SELECT 1 FROM training_runs run LEFT JOIN training_recipes recipe " + "ON recipe.storage_id=run.recipe_storage_id LEFT JOIN training_datasets dataset " + "ON dataset.storage_id=run.dataset_storage_id WHERE recipe.storage_id IS NULL " + "OR dataset.storage_id IS NULL OR recipe.reviewer_id!=run.reviewer_id " + "OR recipe.dataset_storage_id!=run.dataset_storage_id " + "OR run.total_steps!=json_extract(recipe.recipe_json,'$.total_steps') LIMIT 1" + ).fetchone() + if invalid is not None: + raise sqlite3.IntegrityError("v34 training runs require exact scientific migration") + + +def _validate_v35_legacy_run_mapping(connection: sqlite3.Connection) -> None: + invented_terminal = connection.execute( + "SELECT 1 FROM training_runs WHERE lifecycle_version=0 AND state!='created' LIMIT 1" + ).fetchone() + if invented_terminal is not None: + raise sqlite3.IntegrityError("legacy non-created training runs are non-runnable") + if connection.execute("SELECT 1 FROM training_checkpoints LIMIT 1").fetchone() is not None: + raise sqlite3.IntegrityError("v35 checkpoints require explicit scientific re-publication") + + +def _validate_v36_checkpoint_publication(connection: sqlite3.Connection) -> None: + building = connection.execute( + "SELECT count(*) FROM training_checkpoints WHERE publication_state!='finalized'" + ).fetchone()[0] + unmatched_cancellations = connection.execute( + "SELECT count(*) FROM training_cancellation_evidence evidence WHERE NOT EXISTS(" + "SELECT 1 FROM training_cancellation_requests request WHERE request.run_id=evidence.run_id " + "AND request.reviewer_id=evidence.reviewer_id " + "AND request.requested_step=evidence.requested_step)" + ).fetchone()[0] + if building: + raise sqlite3.IntegrityError("v36 building checkpoints cannot migrate") + if unmatched_cancellations: + raise sqlite3.IntegrityError("v36 cancellation evidence lacks an exact request") + rows = connection.execute( + "SELECT checkpoint.*,run.total_steps,run.reviewer_id AS run_reviewer," + "run.recipe_storage_id AS run_recipe_storage," + "run.dataset_storage_id AS run_dataset_storage,recipe.id AS recipe_id," + "recipe.dataset_digest,recipe.character_id,recipe.recipe_json " + "FROM training_checkpoints checkpoint JOIN training_runs run ON run.id=checkpoint.run_id " + "JOIN training_recipes recipe ON recipe.storage_id=run.recipe_storage_id " + "WHERE checkpoint.publication_state='finalized' ORDER BY checkpoint.run_id,checkpoint.step" + ).fetchall() + previous_by_run: dict[str, sqlite3.Row] = {} + finalized_count_by_run: dict[str, int] = {} + try: + for row in rows: + manifest_json = str(row["manifest_json"]) + evidence_json = str(row["evidence_json"]) + manifest = json.loads(manifest_json) + evidence = json.loads(evidence_json) + if _training_checkpoint_json_is_exact(manifest_json, evidence_json) != 1: + raise ValueError + if _canonical_json_digest(manifest_json) != row["manifest_digest"]: + raise ValueError + if _canonical_json_digest(evidence_json) != row["evidence_digest"]: + raise ValueError + fractions = json.loads(str(row["recipe_json"]))["checkpoint_plan"]["fractions"] + ordinal = fractions.index(row["fraction"]) + expected_step = checkpoint_steps(int(row["total_steps"]))[ordinal] + run_id = str(row["run_id"]) + finalized_count = finalized_count_by_run.get(run_id, 0) + previous = previous_by_run.get(run_id) + expected_parent = None if ordinal == 0 else str(previous["id"]) if previous else None + expected_parent_digest = None if previous is None else str(previous["evidence_digest"]) + identity = ( + ordinal == finalized_count + and (previous is None) == (ordinal == 0) + and row["reviewer_id"] == row["run_reviewer"] + and row["recipe_storage_id"] == row["run_recipe_storage"] + and row["dataset_storage_id"] == row["run_dataset_storage"] + and row["step"] == expected_step + and row["parent_checkpoint_id"] == expected_parent + and manifest["run_id"] == row["run_id"] + and manifest["reviewer_id"] == row["reviewer_id"] + and manifest["recipe_id"] == row["recipe_id"] + and manifest["dataset_digest"] == row["dataset_digest"] + and manifest["character_id"] == row["character_id"] + and manifest["step"] == row["step"] + and manifest["fraction"] == row["fraction"] + and manifest["parent_checkpoint_id"] == expected_parent + and manifest["artifact_digest"] == row["artifact_digest"] + and evidence["run_id"] == row["run_id"] + and evidence["recipe_digest"] == row["recipe_id"] + and evidence["dataset_digest"] == row["dataset_digest"] + and evidence["step"] == row["step"] + and evidence["fraction"] == row["fraction"] + and evidence["parent_checkpoint_digest"] == expected_parent_digest + and evidence["safetensors_digest"] == row["artifact_digest"] + ) + shared_fields = ( + "backend_digest", + "runtime_digest", + "material_digest", + "source_digest", + "optimizer_state_digest", + "trainer_state_digest", + "telemetry_digest", + "telemetry_sequence_start", + "telemetry_sequence_end", + "trainable_targets_digest", + "frozen_parameters_digest", + "finite_gradient_proof_digest", + "cancellation_boundary", + "inventory", + "safetensors_size_bytes", + ) + identity = identity and all( + manifest[field] == evidence[field] for field in shared_fields + ) + calibration = connection.execute( + "SELECT 1 FROM training_calibration_evidence WHERE run_id=? AND succeeded=1 " + "AND backend_digest=? AND runtime_digest=? AND material_digest=? AND source_digest=?", + ( + row["run_id"], + evidence["backend_digest"], + evidence["runtime_digest"], + evidence["material_digest"], + evidence["source_digest"], + ), + ).fetchone() + telemetry_rows = connection.execute( + "SELECT sequence,step,payload_json FROM training_telemetry WHERE run_id=? " + "AND sequence BETWEEN ? AND ? ORDER BY sequence", + ( + row["run_id"], + evidence["telemetry_sequence_start"], + evidence["telemetry_sequence_end"], + ), + ).fetchall() + telemetry_payload = [ + { + "sequence": item["sequence"], + "step": item["step"], + "payload": json.loads(str(item["payload_json"])), + } + for item in telemetry_rows + ] + expected_telemetry_digest = ( + "sha256:" + + hashlib.sha256(canonical_json(cast(Any, telemetry_payload)).encode()).hexdigest() + ) + identity = ( + identity + and calibration is not None + and len(telemetry_rows) + == evidence["telemetry_sequence_end"] - evidence["telemetry_sequence_start"] + 1 + and evidence["telemetry_digest"] == expected_telemetry_digest + ) + if not identity or (ordinal > 0 and previous is None): + raise ValueError + inventory = connection.execute( + "SELECT relative_path,artifact_digest,size_bytes,role " + "FROM training_checkpoint_inventory WHERE checkpoint_id=? ORDER BY ordinal", + (row["id"],), + ).fetchall() + actual_inventory = [ + { + "relative_path": item["relative_path"], + "digest": item["artifact_digest"], + "size_bytes": item["size_bytes"], + "role": item["role"], + } + for item in inventory + ] + if actual_inventory != evidence["inventory"] or len(actual_inventory) != 7: + raise ValueError + previous_by_run[run_id] = row + finalized_count_by_run[run_id] = finalized_count + 1 + except (AttributeError, KeyError, TypeError, ValueError, json.JSONDecodeError): + raise sqlite3.IntegrityError("v36 finalized checkpoints are not canonical") from None + + +def _validate_v40_checkpoint_artifact_bindings(connection: sqlite3.Connection) -> None: + rows = connection.execute( + "SELECT inventory.checkpoint_id,inventory.ordinal,count(artifact.id) candidates " + "FROM training_checkpoint_inventory inventory LEFT JOIN artifacts artifact " + "ON 'sha256:'||artifact.sha256=inventory.artifact_digest " + "AND artifact.byte_size=inventory.size_bytes " + "GROUP BY inventory.checkpoint_id,inventory.ordinal HAVING candidates!=1" + ).fetchall() + if rows: + raise sqlite3.IntegrityError( + "v40 checkpoint inventory artifact identities are not uniquely derivable" + ) + + +def _migration_statements(migration: str) -> tuple[str, ...]: + statements: list[str] = [] + pending = "" + for line in migration.splitlines(keepends=True): + pending += line + if sqlite3.complete_statement(pending): + statements.append(pending.strip()) + pending = "" + if pending.strip(): + raise ValueError("incomplete migration statement") + return tuple(statements) + + +def _validate_training_snapshot_lineage_history( + connection: sqlite3.Connection, + table: Literal["training_dataset_head_events", "training_dataset_snapshots"], +) -> None: + exists = connection.execute( + "SELECT 1 FROM sqlite_master WHERE name=? AND type IN ('table','view')", (table,) + ).fetchone() + if exists is None: + return + conflict = connection.execute( + f"SELECT dataset_storage_id FROM {table} GROUP BY dataset_storage_id " + "HAVING count(DISTINCT lineage_id)>1 ORDER BY dataset_storage_id LIMIT 1" + ).fetchone() + if conflict is not None: + raise sqlite3.IntegrityError("training snapshot appears in multiple dataset lineages") + + +def _validate_v27_training_snapshot_bindings(connection: sqlite3.Connection) -> None: + contradiction = connection.execute( + "SELECT history.dataset_storage_id FROM training_dataset_snapshots history " + "LEFT JOIN training_dataset_snapshot_lineages binding " + "ON binding.dataset_storage_id=history.dataset_storage_id " + "GROUP BY history.dataset_storage_id HAVING binding.dataset_storage_id IS NULL " + "OR count(DISTINCT history.lineage_id)!=1 " + "OR min(history.lineage_id)!=min(binding.lineage_id) LIMIT 1" + ).fetchone() + orphan = connection.execute( + "SELECT binding.dataset_storage_id FROM training_dataset_snapshot_lineages binding " + "LEFT JOIN training_dataset_snapshots history " + "ON history.dataset_storage_id=binding.dataset_storage_id " + "WHERE history.dataset_storage_id IS NULL LIMIT 1" + ).fetchone() + if contradiction is not None or orphan is not None: + raise sqlite3.IntegrityError("training snapshot lineage history is inconsistent") + + +def _validate_v29_training_curation_ledgers(connection: sqlite3.Connection) -> None: + conflict = connection.execute( + "SELECT import_id FROM training_curation_confirmations GROUP BY import_id " + "HAVING count(*)>1 OR min(version)!=1 OR max(version)!=1 LIMIT 1" + ).fetchone() + if conflict is not None: + raise sqlite3.IntegrityError("v29 training confirmation ledger is incompatible") + heads = connection.execute( + "SELECT lineage.id,lineage.reviewer_id,lineage.character_id,dataset.manifest_json " + "FROM training_dataset_lineages lineage JOIN training_datasets dataset " + "ON dataset.storage_id=lineage.current_dataset_storage_id" + ).fetchall() + for head in heads: + if ( + _training_ledger_snapshot_is_complete( + connection, + head["manifest_json"], + head["id"], + head["reviewer_id"], + head["character_id"], + ) + != 1 + ): + raise sqlite3.IntegrityError("v29 training dataset ledger is incompatible") + + +def _rekey_character_pack_drafts_for_duplicate_provenance( + connection: sqlite3.Connection, +) -> None: + rows = connection.execute( + "SELECT pack.storage_id, pack.id, pack.character_id, " + "pack.supersedes_storage_id, predecessor.id AS supersedes_pack_id " + "FROM character_packs AS pack " + "LEFT JOIN character_packs AS predecessor " + "ON predecessor.storage_id=pack.supersedes_storage_id " + "WHERE pack.state='draft' ORDER BY pack.storage_id" + ).fetchall() + drafts: dict[str, tuple[str, CharacterPackDraft, str | None]] = {} + for row in rows: + storage_id = str(row["storage_id"]) + old_id = str(row["id"]) + reference_rows = connection.execute( + "SELECT reference.*, annotation.* FROM character_references AS reference " + "JOIN character_annotations AS annotation USING(pack_id, reference_id) " + "WHERE reference.pack_id=? ORDER BY reference.ordinal", + (storage_id,), + ).fetchall() + warning_rows = connection.execute( + "SELECT * FROM character_pack_warnings WHERE pack_id=? " + "ORDER BY left_reference_id, right_reference_id", + (storage_id,), + ).fetchall() + draft = CharacterPackDraft.model_validate( + { + "character_id": str(row["character_id"]), + "references": [ + { + "reference_id": str(reference["reference_id"]), + "image_sha256": str(reference["image_sha256"]), + "media_type": str(reference["media_type"]), + "dhash_hex": str(reference["dhash_hex"]), + "disposition": str(reference["disposition"]), + "annotation": { + "framing": str(reference["framing"]), + "head_angle": str(reference["head_angle"]), + "body_visibility": str(reference["body_visibility"]), + "expression": str(reference["expression"]), + "lighting": str(reference["lighting"]), + "occlusion": str(reference["occlusion"]), + "identity_confirmed": bool(reference["identity_confirmed"]), + }, + } + for reference in reference_rows + ], + "duplicate_warnings": [ + { + "left_reference_id": str(warning["left_reference_id"]), + "right_reference_id": str(warning["right_reference_id"]), + "distance": int(warning["distance"]), + "resolution": str(warning["resolution"]), + } + for warning in warning_rows + ], + "supersedes_pack_id": ( + None if row["supersedes_pack_id"] is None else str(row["supersedes_pack_id"]) + ), + } + ) + predecessor_storage_id = ( + None if row["supersedes_storage_id"] is None else str(row["supersedes_storage_id"]) + ) + drafts[storage_id] = (old_id, draft, predecessor_storage_id) + + resolved: dict[str, str] = {} + visiting: set[str] = set() + + def resolve_identity(storage_id: str) -> str: + if storage_id in resolved: + return resolved[storage_id] + if storage_id in visiting: + raise RuntimeError("character pack identity migration lineage cycle") + visiting.add(storage_id) + _, draft, predecessor_storage_id = drafts[storage_id] + if predecessor_storage_id in drafts: + draft = draft.model_copy( + update={"supersedes_pack_id": resolve_identity(predecessor_storage_id)} + ) + new_id = character_pack_id(draft) + visiting.remove(storage_id) + resolved[storage_id] = new_id + return new_id + + proposals = [ + (storage_id, old_id, resolve_identity(storage_id)) + for storage_id, (old_id, _, _) in drafts.items() + ] + destinations: dict[str, str] = {} + for storage_id, _, new_id in proposals: + destination = destinations.setdefault(new_id, storage_id) + if destination != storage_id: + raise RuntimeError("character pack identity migration destination collision") + + for storage_id, old_id, new_id in proposals: + pack_collision = connection.execute( + "SELECT storage_id FROM character_packs WHERE id=? AND storage_id!=?", + (new_id, storage_id), + ).fetchone() + if pack_collision is not None: + raise RuntimeError("character pack identity migration pack collision") + for handle in {old_id, new_id}: + alias_collision = connection.execute( + "SELECT pack_storage_id FROM character_pack_aliases " + "WHERE alias_id=? AND pack_storage_id!=?", + (handle, storage_id), + ).fetchone() + if alias_collision is not None: + raise RuntimeError("character pack identity migration alias collision") + + for storage_id, old_id, new_id in proposals: + if new_id == old_id: + continue + updated = connection.execute( + "UPDATE character_packs SET id=? WHERE storage_id=? AND id=? AND state='draft'", + (new_id, storage_id, old_id), + ) + if updated.rowcount != 1: + raise RuntimeError("character pack identity migration lost draft") + alias = connection.execute( + "SELECT pack_storage_id FROM character_pack_aliases WHERE alias_id=?", + (old_id,), + ).fetchone() + if alias is None: + connection.execute( + "INSERT INTO character_pack_aliases(alias_id, pack_storage_id) VALUES (?, ?)", + (old_id, storage_id), + ) + + +def _backfill_validation_experiment_bindings(connection: sqlite3.Connection) -> None: + rows = connection.execute( + "SELECT trial.campaign_id, trial.trial_id, " + "COALESCE((SELECT event.new_state FROM validation_events AS event " + "WHERE event.campaign_id = trial.campaign_id " + "ORDER BY event.sequence DESC LIMIT 1), 'draft') AS campaign_state " + "FROM validation_trials AS trial WHERE trial.hidden_repeat = 0 " + "ORDER BY trial.campaign_id, trial.ordinal" + ).fetchall() + completed_states = { + "ready", + "judging", + "analyzing", + "qualified", + "not_qualified", + "inconclusive", + } + for row in rows: + campaign_id = str(row["campaign_id"]) + trial_id = str(row["trial_id"]) + experiment_id = validation_experiment_id(campaign_id, trial_id) + experiment = connection.execute( + "SELECT schema_version FROM experiments WHERE id = ?", (experiment_id,) + ).fetchone() + if experiment is None: + if str(row["campaign_state"]) in completed_states: + raise sqlite3.IntegrityError( + "completed validation trial does not match a deterministic experiment" + ) + continue + if str(experiment["schema_version"]) != "validation-controlled.v1": + raise sqlite3.IntegrityError( + "validation experiment identity does not match controlled evidence" + ) + connection.execute( + "INSERT INTO validation_experiment_bindings(campaign_id, trial_id, experiment_id) " + "VALUES (?, ?, ?)", + (campaign_id, trial_id, experiment_id), + ) + + +def _repair_validation_repeat_review_bindings(connection: sqlite3.Connection) -> None: + connection.execute("DROP TRIGGER IF EXISTS guard_validation_trial_bindings_update") + connection.execute( + "UPDATE validation_trial_bindings AS binding SET review_id=(" + "SELECT review.id FROM validation_trials AS trial " + "JOIN validation_campaigns AS campaign ON campaign.id=trial.campaign_id " + "JOIN review_sessions AS session ON session.experiment_id=binding.experiment_id " + "AND session.reviewer_id=validation_session_reviewer(" + "binding.campaign_id, binding.trial_id, campaign.reviewer_id, 1) " + "JOIN review_revisions AS review ON review.session_id=session.id " + "WHERE trial.campaign_id=binding.campaign_id AND trial.trial_id=binding.trial_id " + "AND trial.hidden_repeat=1 ORDER BY review.revision DESC LIMIT 1) " + "WHERE binding.reused_from_trial_id IS NOT NULL" + ) + missing = connection.execute( + "SELECT 1 FROM validation_trial_bindings WHERE reused_from_trial_id IS NOT NULL " + "AND review_id IS NULL LIMIT 1" + ).fetchone() + if missing is not None: + raise sqlite3.IntegrityError("validation repeat review binding cannot be repaired") + connection.execute( + "CREATE TRIGGER guard_validation_trial_bindings_update " + "BEFORE UPDATE ON validation_trial_bindings BEGIN " + "SELECT RAISE(ABORT, 'validation evidence is append-only'); END" + ) + + +def _validation_session_reviewer_sql( + campaign_id: object, trial_id: object, reviewer_id: object, hidden_repeat: object +) -> str: + if not all(isinstance(value, str) for value in (campaign_id, trial_id, reviewer_id)): + raise ValueError("validation session identity requires text inputs") + if not bool(hidden_repeat): + return str(reviewer_id) + payload = f"validation-review-session.v1\x00{campaign_id}\x00{trial_id}\x00{reviewer_id}" + return "validation_reviewer_" + hashlib.sha256(payload.encode()).hexdigest() + + +def _adaptive_manifest_json_is_valid(value: object) -> int: + if not isinstance(value, str): + return 0 + try: + manifest = Manifest.model_validate_json(value) + except (TypeError, ValueError): + return 0 + return int( + manifest.schema_version == "manifest.v1" + and manifest.protocol_version == "ab.v1" + and value == canonical_json(manifest) + ) + + +def _adaptive_optimizer_config_json_is_valid(value: object) -> int: + if not isinstance(value, str): + return 0 + try: + config = OptimizerConfig.model_validate_json(value) + except (TypeError, ValueError): + return 0 + return int(value == canonical_json(config)) + + +def _adaptive_observation_digest( + connection: sqlite3.Connection, + study_id: object, + schema_version: object, + ordinal: object, + optimizer_step_ordinal: object, + value_a: object, + value_b: object, + experiment_id: object, + review_id: object, + outcome: object, + created_at: object, +) -> str | None: + try: + if ( + not isinstance(study_id, str) + or schema_version != "adaptive-observation.v1" + or type(ordinal) is not int + or type(optimizer_step_ordinal) is not int + or not isinstance(value_a, (int, float)) + or not isinstance(value_b, (int, float)) + or not isinstance(experiment_id, str) + or not isinstance(review_id, str) + or not isinstance(outcome, str) + or not isinstance(created_at, str) + or ordinal <= 0 + or optimizer_step_ordinal != ordinal + ): + return None + observation_created_at = datetime.fromisoformat(created_at) + if observation_created_at.utcoffset() is None or not created_at.endswith("+00:00"): + return None + pair = CandidatePair(value_a=float(value_a), value_b=float(value_b)) + if pair.value_a >= pair.value_b: + return None + preference_outcome = PreferenceOutcome(outcome) + row = connection.execute( + "SELECT " + "study.base_manifest_json, " + "optimizer.output_digest AS optimizer_output_digest, " + "optimizer.selected_pair_json AS optimizer_selected_pair_json, " + "optimizer.stop_reason AS optimizer_stop_reason, " + "optimizer.comparison_count AS optimizer_comparison_count, " + "review.id AS review_id, review.session_id, review.preferred_variant_id, " + "review.display_preference, review.notes, " + "review.created_at AS review_created_at, review.revision, " + "review.supersedes_review_id, " + "session.experiment_id, session.reviewer_id, " + "session.created_at AS session_created_at, " + "session.left_variant_id, session.right_variant_id, " + "session.left_job_id, session.right_job_id, " + "session.left_artifact_id, session.right_artifact_id, " + "session.left_artifact_sha256, session.right_artifact_sha256, " + "experiment.schema_version AS experiment_schema_version, " + "experiment.variable_path, experiment.changed_paths_json, " + "left_variant.experiment_id AS left_variant_experiment_id, " + "left_variant.label AS left_label, " + "left_variant.manifest_json AS left_manifest_json, " + "right_variant.experiment_id AS right_variant_experiment_id, " + "right_variant.label AS right_label, " + "right_variant.manifest_json AS right_manifest_json, " + "left_job.variant_id AS left_job_variant_id, " + "left_job.state AS left_job_state, " + "right_job.variant_id AS right_job_variant_id, " + "right_job.state AS right_job_state, " + "left_artifact.sha256 AS left_persisted_sha256, " + "right_artifact.sha256 AS right_persisted_sha256 " + "FROM adaptive_studies AS study " + "JOIN adaptive_optimizer_steps AS optimizer " + "ON optimizer.study_id = study.id AND optimizer.ordinal = ? " + "JOIN review_revisions AS review ON review.id = ? " + "JOIN review_sessions AS session ON session.id = review.session_id " + "JOIN experiments AS experiment ON experiment.id = session.experiment_id " + "JOIN variants AS left_variant ON left_variant.id = session.left_variant_id " + "JOIN variants AS right_variant ON right_variant.id = session.right_variant_id " + "JOIN jobs AS left_job ON left_job.id = session.left_job_id " + "JOIN jobs AS right_job ON right_job.id = session.right_job_id " + "JOIN artifacts AS left_artifact ON left_artifact.id = session.left_artifact_id " + "JOIN artifacts AS right_artifact ON right_artifact.id = session.right_artifact_id " + "JOIN job_artifacts AS left_link ON left_link.job_id = session.left_job_id " + "AND left_link.artifact_id = session.left_artifact_id " + "JOIN job_artifacts AS right_link ON right_link.job_id = session.right_job_id " + "AND right_link.artifact_id = session.right_artifact_id " + "WHERE study.id = ? AND session.experiment_id = ? " + "AND review.id = (" + "SELECT latest.id FROM review_revisions AS latest " + "WHERE latest.session_id = session.id ORDER BY latest.revision DESC LIMIT 1" + ")", + (optimizer_step_ordinal, review_id, study_id, experiment_id), + ).fetchone() + if row is None: + return None + base_manifest = Manifest.model_validate_json(str(row["base_manifest_json"])) + left_manifest = Manifest.model_validate_json(str(row["left_manifest_json"])) + right_manifest = Manifest.model_validate_json(str(row["right_manifest_json"])) + selected_pair = CandidatePair.model_validate_json(str(row["optimizer_selected_pair_json"])) + if ( + str(row["optimizer_stop_reason"]) != "continue" + or int(row["optimizer_comparison_count"]) != ordinal - 1 + or selected_pair != pair + or str(row["experiment_schema_version"]) != "manifest.v1" + or str(row["variable_path"]) != "guidance_scale" + or str(row["changed_paths_json"]) != '["guidance_scale"]' + or str(row["left_variant_experiment_id"]) != experiment_id + or str(row["right_variant_experiment_id"]) != experiment_id + or str(row["left_job_variant_id"]) != str(row["left_variant_id"]) + or str(row["right_job_variant_id"]) != str(row["right_variant_id"]) + or str(row["left_job_state"]) != "completed" + or str(row["right_job_state"]) != "completed" + or str(row["left_persisted_sha256"]) != str(row["left_artifact_sha256"]) + or str(row["right_persisted_sha256"]) != str(row["right_artifact_sha256"]) + or not _manifest_matches_study(left_manifest, base_manifest) + or not _manifest_matches_study(right_manifest, base_manifest) + or min(left_manifest.guidance_scale, right_manifest.guidance_scale) != pair.value_a + or max(left_manifest.guidance_scale, right_manifest.guidance_scale) != pair.value_b + or _review_outcome(row, left_manifest, right_manifest, pair) is not preference_outcome + ): + return None + return adaptive_observation_provenance_digest( + observation={ + "study_id": study_id, + "schema_version": schema_version, + "ordinal": ordinal, + "optimizer_step_ordinal": optimizer_step_ordinal, + "pair": pair.model_dump(mode="json"), + "experiment_id": experiment_id, + "review_id": review_id, + "outcome": outcome, + "created_at": created_at, + }, + optimizer_output_digest=str(row["optimizer_output_digest"]), + experiment={ + "id": str(row["experiment_id"]), + "schema_version": str(row["experiment_schema_version"]), + "variable_path": str(row["variable_path"]), + "changed_paths": ["guidance_scale"], + }, + review={ + "id": str(row["review_id"]), + "session_id": str(row["session_id"]), + "reviewer_id": str(row["reviewer_id"]), + "revision": int(row["revision"]), + "supersedes_review_id": row["supersedes_review_id"], + "display_preference": str(row["display_preference"]), + "preferred_variant_id": row["preferred_variant_id"], + "notes": row["notes"], + "session_created_at": str(row["session_created_at"]), + "review_created_at": str(row["review_created_at"]), + }, + display_mapping={ + "left": _display_provenance(row, left_manifest, "left"), + "right": _display_provenance(row, right_manifest, "right"), + }, + ) + except (sqlite3.DatabaseError, TypeError, ValueError): + return None + + +def _manifest_matches_study(candidate: Manifest, base: Manifest) -> bool: + payload = candidate.model_dump(mode="json") + payload["guidance_scale"] = base.guidance_scale + return payload == base.model_dump(mode="json") + + +def _review_outcome( + row: sqlite3.Row, + left_manifest: Manifest, + right_manifest: Manifest, + pair: CandidatePair, +) -> PreferenceOutcome | None: + preference = str(row["display_preference"]) + preferred_variant_id = row["preferred_variant_id"] + if preference == "tie": + return PreferenceOutcome.TIE if preferred_variant_id is None else None + position = "left" if preference == "left" else "right" if preference == "right" else None + if position is None or str(preferred_variant_id) != str(row[f"{position}_variant_id"]): + return None + preferred_value = ( + left_manifest.guidance_scale if position == "left" else right_manifest.guidance_scale + ) + return PreferenceOutcome.A if preferred_value == pair.value_a else PreferenceOutcome.B + + +def _display_provenance(row: sqlite3.Row, manifest: Manifest, position: str) -> dict[str, object]: + return { + "variant_id": str(row[f"{position}_variant_id"]), + "variant_label": str(row[f"{position}_label"]), + "manifest": manifest.model_dump(mode="json"), + "job_id": str(row[f"{position}_job_id"]), + "artifact_id": str(row[f"{position}_artifact_id"]), + "artifact_sha256": str(row[f"{position}_artifact_sha256"]), + } + + +class _ClosingConnection(sqlite3.Connection): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> Literal[False]: + try: + super().__exit__(exc_type, exc_value, traceback) + finally: + self.close() + return False + + +class Database: + def __init__(self, path: Path, *, read_only: bool = False) -> None: + self.path = path + self.read_only = read_only + + def connect(self) -> sqlite3.Connection: + target = f"file:{self.path}?mode=ro" if self.read_only else self.path + connection = sqlite3.connect( + target, + isolation_level=None, + uri=self.read_only, + factory=_ClosingConnection, + ) + connection.row_factory = sqlite3.Row + connection.create_function( + "adaptive_manifest_json_is_valid", + 1, + _adaptive_manifest_json_is_valid, + deterministic=True, + ) + connection.create_function( + "training_operation_request_identity_is_valid", + 3, + lambda kind, run_id, payload: _training_operation_request_identity_is_valid( + connection, kind, run_id, payload + ), + ) + connection.create_function( + "training_operation_completion_is_derived", + 1, + lambda operation_id: _training_operation_completion_is_derived( + connection, operation_id + ), + ) + connection.create_function( + "training_operation_evidence_digest", + 3, + lambda operation_id, kind, evidence_id: _training_operation_evidence_digest( + connection, operation_id, kind, evidence_id + ), + ) + connection.create_function( + "training_operation_evidence_link_is_valid", + 6, + lambda operation_id, attempt, claim_token, kind, evidence_id, digest: ( + _training_operation_evidence_link_is_valid( + connection, operation_id, attempt, claim_token, kind, evidence_id, digest + ) + ), + ) + connection.create_function( + "training_operation_receipt_links_are_exact", + 2, + lambda operation_id, receipt: _training_operation_receipt_links_are_exact( + connection, operation_id, receipt + ), + ) + connection.create_function( + "adaptive_optimizer_config_json_is_valid", + 1, + _adaptive_optimizer_config_json_is_valid, + deterministic=True, + ) + connection.create_function( + "adaptive_observation_provenance_digest", + 10, + lambda *arguments: _adaptive_observation_digest(connection, *arguments), + ) + connection.create_function( + "validation_session_reviewer", + 4, + _validation_session_reviewer_sql, + deterministic=True, + ) + connection.create_function( + "character_benchmark_reviewer_id", + 3, + character_benchmark_reviewer_id, + deterministic=True, + ) + connection.create_function( + "character_benchmark_experiment_id", + 2, + character_benchmark_experiment_id, + deterministic=True, + ) + connection.create_function( + "training_dataset_identity_is_valid", + 3, + lambda storage_id, digest, manifest: _training_dataset_identity_is_valid( + connection, storage_id, digest, manifest + ), + ) + connection.create_function( + "training_recipe_identity_is_valid", + 3, + lambda dataset_storage_id, recipe_id, recipe_json: _training_recipe_identity_is_valid( + connection, dataset_storage_id, recipe_id, recipe_json + ), + ) + connection.create_function( + "training_draft_identity_is_valid", + 3, + lambda storage_id, identity, manifest: _training_draft_identity_is_valid( + connection, storage_id, identity, manifest + ), + ) + connection.create_function( + "training_draft_manifest_is_valid", + 7, + _training_draft_manifest_is_valid, + deterministic=True, + ) + connection.create_function( + "canonical_json_digest", 1, _canonical_json_digest, deterministic=True + ) + connection.create_function( + "canonical_json_is_exact", 1, _canonical_json_is_exact, deterministic=True + ) + connection.create_function( + "training_operation_json_is_exact", + 3, + _training_operation_json_is_exact, + deterministic=True, + ) + connection.create_function( + "training_operation_lifecycle_is_valid", + 2, + _training_operation_lifecycle_is_valid, + deterministic=True, + ) + connection.create_function( + "training_operation_completion_target_is_valid", + 2, + _training_operation_completion_target_is_valid, + deterministic=True, + ) + connection.create_function( + "training_import_identity_is_valid", + 2, + _training_import_identity_is_valid, + deterministic=True, + ) + connection.create_function( + "training_confirmation_identity_is_valid", + 2, + _training_confirmation_identity_is_valid, + deterministic=True, + ) + connection.create_function( + "training_ledger_snapshot_is_complete", + 4, + lambda manifest, lineage, reviewer, character: _training_ledger_snapshot_is_complete( + connection, manifest, lineage, reviewer, character + ), + ) + connection.create_function( + "training_ledger_transition_is_valid", + 3, + lambda old_storage, new_storage, state: _training_ledger_transition_is_valid( + connection, old_storage, new_storage, state + ), + ) + connection.create_function( + "training_checkpoint_step", 2, _training_checkpoint_step, deterministic=True + ) + connection.create_function( + "training_checkpoint_json_is_exact", + 2, + _training_checkpoint_json_is_exact, + deterministic=True, + ) + connection.create_function( + "sha256_digest_is_valid", + 1, + lambda value: int(_sha256_digest_is_valid(value)), + deterministic=True, + ) + connection.create_function( + "character_benchmark_provenance_is_valid", + 1, + lambda campaign_id: _character_benchmark_provenance_is_valid(connection, campaign_id), + ) + connection.execute("PRAGMA foreign_keys=ON") + if not self.read_only: + connection.execute("PRAGMA journal_mode=WAL") + return connection + + def migrate(self) -> None: + if self.read_only: + raise RuntimeError("read-only database cannot be migrated") + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + connection.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations(" + "version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL)" + ) + applied = { + int(row["version"]) + for row in connection.execute("SELECT version FROM schema_migrations") + } + for version, migration in _MIGRATIONS.items(): + if version in applied: + continue + if version == 13: + _repair_validation_repeat_review_bindings(connection) + if version == 26: + _validate_training_snapshot_lineage_history( + connection, "training_dataset_head_events" + ) + if version == 27: + _validate_training_snapshot_lineage_history( + connection, "training_dataset_snapshots" + ) + if version == 28: + _validate_v27_training_snapshot_bindings(connection) + if version == 30: + _validate_v29_training_curation_ledgers(connection) + if version == 32: + _validate_v31_training_recipe_materials(connection) + if version == 33: + _validate_v32_training_recipe_materials(connection) + if version == 34: + _validate_v33_training_recipe_materials(connection) + if version == 35: + _validate_v34_training_runs(connection) + if version == 36: + _validate_v35_legacy_run_mapping(connection) + if version == 37: + _validate_v36_checkpoint_publication(connection) + if version == 41: + _validate_v40_checkpoint_artifact_bindings(connection) + if version == 44: + _apply_migration_44(connection) + else: + for statement in _migration_statements(migration): + if statement.strip() and not statement.lstrip().startswith( + "CREATE TABLE schema_migrations" + ): + connection.execute(statement) + if version == 12: + _backfill_validation_experiment_bindings(connection) + if version == 21: + _rekey_character_pack_drafts_for_duplicate_provenance(connection) + connection.execute( + "INSERT INTO schema_migrations(version, applied_at) VALUES (?, ?)", + (version, utc_now().isoformat()), + ) + connection.commit() + except Exception: + connection.rollback() + raise diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/dataset_cli.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/dataset_cli.py new file mode 100644 index 0000000..80fc4f7 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/dataset_cli.py @@ -0,0 +1,1324 @@ +"""Hardened, stable CLI boundary for local dataset compilation services.""" + +from __future__ import annotations + +import errno +import hashlib +import json +import os +import signal +import sqlite3 +import stat +import struct +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from enum import IntEnum +from pathlib import Path, PurePosixPath +from typing import Any, Literal, Protocol, cast + +import typer +from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator + +from video_lab.artifacts import ( + Artifact, + ArtifactError, + ArtifactIntegrityError, + ArtifactStore, + VerifiedArtifactHandle, +) +from video_lab.curated_models import ( + CuratedDatasetManifestV1, + SequenceTransformRecipeV1, + StructuredCaptionV1, +) +from video_lab.curated_source import CuratedSourceLease +from video_lab.curated_splits import assign_identity_aware_splits +from video_lab.curated_validation import ( + CuratedValidationReport, + FrameFinding, + ValidationThresholdsV1, + validate_curated_dataset, +) +from video_lab.database import Database +from video_lab.dataset_compiler import ( + DatasetCompilationError, + DatasetCompilationService, + DatasetCompilerBackend, + TransformReceipt, + temporal_sample_indices, +) +from video_lab.dataset_receipts import ( + CompilationConflict, + CompilationOutput, + DatasetCompilationReceipt, + DatasetCompilationRequest, + InputBinding, +) +from video_lab.domain import canonical_json, utc_now +from video_lab.trainers.wan_batch import WAN_BATCH_TENSORS, validated_safetensors_header_prefix + + +class DatasetExit(IntEnum): + """Stable process contract (sysexits-compatible where practical).""" + + ACCEPTED = 0 + USAGE = 64 + REJECTED = 65 + NEEDS_REVIEW = 66 + CONFLICT = 67 + IN_PROGRESS = 68 + CANCELLED = 69 + INTEGRITY = 70 + INTERNAL = 71 + + +class DatasetCliError(RuntimeError): + def __init__(self, code: str, message: str, exit_code: DatasetExit) -> None: + self.code = code + self.safe_message = message + self.exit_code = exit_code + super().__init__(message) + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + + +class FrameSourceBinding(_FrozenModel): + frame_id: str + artifact_id: str + relative_path: str + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + byte_size: int = Field(gt=0) + media_type: Literal["image/png"] = "image/png" + + +class DatasetValidationRequest(_FrozenModel): + schema_version: Literal["dataset-validation-request.v1"] = "dataset-validation-request.v1" + operation_key: str = Field(pattern=r"^[a-z][a-z0-9_]*$") + manifest: CuratedDatasetManifestV1 + frame_bindings: tuple[FrameSourceBinding, ...] + thresholds: ValidationThresholdsV1 = ValidationThresholdsV1() + human_dispositions: dict[str, Literal["accepted", "rejected"]] = Field(default_factory=dict) + + @model_validator(mode="after") + def validate_bindings(self) -> DatasetValidationRequest: + identities = tuple(binding.frame_id for binding in self.frame_bindings) + if identities != tuple(sorted(identities)) or len(identities) != len(set(identities)): + raise ValueError("frame bindings must be unique and canonically ordered") + return self + + +class InspectRequest(_FrozenModel): + schema_version: Literal["dataset-inspect-request.v1"] = "dataset-inspect-request.v1" + receipt_id: str + + +class ReproduceRequest(_FrozenModel): + schema_version: Literal["dataset-reproduce-request.v1"] = "dataset-reproduce-request.v1" + receipt_id: str + operation_key: str + + +class _WanRecipeV1(_FrozenModel): + schema_version: Literal["wan-curated-compiler-recipe.v1"] + transform_recipe: SequenceTransformRecipeV1 + timestep: int = Field(ge=0) + timestep_boundary: int = Field(gt=0) + latent_channels: int = Field(gt=0) + vae_scaling_factor: float = Field(gt=0) + + +class DatasetResultEnvelope(_FrozenModel): + schema_version: Literal["dataset-cli-result.v1"] = "dataset-cli-result.v1" + ok: bool + data: object | None = None + error: dict[str, str] | None = None + + +class InspectionArtifact(_FrozenModel): + role: str + artifact_id: str + sha256: str + byte_size: int + media_type: str + document: dict[str, object] | None = None + + +class _WanSourceV1(_FrozenModel): + schema_version: Literal["wan-curated-source.v1"] + shape: tuple[int, int, int, int] + dtype: Literal["uint8"] + frames_base64: str + frame_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + transform_receipt: TransformReceipt + transform_receipt_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + caption: StructuredCaptionV1 + caption_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + source_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + validation_evidence: dict[str, str] + selected_sequence_ids: tuple[str, ...] + selected_frame_source_digests: tuple[tuple[str, str], ...] + + +class _WanTensorV1(_FrozenModel): + name: str + dtype: str + rank: int = Field(gt=0) + shape: tuple[int, ...] + byte_offsets: tuple[int, int] + digest: str + finite: Literal[True] + + +class _WanObjectiveV1(_FrozenModel): + target_kind: Literal["flow_noise_minus_clean"] + formula: Literal["target=noise_latents-clean_latents"] + clean_latents_digest: str + noise_latents_digest: str + target_digest: str + + +class _WanScheduleV1(_FrozenModel): + timestep_min: int + timestep_max: int + timestep_boundary: int + selector_rule: Literal["high_if_timestep_gte_boundary_else_low"] + + +class _WanManifestV1(_FrozenModel): + schema_version: Literal["wan-precomputed-training-batch.v1"] + sha256: str + batch_size: int = Field(gt=0) + tensors: tuple[_WanTensorV1, ...] + objective: _WanObjectiveV1 + noise_schedule: _WanScheduleV1 + + @model_validator(mode="after") + def validate_inventory(self) -> _WanManifestV1: + names = tuple(tensor.name for tensor in self.tensors) + if names != tuple(sorted(WAN_BATCH_TENSORS)) or len(names) != len(set(names)): + raise ValueError("tensor inventory must be exact and canonical") + return self + + +class _WanProvenanceV1(_FrozenModel): + schema_version: Literal["wan-curated-batch-provenance.v1"] + source_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + source_document_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + frame_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + transform_receipt_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + caption_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + rendered_caption: str + token_ids: tuple[tuple[int, ...], ...] + recipe_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + component_digests: dict[str, dict[str, str]] + seed: int + batch_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + manifest_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + + +class DatasetValidationReceipt(_FrozenModel): + schema_version: Literal["dataset-validation-receipt.v1"] = "dataset-validation-receipt.v1" + receipt_id: str + operation_key: str + request_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + state: Literal["accepted", "rejected", "needs_review"] + report_artifact: InspectionArtifact + report: CuratedValidationReport + created_at: str + + +class DatasetInspectionView(_FrozenModel): + schema_version: Literal["dataset-inspection-view.v1"] = "dataset-inspection-view.v1" + receipt: DatasetCompilationReceipt + source: InspectionArtifact + recipe: InspectionArtifact + outputs: tuple[InspectionArtifact, ...] + transform_receipt: TransformReceipt + tensor_inventory: tuple[_WanTensorV1, ...] + provenance: _WanProvenanceV1 + reproduction: dict[str, str | None] + + +class CompilationService(Protocol): + def validate(self, request: DatasetCompilationRequest) -> str: ... + def compile(self, request: DatasetCompilationRequest) -> DatasetCompilationReceipt: ... + def inspect(self, receipt_id: str) -> DatasetCompilationReceipt: ... + def reproduce(self, receipt_id: str, *, operation_key: str) -> DatasetCompilationReceipt: ... + + +class DatasetApplicationService: + def __init__( + self, + compilation: DatasetCompilationService | None = None, + *, + database: Database | None = None, + artifacts: ArtifactStore | None = None, + cancelled: Callable[[], bool] = lambda: False, + ) -> None: + self.compilation = compilation + self.database = database + self.artifacts = artifacts + self.cancelled = cancelled + + def validate( + self, request: DatasetValidationRequest, *, source_root: Path + ) -> DatasetValidationReceipt: + if self.database is None or self.artifacts is None: + raise DatasetCompilationError("validation persistence is unavailable") + request_json = canonical_json(request.model_dump(mode="json")) + request_digest = hashlib.sha256(request_json.encode()).hexdigest() + replay = self._validation_replay(request.operation_key, request_digest) + if replay is not None: + return replay + if self.cancelled(): + raise DatasetCompilationError("validation cancelled before source decoding") + frames = { + frame.frame_id: frame + for sequence in request.manifest.sequences + for frame in sequence.frames + } + bindings = {binding.frame_id: binding for binding in request.frame_bindings} + if set(bindings) != set(frames): + raise DatasetCompilationError("frame bindings do not match manifest") + artifacts: list[Artifact] = [] + for frame_id in sorted(frames): + frame, binding = frames[frame_id], bindings[frame_id] + if binding.relative_path != frame.source_path or binding.sha256 != frame.source_digest: + raise DatasetCompilationError("frame binding conflicts with manifest") + artifacts.append( + Artifact( + id=binding.artifact_id, + job_id=None, + sha256=binding.sha256, + byte_size=binding.byte_size, + media_type=binding.media_type, + relative_path=binding.relative_path, + created_at="1970-01-01T00:00:00+00:00", + ) + ) + lease = CuratedSourceLease.open(root=source_root, expected=tuple(artifacts)) + try: + decoded = {} + for frame_id in sorted(frames): + if self.cancelled(): + raise DatasetCompilationError("validation cancelled at frame boundary") + decoded[frame_id] = lease.read_frame(frames[frame_id]) + lease.reverify() + finally: + lease.close() + if self.cancelled(): + raise DatasetCompilationError("validation cancelled before metric work") + try: + report = validate_curated_dataset( + request.manifest, + decoded, + request.thresholds, + human_dispositions=request.human_dispositions, + cancelled=self.cancelled, + ) + except ValueError as error: + if "cancelled" in str(error): + raise DatasetCompilationError("validation cancelled during metric work") from error + raise DatasetCompilationError("validation evidence was rejected") from error + if report.split_assignment_evidence is None: + raise DatasetCompilationError("complete split evidence is required") + try: + assign_identity_aware_splits( + request.manifest, + request.manifest.split_policy, + decoded_frames=decoded, + evidence=report.split_assignment_evidence, + enforce_declared=True, + cancelled=self.cancelled, + ) + except ValueError as error: + raise DatasetCompilationError("declared dataset splits were rejected") from error + if self.cancelled(): + raise DatasetCompilationError("validation cancelled before publication") + return self._publish_validation(request, request_json, request_digest, report) + + def _validation_replay( + self, operation_key: str, request_digest: str + ) -> DatasetValidationReceipt | None: + assert self.database is not None and self.artifacts is not None + with self.database.connect() as connection: + row = connection.execute( + "SELECT operation_key,request_digest,state,report_artifact_id,receipt_id," + "receipt_digest,receipt_json FROM curated_validation_operations " + "WHERE operation_key=?", + (operation_key,), + ).fetchone() + if row is None: + return None + if str(row["request_digest"]) != request_digest: + raise CompilationConflict("validation operation key conflict") + receipt = DatasetValidationReceipt.model_validate_json( + str(row["receipt_json"]), strict=True + ) + receipt_json = canonical_json(receipt.model_dump(mode="json")) + if ( + receipt.operation_key != operation_key + or receipt.request_digest != request_digest + or receipt.receipt_id != str(row["receipt_id"]) + or receipt.state != str(row["state"]) + or receipt.report_artifact.artifact_id != str(row["report_artifact_id"]) + or hashlib.sha256(receipt_json.encode()).hexdigest() + != str(row["receipt_digest"]) + or receipt.receipt_id != "dvr_" + request_digest[:32] + ): + raise DatasetCompilationError("validation receipt authority mismatch") + binding = receipt.report_artifact + with self.database.connect() as connection: + with self.artifacts.open_verified_with_connection( + connection, + binding.artifact_id, + sha256=binding.sha256, + byte_size=binding.byte_size, + media_type=binding.media_type, + cancelled=self.cancelled, + ) as handle: + raw = _read_exact_bounded(handle, binding.byte_size, _MAX_INSPECTION_DOCUMENT_BYTES) + self.artifacts.reverify_open_handle( + connection, handle, cancelled=self.cancelled + ) + if raw != canonical_json(receipt.report.model_dump(mode="json")).encode(): + raise DatasetCompilationError("validation report artifact mismatch") + return receipt + + def _publish_validation( + self, + request: DatasetValidationRequest, + request_json: str, + request_digest: str, + report: CuratedValidationReport, + ) -> DatasetValidationReceipt: + assert self.database is not None and self.artifacts is not None + report_bytes = canonical_json(report.model_dump(mode="json")).encode() + staged = self.artifacts.stage_bytes( + report_bytes, "application/vnd.video-lab.curated-validation+json" + ) + now = utc_now().isoformat() + + def persist( + connection: sqlite3.Connection, committed: tuple[Artifact, ...] + ) -> DatasetValidationReceipt: + artifact = committed[0] + binding = InspectionArtifact( + role="validation_report", + artifact_id=artifact.id, + sha256=artifact.sha256, + byte_size=artifact.byte_size, + media_type=artifact.media_type, + ) + receipt = DatasetValidationReceipt( + receipt_id="dvr_" + request_digest[:32], + operation_key=request.operation_key, + request_digest=request_digest, + state=report.state, + report_artifact=binding, + report=report, + created_at=now, + ) + receipt_json = canonical_json(receipt.model_dump(mode="json")) + connection.execute( + "INSERT INTO curated_validation_operations(operation_key,request_digest," + "request_json,state,report_artifact_id,receipt_id,receipt_digest,receipt_json," + "created_at) VALUES(?,?,?,?,?,?,?,?,?)", + ( + request.operation_key, + request_digest, + request_json, + report.state, + artifact.id, + receipt.receipt_id, + hashlib.sha256(receipt_json.encode()).hexdigest(), + receipt_json, + now, + ), + ) + return receipt + + try: + return self.artifacts.commit_batch((staged,), persist) + except sqlite3.IntegrityError: + self.artifacts.discard(staged) + replay = self._validation_replay(request.operation_key, request_digest) + if replay is None: + raise + return replay + + def inspect(self, receipt_id: str) -> DatasetInspectionView: + try: + return self._inspect_authoritative(receipt_id) + except ArtifactError as error: + if self.cancelled(): + raise DatasetCompilationError( + "inspection cancelled during integrity verification" + ) from error + raise + + def compile(self, request: DatasetCompilationRequest) -> DatasetCompilationReceipt: + if self.compilation is None or self.database is None or self.artifacts is None: + raise DatasetCompilationError("compilation service is unavailable") + with self.database.connect() as connection: + with self.artifacts.open_verified_with_connection( + connection, + request.source.artifact_id, + sha256=request.source.sha256, + byte_size=request.source.byte_size, + media_type=request.source.media_type, + cancelled=self.cancelled, + ) as handle: + raw = _read_exact_bounded( + handle, request.source.byte_size, _MAX_INSPECTION_DOCUMENT_BYTES + ) + self.artifacts.reverify_open_handle( + connection, handle, cancelled=self.cancelled + ) + source = _WanSourceV1.model_validate_json(raw, strict=True) + self._verify_validation_evidence(source) + try: + return self.compilation.compile(request) + except ValueError as error: + raise DatasetCompilationError("compiler authority was rejected") from error + + def reproduce( + self, receipt_id: str, *, operation_key: str + ) -> DatasetCompilationReceipt: + if self.compilation is None: + raise DatasetCompilationError("reproduction service is unavailable") + self.inspect(receipt_id) + return self.compilation.reproduce(receipt_id, operation_key=operation_key) + + def _verify_validation_evidence(self, source: _WanSourceV1) -> None: + assert self.database is not None + evidence = source.validation_evidence + if set(evidence) != { + "complete_manifest_digest", + "dispositions_digest", + "report_digest", + "request_digest", + "split_evidence_digest", + "validation_receipt_id", + }: + raise DatasetCompilationError("validation evidence inventory was rejected") + with self.database.connect() as connection: + validation_row = connection.execute( + "SELECT request_json,receipt_json FROM curated_validation_operations " + "WHERE receipt_id=?", + (evidence["validation_receipt_id"],), + ).fetchone() + if validation_row is None: + raise DatasetCompilationError("validation receipt was unavailable") + validation_receipt = DatasetValidationReceipt.model_validate_json( + str(validation_row["receipt_json"]), strict=True + ) + validation_request = DatasetValidationRequest.model_validate_json( + str(validation_row["request_json"]), strict=True + ) + durable_validation = self._validation_replay( + validation_receipt.operation_key, evidence["request_digest"] + ) + split_evidence = validation_receipt.report.split_assignment_evidence + if ( + durable_validation != validation_receipt + or split_evidence is None + or evidence["complete_manifest_digest"] != source.source_digest + or evidence["complete_manifest_digest"] != validation_receipt.report.dataset_digest + or evidence["report_digest"] != validation_receipt.report_artifact.sha256 + or evidence["request_digest"] != validation_receipt.request_digest + or evidence["split_evidence_digest"] + != hashlib.sha256(canonical_json(split_evidence).encode()).hexdigest() + or evidence["dispositions_digest"] + != hashlib.sha256( + canonical_json(validation_request.human_dispositions).encode() + ).hexdigest() + ): + raise DatasetCompilationError("validation authority was rejected") + sequence_ids = source.selected_sequence_ids + if len(sequence_ids) != 1: + raise DatasetCompilationError("validation_selection_multiple_sequences_unsupported") + if sequence_ids != tuple(sorted(sequence_ids)): + raise DatasetCompilationError("validation_selection_inventory_invalid") + sequences = { + sequence.sequence_id: sequence for sequence in validation_request.manifest.sequences + } + if any(sequence_id not in sequences for sequence_id in sequence_ids): + raise DatasetCompilationError("validation_selection_sequence_unknown") + selected = tuple(sequences[sequence_id] for sequence_id in sequence_ids) + if any(sequence.split != "train" for sequence in selected): + raise DatasetCompilationError("validation_selection_split_invalid") + captions = tuple(sequence.caption for sequence in selected) + if any(caption != source.caption for caption in captions): + raise DatasetCompilationError("validation_selection_caption_mismatch") + recipe = validation_request.manifest.transform_recipe + if ( + source.transform_receipt.recipe_id != recipe.recipe_id + or source.transform_receipt.seed != recipe.seed + ): + raise DatasetCompilationError("validation_selection_recipe_mismatch") + sequence = selected[0] + sampled = temporal_sample_indices(len(sequence.frames), recipe) + expected_inputs = tuple( + (sequence.frames[index].frame_id, sequence.frames[index].pixel_digest) + for index in sampled + ) + actual_inputs = tuple( + (item.frame_id, item.input_pixel_digest) for item in source.transform_receipt.inputs + ) + if actual_inputs != expected_inputs: + raise DatasetCompilationError("validation_selection_frames_mismatch") + expected_sources = tuple( + (sequence.frames[index].frame_id, sequence.frames[index].source_digest) + for index in sampled + ) + if source.selected_frame_source_digests != expected_sources: + raise DatasetCompilationError("validation_selection_source_digests_mismatch") + selected_frames = {frame_id for frame_id, _ in expected_inputs} + findings = tuple(validation_receipt.report.frame_findings) + tuple( + validation_receipt.report.sequence_findings + ) + for finding in findings: + applies = ( + finding.sequence_id in sequence_ids + or ( + not isinstance(finding, FrameFinding) + and bool(set(finding.related_sequence_ids) & set(sequence_ids)) + ) + or ( + not isinstance(finding, FrameFinding) + and any( + item.frame_id in selected_frames for item in finding.implicated_frames + ) + ) + or (isinstance(finding, FrameFinding) and finding.frame_id in selected_frames) + ) + if ( + applies + and finding.state in {"rejected", "needs_review"} + and finding.human_disposition != "accepted" + ): + raise DatasetCompilationError("validation_selection_finding_unaccepted") + if finding.human_disposition is not None and ( + validation_request.human_dispositions.get(finding.finding_id) + != finding.human_disposition + ): + raise DatasetCompilationError("validation_selection_disposition_stale") + + def _inspect_authoritative(self, receipt_id: str) -> DatasetInspectionView: + if self.compilation is None or self.database is None or self.artifacts is None: + raise DatasetCompilationError("inspection service is unavailable") + receipt = self.compilation.inspect(receipt_id) + if receipt.compiler_id != "wan_curated_v1": + raise DatasetCompilationError("unsupported inspection compiler") + if tuple(output.role for output in receipt.outputs) != ("batch", "manifest", "provenance"): + raise DatasetCompilationError("inspection output roles were rejected") + bindings: tuple[tuple[str, InputBinding | CompilationOutput], ...] = ( + ("source_manifest", receipt.source), + ("transform_recipe", receipt.recipe), + *((output.role, output) for output in receipt.outputs), + ) + if ( + len(bindings) > _MAX_INSPECTION_ARTIFACTS + or sum(binding.byte_size for _, binding in bindings) > _MAX_INSPECTION_TOTAL_BYTES + ): + raise DatasetCompilationError("inspection aggregate budget exceeded") + inspected: list[InspectionArtifact] = [] + document_bytes: dict[str, bytes] = {} + with self.database.connect() as connection: + for role, binding in bindings: + if self.cancelled(): + raise DatasetCompilationError("inspection cancelled at artifact boundary") + expected_media = { + "source_manifest": "application/json", + "transform_recipe": "application/json", + "batch": "application/vnd.safetensors", + "manifest": "application/json", + "provenance": "application/json", + }[role] + if binding.media_type != expected_media: + raise DatasetCompilationError("inspection artifact media type was rejected") + with self.artifacts.open_verified_with_connection( + connection, + binding.artifact_id, + sha256=binding.sha256, + byte_size=binding.byte_size, + media_type=binding.media_type, + cancelled=self.cancelled, + ) as handle: + if role == "batch": + batch_prefix = _read_safetensors_prefix(handle, binding.byte_size) + raw = b"" + else: + raw = _read_exact_bounded( + handle, binding.byte_size, _MAX_INSPECTION_DOCUMENT_BYTES + ) + document_bytes[role] = raw + self.artifacts.reverify_open_handle( + connection, handle, cancelled=self.cancelled + ) + document = _bounded_canonical_document(raw, binding.media_type) if raw else None + inspected.append( + InspectionArtifact( + role=role, + artifact_id=binding.artifact_id, + sha256=binding.sha256, + byte_size=binding.byte_size, + media_type=binding.media_type, + document=document, + ) + ) + if self.cancelled(): + raise DatasetCompilationError("inspection cancelled before result publication") + try: + source_document = cast(dict[str, Any], inspected[0].document) + recipe_document = cast(dict[str, Any], inspected[1].document) + manifest_document = cast(dict[str, Any], inspected[3].document) + provenance_document = cast(dict[str, Any], inspected[4].document) + source = _WanSourceV1.model_validate_json( + canonical_json(source_document), strict=True + ) + self._verify_validation_evidence(source) + _WanRecipeV1.model_validate_json( + canonical_json(recipe_document), strict=True + ) + manifest = _WanManifestV1.model_validate_json( + canonical_json(manifest_document), strict=True + ) + provenance = _WanProvenanceV1.model_validate_json( + canonical_json(provenance_document), strict=True + ) + header = validated_safetensors_header_prefix( + batch_prefix, + total_size=inspected[2].byte_size, + expected_names=WAN_BATCH_TENSORS, + batch_size=manifest.batch_size, + ) + if set(header) != {tensor.name for tensor in manifest.tensors}: + raise ValueError + manifest_header = { + tensor.name: { + "dtype": tensor.dtype, + "shape": list(tensor.shape), + "data_offsets": list(tensor.byte_offsets), + } + for tensor in manifest.tensors + } + if header != manifest_header: + raise ValueError + batch_binding = receipt.outputs[0] + manifest_binding = receipt.outputs[1] + if ( + manifest.sha256 != "sha256:" + batch_binding.sha256 + or provenance.batch_digest != batch_binding.sha256 + or provenance.manifest_digest != manifest_binding.sha256 + or provenance.source_document_digest != receipt.source.sha256 + or provenance.recipe_digest != receipt.recipe.sha256 + or provenance.source_digest != source.source_digest + or provenance.frame_digest != source.frame_digest + or provenance.transform_receipt_digest != source.transform_receipt_digest + or provenance.caption_digest != source.caption_digest + or source.transform_receipt_digest + != hashlib.sha256( + canonical_json(source.transform_receipt.model_dump(mode="json")).encode() + ).hexdigest() + or source.caption_digest + != hashlib.sha256( + canonical_json(source.caption.model_dump(mode="json")).encode() + ).hexdigest() + or hashlib.sha256(document_bytes["manifest"]).hexdigest() + != manifest_binding.sha256 + or hashlib.sha256(document_bytes["provenance"]).hexdigest() + != receipt.outputs[2].sha256 + or hashlib.sha256(document_bytes["source_manifest"]).hexdigest() + != receipt.source.sha256 + or hashlib.sha256(document_bytes["transform_recipe"]).hexdigest() + != receipt.recipe.sha256 + or provenance.seed != source.transform_receipt.seed + ): + raise ValueError + with self.database.connect() as connection: + with self.artifacts.open_verified_with_connection( + connection, + batch_binding.artifact_id, + sha256=batch_binding.sha256, + byte_size=batch_binding.byte_size, + media_type=batch_binding.media_type, + cancelled=self.cancelled, + ) as handle: + tensor_digests = _stream_tensor_digests( + handle, manifest, cancelled=self.cancelled + ) + if self.cancelled(): + raise DatasetCompilationError( + "inspection cancelled before post-stream verification" + ) + self.artifacts.reverify_open_handle( + connection, handle, cancelled=self.cancelled + ) + if tensor_digests != {tensor.name: tensor.digest for tensor in manifest.tensors}: + raise ValueError + except (ValidationError, ValueError, TypeError) as error: + raise DatasetCompilationError("inspection evidence was rejected") from error + return DatasetInspectionView( + receipt=receipt, + source=inspected[0], + recipe=inspected[1], + outputs=tuple(inspected[2:]), + transform_receipt=source.transform_receipt, + tensor_inventory=manifest.tensors, + provenance=provenance, + reproduction={ + "receipt_id": receipt.receipt_id, + "reproduces_receipt_id": receipt.reproduces_receipt_id, + "request_digest": receipt.request_digest, + }, + ) + + +_MAX_REQUEST_BYTES = 1024 * 1024 +_MAX_JSON_DEPTH = 32 +_MAX_INSPECTION_DOCUMENT_BYTES = 16 * 1024 * 1024 +_MAX_INSPECTION_ARTIFACTS = 32 +_MAX_INSPECTION_TOTAL_BYTES = 16 * 1024 * 1024 * 1024 +_MAX_REQUEST_PATH_COMPONENTS = 32 +_MAX_RETAINED_REQUEST_FDS = 33 + + +def _pairs(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + +def _depth(value: object, level: int = 0) -> int: + if level > _MAX_JSON_DEPTH: + raise ValueError("JSON nesting limit exceeded") + if isinstance(value, dict): + for item in value.values(): + _depth(item, level + 1) + elif isinstance(value, list): + for item in value: + _depth(item, level + 1) + return level + + +def _bounded_canonical_document(raw: bytes, media_type: str) -> dict[str, object] | None: + if "json" not in media_type: + return None + if len(raw) > _MAX_INSPECTION_DOCUMENT_BYTES: + raise DatasetCompilationError("inspection document budget exceeded") + try: + document = json.loads( + raw.decode("utf-8"), + object_pairs_hook=_pairs, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError()), + ) + _depth(document) + if canonical_json(document).encode() != raw: + raise ValueError("noncanonical inspection document") + if not isinstance(document, dict): + raise ValueError("inspection document must be an object") + return cast(dict[str, object], document) + except (UnicodeError, ValueError, json.JSONDecodeError) as error: + raise DatasetCompilationError("inspection document was rejected") from error + + +def _read_exact_bounded(handle: VerifiedArtifactHandle, declared_size: int, limit: int) -> bytes: + if declared_size < 0 or declared_size > limit: + raise DatasetCompilationError("inspection artifact budget exceeded") + raw = handle.read(declared_size + 1) + if len(raw) != declared_size: + raise DatasetCompilationError("inspection artifact was truncated or oversized") + return raw + + +def _read_safetensors_prefix(handle: VerifiedArtifactHandle, declared_size: int) -> bytes: + if declared_size < 9 or declared_size > _MAX_INSPECTION_TOTAL_BYTES: + raise DatasetCompilationError("inspection artifact budget exceeded") + first = handle.read(8) + if len(first) != 8: + raise DatasetCompilationError("inspection artifact was truncated") + header_size = struct.unpack(" _MAX_INSPECTION_DOCUMENT_BYTES: + raise DatasetCompilationError("inspection header budget exceeded") + header = handle.read(header_size) + if len(header) != header_size: + raise DatasetCompilationError("inspection artifact was truncated") + return first + header + + +def _stream_tensor_digests( + handle: VerifiedArtifactHandle, + manifest: _WanManifestV1, + *, + cancelled: Callable[[], bool], +) -> dict[str, str]: + first = handle.read(8) + if len(first) != 8: + raise DatasetCompilationError("inspection artifact was truncated") + header_size = struct.unpack(" tuple[int, int, int, int, int, int, int]: + info = os.fstat(fd) + return ( + info.st_dev, + info.st_ino, + info.st_mode, + info.st_nlink, + info.st_size, + info.st_mtime_ns, + info.st_ctime_ns, + ) + + +def _close_descriptor(descriptor: int, failures: list[str]) -> None: + try: + os.close(descriptor) + except OSError: + failures.append("descriptor_close_failed") + + +def _open_request(root: Path, relative: str) -> bytes: + path = PurePosixPath(relative) + if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts): + raise DatasetCliError("usage_error", "request path is invalid", DatasetExit.USAGE) + if ( + len(path.parts) > _MAX_REQUEST_PATH_COMPONENTS + or len(path.parts) + 1 > _MAX_RETAINED_REQUEST_FDS + ): + raise DatasetCliError( + "integrity_failure", "request traversal budget exceeded", DatasetExit.INTEGRITY + ) + descriptors: list[int] = [] + identities: list[tuple[int, int, int, int, int, int, int]] = [] + close_failures: list[str] = [] + try: + current = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + descriptors.append(current) + identities.append(_identity(current)) + for component in path.parts[:-1]: + current = os.open( + component, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + dir_fd=current, + ) + descriptors.append(current) + identities.append(_identity(current)) + fd = os.open( + path.parts[-1], + os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, + dir_fd=current, + ) + descriptors.append(fd) + before = _identity(fd) + identities.append(before) + info = os.fstat(fd) + if ( + not stat.S_ISREG(info.st_mode) + or info.st_nlink != 1 + or info.st_size > _MAX_REQUEST_BYTES + ): + raise OSError(errno.EINVAL, "unsafe request") + chunks: list[bytes] = [] + remaining = _MAX_REQUEST_BYTES + 1 + while remaining: + chunk = os.read(fd, min(65536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + if len(data) > _MAX_REQUEST_BYTES or any( + _identity(descriptor) != expected + for descriptor, expected in zip(descriptors, identities, strict=True) + ): + raise OSError(errno.EIO, "request changed") + verify = os.dup(descriptors[0]) + try: + for index, component in enumerate(path.parts): + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + if index < len(path.parts) - 1: + flags |= os.O_DIRECTORY + next_fd = os.open(component, flags, dir_fd=verify) + _close_descriptor(verify, close_failures) + verify = next_fd + if _identity(verify) != before: + raise OSError(errno.EIO, "request replaced") + finally: + _close_descriptor(verify, close_failures) + return data + except DatasetCliError: + raise + except OSError as error: + raise DatasetCliError( + "integrity_failure", "request integrity verification failed", DatasetExit.INTEGRITY + ) from error + finally: + for descriptor in reversed(descriptors): + _close_descriptor(descriptor, close_failures) + if close_failures: + raise DatasetCliError( + "integrity_failure", "request descriptor cleanup failed", DatasetExit.INTEGRITY + ) + + +def load_request[T: BaseModel](root: Path, relative: str, model: type[T]) -> T: + raw = _open_request(root, relative) + try: + text = raw.decode("utf-8") + document = json.loads( + text, + object_pairs_hook=_pairs, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError()), + ) + _depth(document) + if canonical_json(document).encode() != raw: + raise ValueError("request is not canonical JSON") + return model.model_validate_json(raw, strict=True) + except (UnicodeError, ValueError, ValidationError, json.JSONDecodeError) as error: + raise DatasetCliError( + "rejected", "request document was rejected", DatasetExit.REJECTED + ) from error + + +class _BindingBackend: + def __init__(self, compiler_id: str) -> None: + self.compiler_id = compiler_id + + def validate_recipe(self, recipe_bytes: bytes) -> None: + try: + document = json.loads( + recipe_bytes.decode("utf-8"), + object_pairs_hook=_pairs, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError()), + ) + _depth(document) + if canonical_json(document).encode() != recipe_bytes: + raise ValueError("recipe is not canonical") + _WanRecipeV1.model_validate_json(recipe_bytes, strict=True) + except (UnicodeError, ValueError, ValidationError, json.JSONDecodeError) as error: + raise DatasetCompilationError("compiler recipe was rejected") from error + + def compile_batch(self, source_bytes: bytes, recipe_bytes: bytes) -> tuple[()]: + raise DatasetCompilationError("validation backend cannot compile") + + +def build_dataset_service( + *, + database_path: Path, + artifact_root: Path, + command: str, + request: DatasetCompilationRequest | None, + component_roots: tuple[Path, Path, Path, Path] | None, + cancelled: Callable[[], bool], +) -> DatasetCompilationService: + database = Database(database_path) + database.migrate() + backends: dict[str, DatasetCompilerBackend] = {} + if command == "validate" and request is not None: + backends[request.compiler_id] = _BindingBackend(request.compiler_id) + elif command in {"compile", "reproduce"}: + if component_roots is None or any(not root.is_dir() for root in component_roots): + raise DatasetCliError( + "rejected", "required local component root is unavailable", DatasetExit.REJECTED + ) + from video_lab.compilers.wan_curated import WanCompilerSnapshot, WanCuratedBatchCompiler + from video_lab.trainers.wan_factory import WanPinnedLocalRoot + + roots = tuple(WanPinnedLocalRoot(path=root, revision="local") for root in component_roots) + backend = WanCuratedBatchCompiler(snapshot=WanCompilerSnapshot(*roots)) + backends[backend.compiler_id] = backend + return DatasetCompilationService( + database, ArtifactStore(database, artifact_root), backends, cancelled=cancelled + ) + + +def build_dataset_application( + *, + database_path: Path, + artifact_root: Path, + command: str, + request: DatasetCompilationRequest | None, + component_roots: tuple[Path, Path, Path, Path] | None, + cancelled: Callable[[], bool], +) -> DatasetApplicationService: + if command == "validate": + database = Database(database_path) + database.migrate() + return DatasetApplicationService( + database=database, + artifacts=ArtifactStore(database, artifact_root), + cancelled=cancelled, + ) + compilation = build_dataset_service( + database_path=database_path, + artifact_root=artifact_root, + command=command, + request=request, + component_roots=component_roots, + cancelled=cancelled, + ) + return DatasetApplicationService( + compilation, + database=compilation.database, + artifacts=compilation.artifacts, + cancelled=cancelled, + ) + + +@contextmanager +def _cancellation() -> Iterator[Callable[[], bool]]: + requested = False + + def handle(_signum: int, _frame: object) -> None: + nonlocal requested + requested = True + + previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + for sig in previous: + signal.signal(sig, handle) + try: + yield lambda: requested + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + + +def _emit( + envelope: DatasetResultEnvelope, *, json_output: bool, human: str, exit_code: DatasetExit +) -> None: + if json_output: + typer.echo(canonical_json(envelope.model_dump(mode="json"))) + elif envelope.ok: + typer.echo(human) + else: + typer.echo( + envelope.error["message"] if envelope.error else "dataset command failed", err=True + ) + if exit_code: + raise typer.Exit(int(exit_code)) + + +def emit_usage_error(*, json_output: bool, message: str) -> None: + _emit( + DatasetResultEnvelope(ok=False, error={"code": "usage_error", "message": message}), + json_output=json_output, + human="", + exit_code=DatasetExit.USAGE, + ) + + +def _human_result(command: str, result: object, status: object) -> str: + if not isinstance(result, dict): + return str(status) + if command == "validate": + codes = result.get("reason_codes") + suffix = "" if not codes else " reasons=" + ",".join(str(code) for code in codes) + return str(status) + suffix + if command == "inspect": + receipt = result.get("receipt") + receipt_id = ( + receipt.get("receipt_id", "verified") if isinstance(receipt, dict) else "verified" + ) + outputs = result.get("outputs") + roles = ( + ",".join(str(item.get("role")) for item in outputs if isinstance(item, dict)) + if isinstance(outputs, list) + else "" + ) + reproduction = result.get("reproduction") + parent = ( + reproduction.get("reproduces_receipt_id") if isinstance(reproduction, dict) else None + ) + return f"accepted receipt={receipt_id} outputs={roles} reproduces={parent or 'none'}" + return str(status) + + +def run_command( + *, + command: Literal["validate", "compile", "inspect", "reproduce"], + database_path: Path, + artifact_root: Path, + request_root: Path, + request_file: str, + json_output: bool, + source_root: Path | None = None, + component_roots: tuple[Path | None, Path | None, Path | None, Path | None] | None = None, +) -> None: + try: + if command in {"validate", "inspect"} and component_roots is not None: + raise DatasetCliError( + "usage_error", + "component roots are not accepted by this command", + DatasetExit.USAGE, + ) + if component_roots is not None and any(root is None for root in component_roots): + raise DatasetCliError( + "usage_error", + "all component roots are required together", + DatasetExit.USAGE, + ) + exact_roots = cast( + tuple[Path, Path, Path, Path] | None, + component_roots, + ) + model: type[BaseModel] = ( + DatasetValidationRequest + if command == "validate" + else DatasetCompilationRequest + if command == "compile" + else InspectRequest + if command == "inspect" + else ReproduceRequest + ) + parsed = load_request(request_root, request_file, model) + compilation_request = parsed if isinstance(parsed, DatasetCompilationRequest) else None + with _cancellation() as cancelled: + service = build_dataset_application( + database_path=database_path, + artifact_root=artifact_root, + command=command, + request=compilation_request, + component_roots=exact_roots, + cancelled=cancelled, + ) + if command == "validate": + assert isinstance(parsed, DatasetValidationRequest) + if source_root is None: + raise DatasetCliError( + "usage_error", "source root is required", DatasetExit.USAGE + ) + validation_receipt = service.validate(parsed, source_root=source_root) + report = validation_receipt.report + result: object = { + "status": validation_receipt.state, + "validation_receipt": validation_receipt.model_dump(mode="json"), + "reason_codes": sorted( + {item.reason_code for item in report.frame_findings} + | {item.reason_code for item in report.sequence_findings} + ), + } + elif command == "compile": + assert compilation_request is not None + compile_method = getattr(service, "compile", None) + if compile_method is None: + assert service.compilation is not None + compile_method = service.compilation.compile + result = compile_method(compilation_request).model_dump(mode="json") + elif command == "inspect": + assert isinstance(parsed, InspectRequest) + result = service.inspect(parsed.receipt_id).model_dump(mode="json") + else: + assert isinstance(parsed, ReproduceRequest) + reproduce_method = getattr(service, "reproduce", None) + if reproduce_method is None: + assert service.compilation is not None + reproduce_method = service.compilation.reproduce + result = reproduce_method( + parsed.receipt_id, operation_key=parsed.operation_key + ).model_dump(mode="json") + result_status = result.get("status") if isinstance(result, dict) else "accepted" + exit_code = ( + DatasetExit.REJECTED + if result_status == "rejected" + else DatasetExit.NEEDS_REVIEW + if result_status == "needs_review" + else DatasetExit.ACCEPTED + ) + _emit( + DatasetResultEnvelope(ok=True, data=result), + json_output=json_output, + human=_human_result(command, result, result_status), + exit_code=exit_code, + ) + except typer.Exit: + raise + except DatasetCliError as error: + _emit( + DatasetResultEnvelope( + ok=False, error={"code": error.code, "message": error.safe_message} + ), + json_output=json_output, + human="", + exit_code=error.exit_code, + ) + except CompilationConflict: + _emit( + DatasetResultEnvelope( + ok=False, + error={"code": "conflict", "message": "request conflicts with durable evidence"}, + ), + json_output=json_output, + human="", + exit_code=DatasetExit.CONFLICT, + ) + except (ArtifactIntegrityError, ArtifactError, OSError): + _emit( + DatasetResultEnvelope( + ok=False, + error={ + "code": "integrity_failure", + "message": "local evidence integrity verification failed", + }, + ), + json_output=json_output, + human="", + exit_code=DatasetExit.INTEGRITY, + ) + except DatasetCompilationError as error: + message = str(error).lower() + code, status = ( + ("cancelled", DatasetExit.CANCELLED) + if "cancel" in message + else ("in_progress", DatasetExit.IN_PROGRESS) + if "progress" in message + else ("rejected", DatasetExit.REJECTED) + ) + _emit( + DatasetResultEnvelope( + ok=False, error={"code": code, "message": "dataset operation was not accepted"} + ), + json_output=json_output, + human="", + exit_code=status, + ) + except Exception: + _emit( + DatasetResultEnvelope( + ok=False, error={"code": "internal_failure", "message": "dataset command failed"} + ), + json_output=json_output, + human="", + exit_code=DatasetExit.INTERNAL, + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/dataset_compiler.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/dataset_compiler.py new file mode 100644 index 0000000..12426d5 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/dataset_compiler.py @@ -0,0 +1,565 @@ +"""Deterministic transforms and trainer-neutral compilation service.""" + +from __future__ import annotations + +import errno +import hashlib +import sqlite3 +from collections.abc import Callable, Mapping +from contextlib import ExitStack +from dataclasses import dataclass +from typing import Protocol + +import numpy as np +from PIL import Image +from pydantic import BaseModel, ConfigDict, Field + +from video_lab.artifacts import Artifact, ArtifactError, ArtifactStore, StagedArtifact +from video_lab.curated_models import SequenceTransformRecipeV1 +from video_lab.curated_source import CuratedSourceError, DecodedFrame, validate_decoded_frame +from video_lab.database import Database +from video_lab.dataset_receipts import ( + CompilationClaim, + CompilationOutput, + DatasetCompilationReceipt, + DatasetCompilationRepository, + DatasetCompilationRequest, +) +from video_lab.domain import canonical_json, utc_now + + +class DatasetCompilationError(RuntimeError): + """Compilation failed before a complete receipt could be published.""" + + +class _CompilationCancelled(DatasetCompilationError): + pass + + +class _CleanupFailure(DatasetCompilationError): + def __init__(self, failures: tuple[str, ...]) -> None: + self.failures = failures + super().__init__("compilation cleanup failure: " + "; ".join(failures)) + + +@dataclass(frozen=True) +class BackendOutput: + role: str + data: bytes + media_type: str + + +class DatasetCompilerBackend(Protocol): + compiler_id: str + + def validate_recipe(self, recipe_bytes: bytes) -> None: ... + + def compile_batch( + self, source_bytes: bytes, recipe_bytes: bytes + ) -> tuple[BackendOutput, ...]: ... + + +@dataclass(frozen=True) +class TransformInput: + frame_id: str + decoded: DecodedFrame + + +@dataclass(frozen=True) +class TransformLimits: + max_width: int = 4096 + max_height: int = 4096 + max_pixels_per_frame: int = 4096 * 4096 + max_frame_count: int = 256 + max_input_bytes: int = 4 * 1024 * 1024 * 1024 + max_output_bytes: int = 4 * 1024 * 1024 * 1024 + + +DEFAULT_TRANSFORM_LIMITS = TransformLimits() +_MAX_TEMPORAL_FRAME_COUNT = 1_000_000 +_MAX_TEMPORAL_FACTOR = 1_000_000_000 +_MAX_TEMPORAL_PRODUCT = 1_000_000_000_000 + + +class TransformInputReceipt(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + frame_id: str + input_pixel_digest: str + crop_rectangle: tuple[int, int, int, int] + output_shape: tuple[int, int, int] + output_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + + +class TransformReceipt(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + operation_version: str = "sequence-transform.v1" + recipe_id: str + seed: int + inputs: tuple[TransformInputReceipt, ...] + decoded_output_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + + +@dataclass(frozen=True) +class TransformedSequence: + frames: np.ndarray + receipt: TransformReceipt + + +_RESAMPLE = { + "nearest": Image.Resampling.NEAREST, + "bilinear": Image.Resampling.BILINEAR, + "bicubic": Image.Resampling.BICUBIC, + "lanczos": Image.Resampling.LANCZOS, +} + + +def transformed_array_digest(array: np.ndarray) -> str: + header = f"{array.shape}:{array.dtype.str}".encode() + return hashlib.sha256(b"transformed-frame.v1\0" + header + b"\0" + array.tobytes()).hexdigest() + + +def validate_transformed_sequence( + frames: np.ndarray, + receipt: TransformReceipt, + recipe: SequenceTransformRecipeV1, +) -> None: + """Revalidate the exact Task 4 transformed array and receipt authority.""" + expected_shape = ( + recipe.output_frame_count, + recipe.output_height, + recipe.output_width, + 3, + ) + if ( + not isinstance(frames, np.ndarray) + or frames.dtype != np.uint8 + or frames.shape != expected_shape + or receipt.operation_version != "sequence-transform.v1" + or receipt.recipe_id != recipe.recipe_id + or receipt.seed != recipe.seed + or len(receipt.inputs) != recipe.output_frame_count + or receipt.decoded_output_digest != transformed_array_digest(frames) + ): + raise DatasetCompilationError("transformed sequence receipt mismatch") + frame_ids: set[str] = set() + for index, binding in enumerate(receipt.inputs): + frame = frames[index] + left, top, right, bottom = binding.crop_rectangle + if ( + binding.frame_id in frame_ids + or not binding.frame_id + or len(binding.input_pixel_digest) != 64 + or any(character not in "0123456789abcdef" for character in binding.input_pixel_digest) + or not (0 <= left < right and 0 <= top < bottom) + or binding.output_shape != frame.shape + or binding.output_digest != transformed_array_digest(frame) + ): + raise DatasetCompilationError("transformed sequence frame binding mismatch") + frame_ids.add(binding.frame_id) + + +def transform_sequence( + frames: tuple[TransformInput, ...], + recipe: SequenceTransformRecipeV1, + *, + limits: TransformLimits = DEFAULT_TRANSFORM_LIMITS, +) -> TransformedSequence: + if not frames: + raise DatasetCompilationError("transform requires accepted decoded frames") + try: + for item in frames: + validate_decoded_frame(item.decoded) + except CuratedSourceError: + raise DatasetCompilationError("transform requires a canonical decoded frame") from None + output_pixels = recipe.output_width * recipe.output_height + input_bytes = sum( + int(pixels.nbytes) if isinstance(pixels, np.ndarray) else len(pixels) + for pixels in (frame.decoded.pixels for frame in frames) + ) + output_bytes = output_pixels * 3 * recipe.output_frame_count + if ( + recipe.output_width > limits.max_width + or recipe.output_height > limits.max_height + or output_pixels > limits.max_pixels_per_frame + or recipe.output_frame_count > limits.max_frame_count + or input_bytes > limits.max_input_bytes + or output_bytes > limits.max_output_bytes + ): + raise DatasetCompilationError("transform budget exceeded") + selected_indices = temporal_sample_indices(len(frames), recipe) + transformed: list[np.ndarray] = [] + bindings: list[TransformInputReceipt] = [] + for index in selected_indices: + item = frames[index] + if isinstance(item.decoded.pixels, bytes): + source = np.frombuffer(item.decoded.pixels, dtype=np.uint8).reshape( + item.decoded.height, + item.decoded.width, + item.decoded.channels, + ) + else: + source = np.asarray(item.decoded.pixels) + if source.dtype != np.uint8 or source.ndim != 3 or source.shape[2] != 3: + raise DatasetCompilationError("transform input must be owned RGB uint8") + height, width, _ = source.shape + image = Image.fromarray(source, mode="RGB") + if recipe.resize_mode == "center_crop": + target_ratio = recipe.output_width / recipe.output_height + source_ratio = width / height + if source_ratio > target_ratio: + crop_width = max(1, int(height * target_ratio)) + left = (width - crop_width) // 2 + box = (left, 0, left + crop_width, height) + else: + crop_height = max(1, int(width / target_ratio)) + top = (height - crop_height) // 2 + box = (0, top, width, top + crop_height) + output_image = image.crop(box).resize( + (recipe.output_width, recipe.output_height), _RESAMPLE[recipe.interpolation] + ) + else: + box = (0, 0, width, height) + scale = min(recipe.output_width / width, recipe.output_height / height) + resized = image.resize( + (max(1, int(width * scale)), max(1, int(height * scale))), + _RESAMPLE[recipe.interpolation], + ) + output_image = Image.new( + "RGB", (recipe.output_width, recipe.output_height), recipe.padding_color + ) + output_image.paste( + resized, + ( + (recipe.output_width - resized.width) // 2, + (recipe.output_height - resized.height) // 2, + ), + ) + output = np.array(output_image, dtype=np.uint8, copy=True) + output.flags.writeable = False + transformed.append(output) + bindings.append( + TransformInputReceipt( + frame_id=item.frame_id, + input_pixel_digest=item.decoded.pixel_digest, + crop_rectangle=box, + output_shape=output.shape, + output_digest=transformed_array_digest(output), + ) + ) + batch = np.stack(transformed) + batch.flags.writeable = False + receipt = TransformReceipt( + recipe_id=recipe.recipe_id, + seed=recipe.seed, + inputs=tuple(bindings), + decoded_output_digest=transformed_array_digest(batch), + ) + return TransformedSequence(batch, receipt) + + +def temporal_sample_indices( + frame_count: int, recipe: SequenceTransformRecipeV1 +) -> tuple[int, ...]: + """Return Task 4's exact deterministic temporal selection.""" + numerator = recipe.temporal_stride_numerator + denominator = recipe.temporal_stride_denominator + output_count = recipe.output_frame_count + if ( + type(frame_count) is not int + or type(numerator) is not int + or type(denominator) is not int + or type(output_count) is not int + or not 0 < frame_count <= _MAX_TEMPORAL_FRAME_COUNT + or not 0 < numerator <= _MAX_TEMPORAL_FACTOR + or not 0 < denominator <= _MAX_TEMPORAL_FACTOR + or not 0 < output_count <= _MAX_TEMPORAL_FRAME_COUNT + or output_count * numerator > _MAX_TEMPORAL_PRODUCT + ): + raise DatasetCompilationError("temporal recipe authority exceeds bounded integer policy") + selected = tuple((index * numerator) // denominator for index in range(output_count)) + if len(set(selected)) != len(selected) or max(selected) >= frame_count: + raise DatasetCompilationError("temporal recipe exceeds available accepted frames") + return selected + + +class DatasetCompilationService: + def __init__( + self, + database: Database, + artifacts: ArtifactStore, + backends: Mapping[str, DatasetCompilerBackend], + *, + claimant_id: str = "local_compiler", + lease_seconds: int = 300, + cancelled: Callable[[], bool] = lambda: False, + ) -> None: + self.database = database + self.artifacts = artifacts + self.backends = dict(backends) + self.repository = DatasetCompilationRepository(database) + self.claimant_id = claimant_id + self.lease_seconds = lease_seconds + self.cancelled = cancelled + + def validate(self, request: DatasetCompilationRequest) -> str: + backend = self.backends.get(request.compiler_id) + if backend is None or backend.compiler_id != request.compiler_id: + raise DatasetCompilationError("compiler backend is unavailable") + with self.database.connect() as connection, ExitStack() as stack: + source = stack.enter_context( + self.artifacts.open_verified_with_connection( + connection, + request.source.artifact_id, + sha256=request.source.sha256, + byte_size=request.source.byte_size, + media_type=request.source.media_type, + ) + ) + recipe = stack.enter_context( + self.artifacts.open_verified_with_connection( + connection, + request.recipe.artifact_id, + sha256=request.recipe.sha256, + byte_size=request.recipe.byte_size, + media_type=request.recipe.media_type, + ) + ) + recipe_bytes = recipe.read() + backend.validate_recipe(recipe_bytes) + self.artifacts.reverify_open_handle(connection, source) + self.artifacts.reverify_open_handle(connection, recipe) + return request.digest + + def compile(self, request: DatasetCompilationRequest) -> DatasetCompilationReceipt: + return self._compile(request, expected_outputs=None) + + def _compile( + self, + request: DatasetCompilationRequest, + *, + expected_outputs: tuple[tuple[str, str, int, str], ...] | None, + ) -> DatasetCompilationReceipt: + existing = self.repository.begin(request) + if existing.status == "completed" and existing.receipt_id is not None: + return self.inspect(existing.receipt_id) + backend = self.backends.get(request.compiler_id) + if backend is None or backend.compiler_id != request.compiler_id: + raise DatasetCompilationError("compiler backend is unavailable") + with self.database.connect() as connection, ExitStack() as stack: + source = stack.enter_context( + self.artifacts.open_verified_with_connection( + connection, + request.source.artifact_id, + sha256=request.source.sha256, + byte_size=request.source.byte_size, + media_type=request.source.media_type, + ) + ) + recipe = stack.enter_context( + self.artifacts.open_verified_with_connection( + connection, + request.recipe.artifact_id, + sha256=request.recipe.sha256, + byte_size=request.recipe.byte_size, + media_type=request.recipe.media_type, + ) + ) + source_bytes, recipe_bytes = source.read(), recipe.read() + backend.validate_recipe(recipe_bytes) + claim = self.repository.claim( + request.operation_key, + claimant_id=self.claimant_id, + lease_seconds=self.lease_seconds, + ) + if not claim.claimed: + raise DatasetCompilationError("compilation is already in progress") + try: + produced = backend.compile_batch(source_bytes, recipe_bytes) + self._validate_outputs(request, produced) + actual_outputs = tuple( + ( + output.role, + hashlib.sha256(output.data).hexdigest(), + len(output.data), + output.media_type, + ) + for output in sorted(produced, key=lambda item: item.role) + ) + if expected_outputs is not None and actual_outputs != expected_outputs: + raise DatasetCompilationError("reproduction output mismatch") + if self.cancelled(): + raise _CompilationCancelled("compilation cancelled at batch boundary") + self.artifacts.reverify_open_handle(connection, source) + self.artifacts.reverify_open_handle(connection, recipe) + return self._publish(request, claim, produced) + except _CompilationCancelled as error: + evidence = hashlib.sha256(str(error).encode()).hexdigest() + self.repository.cancel(claim, evidence_digest=evidence) + raise + except _CleanupFailure as error: + evidence = hashlib.sha256("\n".join(error.failures).encode()).hexdigest() + self.repository.retryable_failure( + claim, code="cleanup_failed", failure_digest=evidence + ) + raise + except Exception as error: + failure = hashlib.sha256(f"{type(error).__name__}:{error}".encode()).hexdigest() + try: + self.repository.fail(claim, code="compilation_failed", failure_digest=failure) + except Exception as cleanup_error: + raise DatasetCompilationError("compilation cleanup failure") from cleanup_error + raise + + def inspect(self, receipt_id: str) -> DatasetCompilationReceipt: + if self.cancelled(): + raise _CompilationCancelled("inspection cancelled before verification") + receipt = self.repository.receipt(receipt_id) + try: + with self.database.connect() as connection: + for output in receipt.outputs: + if self.cancelled(): + raise _CompilationCancelled("inspection cancelled at artifact boundary") + with self.artifacts.open_verified_with_connection( + connection, + output.artifact_id, + sha256=output.sha256, + byte_size=output.byte_size, + media_type=output.media_type, + cancelled=self.cancelled, + ) as handle: + self.artifacts.reverify_open_handle( + connection, handle, cancelled=self.cancelled + ) + except ArtifactError as error: + if self.cancelled(): + raise _CompilationCancelled("inspection cancelled during verification") from error + raise + return receipt + + def reproduce(self, receipt_id: str, *, operation_key: str) -> DatasetCompilationReceipt: + original = self.inspect(receipt_id) + request = DatasetCompilationRequest( + operation_key=operation_key, + compiler_id=original.compiler_id, + source=original.source, + recipe=original.recipe, + output_roles=tuple(output.role for output in original.outputs), + reproduces_receipt_id=original.receipt_id, + ) + expected = tuple((o.role, o.sha256, o.byte_size, o.media_type) for o in original.outputs) + return self._compile(request, expected_outputs=expected) + + @staticmethod + def _validate_outputs( + request: DatasetCompilationRequest, outputs: tuple[BackendOutput, ...] + ) -> None: + roles = tuple(sorted(output.role for output in outputs)) + if roles != request.output_roles or len(roles) != len(set(roles)): + raise DatasetCompilationError( + "backend output roles are incomplete, extra, or duplicate" + ) + if any( + not output.data or len(output.data) > 1024 * 1024 * 1024 or not output.media_type + for output in outputs + ): + raise DatasetCompilationError("backend output is invalid") + + def _publish( + self, + request: DatasetCompilationRequest, + claim: CompilationClaim, + outputs: tuple[BackendOutput, ...], + ) -> DatasetCompilationReceipt: + staged_list = [] + try: + for output in outputs: + staged_list.append(self.artifacts.stage_bytes(output.data, output.media_type)) + except Exception: + failures = self._discard_all(tuple(staged_list)) + if failures: + raise _CleanupFailure(failures) from None + raise + staged = tuple(staged_list) + by_role = dict(zip((output.role for output in outputs), staged, strict=True)) + now = utc_now().isoformat() + + def persist( + connection: sqlite3.Connection, artifacts: tuple[Artifact, ...] + ) -> DatasetCompilationReceipt: + artifact_by_content = { + (artifact.sha256, artifact.media_type): artifact for artifact in artifacts + } + receipt_outputs = tuple( + CompilationOutput( + role=role, + artifact_id=artifact_by_content[ + (by_role[role].sha256, by_role[role].media_type) + ].id, + sha256=by_role[role].sha256, + byte_size=by_role[role].byte_size, + media_type=by_role[role].media_type, + ) + for role in request.output_roles + ) + receipt_identity = hashlib.sha256( + (request.digest + str(claim.attempt)).encode() + ).hexdigest()[:32] + receipt = DatasetCompilationReceipt( + receipt_id=f"dcr_{receipt_identity}", + operation_id=claim.operation_id, + operation_key=request.operation_key, + request_digest=request.digest, + compiler_id=request.compiler_id, + attempt=claim.attempt, + source=request.source, + recipe=request.recipe, + outputs=receipt_outputs, + reproduces_receipt_id=request.reproduces_receipt_id, + created_at=now, + ) + payload = canonical_json(receipt.model_dump(mode="json")) + self.repository.complete_with_connection( + connection, + claim, + outputs={ + output.role: ( + output.artifact_id, + output.sha256, + output.byte_size, + output.media_type, + ) + for output in receipt_outputs + }, + receipt_digest=hashlib.sha256(payload.encode()).hexdigest(), + receipt_json=payload, + ) + return receipt + + try: + return self.artifacts.commit_batch(staged, persist) + except Exception: + failures = self._discard_all(staged) + if failures: + raise _CleanupFailure(failures) from None + raise + + def _discard_all(self, staged: tuple[StagedArtifact, ...]) -> tuple[str, ...]: + failures = [] + for ordinal, item in enumerate(staged): + try: + self.artifacts.discard(item) + except Exception as error: + if isinstance(error, FileNotFoundError) or ( + isinstance(error, OSError) and error.errno == errno.ENOENT + ): + reason = "discard_missing" + elif isinstance(error, PermissionError) or ( + isinstance(error, OSError) and error.errno in {errno.EACCES, errno.EPERM} + ): + reason = "discard_permission_denied" + elif isinstance(error, OSError): + reason = "discard_io_error" + else: + reason = "discard_unexpected" + failures.append(f"{ordinal}:{reason}") + return tuple(failures) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/dataset_receipts.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/dataset_receipts.py new file mode 100644 index 0000000..4e04ec9 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/dataset_receipts.py @@ -0,0 +1,547 @@ +"""Durable, fenced evidence for trainer-neutral dataset compilation.""" + +from __future__ import annotations + +import hashlib +import sqlite3 +import uuid +from datetime import timedelta +from typing import Literal, cast + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from video_lab.database import Database +from video_lab.domain import canonical_json, utc_now + + +class CompilationConflict(RuntimeError): + """A compilation identity or live claim did not match exactly.""" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + + +class InputBinding(_FrozenModel): + role: Literal["source_manifest", "transform_recipe"] + artifact_id: str = Field(min_length=1) + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + byte_size: int = Field(ge=0) + media_type: str = Field(min_length=1) + + +class DatasetCompilationRequest(_FrozenModel): + schema_version: Literal["dataset-compilation-request.v1"] = "dataset-compilation-request.v1" + operation_key: str = Field(pattern=r"^[a-z][a-z0-9_]*$") + compiler_id: str = Field(pattern=r"^[a-z][a-z0-9_]*$") + source: InputBinding + recipe: InputBinding + output_roles: tuple[str, ...] = Field(min_length=1) + reproduces_receipt_id: str | None = None + + @model_validator(mode="after") + def validate_roles(self) -> DatasetCompilationRequest: + if self.source.role != "source_manifest" or self.recipe.role != "transform_recipe": + raise ValueError("input roles must be exact") + if self.output_roles != tuple(sorted(set(self.output_roles))): + raise ValueError("output roles must be sorted and unique") + return self + + @property + def digest(self) -> str: + payload = canonical_json(self.model_dump(mode="json")).encode() + return hashlib.sha256(payload).hexdigest() + + +class CompilationOutput(_FrozenModel): + role: str + artifact_id: str + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + byte_size: int = Field(ge=0) + media_type: str + + +class DatasetCompilationReceipt(_FrozenModel): + schema_version: Literal["dataset-compilation-receipt.v1"] = "dataset-compilation-receipt.v1" + receipt_id: str + operation_id: str + operation_key: str + request_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + compiler_id: str + attempt: int = Field(gt=0) + source: InputBinding + recipe: InputBinding + outputs: tuple[CompilationOutput, ...] + reproduces_receipt_id: str | None = None + created_at: str + + +class CompilationOperation(_FrozenModel): + operation_id: str + operation_key: str + request: DatasetCompilationRequest + request_digest: str + status: Literal["pending", "running", "retryable", "completed", "failed", "cancelled"] + attempt: int + claim_token: str | None + lease_expires_at: str | None + receipt_id: str | None + cleanup_failures: tuple[CleanupFailureEvidence, ...] = () + + +class CleanupFailureEvidence(_FrozenModel): + attempt: int + claim_token: str + code: str + failure_digest: str + created_at: str + + +class CompilationClaim(_FrozenModel): + operation_id: str + operation_key: str + request_digest: str + claimed: bool + attempt: int + claimant_id: str | None + claim_token: str | None + + +class DatasetCompilationRepository: + def __init__(self, database: Database) -> None: + self.database = database + + def begin(self, request: DatasetCompilationRequest) -> CompilationOperation: + request_json = canonical_json(request.model_dump(mode="json")) + now = utc_now().isoformat() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + existing = connection.execute( + "SELECT request_digest FROM dataset_compilation_operations WHERE operation_key=?", + (request.operation_key,), + ).fetchone() + if existing is not None: + if str(existing["request_digest"]) != request.digest: + raise CompilationConflict("compilation request digest conflict") + connection.commit() + return self.get(request.operation_key) + operation_id = f"dco_{uuid.uuid4().hex}" + connection.execute( + "INSERT INTO dataset_compilation_operations " + "(id,operation_key,compiler_id,request_digest,request_json,status,attempt," + "claimant_id,claim_token,lease_expires_at,receipt_id,created_at,updated_at) " + "VALUES (?,?,?,?,?,'pending',0,NULL,NULL,NULL,NULL,?,?)", + ( + operation_id, + request.operation_key, + request.compiler_id, + request.digest, + request_json, + now, + now, + ), + ) + for binding in (request.source, request.recipe): + connection.execute( + "INSERT INTO dataset_compilation_inputs VALUES (?,?,?,?,?,?,?)", + ( + operation_id, + binding.role, + binding.artifact_id, + binding.sha256, + binding.byte_size, + binding.media_type, + now, + ), + ) + self._event(connection, operation_id, 1, 0, "created", request.digest, now) + connection.commit() + return self.get(request.operation_key) + + def get(self, operation_key: str) -> CompilationOperation: + with self.database.connect() as connection: + row = connection.execute( + "SELECT * FROM dataset_compilation_operations WHERE operation_key=?", + (operation_key,), + ).fetchone() + failures = connection.execute( + "SELECT attempt,claim_token,code,failure_digest,created_at " + "FROM dataset_compilation_failures WHERE operation_id=? AND code='cleanup_failed' " + "ORDER BY attempt", + (row["id"],) if row is not None else ("",), + ).fetchall() + if row is None: + raise KeyError("compilation operation not found") + return self._operation(row, failures) + + def claim( + self, operation_key: str, *, claimant_id: str, lease_seconds: int + ) -> CompilationClaim: + if not claimant_id or lease_seconds <= 0: + raise ValueError("claimant and positive lease are required") + token = uuid.uuid4().hex + now = utc_now() + lease = (now + timedelta(seconds=lease_seconds)).isoformat() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + cursor = connection.execute( + "UPDATE dataset_compilation_operations SET status='running',attempt=attempt+1," + "claimant_id=?,claim_token=?,lease_expires_at=?,updated_at=? WHERE operation_key=? " + "AND (status IN ('pending','retryable') OR (status='running' AND " + "julianday(lease_expires_at)<=julianday('now')))", + (claimant_id, token, lease, now.isoformat(), operation_key), + ) + row = connection.execute( + "SELECT * FROM dataset_compilation_operations WHERE operation_key=?", + (operation_key,), + ).fetchone() + if row is None: + raise KeyError("compilation operation not found") + if cursor.rowcount == 1: + sequence = int( + connection.execute( + "SELECT count(*)+1 FROM dataset_compilation_events WHERE operation_id=?", + (row["id"],), + ).fetchone()[0] + ) + self._event( + connection, + str(row["id"]), + sequence, + int(row["attempt"]), + "claimed", + str(row["request_digest"]), + now.isoformat(), + ) + connection.commit() + return CompilationClaim( + operation_id=str(row["id"]), + operation_key=operation_key, + request_digest=str(row["request_digest"]), + claimed=cursor.rowcount == 1, + attempt=int(row["attempt"]), + claimant_id=claimant_id if cursor.rowcount == 1 else None, + claim_token=token if cursor.rowcount == 1 else None, + ) + + def complete( + self, + claim: CompilationClaim, + *, + outputs: dict[str, tuple[str, str, int, str]], + receipt_digest: str, + receipt_json: str, + ) -> None: + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + self.complete_with_connection( + connection, + claim, + outputs=outputs, + receipt_digest=receipt_digest, + receipt_json=receipt_json, + ) + connection.commit() + + def complete_with_connection( + self, + connection: sqlite3.Connection, + claim: CompilationClaim, + *, + outputs: dict[str, tuple[str, str, int, str]], + receipt_digest: str, + receipt_json: str, + ) -> None: + if not claim.claimed or claim.claim_token is None or claim.claimant_id is None: + raise CompilationConflict("compilation completion requires a live claim") + now = utc_now().isoformat() + row = connection.execute( + "SELECT request_json FROM dataset_compilation_operations WHERE id=? " + "AND status='running' " + "AND attempt=? AND claimant_id=? AND claim_token=? " + "AND julianday(lease_expires_at)>julianday('now')", + (claim.operation_id, claim.attempt, claim.claimant_id, claim.claim_token), + ).fetchone() + if row is None: + raise CompilationConflict("compilation completion claim mismatch") + request = DatasetCompilationRequest.model_validate_json( + str(row["request_json"]), strict=True + ) + if tuple(sorted(outputs)) != request.output_roles: + raise CompilationConflict("compilation output roles mismatch") + receipt = DatasetCompilationReceipt.model_validate_json(receipt_json, strict=True) + expected_outputs = tuple( + CompilationOutput( + role=role, + artifact_id=outputs[role][0], + sha256=outputs[role][1], + byte_size=outputs[role][2], + media_type=outputs[role][3], + ) + for role in request.output_roles + ) + canonical_receipt = canonical_json(receipt.model_dump(mode="json")) + if ( + hashlib.sha256(canonical_receipt.encode()).hexdigest() != receipt_digest + or canonical_receipt != receipt_json + or receipt.operation_id != claim.operation_id + or receipt.operation_key != request.operation_key + or receipt.compiler_id != request.compiler_id + or receipt.attempt != claim.attempt + or receipt.request_digest != claim.request_digest + or receipt.source != request.source + or receipt.recipe != request.recipe + or receipt.outputs != expected_outputs + or receipt.reproduces_receipt_id != request.reproduces_receipt_id + ): + raise CompilationConflict("compilation receipt binding mismatch") + receipt_id = receipt.receipt_id + for role in request.output_roles: + artifact_id, sha256, byte_size, media_type = outputs[role] + connection.execute( + "INSERT INTO dataset_compilation_outputs VALUES (?,?,?,?,?,?,?,?,?)", + ( + receipt_id, + claim.operation_id, + role, + artifact_id, + sha256, + byte_size, + media_type, + claim.attempt, + now, + ), + ) + cursor = connection.execute( + "UPDATE dataset_compilation_operations SET status='completed',claimant_id=NULL," + "claim_token=NULL,lease_expires_at=NULL,receipt_id=?,receipt_digest=?,receipt_json=?," + "updated_at=? WHERE id=? AND status='running' AND attempt=? AND claim_token=? " + "AND julianday(lease_expires_at)>julianday('now')", + ( + receipt_id, + receipt_digest, + receipt_json, + now, + claim.operation_id, + claim.attempt, + claim.claim_token, + ), + ) + if cursor.rowcount != 1: + raise CompilationConflict("compilation completion claim mismatch") + sequence = int( + connection.execute( + "SELECT count(*)+1 FROM dataset_compilation_events WHERE operation_id=?", + (claim.operation_id,), + ).fetchone()[0] + ) + self._event( + connection, + claim.operation_id, + sequence, + claim.attempt, + "completed", + receipt_digest, + now, + ) + if request.reproduces_receipt_id is not None: + connection.execute( + "INSERT INTO dataset_compilation_reproductions VALUES (?,?,?,?)", + (receipt_id, request.reproduces_receipt_id, claim.operation_id, now), + ) + + def fail(self, claim: CompilationClaim, *, code: str, failure_digest: str) -> None: + if not claim.claimed or claim.claim_token is None: + raise CompilationConflict("compilation failure requires a live claim") + now = utc_now().isoformat() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + connection.execute( + "INSERT INTO dataset_compilation_failures VALUES (?,?,?,?,?,?)", + (claim.operation_id, claim.attempt, claim.claim_token, code, failure_digest, now), + ) + cursor = connection.execute( + "UPDATE dataset_compilation_operations SET status='failed',claimant_id=NULL," + "claim_token=NULL,lease_expires_at=NULL,updated_at=? WHERE id=? " + "AND status='running' " + "AND attempt=? AND claim_token=? " + "AND julianday(lease_expires_at)>julianday('now')", + (now, claim.operation_id, claim.attempt, claim.claim_token), + ) + if cursor.rowcount != 1: + raise CompilationConflict("compilation failure claim mismatch") + connection.commit() + + def cancel(self, claim: CompilationClaim, *, evidence_digest: str) -> None: + if not claim.claimed or claim.claim_token is None: + raise CompilationConflict("compilation cancellation requires a live claim") + now = utc_now().isoformat() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + cursor = connection.execute( + "UPDATE dataset_compilation_operations SET status='cancelled',claimant_id=NULL," + "claim_token=NULL,lease_expires_at=NULL,updated_at=? WHERE id=? " + "AND status='running' " + "AND attempt=? AND claim_token=? " + "AND julianday(lease_expires_at)>julianday('now')", + (now, claim.operation_id, claim.attempt, claim.claim_token), + ) + if cursor.rowcount != 1: + raise CompilationConflict("compilation cancellation claim mismatch") + sequence = int( + connection.execute( + "SELECT count(*)+1 FROM dataset_compilation_events WHERE operation_id=?", + (claim.operation_id,), + ).fetchone()[0] + ) + self._event( + connection, + claim.operation_id, + sequence, + claim.attempt, + "cancelled", + evidence_digest, + now, + ) + connection.commit() + + def retryable_failure(self, claim: CompilationClaim, *, code: str, failure_digest: str) -> None: + if not claim.claimed or claim.claim_token is None: + raise CompilationConflict("compilation retryable failure requires a live claim") + now = utc_now().isoformat() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + connection.execute( + "INSERT INTO dataset_compilation_failures VALUES (?,?,?,?,?,?)", + (claim.operation_id, claim.attempt, claim.claim_token, code, failure_digest, now), + ) + cursor = connection.execute( + "UPDATE dataset_compilation_operations SET status='retryable',claimant_id=NULL," + "claim_token=NULL,lease_expires_at=NULL,updated_at=? WHERE id=? " + "AND status='running' AND attempt=? AND claim_token=? " + "AND julianday(lease_expires_at)>julianday('now')", + (now, claim.operation_id, claim.attempt, claim.claim_token), + ) + if cursor.rowcount != 1: + raise CompilationConflict("compilation retryable failure claim mismatch") + sequence = int( + connection.execute( + "SELECT count(*)+1 FROM dataset_compilation_events WHERE operation_id=?", + (claim.operation_id,), + ).fetchone()[0] + ) + self._event( + connection, + claim.operation_id, + sequence, + claim.attempt, + "cleanup_failed", + failure_digest, + now, + ) + connection.commit() + + def receipt(self, receipt_id: str) -> DatasetCompilationReceipt: + with self.database.connect() as connection: + row = connection.execute( + "SELECT * FROM dataset_compilation_operations WHERE receipt_id=? " + "AND status='completed'", + (receipt_id,), + ).fetchone() + inputs = connection.execute( + "SELECT * FROM dataset_compilation_inputs WHERE operation_id=? ORDER BY role", + (row["id"],) if row is not None else ("",), + ).fetchall() + outputs = connection.execute( + "SELECT * FROM dataset_compilation_outputs WHERE receipt_id=? ORDER BY role", + (receipt_id,), + ).fetchall() + if row is None: + raise KeyError("compilation receipt not found") + receipt_json = str(row["receipt_json"]) + receipt = DatasetCompilationReceipt.model_validate_json(receipt_json, strict=True) + request = DatasetCompilationRequest.model_validate_json( + str(row["request_json"]), strict=True + ) + durable_inputs = { + str(item["role"]): InputBinding( + role=cast(Literal["source_manifest", "transform_recipe"], str(item["role"])), + artifact_id=str(item["artifact_id"]), + sha256=str(item["sha256"]), + byte_size=int(item["byte_size"]), + media_type=str(item["media_type"]), + ) + for item in inputs + } + durable_outputs = tuple( + CompilationOutput( + role=str(item["role"]), + artifact_id=str(item["artifact_id"]), + sha256=str(item["sha256"]), + byte_size=int(item["byte_size"]), + media_type=str(item["media_type"]), + ) + for item in outputs + ) + canonical_receipt = canonical_json(receipt.model_dump(mode="json")) + if ( + canonical_receipt != receipt_json + or hashlib.sha256(canonical_receipt.encode()).hexdigest() != str(row["receipt_digest"]) + or receipt.operation_id != str(row["id"]) + or receipt.operation_key != str(row["operation_key"]) + or receipt.compiler_id != str(row["compiler_id"]) + or receipt.attempt != int(row["attempt"]) + or receipt.request_digest != str(row["request_digest"]) + or request.digest != receipt.request_digest + or durable_inputs + != {"source_manifest": receipt.source, "transform_recipe": receipt.recipe} + or durable_outputs != receipt.outputs + ): + raise CompilationConflict("durable compilation receipt mismatch") + return receipt + + @staticmethod + def _event( + connection: sqlite3.Connection, + operation_id: str, + sequence: int, + attempt: int, + kind: str, + digest: str, + created_at: str, + ) -> None: + connection.execute( + "INSERT INTO dataset_compilation_events VALUES (?,?,?,?,?,?)", + (operation_id, sequence, attempt, kind, digest, created_at), + ) + + @staticmethod + def _operation( + row: sqlite3.Row, failures: list[sqlite3.Row] | tuple[sqlite3.Row, ...] = () + ) -> CompilationOperation: + return CompilationOperation( + operation_id=str(row["id"]), + operation_key=str(row["operation_key"]), + request=DatasetCompilationRequest.model_validate_json( + str(row["request_json"]), strict=True + ), + request_digest=str(row["request_digest"]), + status=cast( + Literal["pending", "running", "retryable", "completed", "failed", "cancelled"], + str(row["status"]), + ), + attempt=int(row["attempt"]), + claim_token=row["claim_token"], + lease_expires_at=row["lease_expires_at"], + receipt_id=row["receipt_id"], + cleanup_failures=tuple( + CleanupFailureEvidence( + attempt=int(item["attempt"]), + claim_token=str(item["claim_token"]), + code=str(item["code"]), + failure_digest=str(item["failure_digest"]), + created_at=str(item["created_at"]), + ) + for item in failures + ), + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/domain.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/domain.py new file mode 100644 index 0000000..4751918 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/domain.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import json +from collections.abc import Mapping +from copy import deepcopy +from datetime import UTC, datetime +from enum import StrEnum +from types import MappingProxyType +from typing import Any, Literal, Self +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator + + +class FrozenModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + def model_copy( + self, + *, + update: Mapping[str, Any] | None = None, + deep: bool = False, + ) -> Self: + dumped = self.model_dump(round_trip=True) + payload = deepcopy(dumped) if deep else dumped + if update is not None: + payload.update(update) + return self.__class__.model_validate(payload) + + +class LoraBinding(FrozenModel): + model_id: str + revision: str + checksum: str + weight: float + + +class Manifest(FrozenModel): + schema_version: str = "manifest.v1" + protocol_version: str = "ab.v1" + model_id: str + model_revision: str + model_checksum: str + adapter_id: str + adapter_version: str + runtime_versions: Mapping[str, str] + source_checksums: Mapping[str, str] = Field(default_factory=lambda: MappingProxyType({})) + prompt: str = Field(min_length=1) + negative_prompt: str = "" + seed: int + width: int = Field(gt=0) + height: int = Field(gt=0) + num_frames: int = Field(gt=0) + fps: int = Field(gt=0) + inference_steps: int = Field(gt=0) + guidance_scale: float = Field(ge=0) + scheduler: str + dtype: str + quantization: str + loras: tuple[LoraBinding, ...] = () + backend: str + device: str + memory_strategy: str + hardware_fingerprint: str + os_fingerprint: str + + @field_validator("runtime_versions", "source_checksums") + @classmethod + def freeze_runtime_versions(cls, value: Mapping[str, str]) -> Mapping[str, str]: + return MappingProxyType(dict(value)) + + @field_serializer("runtime_versions", "source_checksums") + def serialize_runtime_versions(self, value: Mapping[str, str]) -> dict[str, str]: + return dict(value) + + +class ExperimentDraft(FrozenModel): + baseline: Manifest + variable_path: str + alternate_value: Any + + +class Variant(FrozenModel): + id: str + experiment_id: str + label: str + manifest: Manifest + + +class ReviewDimension(StrEnum): + PROMPT_ADHERENCE = "prompt_adherence" + VISUAL_QUALITY = "visual_quality" + MOTION_QUALITY = "motion_quality" + TEMPORAL_CONSISTENCY = "temporal_consistency" + CHARACTER_CONSISTENCY = "character_consistency" + + +class DisplayPreference(StrEnum): + LEFT = "left" + RIGHT = "right" + TIE = "tie" + + +class ReviewSession(FrozenModel): + id: str + experiment_id: str + display_positions: tuple[Literal["left", "right"], Literal["left", "right"]] = ( + "left", + "right", + ) + created_at: datetime + + @field_validator("display_positions") + @classmethod + def validate_display_positions( + cls, value: tuple[Literal["left", "right"], Literal["left", "right"]] + ) -> tuple[Literal["left", "right"], Literal["left", "right"]]: + if value != ("left", "right"): + raise ValueError("display positions must be left and right") + return value + + @field_validator("created_at") + @classmethod + def normalize_created_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + +class RubricScore(FrozenModel): + display_position: Literal["left", "right"] + dimension: ReviewDimension + score: int = Field(ge=1, le=5) + + +class ReviewRevision(FrozenModel): + id: str + session_id: str + preferred_variant_id: str | None + display_preference: DisplayPreference + scores: tuple[RubricScore, ...] + notes: str | None + created_at: datetime + revision: int = Field(gt=0) + supersedes_review_id: str | None = None + + @field_validator("created_at") + @classmethod + def normalize_created_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + +class JobState(StrEnum): + QUEUED = "queued" + PREPARING = "preparing" + DOWNLOADING = "downloading" + LOADING = "loading" + GENERATING = "generating" + ENCODING = "encoding" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +class Job(FrozenModel): + id: str + variant_id: str + state: JobState + created_at: datetime + updated_at: datetime + + +class RunAttemptStatus(StrEnum): + ACTIVE = "active" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + INTERRUPTED = "interrupted" + + +def _aware_utc(value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("timestamp must be timezone-aware") + return value.astimezone(UTC) + + +class RunAttempt(FrozenModel): + id: str + job_id: str + number: int = Field(gt=0) + worker_id: str + status: RunAttemptStatus + started_at: datetime + finished_at: datetime | None = None + error_code: str | None = None + error_message: str | None = None + artifact_id: str | None = None + + @field_validator("started_at", "finished_at") + @classmethod + def normalize_timestamps(cls, value: datetime | None) -> datetime | None: + return None if value is None else _aware_utc(value) + + +class RunEvent(FrozenModel): + id: int + job_id: str + attempt_id: str + kind: str + payload: dict[str, object] + created_at: datetime + + @field_validator("created_at") + @classmethod + def normalize_created_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + +class TelemetrySample(FrozenModel): + id: int | None = None + attempt_id: str + recorded_at: datetime + elapsed_seconds: float = Field(ge=0) + process_rss_bytes: int = Field(ge=0) + system_used_bytes: int = Field(ge=0) + system_available_bytes: int = Field(ge=0) + swap_used_bytes: int = Field(ge=0) + thermal_pressure: str | None = None + + @field_validator("recorded_at") + @classmethod + def normalize_recorded_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + +class ClaimedJob(FrozenModel): + job: Job + attempt: RunAttempt + + +def new_id(prefix: str) -> str: + return f"{prefix}_{uuid4().hex}" + + +def utc_now() -> datetime: + return datetime.now(UTC) + + +def canonical_json(value: BaseModel | dict[str, Any]) -> str: + payload = value.model_dump(mode="json") if isinstance(value, BaseModel) else value + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/evidence_service.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/evidence_service.py new file mode 100644 index 0000000..c2f1d02 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/evidence_service.py @@ -0,0 +1,525 @@ +from __future__ import annotations + +import json +import re +import sqlite3 +from typing import Any + +from video_lab.artifacts import ArtifactError, ArtifactStore +from video_lab.database import Database +from video_lab.domain import FrozenModel +from video_lab.repositories import ExperimentRepository + + +class EvidenceNotFound(LookupError): + """The requested persisted experiment does not exist.""" + + +class ScalarRange(FrozenModel): + min: float + max: float + last: float + + +class TelemetryEvidence(FrozenModel): + sample_count: int + elapsed_seconds: ScalarRange + process_rss_bytes: ScalarRange + system_used_bytes: ScalarRange + system_available_bytes: ScalarRange + swap_used_bytes: ScalarRange + last_thermal_pressure: str | None + + +class EventEvidence(FrozenModel): + sequence: int + kind: str + payload: dict[str, object] + created_at: str + + +class AttemptEvidence(FrozenModel): + id: str + number: int + status: str + started_at: str + finished_at: str | None + error_code: str | None + error_message: str | None + artifact_id: str | None + events: tuple[EventEvidence, ...] + telemetry: TelemetryEvidence | None + + +class ArtifactEvidence(FrozenModel): + id: str + sha256: str + byte_size: int + media_type: str + created_at: str + integrity: str + media_url: str | None + + +class JobEvidence(FrozenModel): + id: str + variant_label: str + state: str + created_at: str + updated_at: str + error_code: str | None + error_message: str | None + attempts: tuple[AttemptEvidence, ...] + artifacts: tuple[ArtifactEvidence, ...] + + +class CanonicalDiff(FrozenModel): + path: str + changed_paths: tuple[str, ...] + baseline: Any + alternate: Any + + +class VariantEvidence(FrozenModel): + id: str + label: str + manifest: dict[str, object] + + +class ReviewRevisionEvidence(FrozenModel): + id: str + revision: int + display_preference: str + preferred_variant_label: str | None + notes: str | None + supersedes_review_id: str | None + created_at: str + scores: tuple[dict[str, object], ...] + + +class ReviewEvidence(FrozenModel): + reviewable: bool + reason_codes: tuple[str, ...] + entry_url: str | None + resume_available: bool = False + session_id: str | None + submitted: bool + canonical_reveal: dict[str, object] | None + revisions: tuple[ReviewRevisionEvidence, ...] + + +class ExperimentHistoryItem(FrozenModel): + id: str + sealed: bool + variable_path: str | None + declared_change: str | None + created_at: str + states: tuple[str, ...] + attempt_count: int + evidence_completeness: str + + +class ExperimentHistory(FrozenModel): + schema_version: str = "evidence.v1" + filters: dict[str, str] + items: tuple[ExperimentHistoryItem, ...] + + +class ExperimentEvidence(FrozenModel): + schema_version: str = "evidence.v1" + id: str + sealed: bool + experiment_schema_version: str | None + created_at: str | None + canonical_diff: CanonicalDiff | None + variants: tuple[VariantEvidence, ...] + jobs: tuple[JobEvidence, ...] + states: tuple[str, ...] + attempt_count: int + review: ReviewEvidence + evidence_completeness: str + + +class EvidenceService: + """Read-only projection over immutable laboratory persistence.""" + + def __init__(self, database: Database, artifacts: ArtifactStore) -> None: + self.database = database + self.artifacts = artifacts + self.experiments = ExperimentRepository(database) + + def list( + self, + *, + reviewer_id: str | None = None, + state: str | None = None, + variable: str | None = None, + ) -> ExperimentHistory: + filters = {key: value for key, value in (("state", state), ("variable", variable)) if value} + with self.database.connect() as connection: + rows = connection.execute( + "SELECT e.id, e.variable_path, e.created_at, " + "group_concat(j.state, '|') AS states, count(DISTINCT ra.id) AS attempts " + "FROM experiments e JOIN variants v ON v.experiment_id=e.id " + "JOIN jobs j ON j.variant_id=v.id " + "LEFT JOIN run_attempts ra ON ra.job_id=j.id " + "WHERE NOT EXISTS (SELECT 1 FROM adaptive_study_events AS event " + "WHERE json_extract(event.payload_json, '$.experiment_id') = e.id) " + "AND e.schema_version <> 'validation-controlled.v1' " + "GROUP BY e.id ORDER BY e.created_at DESC, e.id DESC" + ).fetchall() + items: list[ExperimentHistoryItem] = [] + for row in rows: + states = tuple(sorted(set(str(row["states"]).split("|")))) + variable_path = str(row["variable_path"]) + if state and state not in states: + continue + revealed = reviewer_id is not None and self.owner_submitted(str(row["id"]), reviewer_id) + if variable and (not revealed or variable != variable_path): + continue + jobs = self._jobs_for_experiment(str(row["id"])) + items.append( + ExperimentHistoryItem( + id=str(row["id"]), + sealed=not revealed, + variable_path=variable_path if revealed else None, + declared_change=variable_path if revealed else None, + created_at=str(row["created_at"]), + states=states, + attempt_count=int(row["attempts"]), + evidence_completeness="complete" if _complete_pair(jobs) else "partial", + ) + ) + return ExperimentHistory(filters=filters, items=tuple(items)) + + def detail(self, experiment_id: str, reviewer_id: str) -> ExperimentEvidence: + if not self.experiments.is_generic_visible(experiment_id): + raise EvidenceNotFound(f"experiment not found: {experiment_id}") + with self.database.connect() as connection: + experiment = connection.execute( + "SELECT * FROM experiments WHERE id=?", (experiment_id,) + ).fetchone() + if experiment is None: + raise EvidenceNotFound(f"experiment not found: {experiment_id}") + variant_rows = connection.execute( + "SELECT * FROM variants WHERE experiment_id=? ORDER BY label", (experiment_id,) + ).fetchall() + job_rows = connection.execute( + "SELECT j.*, v.label FROM jobs j JOIN variants v ON v.id=j.variant_id " + "WHERE v.experiment_id=? ORDER BY v.label, j.created_at, j.id", + (experiment_id,), + ).fetchall() + session = connection.execute( + "SELECT * FROM review_sessions WHERE experiment_id=? AND reviewer_id=?", + (experiment_id, reviewer_id), + ).fetchone() + variants = tuple(self._variant(row) for row in variant_rows) + manifests = {item.label: item.manifest for item in variants} + variable_path = str(experiment["variable_path"]) + jobs = tuple(self._job(row) for row in job_rows) + review = self._review(experiment_id, session, variants, jobs) + states = tuple(sorted(job.state for job in jobs)) + attempt_count = sum(len(job.attempts) for job in jobs) + complete = "complete" if _complete_pair(jobs) else "partial" + if not review.submitted: + return ExperimentEvidence( + id=experiment_id, + sealed=True, + experiment_schema_version=None, + created_at=None, + canonical_diff=None, + variants=(), + jobs=(), + states=states, + attempt_count=attempt_count, + review=review, + evidence_completeness=complete, + ) + return ExperimentEvidence( + id=experiment_id, + sealed=False, + experiment_schema_version=str(experiment["schema_version"]), + created_at=str(experiment["created_at"]), + canonical_diff=CanonicalDiff( + path=variable_path, + changed_paths=tuple(json.loads(str(experiment["changed_paths_json"]))), + baseline=_value_at(manifests["A"], variable_path), + alternate=_value_at(manifests["B"], variable_path), + ), + variants=variants, + jobs=jobs, + states=states, + attempt_count=attempt_count, + review=review, + evidence_completeness=complete, + ) + + def owner_submitted(self, experiment_id: str, reviewer_id: str) -> bool: + with self.database.connect() as connection: + row = connection.execute( + "SELECT 1 FROM review_sessions s JOIN review_revisions r ON r.session_id=s.id " + "WHERE s.experiment_id=? AND s.reviewer_id=? LIMIT 1", + (experiment_id, reviewer_id), + ).fetchone() + return row is not None + + def _jobs_for_experiment(self, experiment_id: str) -> tuple[JobEvidence, ...]: + with self.database.connect() as connection: + rows = connection.execute( + "SELECT j.*, v.label FROM jobs j JOIN variants v ON v.id=j.variant_id " + "WHERE v.experiment_id=? ORDER BY v.label, j.created_at, j.id", + (experiment_id,), + ).fetchall() + return tuple(self._job(row) for row in rows) + + def _variant(self, row: sqlite3.Row) -> VariantEvidence: + payload = json.loads(str(row["manifest_json"])) + return VariantEvidence( + id=str(row["id"]), label=str(row["label"]), manifest=_safe_mapping(payload) + ) + + def _job(self, row: sqlite3.Row) -> JobEvidence: + job_id = str(row["id"]) + with self.database.connect() as connection: + attempts = connection.execute( + "SELECT * FROM run_attempts WHERE job_id=? ORDER BY number, id", (job_id,) + ).fetchall() + artifact_rows = connection.execute( + "SELECT a.* FROM artifacts a JOIN job_artifacts ja ON ja.artifact_id=a.id " + "WHERE ja.job_id=? ORDER BY ja.created_at, a.id", + (job_id,), + ).fetchall() + return JobEvidence( + id=job_id, + variant_label=str(row["label"]), + state=str(row["state"]), + created_at=str(row["created_at"]), + updated_at=str(row["updated_at"]), + error_code=_optional(row["error_code"]), + error_message=_safe_text(_optional(row["error_message"])), + attempts=tuple(self._attempt(item) for item in attempts), + artifacts=tuple(self._artifact(item) for item in artifact_rows), + ) + + def _attempt(self, row: sqlite3.Row) -> AttemptEvidence: + attempt_id = str(row["id"]) + with self.database.connect() as connection: + events = connection.execute( + "SELECT * FROM run_events WHERE attempt_id=? ORDER BY id", (attempt_id,) + ).fetchall() + samples = connection.execute( + "SELECT * FROM telemetry_samples WHERE attempt_id=? ORDER BY recorded_at, id", + (attempt_id,), + ).fetchall() + return AttemptEvidence( + id=attempt_id, + number=int(row["number"]), + status=str(row["status"]), + started_at=str(row["started_at"]), + finished_at=_optional(row["finished_at"]), + error_code=_optional(row["error_code"]), + error_message=_safe_text(_optional(row["error_message"])), + artifact_id=_optional(row["artifact_id"]), + events=tuple( + EventEvidence( + sequence=int(event["id"]), + kind=str(event["kind"])[:80], + payload=_safe_mapping(json.loads(str(event["payload_json"]))), + created_at=str(event["created_at"]), + ) + for event in events + ), + telemetry=_telemetry(samples), + ) + + def _artifact(self, row: sqlite3.Row) -> ArtifactEvidence: + artifact_id = str(row["id"]) + integrity = "verified" + try: + self.artifacts.resolve(artifact_id) + except ArtifactError as error: + integrity = "missing" if "missing" in str(error).lower() else "corrupt" + media_type = str(row["media_type"]) + return ArtifactEvidence( + id=artifact_id, + sha256=str(row["sha256"]), + byte_size=int(row["byte_size"]), + media_type=media_type, + created_at=str(row["created_at"]), + integrity=integrity, + media_url=f"/media/artifacts/{artifact_id}" + if media_type == "video/mp4" and integrity == "verified" + else None, + ) + + def _review( + self, + experiment_id: str, + session: sqlite3.Row | None, + variants: tuple[VariantEvidence, ...], + jobs: tuple[JobEvidence, ...], + ) -> ReviewEvidence: + reasons: list[str] = [] + if len(jobs) != 2 or any(job.state != "completed" for job in jobs): + reasons.append("jobs_not_completed") + if any(len(job.artifacts) != 1 for job in jobs): + reasons.append("missing_or_duplicate_artifact") + if any( + job.artifacts + and ( + job.artifacts[0].media_type != "video/mp4" + or job.artifacts[0].integrity != "verified" + ) + for job in jobs + ): + reasons.append("artifact_unavailable") + if session is None: + return ReviewEvidence( + reviewable=not reasons, + reason_codes=tuple(reasons), + entry_url=f"/review/{experiment_id}" if not reasons else None, + session_id=None, + submitted=False, + canonical_reveal=None, + revisions=(), + ) + session_id = str(session["id"]) + with self.database.connect() as connection: + revision_rows = connection.execute( + "SELECT * FROM review_revisions WHERE session_id=? ORDER BY revision, id", + (session_id,), + ).fetchall() + scores = { + str(row["id"]): connection.execute( + "SELECT display_position, dimension, score FROM rubric_scores " + "WHERE review_id=? ORDER BY display_position, dimension", + (str(row["id"]),), + ).fetchall() + for row in revision_rows + } + label_by_id = {variant.id: variant.label for variant in variants} + revisions = tuple( + ReviewRevisionEvidence( + id=str(row["id"]), + revision=int(row["revision"]), + display_preference=str(row["display_preference"]), + preferred_variant_label=None + if row["preferred_variant_id"] is None + else label_by_id.get(str(row["preferred_variant_id"])), + notes=_safe_text(_optional(row["notes"])), + supersedes_review_id=_optional(row["supersedes_review_id"]), + created_at=str(row["created_at"]), + scores=tuple( + { + "display_position": str(score["display_position"]), + "dimension": str(score["dimension"]), + "score": int(score["score"]), + } + for score in scores[str(row["id"])] + ), + ) + for row in revision_rows + ) + if not revisions: + return ReviewEvidence( + reviewable=not reasons, + reason_codes=tuple(reasons), + entry_url=f"/review/{experiment_id}" if not reasons else None, + resume_available=not reasons, + session_id=None, + submitted=False, + canonical_reveal=None, + revisions=(), + ) + submitted = bool(revisions) + reveal: dict[str, object] | None = None + if submitted: + reveal = { + "left_variant_label": label_by_id[str(session["left_variant_id"])], + "right_variant_label": label_by_id[str(session["right_variant_id"])], + "latest_review_id": revisions[-1].id, + } + return ReviewEvidence( + reviewable=not reasons, + reason_codes=tuple(reasons), + entry_url=f"/reviews/{session_id}/receipt" if submitted else None, + session_id=session_id, + submitted=submitted, + canonical_reveal=reveal, + revisions=revisions, + ) + + +def _optional(value: object) -> str | None: + return None if value is None else str(value) + + +_LOCAL_PATH = re.compile(r"(?]+/)+[^\s\"'<>]+") + + +def _safe_text(value: str | None) -> str | None: + if value is None: + return None + return _LOCAL_PATH.sub("[local-path]", value)[:500] + + +def _safe_mapping(payload: object) -> dict[str, object]: + if not isinstance(payload, dict): + return {} + result: dict[str, object] = {} + for key, value in payload.items(): + safe = _safe_value(value) + if safe is not _UNSUPPORTED: + result[str(key)] = safe + return result + + +_UNSUPPORTED = object() + + +def _safe_value(value: object) -> object: + if isinstance(value, str): + return _safe_text(value) or "" + if isinstance(value, dict): + return _safe_mapping(value) + if isinstance(value, list): + return [safe for item in value if (safe := _safe_value(item)) is not _UNSUPPORTED] + if value is None or isinstance(value, (bool, int, float)): + return value + return _UNSUPPORTED + + +def _complete_pair(jobs: tuple[JobEvidence, ...]) -> bool: + return len(jobs) == 2 and all( + job.state == "completed" + and len(job.artifacts) == 1 + and job.artifacts[0].media_type == "video/mp4" + and job.artifacts[0].integrity == "verified" + for job in jobs + ) + + +def _value_at(payload: dict[str, object], path: str) -> Any: + value: Any = payload + for part in path.split("."): + value = value[int(part)] if isinstance(value, list) else value[part] + return value + + +def _telemetry(rows: list[sqlite3.Row]) -> TelemetryEvidence | None: + if not rows: + return None + + def values(name: str) -> ScalarRange: + series = [float(row[name]) for row in rows] + return ScalarRange(min=min(series), max=max(series), last=series[-1]) + + return TelemetryEvidence( + sample_count=len(rows), + elapsed_seconds=values("elapsed_seconds"), + process_rss_bytes=values("process_rss_bytes"), + system_used_bytes=values("system_used_bytes"), + system_available_bytes=values("system_available_bytes"), + swap_used_bytes=values("swap_used_bytes"), + last_thermal_pressure=_optional(rows[-1]["thermal_pressure"]), + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility.py new file mode 100644 index 0000000..2b6662b --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility.py @@ -0,0 +1,689 @@ +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Protocol, cast + +from video_lab.animegen_pair_contract import ( + ANIMEGEN_CONTROLLED_GUIDANCE_VALUES, + ANIMEGEN_CONTROLLED_PAIR_SEED, +) +from video_lab.comparison import build_controlled_pair +from video_lab.domain import ExperimentDraft, Manifest, canonical_json, new_id, utc_now +from video_lab.feasibility_models import ( + FeasibilityConfiguration, + FeasibilityRun, + FeasibilityStage, + StageAttempt, + StageEvidence, + StageOutcome, + StageResult, + SupportVerdict, +) +from video_lab.feasibility_repository import FeasibilityRepository +from video_lab.feasibility_telemetry import StageTelemetryCapture +from video_lab.normal_generation import CommittedGenerationEvidence + +_PATH_START = re.compile(r"(? FeasibilityProtocol: + definitions: list[StageDefinition] = [] + previous: FeasibilityStage | None = None + for stage in FeasibilityStage: + definitions.append(StageDefinition(stage, () if previous is None else (previous,))) + previous = stage + return cls(tuple(definitions)) + + +@dataclass(frozen=True) +class StageExecutionContext: + run: FeasibilityRun + attempts: tuple[StageAttempt, ...] + + +@dataclass(frozen=True) +class FeasibilityStatus: + run: FeasibilityRun + attempts: tuple[StageAttempt, ...] + verdict: SupportVerdict + + +class StageExecutor(Protocol): + def execute(self, context: StageExecutionContext, stage: StageDefinition) -> StageResult: ... + + +class NormalGenerationRunner(Protocol): + """Queues and completes a manifest through the normal worker/artifact path.""" + + def run_one(self, manifest: Manifest) -> CommittedGenerationEvidence: ... + def run_pair( + self, + baseline: Manifest, + candidate: Manifest, + *, + variable_path: str, + ) -> tuple[CommittedGenerationEvidence, CommittedGenerationEvidence]: ... + + +class BlindReviewStarter(Protocol): + """Creates the persisted, randomized review identity for a generated pair.""" + + def start(self, experiment_id: str, reviewer_id: str) -> Any: ... + + +class GenerationQualificationExecutor: + """Executes generation feasibility stages using persisted normal-path jobs.""" + + def __init__( + self, + delegate: StageExecutor, + runner: NormalGenerationRunner, + reviews: BlindReviewStarter | None = None, + ) -> None: + self._delegate = delegate + self._runner = runner + self._reviews = reviews + + def preflight(self, context: StageExecutionContext, stage: StageDefinition) -> StageResult: + preflight = getattr(self._delegate, "preflight", None) + if not callable(preflight): + return StageResult.passed() + return cast(StageResult, preflight(context, stage)) + + def execute(self, context: StageExecutionContext, stage: StageDefinition) -> StageResult: + if stage.stage not in { + FeasibilityStage.GENERATE_MINIMUM_1, + FeasibilityStage.GENERATE_MINIMUM_2, + FeasibilityStage.QUALIFY_STANDARD, + FeasibilityStage.VALIDATE_CONTROLLED_PAIR, + }: + return self._delegate.execute(context, stage) + profiles = context.run.configuration.get("manifest_profiles") + if not isinstance(profiles, Mapping): + raise InvalidEvidenceError("persisted manifest profiles are missing") + if stage.stage is FeasibilityStage.VALIDATE_CONTROLLED_PAIR: + standard = Manifest.model_validate(profiles.get("standard")) + baseline = standard.model_copy( + update={ + "seed": ANIMEGEN_CONTROLLED_PAIR_SEED, + "guidance_scale": ANIMEGEN_CONTROLLED_GUIDANCE_VALUES[0], + } + ) + pair = build_controlled_pair( + ExperimentDraft( + baseline=baseline, + variable_path="guidance_scale", + alternate_value=ANIMEGEN_CONTROLLED_GUIDANCE_VALUES[1], + ) + ) + first_committed, second_committed = self._runner.run_pair( + pair.variant_a, pair.variant_b, variable_path=pair.variable_path + ) + experiment_id = first_committed.experiment_id + if experiment_id is None or second_committed.experiment_id != experiment_id: + raise InvalidEvidenceError("controlled pair has no shared persisted experiment") + if self._reviews is None: + raise InvalidEvidenceError("controlled pair has no persisted blind review service") + review = self._reviews.start(experiment_id, f"feasibility:{context.run.id}") + session_id = getattr(review, "session_id", None) + if not isinstance(session_id, str) or not session_id: + raise InvalidEvidenceError("controlled pair has no persisted blind review session") + evidence = validate_controlled_pair_evidence( + pair.variant_a.model_dump(mode="json"), + pair.variant_b.model_dump(mode="json"), + variable_path=pair.variable_path, + ) + return StageResult.passed( + StageEvidence( + details={ + **evidence.model_dump(mode="json")["details"], + "job_ids": ( + first_committed.job_id, + second_committed.job_id, + ), + "artifact_sha256": ( + first_committed.artifact_sha256, + second_committed.artifact_sha256, + ), + "experiment_id": experiment_id, + "variable_path": pair.variable_path, + "pair_seed": ANIMEGEN_CONTROLLED_PAIR_SEED, + "controlled_values": ( + pair.variant_a.guidance_scale, + pair.variant_b.guidance_scale, + ), + "review_identity": { + "experiment_id": experiment_id, + "session_id": session_id, + "state": "blinded_randomized", + }, + } + ) + ) + name = "standard" if stage.stage is FeasibilityStage.QUALIFY_STANDARD else "minimum" + manifest = Manifest.model_validate(profiles.get(name)) + committed_evidence = self._runner.run_one(manifest) + committed = committed_evidence.stage_details() + manifest_hash = hashlib.sha256(canonical_json(manifest).encode()).hexdigest() + repeatability: dict[str, object] = {} + if stage.stage is FeasibilityStage.GENERATE_MINIMUM_2: + first_attempt = next( + ( + attempt + for attempt in reversed(context.attempts) + if attempt.stage is FeasibilityStage.GENERATE_MINIMUM_1 + and attempt.evidence is not None + ), + None, + ) + if first_attempt is None or first_attempt.evidence is None: + raise InvalidEvidenceError("first minimum generation evidence is missing") + from video_lab.adapters.animegen_mps import classify_repeatability + + first_artifact = first_attempt.evidence.details.get("artifact") + second_artifact = committed.get("artifact") + if not isinstance(first_artifact, Mapping) or not isinstance(second_artifact, Mapping): + raise InvalidEvidenceError("committed generation artifact evidence is missing") + first_frames = first_artifact.get("decoded_frame_sha256") + second_frames = second_artifact.get("decoded_frame_sha256") + if not isinstance(first_frames, tuple) or not isinstance(second_frames, tuple): + raise InvalidEvidenceError("decoded frame evidence is missing") + classified = classify_repeatability( + first_artifact_sha256=str(first_artifact.get("sha256", "")), + second_artifact_sha256=str(second_artifact.get("sha256", "")), + first_frame_sha256=cast(tuple[str, ...], first_frames), + second_frame_sha256=cast(tuple[str, ...], second_frames), + manifest_hash=manifest_hash, + source_fingerprint=str(committed.get("source_fingerprint", "")), + runtime_fingerprint=str(committed.get("runtime_fingerprint", "")), + placement_fingerprint=str(committed.get("placement_fingerprint", "")), + second_manifest_hash=manifest_hash, + second_source_fingerprint=str( + first_attempt.evidence.details.get("source_fingerprint", "") + ), + second_runtime_fingerprint=str( + first_attempt.evidence.details.get("runtime_fingerprint", "") + ), + second_placement_fingerprint=str( + first_attempt.evidence.details.get("placement_fingerprint", "") + ), + ) + repeatability = { + "repeatability": { + "classification": classified.classification, + "artifact_sha256": classified.artifact_sha256, + "decoded_frame_sha256": classified.decoded_frame_sha256, + "matching_frame_count": classified.matching_frame_count, + "differing_frame_count": classified.differing_frame_count, + "matching_frame_ratio": classified.matching_frame_ratio, + } + } + return StageResult.passed( + StageEvidence( + details=cast( + Any, + { + **committed, + "canonical_manifest_hash": manifest_hash, + "profile": name, + **repeatability, + }, + ) + ) + ) + + +class ResourceExhaustedError(RuntimeError): + pass + + +class InvalidEvidenceError(RuntimeError): + pass + + +def _canonical_configuration(configuration: Mapping[str, Any]) -> dict[str, Any]: + validated = FeasibilityConfiguration.model_validate(configuration) + return validated.model_dump(mode="json") + + +def canonical_fingerprint(configuration: Mapping[str, Any]) -> str: + payload = json.dumps( + _canonical_configuration(configuration), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode() + return hashlib.sha256(payload).hexdigest() + + +def _sanitize_text(value: str) -> str: + path = _PATH_START.search(value) + if path is not None: + value = value[: path.start()] + "" + value = _CACHE.sub("", value) + value = _CREDENTIAL.sub("", value) + value = _SECRET.sub("", value) + return value[:1000] + + +def sanitize_diagnostic(value: Any, *, key: str | None = None) -> Any: + if isinstance(value, str): + return _sanitize_text(value) + if isinstance(value, Mapping): + return { + _sanitize_text(str(item_key)): sanitize_diagnostic(item, key=str(item_key)) + for item_key, item in value.items() + } + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return [sanitize_diagnostic(item) for item in value] + return value + + +def classify_exception(error: BaseException) -> StageResult: + message = _sanitize_text(str(error) or type(error).__name__) + lowered = str(error).lower() + if isinstance(error, (KeyboardInterrupt, InterruptedError)): + return StageResult( + outcome=StageOutcome.INTERRUPTED, + diagnostic_code="execution_cancelled", + diagnostic_message=message, + ) + if isinstance(error, InvalidEvidenceError): + code = "checksum_mismatch" if "checksum" in lowered else "invalid_evidence" + return StageResult( + outcome=StageOutcome.INVALID_EVIDENCE, diagnostic_code=code, diagnostic_message=message + ) + if isinstance(error, ResourceExhaustedError) or any( + term in lowered + for term in ( + "out of memory", + "allocation failed", + "cannot allocate", + "insufficient free space", + "disk headroom", + "unified-memory headroom", + ) + ): + return StageResult( + outcome=StageOutcome.RESOURCE_EXHAUSTED, + diagnostic_code="resource_exhausted", + diagnostic_message=message, + ) + if ( + "mps" in lowered + and any(term in lowered for term in ("not implemented", "unsupported", "operator")) + ) or "undefined type float8" in lowered: + return StageResult( + outcome=StageOutcome.INCOMPATIBLE, + diagnostic_code="mps_operator_unsupported", + diagnostic_message=message, + ) + return StageResult( + outcome=StageOutcome.UNEXPECTED_FAILURE, + diagnostic_code="unexpected_failure", + diagnostic_message=message, + ) + + +class FeasibilityService: + def __init__( + self, + repository: FeasibilityRepository, + executor: StageExecutor, + protocol: FeasibilityProtocol | None = None, + ) -> None: + self.repository = repository + self.executor = executor + self.protocol = protocol or FeasibilityProtocol.default() + + def start(self, configuration: Mapping[str, Any]) -> FeasibilityRun: + canonical = _canonical_configuration(configuration) + fingerprint = canonical_fingerprint(canonical) + with self.repository.database.connect() as connection: + row = connection.execute( + "SELECT id FROM feasibility_runs WHERE input_fingerprint = ? " + "ORDER BY created_at LIMIT 1", + (fingerprint,), + ).fetchone() + if row is not None: + return self.repository.get_run(str(row["id"])) + now = utc_now() + run = FeasibilityRun( + id=new_id("feasibility"), + adapter_id=str(canonical["adapter_id"]), + protocol_version=str(canonical["protocol_version"]), + input_fingerprint=fingerprint, + configuration=canonical, + created_at=now, + updated_at=now, + ) + return self.repository.create_run(run) + + def status(self, run_id: str) -> FeasibilityStatus: + run = self.repository.get_run(run_id) + attempts = self.repository.list_attempts(run_id) + verdict = derive_display_verdict(attempts) + return FeasibilityStatus(run, attempts, verdict) + + def _persisted_status(self, run_id: str) -> FeasibilityStatus: + status = self.status(run_id) + if status.run.verdict is not status.verdict: + try: + self.repository.set_verdict(run_id, status.verdict) + except ValueError: + if status.verdict is not SupportVerdict.STANDARD_LAB_QUALIFIED: + raise + status = self.status(run_id) + return status + + def resume(self, run_id: str) -> FeasibilityStatus: + self.repository.recover_interrupted(run_id) + current = self.status(run_id) + latest = _latest_attempts(current.attempts) + for definition in self.protocol.stages: + prior = latest.get(definition.stage) + if prior is not None and prior.outcome is StageOutcome.PASSED: + continue + if prior is not None and prior.outcome not in ( + StageOutcome.INTERRUPTED, + StageOutcome.UNEXPECTED_FAILURE, + ): + return self._persisted_status(run_id) + if any( + latest.get(required) is None or latest[required].outcome is not StageOutcome.PASSED + for required in definition.prerequisites + ): + return self._persisted_status(run_id) + active = self.repository.start_attempt(run_id, definition.stage) + telemetry = StageTelemetryCapture.start(definition.stage) + try: + context = StageExecutionContext(current.run, current.attempts) + preflight = getattr(self.executor, "preflight", None) + preflight_result = ( + cast(StageResult, preflight(context, definition)) + if callable(preflight) + and definition.stage + not in { + FeasibilityStage.RESOLVE_SOURCES, + FeasibilityStage.INVENTORY_SOURCES, + FeasibilityStage.STORAGE_PREFLIGHT, + FeasibilityStage.PUBLISH_VERDICT, + } + else StageResult.passed() + ) + if preflight_result.outcome is not StageOutcome.PASSED: + result = preflight_result + else: + executed = self.executor.execute(context, definition) + result = _merge_stage_evidence(preflight_result, executed) + except BaseException as error: + result = classify_exception(error) + result = result.model_copy(update={"evidence": telemetry.merge(result.evidence)}) + result = _sanitize_result(result) + self.repository.finish_attempt( + active.id, + result.outcome, + result.evidence, + result.diagnostic_code, + result.diagnostic_message, + ) + return self._persisted_status(run_id) + return self._persisted_status(run_id) + + +def _merge_stage_evidence(preflight: StageResult, executed: StageResult) -> StageResult: + measurements = { + **preflight.evidence.measurements, + **executed.evidence.measurements, + } + details = { + **preflight.evidence.model_dump(mode="json")["details"], + **executed.evidence.model_dump(mode="json")["details"], + } + return executed.model_copy( + update={"evidence": StageEvidence(measurements=measurements, details=details)} + ) + + +def _sanitize_result(result: StageResult) -> StageResult: + message = ( + None if result.diagnostic_message is None else _sanitize_text(result.diagnostic_message) + ) + details = sanitize_diagnostic(result.evidence.model_dump(mode="json")["details"]) + evidence = StageEvidence(measurements=result.evidence.measurements, details=details) + return result.model_copy(update={"evidence": evidence, "diagnostic_message": message}) + + +def _latest_attempts(attempts: tuple[StageAttempt, ...]) -> dict[FeasibilityStage, StageAttempt]: + latest: dict[FeasibilityStage, StageAttempt] = {} + for attempt in attempts: + if attempt.stage not in latest or attempt.ordinal > latest[attempt.stage].ordinal: + latest[attempt.stage] = attempt + return latest + + +def _valid_generation( + attempt: StageAttempt | None, manifest: str | None = None +) -> tuple[bool, str | None]: + if attempt is None or attempt.outcome is not StageOutcome.PASSED or attempt.evidence is None: + return False, manifest + details = attempt.evidence.details + artifact = details.get("artifact") + frame_hashes = artifact.get("decoded_frame_sha256") if isinstance(artifact, Mapping) else None + duration = artifact.get("duration_seconds") if isinstance(artifact, Mapping) else None + frame_count = artifact.get("num_frames") if isinstance(artifact, Mapping) else None + valid_artifact = ( + isinstance(artifact, Mapping) + and artifact.get("format") == "mp4" + and artifact.get("valid") is True + and artifact.get("codec") in ("h264", "avc1") + and isinstance(duration, int | float) + and float(duration) > 0 + and isinstance(artifact.get("sha256"), str) + and _SHA256.fullmatch(str(artifact["sha256"])) is not None + and isinstance(frame_hashes, Sequence) + and not isinstance(frame_hashes, str | bytes | bytearray) + and len(frame_hashes) == artifact.get("num_frames") + and all(isinstance(item, str) and _SHA256.fullmatch(item) for item in frame_hashes) + and isinstance(artifact.get("width"), int) + and isinstance(artifact.get("height"), int) + and isinstance(frame_count, int) + and isinstance(artifact.get("fps"), int) + and frame_count >= 17 + and (frame_count - 1) % 16 == 0 + ) + value = details.get("canonical_manifest_hash") + valid_manifest = ( + isinstance(value, str) + and _SHA256.fullmatch(value) is not None + and (manifest is None or value == manifest) + ) + fingerprints = tuple( + details.get(key) + for key in ("source_fingerprint", "runtime_fingerprint", "placement_fingerprint") + ) + valid = ( + valid_artifact + and valid_manifest + and all(isinstance(item, str) and _SHA256.fullmatch(item) for item in fingerprints) + and details.get("placement") == "local_hybrid" + and details.get("fallback_violations") in ((), []) + ) + return valid, value if isinstance(value, str) else manifest + + +def derive_display_verdict( + attempts: tuple[StageAttempt, ...] | list[StageAttempt], +) -> SupportVerdict: + """Project stage metadata for display; never grants generation authorization.""" + latest = _latest_attempts(tuple(attempts)) + outcomes = {attempt.outcome for attempt in latest.values()} + if StageOutcome.INVALID_EVIDENCE in outcomes: + return SupportVerdict.INCOMPATIBLE + if StageOutcome.INCOMPATIBLE in outcomes: + return SupportVerdict.INCOMPATIBLE + if StageOutcome.RESOURCE_EXHAUSTED in outcomes: + return SupportVerdict.RESOURCE_LIMITED + first, manifest = _valid_generation(latest.get(FeasibilityStage.GENERATE_MINIMUM_1)) + second, _ = _valid_generation(latest.get(FeasibilityStage.GENERATE_MINIMUM_2), manifest) + if not first or not second: + if latest.get(FeasibilityStage.GENERATE_MINIMUM_2) is not None: + return SupportVerdict.INCOMPATIBLE + return SupportVerdict.PENDING + standard_attempt = latest.get(FeasibilityStage.QUALIFY_STANDARD) + standard, _ = _valid_generation(standard_attempt) + standard_artifact = ( + standard_attempt.evidence.details.get("artifact") + if standard_attempt is not None and standard_attempt.evidence is not None + else None + ) + standard = ( + standard + and isinstance(standard_artifact, Mapping) + and ( + standard_artifact.get("width"), + standard_artifact.get("height"), + standard_artifact.get("num_frames"), + standard_artifact.get("fps"), + ) + == (512, 288, 17, 8) + ) + minimum_details = latest[FeasibilityStage.GENERATE_MINIMUM_1].evidence + standard_details = standard_attempt.evidence if standard_attempt is not None else None + if minimum_details is None or standard_details is None: + standard = False + else: + identity_keys = ( + "source_fingerprint", + "runtime_fingerprint", + "placement_fingerprint", + ) + standard = standard and all( + minimum_details.details.get(key) == standard_details.details.get(key) + for key in identity_keys + ) + pair = latest.get(FeasibilityStage.VALIDATE_CONTROLLED_PAIR) + changed_paths = ( + pair.evidence.details.get("changed_paths") + if pair is not None and pair.evidence is not None + else None + ) + variable_path = ( + pair.evidence.details.get("variable_path") + if pair is not None and pair.evidence is not None + else None + ) + controlled_values = ( + pair.evidence.details.get("controlled_values") + if pair is not None and pair.evidence is not None + else None + ) + pair_seed = ( + pair.evidence.details.get("pair_seed") + if pair is not None and pair.evidence is not None + else None + ) + if ( + standard + and pair is not None + and pair.outcome is StageOutcome.PASSED + and pair.evidence is not None + and pair.evidence.details.get("valid") is True + and isinstance(changed_paths, tuple) + and changed_paths == ("guidance_scale",) + and variable_path == "guidance_scale" + and pair_seed == ANIMEGEN_CONTROLLED_PAIR_SEED + and controlled_values == ANIMEGEN_CONTROLLED_GUIDANCE_VALUES + ): + return SupportVerdict.STANDARD_LAB_QUALIFIED + return SupportVerdict.EXPERIMENTAL_LOCAL_HYBRID + + +def validate_controlled_pair_evidence( + baseline: Mapping[str, Any], + alternate: Mapping[str, Any], + *, + variable_path: str, +) -> StageEvidence: + keys = set(baseline) | set(alternate) + changed = tuple(sorted(key for key in keys if baseline.get(key) != alternate.get(key))) + if changed != (variable_path,): + raise InvalidEvidenceError(f"controlled pair must differ only in {variable_path}") + return StageEvidence(details={"valid": True, "changed_paths": changed}) + + +def build_generation_feasibility_service( + *, + repository: FeasibilityRepository, + delegate: StageExecutor, + experiments: Any, + jobs: Any, + artifacts: Any, + worker: Any, +) -> FeasibilityService: + """Product composition root for resumable generation qualification stages.""" + from video_lab.normal_generation import RepositoryWorkerGenerationRunner + from video_lab.review_service import ReviewService + from video_lab.reviews import ReviewRepository + + runner = RepositoryWorkerGenerationRunner( + experiments=experiments, + jobs=jobs, + artifacts=artifacts, + worker=worker, + ) + reviews = ReviewService( + experiments=experiments, + jobs=jobs, + artifacts=artifacts, + reviews=ReviewRepository(repository.database), + ) + return FeasibilityService( + repository, + GenerationQualificationExecutor(delegate, runner, reviews), + ) + + +__all__ = [ + "FeasibilityProtocol", + "FeasibilityService", + "InvalidEvidenceError", + "ResourceExhaustedError", + "StageDefinition", + "StageExecutionContext", + "StageResult", + "canonical_fingerprint", + "build_generation_feasibility_service", + "classify_exception", + "derive_display_verdict", + "sanitize_diagnostic", + "validate_controlled_pair_evidence", +] diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_models.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_models.py new file mode 100644 index 0000000..d33fbcd --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_models.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from datetime import datetime +from enum import StrEnum +from math import isfinite +from types import MappingProxyType +from typing import Any + +from pydantic import Field, field_serializer, field_validator, model_validator + +from video_lab.domain import FrozenModel, _aware_utc + +type JsonScalar = str | int | float | bool | None +type JsonValue = JsonScalar | Mapping[str, JsonValue] | tuple[JsonValue, ...] + + +class FeasibilityEvidenceError(ValueError): + """Base error for evidence that cannot be frozen or verified.""" + + +class FeasibilityStage(StrEnum): + RESOLVE_SOURCES = "resolve_sources" + INVENTORY_SOURCES = "inventory_sources" + STORAGE_PREFLIGHT = "storage_preflight" + DOWNLOAD_SOURCES = "download_sources" + LOAD_TEXT_STACK = "load_text_stack" + LOAD_VAE = "load_vae" + LOAD_HIGH_TRANSFORMER = "load_high_transformer" + LOAD_LOW_TRANSFORMER = "load_low_transformer" + ASSEMBLE_PIPELINE = "assemble_pipeline" + GENERATE_MINIMUM_1 = "generate_minimum_1" + GENERATE_MINIMUM_2 = "generate_minimum_2" + QUALIFY_STANDARD = "qualify_standard" + VALIDATE_CONTROLLED_PAIR = "validate_controlled_pair" + PUBLISH_VERDICT = "publish_verdict" + + +class StageOutcome(StrEnum): + PASSED = "passed" + INCOMPATIBLE = "incompatible" + RESOURCE_EXHAUSTED = "resource_exhausted" + INTERRUPTED = "interrupted" + INVALID_EVIDENCE = "invalid_evidence" + UNEXPECTED_FAILURE = "unexpected_failure" + + +class SupportVerdict(StrEnum): + PENDING = "pending" + INCOMPATIBLE = "incompatible" + RESOURCE_LIMITED = "resource_limited" + EXPERIMENTAL_LOCAL_HYBRID = "experimental_local_hybrid" + STANDARD_LAB_QUALIFIED = "standard_lab_qualified" + + +def _freeze_json(value: Any) -> JsonValue: + if value is None or isinstance(value, (str, bool, int)): + return value + if isinstance(value, float): + if not isfinite(value): + raise ValueError("JSON numbers must be finite") + return value + if isinstance(value, Mapping): + if any(not isinstance(key, str) for key in value): + raise ValueError("JSON object keys must be strings") + return MappingProxyType({key: _freeze_json(item) for key, item in value.items()}) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return tuple(_freeze_json(item) for item in value) + raise ValueError(f"value is not JSON-compatible: {type(value).__name__}") + + +def _thaw_json(value: JsonValue) -> object: + if isinstance(value, Mapping): + return {key: _thaw_json(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thaw_json(item) for item in value] + return value + + +class StageEvidence(FrozenModel): + measurements: Mapping[str, float] = Field(default_factory=lambda: MappingProxyType({})) + details: Mapping[str, JsonValue] = Field(default_factory=lambda: MappingProxyType({})) + + @field_validator("measurements") + @classmethod + def freeze_measurements(cls, value: Mapping[str, float]) -> Mapping[str, float]: + copied = {str(key): float(measurement) for key, measurement in value.items()} + if any(measurement < 0 or not isfinite(measurement) for measurement in copied.values()): + raise ValueError("measurements must be finite and non-negative") + return MappingProxyType(copied) + + @field_validator("details") + @classmethod + def freeze_details(cls, value: object) -> Mapping[str, JsonValue]: + frozen = _freeze_json(value) + if not isinstance(frozen, Mapping): + raise ValueError("details must be a mapping") + return frozen + + @field_serializer("measurements") + def serialize_measurements(self, value: Mapping[str, float]) -> dict[str, float]: + return dict(value) + + @field_serializer("details") + def serialize_details(self, value: Mapping[str, JsonValue]) -> dict[str, object]: + return {key: _thaw_json(item) for key, item in value.items()} + + +class StageResult(FrozenModel): + outcome: StageOutcome + evidence: StageEvidence = Field(default_factory=StageEvidence) + diagnostic_code: str | None = Field(default=None, min_length=1, max_length=64) + diagnostic_message: str | None = Field(default=None, min_length=1, max_length=1000) + + @classmethod + def passed(cls, evidence: StageEvidence | None = None) -> StageResult: + return cls(outcome=StageOutcome.PASSED, evidence=evidence or StageEvidence()) + + +class FeasibilitySourceIdentity(FrozenModel): + id: str = Field(min_length=1) + revision: str = Field(pattern=r"^[0-9a-f]{40}$") + checksum: str = Field(pattern=r"^sha256:[0-9a-f]{64}$") + + @field_validator("id", "revision", "checksum", mode="before") + @classmethod + def strip_identity(cls, value: object) -> object: + return value.strip() if isinstance(value, str) else value + + +class FeasibilityManifestProfiles(FrozenModel): + minimum: Mapping[str, JsonValue] + standard: Mapping[str, JsonValue] + + @field_validator("minimum", "standard", mode="before") + @classmethod + def freeze_profile(cls, value: object) -> Mapping[str, JsonValue]: + frozen = _freeze_json(_normalize_identity_json(value)) + if not isinstance(frozen, Mapping) or not frozen: + raise ValueError("manifest profiles must be non-empty mappings") + return frozen + + @field_serializer("minimum", "standard") + def serialize_profile(self, value: Mapping[str, JsonValue]) -> dict[str, object]: + return {key: _thaw_json(item) for key, item in value.items()} + + +class FeasibilityConfiguration(FrozenModel): + adapter_id: str = Field(min_length=1) + protocol_version: str = Field(min_length=1) + sources: Mapping[str, FeasibilitySourceIdentity] + runtime_versions: Mapping[str, str] + hardware_fingerprint: str = Field(min_length=1) + os_fingerprint: str = Field(min_length=1) + manifest_profiles: FeasibilityManifestProfiles + memory_strategy: str = Field(min_length=1) + + @field_validator( + "adapter_id", + "protocol_version", + "hardware_fingerprint", + "os_fingerprint", + "memory_strategy", + mode="before", + ) + @classmethod + def strip_identity(cls, value: object) -> object: + return value.strip() if isinstance(value, str) else value + + @field_validator("adapter_id") + @classmethod + def canonicalize_adapter_id(cls, value: str) -> str: + return value.strip().lower() + + @field_validator("sources", mode="before") + @classmethod + def normalize_source_keys(cls, value: object) -> object: + if not isinstance(value, Mapping): + return value + normalized = {_strip_required_string(key): item for key, item in value.items()} + if not normalized: + raise ValueError("sources must contain named exact identities") + return normalized + + @field_validator("sources") + @classmethod + def freeze_sources( + cls, value: Mapping[str, FeasibilitySourceIdentity] + ) -> Mapping[str, FeasibilitySourceIdentity]: + return MappingProxyType(dict(value)) + + @field_validator("runtime_versions", mode="before") + @classmethod + def freeze_runtime_versions(cls, value: object) -> Mapping[str, str]: + if not isinstance(value, Mapping): + raise ValueError("runtime_versions must be a mapping") + copied = { + _strip_required_string(key): _strip_required_string(version) + for key, version in value.items() + } + if not copied: + raise ValueError("runtime_versions must contain named versions") + return MappingProxyType(copied) + + @field_serializer("sources", "runtime_versions") + def serialize_mapping(self, value: Mapping[str, object]) -> dict[str, object]: + return dict(value) + + +def _strip_required_string(value: object) -> str: + if not isinstance(value, str) or not (stripped := value.strip()): + raise ValueError("identity strings must not be blank") + return stripped + + +def _normalize_identity_json(value: object) -> object: + if isinstance(value, str): + return _strip_required_string(value) + if isinstance(value, Mapping): + return { + _strip_required_string(key): ( + "" + if _strip_required_string(key) == "negative_prompt" and item == "" + else _normalize_identity_json(item) + ) + for key, item in value.items() + } + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return [_normalize_identity_json(item) for item in value] + return value + + +class FeasibilityRun(FrozenModel): + id: str = Field(min_length=1) + adapter_id: str = Field(min_length=1) + protocol_version: str = Field(min_length=1) + input_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$") + configuration: Mapping[str, JsonValue] + verdict: SupportVerdict = SupportVerdict.PENDING + created_at: datetime + updated_at: datetime + + @field_validator("configuration") + @classmethod + def freeze_configuration(cls, value: object) -> Mapping[str, JsonValue]: + frozen = _freeze_json(value) + if not isinstance(frozen, Mapping): + raise ValueError("configuration must be a mapping") + return frozen + + @field_validator("created_at", "updated_at") + @classmethod + def normalize_timestamps(cls, value: datetime) -> datetime: + return _aware_utc(value) + + @field_serializer("configuration") + def serialize_configuration(self, value: Mapping[str, JsonValue]) -> dict[str, object]: + return {key: _thaw_json(item) for key, item in value.items()} + + @model_validator(mode="after") + def validate_timestamp_order(self) -> FeasibilityRun: + if self.updated_at < self.created_at: + raise ValueError("updated_at must not precede created_at") + return self + + +class StageAttempt(FrozenModel): + id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + stage: FeasibilityStage + ordinal: int = Field(gt=0) + input_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$") + outcome: StageOutcome | None = None + started_at: datetime + finished_at: datetime | None = None + evidence: StageEvidence | None = None + diagnostic_code: str | None = Field(default=None, min_length=1, max_length=64) + diagnostic_message: str | None = Field(default=None, min_length=1, max_length=1000) + + @field_validator("started_at", "finished_at") + @classmethod + def normalize_timestamps(cls, value: datetime | None) -> datetime | None: + return None if value is None else _aware_utc(value) + + @model_validator(mode="after") + def validate_terminal_state(self) -> StageAttempt: + terminal_values = ( + self.outcome, + self.finished_at, + self.evidence, + self.diagnostic_code, + self.diagnostic_message, + ) + if self.outcome is None and any(value is not None for value in terminal_values[1:]): + raise ValueError("active attempts cannot contain terminal evidence") + if self.outcome is not None and self.finished_at is None: + raise ValueError("finished attempts require finished_at") + if self.finished_at is not None and self.finished_at < self.started_at: + raise ValueError("finished_at must not precede started_at") + return self diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_projection.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_projection.py new file mode 100644 index 0000000..c85964a --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_projection.py @@ -0,0 +1,531 @@ +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import asdict +from typing import cast + +from video_lab.artifacts import ArtifactError +from video_lab.feasibility import ( + FeasibilityProtocol, + FeasibilityStatus, + derive_display_verdict, + sanitize_diagnostic, +) +from video_lab.feasibility_models import FeasibilityStage, StageAttempt, StageOutcome +from video_lab.feasibility_repository import FeasibilityRepository + +_SCHEMA = "feasibility-evidence.v1" +_GENERATION_STAGES = { + FeasibilityStage.GENERATE_MINIMUM_1, + FeasibilityStage.GENERATE_MINIMUM_2, + FeasibilityStage.QUALIFY_STANDARD, + FeasibilityStage.VALIDATE_CONTROLLED_PAIR, +} + + +def normalize_reviewer_id(reviewer_id: str | None) -> str | None: + if reviewer_id is None: + return None + normalized = reviewer_id.strip() + return normalized or None + + +def next_feasibility_stage(attempts: Sequence[StageAttempt]) -> str | None: + latest = {attempt.stage: attempt for attempt in attempts} + for definition in FeasibilityProtocol.default().stages: + attempt = latest.get(definition.stage) + if attempt is None: + return definition.stage.value + if attempt.outcome is StageOutcome.PASSED: + continue + if attempt.outcome in (StageOutcome.INTERRUPTED, StageOutcome.UNEXPECTED_FAILURE): + return definition.stage.value + return None + return None + + +class FeasibilityEvidenceProjector: + """Builds one path-neutral, read-only phase projection for CLI, web, and export.""" + + def __init__(self, repository: FeasibilityRepository) -> None: + self.repository = repository + + def for_run( + self, + run_id: str, + *, + reviewer_id: str | None = None, + completed_attempt: StageAttempt | None = None, + ) -> dict[str, object]: + run = self.repository.get_run(run_id) + attempts = self.repository.list_attempts(run_id) + status = FeasibilityStatus(run, attempts, derive_display_verdict(attempts)) + return self.project( + status, + reviewer_id=reviewer_id, + completed_attempt=completed_attempt, + ) + + def project( + self, + status: FeasibilityStatus, + *, + reviewer_id: str | None = None, + completed_attempt: StageAttempt | None = None, + ) -> dict[str, object]: + attempts = status.attempts + reviewer = normalize_reviewer_id(reviewer_id) + authorization = self.repository.generation_authorization_bundle(status.run.adapter_id) + authorized = ( + authorization + if authorization is not None and authorization.run_id == status.run.id + else None + ) + if not self._review_submitted(attempts, reviewer): + return self._sealed_projection( + status, + authorized=authorized is not None, + completed_attempt=completed_attempt, + ) + resources = { + attempt.stage.value: attempt.evidence.details["preassembly_preflight"] + for attempt in attempts + if attempt.evidence is not None and "preassembly_preflight" in attempt.evidence.details + } + projected: dict[str, object] = { + "schema_version": _SCHEMA, + "projection_state": "revealed", + "run": status.run.model_dump(mode="json"), + "attempts": [attempt.model_dump(mode="json") for attempt in attempts], + "completed_attempt": ( + None if completed_attempt is None else completed_attempt.model_dump(mode="json") + ), + "next_stage": next_feasibility_stage(attempts), + "verdict": status.verdict.value, + "gates": [ + self._gate(attempts, definition.stage) + for definition in FeasibilityProtocol.default().stages + ], + "resources": {"preassembly_checks": resources}, + "artifacts": self._artifacts(status.run.id, attempts), + "controlled_pair": self._controlled_pair(attempts, reviewer), + "provenance": self._provenance(attempts), + "conclusions": { + "display_verdict": status.verdict.value, + "persisted_verdict": status.run.verdict.value, + "authorized": authorized is not None, + "authorization": ( + None + if authorized is None + else { + "run_id": authorized.run_id, + "verdict": authorized.verdict.value, + "measured_profile": authorized.measured_profile, + "canonical_manifest_hash": authorized.canonical_manifest_hash, + "source_fingerprint": authorized.source_fingerprint, + "runtime_fingerprint": authorized.runtime_fingerprint, + "placement_fingerprint": authorized.placement_fingerprint, + "authorized_manifest": authorized.authorized_manifest.model_dump( + mode="json" + ), + "resolved_sources": { + key: asdict(value) for key, value in authorized.resolved_sources.items() + }, + } + ), + }, + } + return cast(dict[str, object], sanitize_diagnostic(projected)) + + def _sealed_projection( + self, + status: FeasibilityStatus, + *, + authorized: bool, + completed_attempt: StageAttempt | None, + ) -> dict[str, object]: + attempts = status.attempts + projected: dict[str, object] = { + "schema_version": _SCHEMA, + "projection_state": "sealed", + "run": { + "id": status.run.id, + "adapter_id": status.run.adapter_id, + "protocol_version": status.run.protocol_version, + "verdict": status.run.verdict.value, + "created_at": status.run.created_at.isoformat(), + "updated_at": status.run.updated_at.isoformat(), + }, + "attempts": [self._sealed_attempt(attempt) for attempt in attempts], + "completed_attempt": ( + None if completed_attempt is None else self._sealed_attempt(completed_attempt) + ), + "next_stage": next_feasibility_stage(attempts), + "verdict": status.verdict.value, + "gates": [ + self._sealed_gate(attempts, definition.stage) + for definition in FeasibilityProtocol.default().stages + ], + "resources": {"state": "sealed"}, + "artifacts": self._sealed_artifacts(status.run.id, attempts), + "controlled_pair": self._sealed_controlled_pair(attempts), + "provenance": {"state": "sealed"}, + "conclusions": { + "display_verdict": status.verdict.value, + "persisted_verdict": status.run.verdict.value, + "authorized": authorized, + "authorization": None, + "authorization_state": "sealed", + }, + } + return cast(dict[str, object], sanitize_diagnostic(projected)) + + @staticmethod + def _sealed_attempt(attempt: StageAttempt) -> dict[str, object]: + return { + "stage": attempt.stage.value, + "ordinal": attempt.ordinal, + "outcome": None if attempt.outcome is None else attempt.outcome.value, + "started_at": attempt.started_at.isoformat(), + "finished_at": ( + None if attempt.finished_at is None else attempt.finished_at.isoformat() + ), + } + + @classmethod + def _sealed_gate( + cls, attempts: Sequence[StageAttempt], stage: FeasibilityStage + ) -> dict[str, object]: + attempt = next((item for item in reversed(attempts) if item.stage is stage), None) + if attempt is None: + return {"stage": stage.value, "outcome": "pending", "attempt": None} + return { + "stage": stage.value, + "outcome": None if attempt.outcome is None else attempt.outcome.value, + "attempt": cls._sealed_attempt(attempt), + } + + @staticmethod + def _gate(attempts: Sequence[StageAttempt], stage: FeasibilityStage) -> dict[str, object]: + attempt = next((item for item in reversed(attempts) if item.stage is stage), None) + if attempt is None: + return {"stage": stage.value, "outcome": "pending", "attempt": None} + return { + "stage": stage.value, + "outcome": None if attempt.outcome is None else attempt.outcome.value, + "attempt": attempt.model_dump(mode="json"), + } + + @staticmethod + def _pair_details(attempts: Sequence[StageAttempt]) -> Mapping[str, object] | None: + pair = next( + ( + attempt + for attempt in reversed(attempts) + if attempt.stage is FeasibilityStage.VALIDATE_CONTROLLED_PAIR + and attempt.evidence is not None + ), + None, + ) + if pair is None or pair.evidence is None: + return None + return cast(Mapping[str, object], pair.evidence.details) + + def _review_submitted( + self, + attempts: Sequence[StageAttempt], + reviewer_id: str | None, + ) -> bool: + if reviewer_id is None: + return False + details = self._pair_details(attempts) + experiment_id = None if details is None else details.get("experiment_id") + if not isinstance(experiment_id, str): + return False + with self.repository.database.connect() as connection: + row = connection.execute( + "SELECT review_revisions.id FROM review_revisions " + "JOIN review_sessions ON review_sessions.id = review_revisions.session_id " + "JOIN rubric_scores ON rubric_scores.review_id = review_revisions.id " + "WHERE review_sessions.experiment_id = ? AND review_sessions.reviewer_id = ? " + "GROUP BY review_revisions.id HAVING COUNT(*) = 10 " + "ORDER BY review_revisions.created_at, review_revisions.revision LIMIT 1", + (experiment_id, reviewer_id), + ).fetchone() + return row is not None + + def _sealed_controlled_pair(self, attempts: Sequence[StageAttempt]) -> dict[str, object] | None: + details = self._pair_details(attempts) + if details is None: + return None + identity = details.get("review_identity") + session_id = identity.get("session_id") if isinstance(identity, Mapping) else None + if not isinstance(session_id, str) or self._phase_session(session_id) is None: + return None + return { + "display_positions": ["left", "right"], + "review": { + "state": "blinded_randomized", + "submitted": False, + "revisions": [], + }, + } + + def _sealed_artifacts( + self, run_id: str, attempts: Sequence[StageAttempt] + ) -> list[dict[str, str]]: + details = self._pair_details(attempts) + identity = None if details is None else details.get("review_identity") + session_id = identity.get("session_id") if isinstance(identity, Mapping) else None + if not isinstance(session_id, str) or self._phase_session(session_id) is None: + return [] + return [ + { + "display_position": position, + "media_url": f"/media/feasibility/{run_id}/{position}", + } + for position in ("left", "right") + ] + + def media_artifact_id( + self, + run_id: str, + handle: str, + *, + reviewer_id: str | None = None, + ) -> str | None: + attempts = self.repository.list_attempts(run_id) + reviewer = normalize_reviewer_id(reviewer_id) + details = self._pair_details(attempts) + identity = None if details is None else details.get("review_identity") + session_id = identity.get("session_id") if isinstance(identity, Mapping) else None + session = self._phase_session(session_id) if isinstance(session_id, str) else None + if handle in ("left", "right") and session is not None: + return str(session[f"{handle}_artifact_id"]) + if not self._review_submitted(attempts, reviewer): + return None + projected = self._artifacts(run_id, attempts) + allowed = { + str(item["artifact_id"]) + for item in projected + if isinstance(item.get("artifact_id"), str) + } + return handle if handle in allowed else None + + def _phase_session(self, session_id: str) -> Mapping[str, object] | None: + with self.repository.database.connect() as connection: + row = connection.execute( + "SELECT * FROM review_sessions WHERE id = ? AND reviewer_id LIKE 'feasibility:%'", + (session_id,), + ).fetchone() + return None if row is None else cast(Mapping[str, object], row) + + def _artifacts(self, run_id: str, attempts: Sequence[StageAttempt]) -> list[dict[str, object]]: + jobs: list[tuple[FeasibilityStage, str, str]] = [] + for attempt in attempts: + if ( + attempt.stage not in _GENERATION_STAGES + or attempt.outcome is not StageOutcome.PASSED + or attempt.evidence is None + ): + continue + details = attempt.evidence.details + job_id = details.get("job_id") + if isinstance(job_id, str): + jobs.append((attempt.stage, "output", job_id)) + job_ids = details.get("job_ids") + if isinstance(job_ids, tuple): + jobs.extend( + (attempt.stage, f"candidate_{index + 1}", value) + for index, value in enumerate(job_ids) + if isinstance(value, str) + ) + projected: list[dict[str, object]] = [] + for stage, slot, job_id in jobs: + registered = ( + () + if self.repository.artifacts is None + else self.repository.artifacts.list_for_job(job_id) + ) + if len(registered) != 1: + continue + artifact = registered[0] + integrity = "unverified" + if self.repository.artifacts is not None: + try: + self.repository.artifacts.resolve(artifact.id) + except ArtifactError: + integrity = "corrupt" + else: + integrity = "verified" + projected.append( + { + "stage": stage.value, + "slot": slot, + "job_id": job_id, + "artifact_id": artifact.id, + "sha256": artifact.sha256, + "byte_size": artifact.byte_size, + "media_type": artifact.media_type, + "integrity": integrity, + "media_url": f"/media/feasibility/{run_id}/{artifact.id}", + "probe": self._completion_probe(job_id), + } + ) + return projected + + def _completion_probe(self, job_id: str) -> Mapping[str, object] | None: + with self.repository.database.connect() as connection: + row = connection.execute( + "SELECT payload_json FROM run_events WHERE job_id = ? " + "AND kind = 'job_completed' ORDER BY id DESC LIMIT 1", + (job_id,), + ).fetchone() + if row is None: + return None + payload = json.loads(str(row["payload_json"])) + if not isinstance(payload, Mapping): + return None + return cast(Mapping[str, object], payload) + + def _controlled_pair( + self, + attempts: Sequence[StageAttempt], + reviewer_id: str | None, + ) -> dict[str, object] | None: + pair = next( + ( + attempt + for attempt in reversed(attempts) + if attempt.stage is FeasibilityStage.VALIDATE_CONTROLLED_PAIR + and attempt.evidence is not None + ), + None, + ) + if pair is None or pair.evidence is None: + return None + details = pair.evidence.details + experiment_id = details.get("experiment_id") + review = self._review( + experiment_id if isinstance(experiment_id, str) else None, + reviewer_id, + ) + return { + "experiment_id": details.get("experiment_id"), + "variable_path": details.get("variable_path"), + "changed_paths": details.get("changed_paths"), + "controlled_values": details.get("controlled_values"), + "pair_seed": details.get("pair_seed"), + "job_ids": details.get("job_ids"), + "artifact_sha256": details.get("artifact_sha256"), + "review": review, + } + + def _review( + self, + experiment_id: str | None, + reviewer_id: str | None, + ) -> dict[str, object]: + empty: dict[str, object] = { + "session_id": None, + "state": "missing", + "revisions": [], + } + if experiment_id is None or reviewer_id is None: + return empty + with self.repository.database.connect() as connection: + session = connection.execute( + "SELECT * FROM review_sessions WHERE experiment_id = ? AND reviewer_id = ?", + (experiment_id, reviewer_id), + ).fetchone() + if session is None: + return empty + revisions = connection.execute( + "SELECT * FROM review_revisions WHERE session_id = ? " + "ORDER BY review_revisions.created_at, review_revisions.revision", + (str(session["id"]),), + ).fetchall() + projected_revisions: list[dict[str, object]] = [] + for revision in revisions: + scores = connection.execute( + "SELECT display_position, dimension, score FROM rubric_scores " + "WHERE review_id = ? ORDER BY display_position, dimension", + (str(revision["id"]),), + ).fetchall() + if len(scores) != 10: + continue + projected_revisions.append( + { + "id": str(revision["id"]), + "session_id": str(revision["session_id"]), + "revision": int(revision["revision"]), + "display_preference": str(revision["display_preference"]), + "preferred_variant_id": revision["preferred_variant_id"], + "notes": revision["notes"], + "created_at": str(revision["created_at"]), + "supersedes_review_id": revision["supersedes_review_id"], + "scores": [dict(score) for score in scores], + "display_mapping": { + position: { + "variant_id": str(session[f"{position}_variant_id"]), + "job_id": str(session[f"{position}_job_id"]), + "artifact_id": str(session[f"{position}_artifact_id"]), + "artifact_sha256": str(session[f"{position}_artifact_sha256"]), + } + for position in ("left", "right") + }, + } + ) + return { + "session_id": str(session["id"]), + "state": "submitted" if projected_revisions else "blinded_randomized", + "experiment_id": str(session["experiment_id"]), + "left": { + "variant_id": str(session["left_variant_id"]), + "job_id": str(session["left_job_id"]), + "artifact_id": str(session["left_artifact_id"]), + "artifact_sha256": str(session["left_artifact_sha256"]), + }, + "right": { + "variant_id": str(session["right_variant_id"]), + "job_id": str(session["right_job_id"]), + "artifact_id": str(session["right_artifact_id"]), + "artifact_sha256": str(session["right_artifact_sha256"]), + }, + "revisions": projected_revisions, + } + + @staticmethod + def _provenance(attempts: Sequence[StageAttempt]) -> dict[str, object]: + inventory = None + identities: dict[str, dict[str, object]] = {} + repeatability = None + for attempt in attempts: + if attempt.evidence is None: + continue + details = attempt.evidence.details + if attempt.stage is FeasibilityStage.DOWNLOAD_SOURCES: + inventory = details.get("inventory") + if attempt.stage in _GENERATION_STAGES: + identities[attempt.stage.value] = { + key: details.get(key) + for key in ( + "canonical_manifest_hash", + "source_fingerprint", + "runtime_fingerprint", + "placement_fingerprint", + "placement", + "fallback_violations", + ) + } + if attempt.stage is FeasibilityStage.GENERATE_MINIMUM_2: + repeatability = details.get("repeatability") + return { + "source_inventory": inventory, + "generation_identities": identities, + "repeatability": repeatability, + } + + +__all__ = ["FeasibilityEvidenceProjector", "next_feasibility_stage"] diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_repository.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_repository.py new file mode 100644 index 0000000..dcc9f0a --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_repository.py @@ -0,0 +1,900 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime +from typing import Any, cast + +from video_lab.animegen_pair_contract import ( + ANIMEGEN_CONTROLLED_GUIDANCE_VALUES, + ANIMEGEN_CONTROLLED_PAIR_SEED, +) +from video_lab.artifacts import ArtifactError, ArtifactStore +from video_lab.database import Database +from video_lab.domain import Manifest, canonical_json, new_id, utc_now +from video_lab.feasibility_models import ( + FeasibilityRun, + FeasibilityStage, + StageAttempt, + StageEvidence, + StageOutcome, + SupportVerdict, +) +from video_lab.runtime_identity import runtime_fingerprint +from video_lab.source_inventory import ResolvedSource, SourceFile + + +class FeasibilityRunNotFound(LookupError): + pass + + +class StageAttemptNotFound(LookupError): + pass + + +class ActiveStageAttemptError(RuntimeError): + pass + + +class StageAttemptFinishedError(RuntimeError): + pass + + +@dataclass(frozen=True) +class GenerationAuthorizationBundle: + run_id: str + verdict: SupportVerdict + measured_profile: tuple[int, int, int, int] + canonical_manifest_hash: str + source_fingerprint: str + runtime_fingerprint: str + placement_fingerprint: str + resolved_sources: Mapping[str, ResolvedSource] + authorized_manifest: Manifest + + +class FeasibilityRepository: + def __init__(self, database: Database, artifacts: ArtifactStore | None = None) -> None: + self.database = database + self.artifacts = artifacts + + def create_run(self, run: FeasibilityRun) -> FeasibilityRun: + validated = FeasibilityRun.model_validate(run.model_dump(round_trip=True)) + with self.database.connect() as connection: + connection.execute( + "INSERT INTO feasibility_runs(" + "id, adapter_id, protocol_version, input_fingerprint, configuration_json, " + "verdict, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + validated.id, + validated.adapter_id, + validated.protocol_version, + validated.input_fingerprint, + canonical_json(validated.model_dump(mode="json")["configuration"]), + validated.verdict.value, + validated.created_at.isoformat(), + validated.updated_at.isoformat(), + ), + ) + return validated + + def get_run(self, run_id: str) -> FeasibilityRun: + with self.database.connect() as connection: + row = connection.execute( + "SELECT * FROM feasibility_runs WHERE id = ?", (run_id,) + ).fetchone() + if row is None: + raise FeasibilityRunNotFound(f"feasibility run not found: {run_id}") + return _run_from_row(row) + + def set_verdict(self, run_id: str, verdict: SupportVerdict) -> None: + if verdict in ( + SupportVerdict.EXPERIMENTAL_LOCAL_HYBRID, + SupportVerdict.STANDARD_LAB_QUALIFIED, + ): + from video_lab.feasibility import derive_display_verdict + + attempts = self.list_attempts(run_id) + if derive_display_verdict(attempts) is not verdict: + raise ValueError("support verdict is not proven by immutable stage evidence") + if verdict is SupportVerdict.STANDARD_LAB_QUALIFIED: + run = self.get_run(run_id) + with self.database.connect() as connection: + if not self._validated_standard_pair(connection, run, attempts): + raise ValueError( + "standard support verdict is not proven by persisted " + "generation evidence" + ) + with self.database.connect() as connection: + connection.execute( + "UPDATE feasibility_runs SET verdict = ?, updated_at = ? WHERE id = ?", + (verdict.value, utc_now().isoformat(), run_id), + ) + + def latest_authorized_profile( + self, adapter_id: str + ) -> tuple[SupportVerdict, tuple[int, int, int]] | None: + with self.database.connect() as connection: + rows = connection.execute( + "SELECT * FROM feasibility_runs WHERE adapter_id = ? " + "AND verdict IN (?, ?) ORDER BY updated_at DESC, id DESC", + ( + adapter_id, + SupportVerdict.EXPERIMENTAL_LOCAL_HYBRID.value, + SupportVerdict.STANDARD_LAB_QUALIFIED.value, + ), + ).fetchall() + from video_lab.feasibility import derive_display_verdict + + for row in rows: + run = _run_from_row(row) + all_attempts = self.list_attempts(run.id) + derived = derive_display_verdict(all_attempts) + if derived not in ( + SupportVerdict.EXPERIMENTAL_LOCAL_HYBRID, + SupportVerdict.STANDARD_LAB_QUALIFIED, + ): + continue + profiles = run.configuration.get("manifest_profiles") + if not isinstance(profiles, Mapping): + continue + name = "standard" if derived is SupportVerdict.STANDARD_LAB_QUALIFIED else "minimum" + profile = profiles.get(name) + if not isinstance(profile, Mapping): + continue + width, height, frames = ( + profile.get("width"), + profile.get("height"), + profile.get("num_frames", profile.get("frames")), + ) + if isinstance(width, int) and isinstance(height, int) and isinstance(frames, int): + stage = ( + FeasibilityStage.QUALIFY_STANDARD + if derived is SupportVerdict.STANDARD_LAB_QUALIFIED + else FeasibilityStage.GENERATE_MINIMUM_2 + ) + profile_attempts = self.list_attempts(run.id, stage) + evidence = profile_attempts[-1].evidence if profile_attempts else None + if evidence is None: + continue + artifact = evidence.details.get("artifact") + if not isinstance(artifact, Mapping): + continue + if ( + artifact.get("width"), + artifact.get("height"), + artifact.get("num_frames"), + ) != (width, height, frames): + continue + configured_fps = profile.get("fps") + if isinstance(configured_fps, int) and artifact.get("fps") != configured_fps: + continue + try: + configured_manifest = Manifest.model_validate(profile) + except ValueError: + configured_manifest = None + if configured_manifest is not None: + expected_hash = hashlib.sha256( + canonical_json(configured_manifest).encode() + ).hexdigest() + if evidence.details.get("canonical_manifest_hash") != expected_hash: + continue + if derived is SupportVerdict.STANDARD_LAB_QUALIFIED: + with self.database.connect() as connection: + if not self._validated_standard_pair(connection, run, all_attempts): + continue + return derived, (width, height, frames) + return None + + def generation_authorization_bundle( + self, adapter_id: str + ) -> GenerationAuthorizationBundle | None: + from video_lab.feasibility import derive_display_verdict + + with self.database.connect() as connection: + connection.execute("BEGIN") + rows = connection.execute( + "SELECT * FROM feasibility_runs WHERE adapter_id = ? " + "ORDER BY updated_at DESC, id DESC", + (adapter_id,), + ).fetchall() + for row in rows: + run = _run_from_row(row) + attempt_rows = connection.execute( + "SELECT * FROM feasibility_stage_attempts WHERE run_id = ? " + "ORDER BY started_at, stage, ordinal, id", + (run.id,), + ).fetchall() + attempts = tuple(_attempt_from_row(item) for item in attempt_rows) + derived = derive_display_verdict(attempts) + if run.verdict is not derived or derived not in ( + SupportVerdict.EXPERIMENTAL_LOCAL_HYBRID, + SupportVerdict.STANDARD_LAB_QUALIFIED, + ): + continue + stage = ( + FeasibilityStage.QUALIFY_STANDARD + if derived is SupportVerdict.STANDARD_LAB_QUALIFIED + else FeasibilityStage.GENERATE_MINIMUM_2 + ) + generation = next( + ( + item + for item in reversed(attempts) + if item.stage is stage and item.evidence is not None + ), + None, + ) + download = next( + ( + item + for item in reversed(attempts) + if item.stage is FeasibilityStage.DOWNLOAD_SOURCES + and item.outcome is StageOutcome.PASSED + and item.evidence is not None + ), + None, + ) + if generation is None or generation.evidence is None or download is None: + continue + artifact = generation.evidence.details.get("artifact") + if not isinstance(artifact, Mapping): + continue + measured = tuple( + artifact.get(key) for key in ("width", "height", "num_frames", "fps") + ) + if not all(isinstance(value, int) for value in measured): + continue + profiles = run.configuration.get("manifest_profiles") + profile_name = ( + "standard" if derived is SupportVerdict.STANDARD_LAB_QUALIFIED else "minimum" + ) + if not isinstance(profiles, Mapping): + continue + try: + authorized_manifest = Manifest.model_validate(profiles.get(profile_name)) + except ValueError: + continue + expected_hash = hashlib.sha256( + canonical_json(authorized_manifest).encode() + ).hexdigest() + if ( + measured + != ( + authorized_manifest.width, + authorized_manifest.height, + authorized_manifest.num_frames, + authorized_manifest.fps, + ) + or generation.evidence.details.get("profile") != profile_name + ): + continue + sources = _resolved_sources_from_attempt(download) + if sources is None: + continue + manifest_hash = generation.evidence.details.get("canonical_manifest_hash") + fingerprints = tuple( + generation.evidence.details.get(key) + for key in ( + "source_fingerprint", + "runtime_fingerprint", + "placement_fingerprint", + ) + ) + if not isinstance(manifest_hash, str) or not all( + isinstance(value, str) for value in fingerprints + ): + continue + if manifest_hash != expected_hash: + continue + standard_unproven = ( + derived is SupportVerdict.STANDARD_LAB_QUALIFIED + and not self._validated_standard_pair(connection, run, attempts) + ) + if standard_unproven: + continue + connection.commit() + return GenerationAuthorizationBundle( + run_id=run.id, + verdict=derived, + measured_profile=measured, + canonical_manifest_hash=manifest_hash, + source_fingerprint=cast(str, fingerprints[0]), + runtime_fingerprint=cast(str, fingerprints[1]), + placement_fingerprint=cast(str, fingerprints[2]), + resolved_sources=sources, + authorized_manifest=authorized_manifest, + ) + connection.commit() + return None + + def _validated_standard_pair( + self, + connection: sqlite3.Connection, + run: FeasibilityRun, + attempts: tuple[StageAttempt, ...], + ) -> bool: + """Validate standard qualification against persisted jobs, bytes, probes, and pins.""" + if self.artifacts is None: + return False + profiles = run.configuration.get("manifest_profiles") + if not isinstance(profiles, Mapping): + return False + try: + standard = Manifest.model_validate(profiles.get("standard")) + except ValueError: + return False + if ( + standard.prompt, + standard.negative_prompt, + standard.seed, + standard.width, + standard.height, + standard.num_frames, + standard.fps, + standard.inference_steps, + standard.guidance_scale, + standard.dtype, + standard.quantization, + standard.scheduler, + standard.memory_strategy, + ) != ( + "anime character walking", + "", + 7, + 512, + 288, + 17, + 8, + 4, + 1.0, + "bfloat16", + "bf16", + "flow-match-euler-shift-3", + "model_cpu_offload", + ): + return False + latest = _latest_attempts(attempts) + standard_attempt = latest.get(FeasibilityStage.QUALIFY_STANDARD) + pair_attempt = latest.get(FeasibilityStage.VALIDATE_CONTROLLED_PAIR) + if ( + standard_attempt is None + or standard_attempt.evidence is None + or standard_attempt.outcome is not StageOutcome.PASSED + or pair_attempt is None + or pair_attempt.evidence is None + or pair_attempt.outcome is not StageOutcome.PASSED + ): + return False + standard_details = standard_attempt.evidence.details + identities = tuple( + standard_details.get(key) + for key in ("source_fingerprint", "runtime_fingerprint", "placement_fingerprint") + ) + if not all(isinstance(value, str) and len(value) == 64 for value in identities): + return False + standard_job_id = standard_details.get("job_id") + standard_sha256 = standard_details.get("artifact_sha256") + if not isinstance(standard_job_id, str) or not isinstance(standard_sha256, str): + return False + if not self._validated_generation_job( + connection, + standard_job_id, + standard_sha256, + standard, + cast(tuple[str, str, str], identities), + ): + return False + + details = pair_attempt.evidence.details + if ( + details.get("valid") is not True + or details.get("changed_paths") != ("guidance_scale",) + or details.get("variable_path") != "guidance_scale" + or details.get("pair_seed") != ANIMEGEN_CONTROLLED_PAIR_SEED + or details.get("controlled_values") != ANIMEGEN_CONTROLLED_GUIDANCE_VALUES + ): + return False + experiment_id = details.get("experiment_id") + job_ids = details.get("job_ids") + artifact_sha256 = details.get("artifact_sha256") + review_identity = details.get("review_identity") + if ( + not isinstance(experiment_id, str) + or not isinstance(job_ids, tuple) + or len(job_ids) != 2 + or not all(isinstance(value, str) for value in job_ids) + or not isinstance(artifact_sha256, tuple) + or len(artifact_sha256) != 2 + or not all(isinstance(value, str) for value in artifact_sha256) + or not isinstance(review_identity, Mapping) + ): + return False + session_id = review_identity.get("session_id") + if ( + review_identity.get("experiment_id") != experiment_id + or review_identity.get("state") != "blinded_randomized" + or not isinstance(session_id, str) + ): + return False + experiment = connection.execute( + "SELECT * FROM experiments WHERE id = ?", (experiment_id,) + ).fetchone() + if ( + experiment is None + or str(experiment["schema_version"]) != standard.schema_version + or str(experiment["variable_path"]) != "guidance_scale" + or json.loads(str(experiment["changed_paths_json"])) != ["guidance_scale"] + ): + return False + variants = connection.execute( + "SELECT variants.*, jobs.id AS job_id, jobs.state FROM variants " + "JOIN jobs ON jobs.variant_id = variants.id " + "WHERE variants.experiment_id = ? ORDER BY variants.label", + (experiment_id,), + ).fetchall() + if len(variants) != 2 or tuple(str(row["label"]) for row in variants) != ("A", "B"): + return False + expected_manifests = ( + standard.model_copy( + update={ + "seed": ANIMEGEN_CONTROLLED_PAIR_SEED, + "guidance_scale": ANIMEGEN_CONTROLLED_GUIDANCE_VALUES[0], + } + ), + standard.model_copy( + update={ + "seed": ANIMEGEN_CONTROLLED_PAIR_SEED, + "guidance_scale": ANIMEGEN_CONTROLLED_GUIDANCE_VALUES[1], + } + ), + ) + if tuple(str(row["job_id"]) for row in variants) != job_ids: + return False + for index, (row, manifest) in enumerate(zip(variants, expected_manifests, strict=True)): + if str(row["state"]) != "completed": + return False + try: + persisted_manifest = Manifest.model_validate_json(str(row["manifest_json"])) + except ValueError: + return False + if persisted_manifest != manifest or not self._validated_generation_job( + connection, + cast(str, job_ids[index]), + cast(str, artifact_sha256[index]), + manifest, + cast(tuple[str, str, str], identities), + ): + return False + session = connection.execute( + "SELECT * FROM review_sessions WHERE id = ?", (session_id,) + ).fetchone() + if session is None: + return False + pins = { + ( + str(session["left_variant_id"]), + str(session["left_job_id"]), + str(session["left_artifact_id"]), + str(session["left_artifact_sha256"]), + ), + ( + str(session["right_variant_id"]), + str(session["right_job_id"]), + str(session["right_artifact_id"]), + str(session["right_artifact_sha256"]), + ), + } + expected_pins: set[tuple[str, str, str, str]] = set() + for row, job_id, digest in zip(variants, job_ids, artifact_sha256, strict=True): + artifacts = self.artifacts.list_for_job(cast(str, job_id)) + if len(artifacts) != 1: + return False + expected_pins.add( + (str(row["id"]), cast(str, job_id), artifacts[0].id, cast(str, digest)) + ) + return ( + str(session["experiment_id"]) == experiment_id + and str(session["reviewer_id"]) == f"feasibility:{run.id}" + and pins == expected_pins + ) + + def _validated_generation_job( + self, + connection: sqlite3.Connection, + job_id: str, + artifact_sha256: str, + manifest: Manifest, + identities: tuple[str, str, str], + ) -> bool: + if self.artifacts is None: + return False + job = connection.execute( + "SELECT jobs.state, variants.manifest_json FROM jobs " + "JOIN variants ON variants.id = jobs.variant_id WHERE jobs.id = ?", + (job_id,), + ).fetchone() + if job is None or str(job["state"]) != "completed": + return False + try: + persisted_manifest = Manifest.model_validate_json(str(job["manifest_json"])) + except ValueError: + return False + if persisted_manifest != manifest: + return False + attempts = connection.execute( + "SELECT * FROM run_attempts WHERE job_id = ? ORDER BY number", (job_id,) + ).fetchall() + if ( + len(attempts) != 1 + or str(attempts[0]["status"]) != "completed" + or attempts[0]["artifact_id"] is None + ): + return False + registered = self.artifacts.list_for_job(job_id) + if ( + len(registered) != 1 + or registered[0].media_type != "video/mp4" + or registered[0].id != str(attempts[0]["artifact_id"]) + or registered[0].sha256 != artifact_sha256 + ): + return False + try: + self.artifacts.resolve(registered[0].id) + except ArtifactError: + return False + events = connection.execute( + "SELECT * FROM run_events WHERE job_id = ? AND attempt_id = ? " + "AND kind = 'job_completed' ORDER BY id", + (job_id, str(attempts[0]["id"])), + ).fetchall() + if len(events) != 1: + return False + try: + payload = json.loads(str(events[0]["payload_json"])) + except (TypeError, json.JSONDecodeError): + return False + decoded = payload.get("decoded_frame_sha256") + aggregate = payload.get("decoded_rgb_aggregate_sha256") + if ( + payload.get("artifact_id") != registered[0].id + or payload.get("artifact_sha256") != artifact_sha256 + or payload.get("codec") not in ("h264", "avc1") + or not isinstance(payload.get("duration_seconds"), int | float) + or float(payload["duration_seconds"]) <= 0 + or ( + payload.get("width"), + payload.get("height"), + payload.get("frame_count"), + payload.get("fps"), + ) + != (manifest.width, manifest.height, manifest.num_frames, manifest.fps) + or not isinstance(decoded, list) + or len(decoded) != manifest.num_frames + or not all(isinstance(item, str) and len(item) == 64 for item in decoded) + or not isinstance(aggregate, str) + or len(aggregate) != 64 + ): + return False + runtime_versions = payload.get("runtime_versions") + telemetry = payload.get("adapter_telemetry") + if not isinstance(runtime_versions, Mapping) or not isinstance(telemetry, Mapping): + return False + observed = telemetry.get("observed_configuration") + if not isinstance(observed, Mapping): + return False + source, runtime, placement = identities + expected_observed = { + "scheduler": manifest.scheduler, + "dtype": manifest.dtype, + "quantization": manifest.quantization, + "compute_dtype": manifest.dtype, + "storage_dtype": ("bfloat16" if manifest.quantization == "bf16" else "float8_e4m3fn"), + "loras": [item.model_dump(mode="json") for item in manifest.loras], + "memory_strategy": manifest.memory_strategy, + "execution_placement": "local_hybrid:mps", + "source_fingerprint": source, + "placement_fingerprint": placement, + } + return ( + runtime_fingerprint(cast(Mapping[str, str], runtime_versions)) == runtime + and all(observed.get(key) == value for key, value in expected_observed.items()) + and telemetry.get("fallback_violations", []) == [] + ) + + def latest_resolved_sources(self, adapter_id: str) -> dict[str, ResolvedSource] | None: + with self.database.connect() as connection: + rows = connection.execute( + "SELECT id FROM feasibility_runs WHERE adapter_id = ? " + "ORDER BY updated_at DESC, id DESC", + (adapter_id,), + ).fetchall() + for row in rows: + from video_lab.feasibility import derive_display_verdict + + if derive_display_verdict(self.list_attempts(str(row["id"]))) not in ( + SupportVerdict.EXPERIMENTAL_LOCAL_HYBRID, + SupportVerdict.STANDARD_LAB_QUALIFIED, + ): + continue + attempts = self.list_attempts(str(row["id"]), FeasibilityStage.DOWNLOAD_SOURCES) + passed = next( + ( + attempt + for attempt in reversed(attempts) + if attempt.outcome is StageOutcome.PASSED and attempt.evidence is not None + ), + None, + ) + if passed is not None: + resolved = _resolved_sources_from_attempt(passed) + if resolved is not None: + return resolved + return None + + def start_attempt( + self, + run_id: str, + stage: FeasibilityStage, + input_fingerprint: str | None = None, + ) -> StageAttempt: + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + run = connection.execute( + "SELECT input_fingerprint FROM feasibility_runs WHERE id = ?", (run_id,) + ).fetchone() + if run is None: + raise FeasibilityRunNotFound(f"feasibility run not found: {run_id}") + run_fingerprint = str(run["input_fingerprint"]) + fingerprint = run_fingerprint if input_fingerprint is None else input_fingerprint + if run_fingerprint != fingerprint: + raise ValueError("attempt input fingerprint must match its run") + active = connection.execute( + "SELECT 1 FROM feasibility_stage_attempts " + "WHERE run_id = ? AND stage = ? AND outcome IS NULL", + (run_id, stage.value), + ).fetchone() + if active is not None: + raise ActiveStageAttemptError( + f"active attempt already exists for {run_id}/{stage.value}" + ) + ordinal = int( + connection.execute( + "SELECT COALESCE(MAX(ordinal), 0) + 1 " + "FROM feasibility_stage_attempts WHERE run_id = ? AND stage = ?", + (run_id, stage.value), + ).fetchone()[0] + ) + attempt = StageAttempt( + id=new_id("feasibility_attempt"), + run_id=run_id, + stage=stage, + ordinal=ordinal, + input_fingerprint=fingerprint, + started_at=utc_now(), + ) + connection.execute( + "INSERT INTO feasibility_stage_attempts(" + "id, run_id, stage, ordinal, input_fingerprint, started_at) " + "VALUES (?, ?, ?, ?, ?, ?)", + ( + attempt.id, + attempt.run_id, + attempt.stage.value, + attempt.ordinal, + attempt.input_fingerprint, + attempt.started_at.isoformat(), + ), + ) + connection.commit() + except Exception: + connection.rollback() + raise + return attempt + + def finish_attempt( + self, + attempt_id: str, + outcome: StageOutcome, + evidence: StageEvidence, + diagnostic_code: str | None = None, + diagnostic_message: str | None = None, + ) -> StageAttempt: + finished_at = utc_now() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + row = connection.execute( + "SELECT * FROM feasibility_stage_attempts WHERE id = ?", (attempt_id,) + ).fetchone() + if row is None: + raise StageAttemptNotFound(f"stage attempt not found: {attempt_id}") + if row["outcome"] is not None: + raise StageAttemptFinishedError(f"stage attempt already finished: {attempt_id}") + candidate = _attempt_from_row(row).model_copy( + update={ + "outcome": outcome, + "finished_at": finished_at, + "evidence": evidence, + "diagnostic_code": diagnostic_code, + "diagnostic_message": diagnostic_message, + } + ) + cursor = connection.execute( + "UPDATE feasibility_stage_attempts SET outcome = ?, finished_at = ?, " + "evidence_json = ?, diagnostic_code = ?, diagnostic_message = ? " + "WHERE id = ? AND outcome IS NULL", + ( + candidate.outcome.value if candidate.outcome is not None else None, + candidate.finished_at.isoformat() if candidate.finished_at else None, + canonical_json(evidence), + candidate.diagnostic_code, + candidate.diagnostic_message, + attempt_id, + ), + ) + if cursor.rowcount != 1: + raise StageAttemptFinishedError(f"stage attempt already finished: {attempt_id}") + connection.commit() + except Exception: + connection.rollback() + raise + return candidate + + def list_attempts( + self, run_id: str, stage: FeasibilityStage | None = None + ) -> tuple[StageAttempt, ...]: + with self.database.connect() as connection: + exists = connection.execute( + "SELECT 1 FROM feasibility_runs WHERE id = ?", (run_id,) + ).fetchone() + if exists is None: + raise FeasibilityRunNotFound(f"feasibility run not found: {run_id}") + query = "SELECT * FROM feasibility_stage_attempts WHERE run_id = ?" + parameters: tuple[str, ...] = (run_id,) + if stage is not None: + query += " AND stage = ?" + parameters = (run_id, stage.value) + query += " ORDER BY started_at, stage, ordinal, id" + rows = connection.execute(query, parameters).fetchall() + return tuple(_attempt_from_row(row) for row in rows) + + def recover_interrupted(self, run_id: str | None = None) -> tuple[StageAttempt, ...]: + finished_at = utc_now() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + if run_id is not None: + exists = connection.execute( + "SELECT 1 FROM feasibility_runs WHERE id = ?", (run_id,) + ).fetchone() + if exists is None: + raise FeasibilityRunNotFound(f"feasibility run not found: {run_id}") + query = "SELECT * FROM feasibility_stage_attempts WHERE outcome IS NULL" + parameters: tuple[str, ...] = () + if run_id is not None: + query += " AND run_id = ?" + parameters = (run_id,) + query += " ORDER BY started_at, stage, ordinal, id" + rows = connection.execute(query, parameters).fetchall() + evidence = StageEvidence() + recovered: list[StageAttempt] = [] + for row in rows: + attempt = _attempt_from_row(row).model_copy( + update={ + "outcome": StageOutcome.INTERRUPTED, + "finished_at": finished_at, + "evidence": evidence, + "diagnostic_code": "process_interrupted", + "diagnostic_message": "active attempt interrupted before completion", + } + ) + connection.execute( + "UPDATE feasibility_stage_attempts SET outcome = ?, finished_at = ?, " + "evidence_json = ?, diagnostic_code = ?, diagnostic_message = ? " + "WHERE id = ? AND outcome IS NULL", + ( + attempt.outcome.value if attempt.outcome is not None else None, + finished_at.isoformat(), + canonical_json(evidence), + attempt.diagnostic_code, + attempt.diagnostic_message, + attempt.id, + ), + ) + recovered.append(attempt) + connection.commit() + except Exception: + connection.rollback() + raise + return tuple(recovered) + + +def _latest_attempts(attempts: tuple[StageAttempt, ...]) -> dict[FeasibilityStage, StageAttempt]: + latest: dict[FeasibilityStage, StageAttempt] = {} + for attempt in attempts: + if attempt.stage not in latest or attempt.ordinal > latest[attempt.stage].ordinal: + latest[attempt.stage] = attempt + return latest + + +def _run_from_row(row: sqlite3.Row) -> FeasibilityRun: + return FeasibilityRun( + id=str(row["id"]), + adapter_id=str(row["adapter_id"]), + protocol_version=str(row["protocol_version"]), + input_fingerprint=str(row["input_fingerprint"]), + configuration=json.loads(str(row["configuration_json"])), + verdict=SupportVerdict(str(row["verdict"])), + created_at=datetime.fromisoformat(str(row["created_at"])), + updated_at=datetime.fromisoformat(str(row["updated_at"])), + ) + + +def _attempt_from_row(row: sqlite3.Row) -> StageAttempt: + outcome = row["outcome"] + evidence_json = row["evidence_json"] + return StageAttempt( + id=str(row["id"]), + run_id=str(row["run_id"]), + stage=FeasibilityStage(str(row["stage"])), + ordinal=int(row["ordinal"]), + input_fingerprint=str(row["input_fingerprint"]), + outcome=None if outcome is None else StageOutcome(str(outcome)), + started_at=datetime.fromisoformat(str(row["started_at"])), + finished_at=( + None if row["finished_at"] is None else datetime.fromisoformat(str(row["finished_at"])) + ), + evidence=( + None if evidence_json is None else StageEvidence.model_validate_json(str(evidence_json)) + ), + diagnostic_code=(None if row["diagnostic_code"] is None else str(row["diagnostic_code"])), + diagnostic_message=( + None if row["diagnostic_message"] is None else str(row["diagnostic_message"]) + ), + ) + + +def _resolved_sources_from_attempt( + attempt: StageAttempt, +) -> dict[str, ResolvedSource] | None: + evidence = attempt.evidence + inventory = None if evidence is None else evidence.details.get("inventory") + values = inventory.get("sources") if isinstance(inventory, Mapping) else None + if not isinstance(values, tuple): + return None + resolved: dict[str, ResolvedSource] = {} + for value in values: + if not isinstance(value, Mapping): + return None + files_value = value.get("files") + if not isinstance(files_value, tuple): + return None + files = tuple( + SourceFile( + relative_path=str(item["relative_path"]), + size=int(cast(Any, item["size"])), + upstream_sha256=( + None if item.get("upstream_sha256") is None else str(item["upstream_sha256"]) + ), + local_sha256=( + None if item.get("local_sha256") is None else str(item["local_sha256"]) + ), + ) + for item in files_value + if isinstance(item, Mapping) + ) + source = ResolvedSource( + component=str(value["component"]), + repo_id=str(value["repo_id"]), + requested_revision=str(value["requested_revision"]), + revision=str(value["revision"]), + license=str(value["license"]), + files=files, + ) + resolved[source.component] = source + return resolved if len(resolved) == len(values) else None diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_telemetry.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_telemetry.py new file mode 100644 index 0000000..fd74075 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/feasibility_telemetry.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import importlib +import os +import resource +import sys +from dataclasses import dataclass +from time import monotonic +from typing import Any, cast + +from video_lab.feasibility_models import FeasibilityStage, StageEvidence + +_INFERENCE_STAGES = { + FeasibilityStage.LOAD_TEXT_STACK, + FeasibilityStage.LOAD_VAE, + FeasibilityStage.LOAD_HIGH_TRANSFORMER, + FeasibilityStage.LOAD_LOW_TRANSFORMER, + FeasibilityStage.ASSEMBLE_PIPELINE, + FeasibilityStage.GENERATE_MINIMUM_1, + FeasibilityStage.GENERATE_MINIMUM_2, + FeasibilityStage.QUALIFY_STANDARD, + FeasibilityStage.VALIDATE_CONTROLLED_PAIR, +} +_MPS_RUNTIME: object | None = None + + +def mark_mps_context_ready(torch_runtime: object) -> None: + global _MPS_RUNTIME + _MPS_RUNTIME = torch_runtime + + +def _unavailable(reason: str) -> dict[str, object]: + return {"availability": reason} + + +def _process_snapshot() -> dict[str, object]: + try: + psutil = importlib.import_module("psutil") + process = psutil.Process() + virtual = psutil.virtual_memory() + swap = psutil.swap_memory() + return { + "rss_bytes": int(process.memory_info().rss), + "system_used_bytes": int(virtual.used), + "system_available_bytes": int(virtual.available), + "swap_used_bytes": int(swap.used), + "availability": "available", + } + except Exception as error: + return _unavailable(type(error).__name__) + + +def _rss_peak() -> tuple[int | None, str]: + try: + value = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + if not sys.platform.startswith("darwin"): + value *= 1024 + return value, "available" + except Exception as error: + return None, type(error).__name__ + + +def _mps_snapshot(stage: FeasibilityStage) -> dict[str, object]: + if stage not in _INFERENCE_STAGES: + return _unavailable("not_applicable") + if _MPS_RUNTIME is None: + return _unavailable("context_not_initialized") + try: + mps = cast(Any, _MPS_RUNTIME).mps + return { + "allocated": int(mps.current_allocated_memory()), + "driver": int(mps.driver_allocated_memory()), + "availability": "available", + } + except Exception as error: + return _unavailable(type(error).__name__) + + +@dataclass +class StageTelemetryCapture: + stage: FeasibilityStage + started: float + before_process: dict[str, object] + before_mps: dict[str, object] + + @classmethod + def start(cls, stage: FeasibilityStage) -> StageTelemetryCapture: + return cls(stage, monotonic(), _process_snapshot(), _mps_snapshot(stage)) + + def merge(self, evidence: StageEvidence) -> StageEvidence: + after_process = _process_snapshot() + after_mps = _mps_snapshot(self.stage) + peak_rss, peak_availability = _rss_peak() + samples = (self.before_process, after_process) + before_allocated = self.before_mps.get("allocated") + after_allocated = after_mps.get("allocated") + before_driver = self.before_mps.get("driver") + after_driver = after_mps.get("driver") + + def peak(first: object, second: object) -> int | None: + values = [value for value in (first, second) if isinstance(value, int)] + return max(values) if values else None + + current_rss = after_process.get("rss_bytes") + telemetry: dict[str, Any] = { + "pid": os.getpid(), + "sample_count": len(samples), + "samples": samples, + "rss_bytes": { + "current": current_rss if isinstance(current_rss, int) else None, + "peak": peak_rss, + "availability": peak_availability, + }, + "mps_bytes": { + "allocated_before": before_allocated if isinstance(before_allocated, int) else None, + "allocated_endpoint_max": peak(before_allocated, after_allocated), + "allocated_after": after_allocated if isinstance(after_allocated, int) else None, + "driver_before": before_driver if isinstance(before_driver, int) else None, + "driver_endpoint_max": peak(before_driver, after_driver), + "driver_after": after_driver if isinstance(after_driver, int) else None, + "availability": str(after_mps.get("availability", "unavailable")), + }, + } + measurements = dict(evidence.measurements) + measurements["stage_wall_seconds"] = max(0.0, monotonic() - self.started) + details = evidence.model_dump(mode="json")["details"] + details["telemetry"] = telemetry + return StageEvidence(measurements=measurements, details=details) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/host_canaries/__init__.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/host_canaries/__init__.py new file mode 100644 index 0000000..e6902be --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/host_canaries/__init__.py @@ -0,0 +1,29 @@ +"""Strictly local host-canary orchestration.""" + +from video_lab.host_canaries.wan_curated import ( + WanProductionCanaryExecutionV2, + run_wan_curated_host_canary, +) +from video_lab.host_canaries.wan_qualification import ( + WAN_PRODUCTION_AUTHORITY_NAMES, + WanProductionAuthorityName, + WanQualificationCandidateV1, + WanQualificationManifestV1, + WanQualificationRejectionV1, + WanQualificationSearchV1, + discover_wan_production_authorities, + load_wan_qualification_manifest, +) + +__all__ = ( + "WAN_PRODUCTION_AUTHORITY_NAMES", + "WanProductionAuthorityName", + "WanProductionCanaryExecutionV2", + "WanQualificationCandidateV1", + "WanQualificationManifestV1", + "WanQualificationRejectionV1", + "WanQualificationSearchV1", + "discover_wan_production_authorities", + "load_wan_qualification_manifest", + "run_wan_curated_host_canary", +) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/host_canaries/wan_curated.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/host_canaries/wan_curated.py new file mode 100644 index 0000000..acf007d --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/host_canaries/wan_curated.py @@ -0,0 +1,1046 @@ +"""Sequential, strictly local Wan production canary orchestration.""" + +from __future__ import annotations + +import base64 +import fcntl +import gc +import hashlib +import importlib.metadata +import os +import platform +import shutil +import stat +import uuid +from collections.abc import Callable +from contextlib import ExitStack +from dataclasses import asdict, dataclass, replace +from functools import partial +from pathlib import Path +from typing import Any, Literal, Protocol, cast + +from pydantic import ValidationError + +from video_lab.artifacts import ArtifactStore +from video_lab.database import Database +from video_lab.dataset_receipts import DatasetCompilationRepository +from video_lab.domain import canonical_json +from video_lab.local_authority import ( + RetainedDirectoryAuthority, + RetainedSqliteAuthority, +) +from video_lab.trainer_protocol import LoRAConfiguration +from video_lab.trainers.wan_branch_loader import ( + WanBranchLoadRequest, + WanMpsPreparedFactory, + inventory_wan_branch, +) +from video_lab.trainers.wan_branch_runtime import ( + WanMpsBranchOperations, + is_recognized_mps_oom, +) +from video_lab.trainers.wan_factory import _is_mps_device +from video_lab.trainers.wan_mps import WanTrainingSourceLease, WanTrainingSources +from video_lab.trainers.wan_production_models import ( + WanBranchCanaryResultV1, + WanBranchNumericalProbeV2, + WanBranchRecipeV1, + WanCanaryPhase, + WanCapacityResultV2, + WanComponentInventoryV1, + WanProductionCanaryReceiptV2, + WanProductionCanaryRequestV2, + WanProductionCanaryUnavailableV2, + WanTransformerBranch, + wan_branch_recipe_digest, + wan_inventory_digest, + wan_inventory_family_digest, + wan_recipe_family_digest, + wan_required_tensor_bytes, +) + + +class WanCanaryCancelled(RuntimeError): + """Cancellation observed at a declared lifecycle boundary.""" + + +class WanCapacityExceeded(RuntimeError): + def __init__(self, result: WanCapacityResultV2) -> None: + self.result = result + super().__init__("wan_mps_capacity_exceeded") + + +@dataclass(frozen=True, slots=True) +class WanProductionCanaryExecutionV2: + """Concrete local authorities required to execute, restore, and publish.""" + + request: WanProductionCanaryRequestV2 + branch_load_request: WanBranchLoadRequest + training_sources: WanTrainingSources + training_source_lease: WanTrainingSourceLease + compilation_receipt_id: str + checkpoint_root: Path + output_root: Path + + def __post_init__(self) -> None: + if ( + type(self.request) is not WanProductionCanaryRequestV2 + or type(self.branch_load_request) is not WanBranchLoadRequest + or type(self.training_sources) is not WanTrainingSources + or type(self.training_source_lease) is not WanTrainingSourceLease + or self.training_source_lease.sources != self.training_sources + or type(self.compilation_receipt_id) is not str + or not self.compilation_receipt_id + ): + raise ValueError("wan_canary_execution_authority_invalid") + for root in (self.checkpoint_root, self.output_root): + if not root.is_absolute() or not root.is_dir() or root.is_symlink(): + raise ValueError("wan_canary_execution_authority_invalid") + if self.checkpoint_root == self.output_root: + raise ValueError("wan_canary_execution_authority_invalid") + + +def require_strict_wan_production_execution( + execution: object, +) -> WanProductionCanaryExecutionV2: + if type(execution) is not WanProductionCanaryExecutionV2: + raise ValueError("wan_canary_execution_authority_invalid") + try: + request = WanProductionCanaryRequestV2.model_validate( + execution.request.model_dump(mode="python"), strict=True + ) + except (AttributeError, ValidationError, TypeError, ValueError): + raise ValueError("wan_canary_execution_authority_invalid") from None + if request != execution.request: + raise ValueError("wan_canary_execution_authority_invalid") + return replace(execution, request=request) + + +class _Preflight(Protocol): + def __call__(self) -> None: ... + + +@dataclass(frozen=True, slots=True) +class _WanCanaryHooks: + preflight: _Preflight + inventory: Callable[ + [WanTransformerBranch, WanComponentInventoryV1], + WanComponentInventoryV1 | WanProductionCanaryUnavailableV2, + ] + device_preflight: Callable[[], None] + load: Callable[[WanTransformerBranch, Literal["trained", "restored"]], object] + step: Callable[[object, WanBranchRecipeV1], object] + checkpoint: Callable[[object], object] + restore: Callable[[object, WanBranchRecipeV1], None] + continuity: Callable[ + [object, WanBranchRecipeV1], + WanBranchCanaryResultV1 | _WanContinuityOutcome, + ] + close: Callable[[object], object] + memory_released: Callable[[WanTransformerBranch, Literal["trained", "restored"], int], int] + publish: Callable[[WanProductionCanaryReceiptV2], WanProductionCanaryReceiptV2] + capacity_observation: Callable[[], tuple[int, int]] + capacity_cleanup: Callable[[], None] + + +class _CloseOnce: + def __init__(self, close: Callable[[object], object], handle: object) -> None: + self._close, self._handle = close, handle + + def __call__(self) -> None: + handle, self._handle = self._handle, None + if handle is not None: + self._close(handle) + + +@dataclass(frozen=True, slots=True) +class _WanContinuityOutcome: + result: WanBranchCanaryResultV1 + probe: WanBranchNumericalProbeV2 + + +@dataclass(frozen=True, slots=True) +class _RetainedReadonlyDatabase: + authority: RetainedSqliteAuthority + + @property + def path(self) -> Path: + return self.authority.path + + def connect(self) -> Any: + return self.authority.connect_readonly() + + +def _paths_overlap(left: Path, right: Path) -> bool: + return left == right or left.is_relative_to(right) or right.is_relative_to(left) + + +def _validate_execution_roots( + execution: WanProductionCanaryExecutionV2, + output: RetainedDirectoryAuthority, + checkpoint: RetainedDirectoryAuthority, +) -> None: + model_root = execution.branch_load_request.model_root.path + source_files = tuple(item.path for item in execution.training_source_lease.sources.files) + protected = (model_root, *(item.parent for item in source_files), *source_files) + if _paths_overlap(output.path, checkpoint.path) or any( + _paths_overlap(root.path, source) + for root in (output, checkpoint) + for source in protected + ): + raise ValueError("wan_canary_root_overlap") + try: + source_roots = {item.parent for item in source_files} + if any( + os.stat(path, follow_symlinks=False).st_mode & 0o222 + for path in (*source_files, *source_roots, model_root) + ): + raise ValueError + except (OSError, ValueError): + raise ValueError("wan_canary_source_writable") from None + execution.training_source_lease.reverify() + output.reverify() + checkpoint.reverify() + + +def _guard_hooks( + hooks: _WanCanaryHooks, + execution: WanProductionCanaryExecutionV2, + output: RetainedDirectoryAuthority, + checkpoint: RetainedDirectoryAuthority, +) -> _WanCanaryHooks: + def guard(operation: Callable[..., Any]) -> Callable[..., Any]: + def checked(*args: Any, **kwargs: Any) -> Any: + _validate_execution_roots(execution, output, checkpoint) + result = operation(*args, **kwargs) + _validate_execution_roots(execution, output, checkpoint) + return result + + return checked + + return _WanCanaryHooks( + preflight=guard(hooks.preflight), + inventory=guard(hooks.inventory), + device_preflight=guard(hooks.device_preflight), + load=guard(hooks.load), + step=guard(hooks.step), + checkpoint=guard(hooks.checkpoint), + restore=guard(hooks.restore), + continuity=guard(hooks.continuity), + close=guard(hooks.close), + memory_released=guard(hooks.memory_released), + publish=guard(hooks.publish), + capacity_observation=guard(hooks.capacity_observation), + capacity_cleanup=guard(hooks.capacity_cleanup), + ) + + +def _selected(request: WanBranchLoadRequest, branch: WanTransformerBranch) -> WanBranchLoadRequest: + selected = replace(request, branch=branch) + object.__setattr__(selected, "_authority_signature", request._authority_signature) + object.__setattr__(selected, "_snapshot_parent_identity", request._snapshot_parent_identity) + return selected + + +def _numerical_probe( + branch: WanTransformerBranch, + recipe: WanBranchRecipeV1, + mapping: object, +) -> WanBranchNumericalProbeV2: + if type(mapping) is not dict or set(mapping) != {"probe.output"}: + raise ValueError("wan_canary_probe_invalid") + tensor = cast(dict[str, Any], mapping)["probe.output"] + if not all(hasattr(tensor, attribute) for attribute in ("detach", "to", "reshape", "tolist")): + raise ValueError("wan_canary_probe_invalid") + observed = tensor.detach() + dtype = str(observed.dtype).removeprefix("torch.") + if dtype not in {"bfloat16", "float32"}: + raise ValueError("wan_canary_probe_invalid") + shape = tuple(int(size) for size in observed.shape) + values = observed.to("cpu").reshape(-1).tolist() + return WanBranchNumericalProbeV2( + branch=branch, + recipe_digest=wan_branch_recipe_digest(recipe), + tensor_name="probe.output", + shape=shape, + dtype=cast(Literal["bfloat16", "float32"], dtype), + values=tuple(float(value) for value in values), + ) + + +def _compare_numerical_probes( + reference: WanBranchNumericalProbeV2, + candidate: WanBranchNumericalProbeV2, + recipe: WanBranchRecipeV1, +) -> tuple[float, float]: + if ( + reference.branch != candidate.branch + or reference.recipe_digest != candidate.recipe_digest + or reference.tensor_name != candidate.tensor_name + or reference.shape != candidate.shape + or reference.dtype != candidate.dtype + or reference.recipe_digest != wan_branch_recipe_digest(recipe) + ): + raise ValueError("wan_canary_continuity_mismatch") + maximum_absolute = 0.0 + maximum_relative = 0.0 + for expected, observed in zip(reference.values, candidate.values, strict=True): + absolute = abs(observed - expected) + relative = absolute / max(abs(expected), recipe.tolerance.atol) + maximum_absolute = max(maximum_absolute, absolute) + maximum_relative = max(maximum_relative, relative) + if absolute > recipe.tolerance.atol + recipe.tolerance.rtol * abs(expected): + raise ValueError("wan_canary_continuity_mismatch") + return maximum_absolute, maximum_relative + + +def _acquire_checkpoint_digest_lock( + root_descriptor: int, final_name: str +) -> tuple[int, int]: + """Retain a stable per-digest flock in the app-owned checkpoint root. + + Lock files intentionally outlive callers. Unlinking one could split current + waiters on the old inode from a later publisher locking a newly created inode. + """ + + lock_directory = -1 + lock_descriptor = -1 + try: + try: + os.mkdir(".checkpoint-locks", mode=0o700, dir_fd=root_descriptor) + except FileExistsError: + pass + lock_directory = os.open( + ".checkpoint-locks", + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=root_descriptor, + ) + if not stat.S_ISDIR(os.fstat(lock_directory).st_mode): + raise OSError("checkpoint lock namespace is not a directory") + lock_descriptor = os.open( + f"{final_name}.lock", + os.O_RDWR + | os.O_CREAT + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + 0o600, + dir_fd=lock_directory, + ) + if not stat.S_ISREG(os.fstat(lock_descriptor).st_mode): + raise OSError("checkpoint digest lock is not a regular file") + fcntl.flock(lock_descriptor, fcntl.LOCK_EX) + return lock_directory, lock_descriptor + except OSError: + if lock_descriptor >= 0: + os.close(lock_descriptor) + if lock_directory >= 0: + os.close(lock_directory) + raise ValueError("wan_canary_checkpoint_lock_failed") from None + + +def _publish_checkpoint( + root: Path, + branch: WanTransformerBranch, + checkpoint: Any, + *, + root_authority: RetainedDirectoryAuthority, +) -> None: + payload = canonical_json( + { + "manifest": asdict(checkpoint.manifest), + "weights": base64.b64encode(checkpoint.weights).decode("ascii"), + "optimizer": dict(checkpoint.optimizer), + "cpu_rng_state": base64.b64encode(checkpoint.cpu_rng_state).decode("ascii"), + "mps_rng_state": base64.b64encode(checkpoint.mps_rng_state).decode("ascii"), + } + ).encode() + digest = hashlib.sha256(payload).hexdigest() + final_name = f"{branch}-{digest}.checkpoint.json" + temporary_name = f".{final_name}.{uuid.uuid4().hex}.tmp" + if root != root_authority.path: + raise ValueError(root_authority.error_code) + try: + root_descriptor = os.dup(root_authority.descriptor) + except OSError: + raise ValueError(root_authority.error_code) from None + lock_directory = -1 + lock_descriptor = -1 + descriptor = -1 + final_created = False + temporary_created = False + staged_identity: tuple[int, int] | None = None + try: + lock_directory, lock_descriptor = _acquire_checkpoint_digest_lock( + root_descriptor, final_name + ) + root_authority.reverify() + try: + descriptor = os.open( + final_name, + os.O_RDONLY | os.O_NOFOLLOW, + dir_fd=root_descriptor, + ) + except FileNotFoundError: + descriptor = os.open( + temporary_name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=root_descriptor, + ) + temporary_created = True + staged = os.fstat(descriptor) + if not stat.S_ISREG(staged.st_mode): + raise ValueError("wan_canary_checkpoint_integrity_failed") from None + staged_identity = (staged.st_dev, staged.st_ino) + remaining = memoryview(payload) + while remaining: + written = os.write(descriptor, remaining) + if written <= 0: + raise OSError("wan_canary_checkpoint_write_failed") from None + remaining = remaining[written:] + os.fsync(descriptor) + os.close(descriptor) + descriptor = -1 + try: + os.link( + temporary_name, + final_name, + src_dir_fd=root_descriptor, + dst_dir_fd=root_descriptor, + follow_symlinks=False, + ) + final_created = True + except FileExistsError: + descriptor = os.open( + final_name, + os.O_RDONLY | os.O_NOFOLLOW, + dir_fd=root_descriptor, + ) + observed = os.pread(descriptor, len(payload) + 1, 0) + if not stat.S_ISREG(os.fstat(descriptor).st_mode) or observed != payload: + raise ValueError("wan_canary_checkpoint_integrity_failed") from None + os.unlink(temporary_name, dir_fd=root_descriptor) + temporary_created = False + os.fsync(root_descriptor) + else: + observed = os.pread(descriptor, len(payload) + 1, 0) + if not stat.S_ISREG(os.fstat(descriptor).st_mode) or observed != payload: + raise ValueError("wan_canary_checkpoint_integrity_failed") + if descriptor >= 0: + os.close(descriptor) + descriptor = -1 + root_authority.reverify() + except BaseException as primary: + cleanup_errors: list[BaseException] = [] + if descriptor >= 0: + try: + os.close(descriptor) + except BaseException as cleanup_error: + cleanup_errors.append(cleanup_error) + descriptor = -1 + removed = False + if final_created: + try: + observed_final = os.stat( + final_name, + dir_fd=root_descriptor, + follow_symlinks=False, + ) + if (observed_final.st_dev, observed_final.st_ino) != staged_identity: + raise OSError("checkpoint publication ownership changed") + os.unlink(final_name, dir_fd=root_descriptor) + removed = True + except FileNotFoundError: + pass + except BaseException as cleanup_error: + cleanup_errors.append(cleanup_error) + if temporary_created: + try: + os.unlink(temporary_name, dir_fd=root_descriptor) + except FileNotFoundError: + pass + except BaseException as cleanup_error: + cleanup_errors.append(cleanup_error) + else: + removed = True + if removed: + try: + os.fsync(root_descriptor) + except BaseException as cleanup_error: + cleanup_errors.append(cleanup_error) + if cleanup_errors: + primary_message = str(primary) + normalized_primary = ( + ValueError(primary_message) + if isinstance(primary, ValueError) + and primary_message.startswith("wan_canary_") + and primary_message.replace("_", "").isalnum() + else ValueError("wan_canary_checkpoint_publication_failed") + ) + normalized_rollbacks = tuple( + ValueError("wan_canary_checkpoint_rollback_operation_failed") + for _error in cleanup_errors + ) + raise ValueError("wan_canary_checkpoint_rollback_failed") from BaseExceptionGroup( + "wan_canary_checkpoint_rollback_errors", + (normalized_primary, *normalized_rollbacks), + ) + raise + finally: + if lock_descriptor >= 0: + try: + os.close(lock_descriptor) + except OSError: + pass + if lock_directory >= 0: + try: + os.close(lock_directory) + except OSError: + pass + try: + os.close(root_descriptor) + except OSError: + pass + + +def _production_hooks( + execution: WanProductionCanaryExecutionV2, + database: Database, + artifacts: ArtifactStore, + *, + importer: Callable[[str], object] | None = None, + cancelled: Callable[[], bool] = lambda: False, + output_authority: RetainedDirectoryAuthority | None = None, + checkpoint_authority: RetainedDirectoryAuthority | None = None, + database_authority: RetainedSqliteAuthority | None = None, +) -> _WanCanaryHooks: + state: dict[str, object] = {} + library_importer = importer or importlib.import_module + + def preflight() -> None: + if artifacts.root != execution.output_root.resolve(): + raise ValueError("wan_canary_output_authority_invalid") + if platform.system() != "Darwin": + raise ValueError("wan_canary_host_unsupported") + if dict(execution.request.hardware_identity) != { + "device": "mps", "machine": platform.machine(), "system": platform.system() + }: + raise ValueError("wan_canary_hardware_identity_stale") + try: + observed_versions = tuple( + (name, importlib.metadata.version(name)) + for name, _ in execution.request.dependency_versions + ) + except importlib.metadata.PackageNotFoundError: + raise ValueError("wan_canary_dependency_unavailable") from None + if observed_versions != execution.request.dependency_versions: + raise ValueError("wan_canary_dependency_version_stale") + if any( + shutil.disk_usage(root).free < execution.request.required_artifact_bytes + for root in (execution.checkpoint_root, execution.output_root) + ): + raise ValueError("wan_canary_artifact_capacity_insufficient") + receipt_database = ( + database + if database_authority is None + else cast(Database, _RetainedReadonlyDatabase(database_authority)) + ) + receipt = DatasetCompilationRepository(receipt_database).receipt( + execution.compilation_receipt_id + ) + digest = "sha256:" + hashlib.sha256( + canonical_json(receipt.model_dump(mode="json")).encode() + ).hexdigest() + if digest != execution.request.compilation_receipt_digest: + raise ValueError("wan_canary_compilation_receipt_stale") + batch_source = next( + ( + item + for item in execution.training_source_lease.sources.files + if item.name == "batch_tensors" + ), + None, + ) + if batch_source is None or batch_source.digest != execution.request.batch_digest: + raise ValueError("wan_canary_batch_authority_stale") + + def device_preflight() -> None: + torch = cast(Any, library_importer("torch")) + if not torch.backends.mps.is_built() or not torch.backends.mps.is_available(): + raise ValueError("wan_canary_mps_unavailable") + probe: Any = None + try: + probe = torch.empty(1, device="mps", dtype=torch.bfloat16) + if not _is_mps_device(probe.device) or str(probe.dtype) not in { + "bfloat16", "torch.bfloat16" + }: + raise ValueError("wan_canary_bf16_unsupported") + except ValueError: + raise + except BaseException: + raise ValueError("wan_canary_bf16_unsupported") from None + finally: + probe = None + try: + torch.mps.synchronize() + torch.mps.empty_cache() + except BaseException: + raise ValueError("wan_canary_bf16_probe_cleanup_failed") from None + state["torch"] = torch + + def inventory( + branch: WanTransformerBranch, expected: WanComponentInventoryV1 + ) -> WanComponentInventoryV1 | WanProductionCanaryUnavailableV2: + result = inventory_wan_branch(_selected(execution.branch_load_request, branch)) + if not result.available or result.inventory is None: + return WanProductionCanaryUnavailableV2( + schema_version="wan-mps-production-canary-unavailable.v2", + branch=branch, + missing_roles=result.missing_roles, + ) + if result.inventory != expected: + raise ValueError("wan_canary_inventory_stale") + return result.inventory + + def load(branch: WanTransformerBranch, role: Literal["trained", "restored"]) -> object: + recipe = execution.request.recipes[0 if branch == "high_noise" else 1] + factory = WanMpsPreparedFactory( + _selected(execution.branch_load_request, branch), importer=library_importer + ) + prepared = factory.prepare_branch( + execution.training_source_lease, + object(), + recipe, + cancelled=cancelled, + ) + operations = cast(WanMpsBranchOperations, prepared.handle) + try: + operations.inject_lora( + targets=operations.eligible_lora_modules(), + configuration=LoRAConfiguration(rank=2, alpha=recipe.alpha, dropout=0.0), + seed=recipe.seed, + device="mps", + dtype="bfloat16", + ) + return operations + except BaseException as primary: + try: + operations.close() + except BaseException as cleanup: + raise BaseExceptionGroup( + "wan_branch_acquisition_cleanup_failed", (primary, cleanup) + ) from None + raise + + def step(handle: object, recipe: WanBranchRecipeV1) -> object: + operations = cast(WanMpsBranchOperations, handle) + result = operations.train_step(step=1, total_steps=1) + state[f"{recipe.branch}.reference"] = _numerical_probe( + recipe.branch, recipe, operations.continuity_probe() + ) + return result + + def checkpoint(handle: object) -> object: + operations = cast(WanMpsBranchOperations, handle) + checkpoint = operations.checkpoint_state_v2(parent_checkpoint_id=None) + authority = checkpoint_authority + owns_authority = authority is None + if authority is None: + authority = RetainedDirectoryAuthority.capture( + execution.checkpoint_root, + error_code="wan_canary_checkpoint_root_changed", + ) + try: + _publish_checkpoint( + execution.checkpoint_root, + operations.recipe.branch, + checkpoint, + root_authority=authority, + ) + finally: + if owns_authority: + authority.close() + state[f"{operations.recipe.branch}.checkpoint"] = checkpoint + return checkpoint + + def restore(handle: object, recipe: WanBranchRecipeV1) -> None: + operations = cast(WanMpsBranchOperations, handle) + checkpoint = cast(Any, state[f"{recipe.branch}.checkpoint"]) + operations.restore_checkpoint_v2(checkpoint, checkpoint.manifest) + + def continuity( + handle: object, recipe: WanBranchRecipeV1 + ) -> _WanContinuityOutcome: + operations = cast(WanMpsBranchOperations, handle) + checkpoint = cast(Any, state[f"{recipe.branch}.checkpoint"]) + probe = _numerical_probe( + recipe.branch, recipe, operations.continuity_probe() + ) + maximum_absolute, maximum_relative = _compare_numerical_probes( + cast(WanBranchNumericalProbeV2, state[f"{recipe.branch}.reference"]), + probe, + recipe, + ) + return _WanContinuityOutcome( + result=WanBranchCanaryResultV1( + schema_version="wan-mps-branch-canary-result.v1", + branch=recipe.branch, + recipe_digest=wan_branch_recipe_digest(recipe), + output_digest="sha256:" + hashlib.sha256(checkpoint.weights).hexdigest(), + max_absolute_error=maximum_absolute, + max_relative_error=maximum_relative, + ), + probe=probe, + ) + + def close(handle: object) -> object: + cast(WanMpsBranchOperations, handle).close() + return None + + def memory_released( + branch: WanTransformerBranch, role: Literal["trained", "restored"], bound: int + ) -> int: + del branch, role + torch = cast(Any, state["torch"]) + torch.mps.synchronize() + gc.collect() + torch.mps.empty_cache() + residual = int(torch.mps.current_allocated_memory()) + if residual > bound: + raise ValueError("wan_canary_memory_release_failed") + return residual + + def publish(receipt: WanProductionCanaryReceiptV2) -> WanProductionCanaryReceiptV2: + payload = canonical_json(receipt.model_dump(mode="json")).encode() + if output_authority is not None and type(artifacts) is ArtifactStore: + artifacts.commit_bytes_at_retained_root( + payload, + "application/vnd.video-lab.wan-canary+json", + output_authority.descriptor, + reverify=output_authority.reverify, + ) + return receipt + staged = artifacts.stage_bytes(payload, "application/vnd.video-lab.wan-canary+json") + try: + artifacts.commit(staged, None) + except BaseException: + artifacts.discard(staged) + raise + return receipt + + def capacity() -> tuple[int, int]: + torch = cast(Any, state["torch"]) + return ( + int(torch.mps.driver_allocated_memory()), + int(torch.mps.recommended_max_memory()), + ) + + def capacity_cleanup() -> None: + torch = cast(Any, state["torch"]) + torch.mps.synchronize() + gc.collect() + torch.mps.empty_cache() + + return _WanCanaryHooks( + preflight=preflight, + inventory=inventory, + device_preflight=device_preflight, + load=load, + step=step, + checkpoint=checkpoint, + restore=restore, + continuity=continuity, + close=close, + memory_released=memory_released, + publish=publish, + capacity_observation=capacity, + capacity_cleanup=capacity_cleanup, + ) + + +def _cancel(cancelled: Callable[[], bool]) -> None: + if cancelled(): + raise WanCanaryCancelled("wan_canary_cancelled") + + +def _capacity( + request: WanProductionCanaryRequestV2, + branch: WanTransformerBranch, + phase: WanCanaryPhase, + hooks: _WanCanaryHooks, + before: int, + after_failure: int, + after_cleanup: int, + recommended: int, +) -> WanCapacityResultV2: + del hooks + index = 0 if branch == "high_noise" else 1 + return WanCapacityResultV2( + schema_version="wan-mps-capacity-result.v2", branch=branch, phase=phase, + device="mps", dtype="bfloat16", + requested_tensor_envelope=request.requested_tensor_envelope, + driver_allocated_before_bytes=before, + driver_allocated_after_failure_bytes=after_failure, + driver_allocated_after_cleanup_bytes=after_cleanup, + recommended_maximum_bytes=recommended, + required_memory_bytes=wan_required_tensor_bytes(request.requested_tensor_envelope), + recipe_digest=wan_branch_recipe_digest(request.recipes[index]), + inventory_digest=wan_inventory_digest(request.inventories[index]), fits=False, + classification="recognized_mps_allocation_failure", + ) + + +class _ObservedMpsOom(RuntimeError): + def __init__( + self, + phase: WanCanaryPhase, + before: int, + after_failure: int, + recommended: int, + ) -> None: + self.phase = phase + self.before = before + self.after_failure = after_failure + self.recommended = recommended + super().__init__("wan_capacity_observed_mps_oom") + + +def _observed_call( + hooks: _WanCanaryHooks, + phase: WanCanaryPhase, + operation: Callable[[], Any], +) -> Any: + try: + before, recommended = hooks.capacity_observation() + except BaseException: + raise RuntimeError("wan_capacity_observation_unavailable") from None + try: + return operation() + except BaseException as error: + if not is_recognized_mps_oom(error): + raise + try: + after_failure, observed_recommended = hooks.capacity_observation() + except BaseException: + raise RuntimeError("wan_capacity_observation_unavailable") from error + if observed_recommended != recommended: + raise RuntimeError("wan_capacity_observation_unavailable") from error + raise _ObservedMpsOom(phase, before, after_failure, recommended) from None + + +def run_wan_curated_host_canary( + *, + execution: WanProductionCanaryExecutionV2, + database: Database, + artifacts: ArtifactStore, + cancelled: Callable[[], bool] = lambda: False, + _hooks: _WanCanaryHooks | None = None, + _importer: Callable[[str], object] | None = None, +) -> WanProductionCanaryReceiptV2 | WanProductionCanaryUnavailableV2: + """Run each trained and fresh-restored branch without overlapping residency.""" + + execution = require_strict_wan_production_execution(execution) + if os.environ.get("PYTORCH_ENABLE_MPS_FALLBACK", "0") != "0": + raise ValueError("wan_mps_fallback_forbidden") + if _hooks is not None: + return _run_wan_curated_host_canary( + execution=execution, + hooks=_hooks, + cancelled=cancelled, + hooks_are_injected=True, + ) + try: + lease = execution.training_source_lease + except AttributeError: + raise ValueError("wan_canary_source_lease_required") from None + if not hasattr(lease, "reverify") or not hasattr(lease, "sources"): + raise ValueError("wan_canary_source_lease_required") + with ExitStack() as authorities: + output = RetainedDirectoryAuthority.capture( + execution.output_root, + error_code="wan_canary_output_root_changed", + ) + authorities.callback(output.close) + checkpoint = RetainedDirectoryAuthority.capture( + execution.checkpoint_root, + error_code="wan_canary_checkpoint_root_changed", + ) + authorities.callback(checkpoint.close) + database_authority = RetainedSqliteAuthority.capture(database.path) + authorities.callback(database_authority.close) + _validate_execution_roots(execution, output, checkpoint) + hooks = _production_hooks( + execution, + database, + artifacts, + importer=_importer, + cancelled=cancelled, + output_authority=output, + checkpoint_authority=checkpoint, + database_authority=database_authority, + ) + return _run_wan_curated_host_canary( + execution=execution, + hooks=_guard_hooks(hooks, execution, output, checkpoint), + cancelled=cancelled, + hooks_are_injected=False, + ) + + +def _run_wan_curated_host_canary( + *, + execution: WanProductionCanaryExecutionV2, + hooks: _WanCanaryHooks, + cancelled: Callable[[], bool], + hooks_are_injected: bool, +) -> WanProductionCanaryReceiptV2 | WanProductionCanaryUnavailableV2: + hooks.preflight() + request = execution.request + results: list[WanBranchCanaryResultV1] = [] + probes: list[WanBranchNumericalProbeV2] = [] + releases: list[tuple[str, int]] = [] + branches: tuple[WanTransformerBranch, WanTransformerBranch] = ( + "high_noise", "low_noise" + ) + + for index, branch in enumerate(branches): + _cancel(cancelled) + inventory = hooks.inventory(branch, request.inventories[index]) + if isinstance(inventory, WanProductionCanaryUnavailableV2): + return inventory + if inventory != request.inventories[index]: + raise ValueError("wan_canary_inventory_stale") + + _cancel(cancelled) + hooks.device_preflight() + + for index, branch in enumerate(branches): + recipe = request.recipes[index] + try: + with ExitStack() as cleanup: + _cancel(cancelled) + trained = _observed_call( + hooks, + "load_trained", + partial(hooks.load, branch, "trained"), + ) + close_trained = _CloseOnce(hooks.close, trained) + cleanup.callback(close_trained) + _cancel(cancelled) + _observed_call( + hooks, + "step", + partial(hooks.step, trained, recipe), + ) + _cancel(cancelled) + _observed_call( + hooks, + "checkpoint", + partial(hooks.checkpoint, trained), + ) + _cancel(cancelled) + close_trained() + _cancel(cancelled) + residual = hooks.memory_released( + branch, "trained", request.memory_release_bound_bytes + ) + releases.append((f"{branch}.trained", residual)) + _cancel(cancelled) + restored = _observed_call( + hooks, + "load_fresh_restore", + partial(hooks.load, branch, "restored"), + ) + close_restored = _CloseOnce(hooks.close, restored) + cleanup.callback(close_restored) + _cancel(cancelled) + _observed_call( + hooks, + "restore", + partial(hooks.restore, restored, recipe), + ) + _cancel(cancelled) + outcome = _observed_call( + hooks, + "continuity", + partial(hooks.continuity, restored, recipe), + ) + if type(outcome) is _WanContinuityOutcome: + result = outcome.result + probe = outcome.probe + elif hooks_are_injected and type(outcome) is WanBranchCanaryResultV1: + result = outcome + probe = WanBranchNumericalProbeV2( + branch=recipe.branch, + recipe_digest=result.recipe_digest, + tensor_name="probe.output", + shape=(2,), + dtype="float32", + values=(result.max_absolute_error, result.max_relative_error), + ) + else: + raise ValueError("wan_canary_probe_invalid") + if ( + result.branch != branch + or result.recipe_digest != wan_branch_recipe_digest(recipe) + ): + raise ValueError("wan_canary_continuity_mismatch") + results.append(result) + probes.append(probe) + _cancel(cancelled) + close_restored() + _cancel(cancelled) + residual = hooks.memory_released( + branch, "restored", request.memory_release_bound_bytes + ) + releases.append((f"{branch}.restored", residual)) + except _ObservedMpsOom as error: + try: + hooks.capacity_cleanup() + after_cleanup, after_recommended = hooks.capacity_observation() + except BaseException: + raise RuntimeError("wan_capacity_observation_unavailable") from error + if after_recommended != error.recommended: + raise RuntimeError("wan_capacity_observation_unavailable") from error + raise WanCapacityExceeded( + _capacity( + request, + branch, + error.phase, + hooks, + error.before, + error.after_failure, + after_cleanup, + error.recommended, + ) + ) from None + _cancel(cancelled) + receipt = WanProductionCanaryReceiptV2( + schema_version="wan-mps-production-canary-receipt.v2", + branch_results=(results[0], results[1]), + branch_probes=(probes[0], probes[1]), + compilation_receipt_digest=request.compilation_receipt_digest, + batch_digest=request.batch_digest, + inventory_family_digest=wan_inventory_family_digest(request.inventories), + recipe_family_digest=wan_recipe_family_digest(request.recipes), + primary_run_id=request.primary_run_id, + independent_run_id=request.independent_run_id, + dependency_versions=request.dependency_versions, + hardware_identity=request.hardware_identity, + memory_release_bytes=(releases[0], releases[1], releases[2], releases[3]), + cleanup_state="cleaned", + ) + return hooks.publish(receipt) + + +__all__ = ( + "WanCanaryCancelled", + "WanCapacityExceeded", + "WanProductionCanaryExecutionV2", + "WanProductionCanaryUnavailableV2", + "require_strict_wan_production_execution", + "run_wan_curated_host_canary", +) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/host_canaries/wan_qualification.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/host_canaries/wan_qualification.py new file mode 100644 index 0000000..7516711 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/host_canaries/wan_qualification.py @@ -0,0 +1,1555 @@ +"""Bounded, read-only discovery of explicit local Wan production authorities.""" + +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +import os +import re +import sqlite3 +import stat +from collections.abc import Iterator, Mapping +from copy import deepcopy +from dataclasses import dataclass +from pathlib import Path, PurePath, PureWindowsPath +from typing import Annotated, Any, Literal, NoReturn, Self, cast + +from pydantic import ConfigDict, Field, StrictInt, field_validator, model_validator + +from video_lab.dataset_receipts import ( + CompilationOutput, + DatasetCompilationReceipt, + DatasetCompilationRequest, + InputBinding, +) +from video_lab.domain import FrozenModel, canonical_json +from video_lab.local_authority import RetainedDirectoryAuthority, RetainedSqliteAuthority +from video_lab.trainers.wan_production_models import ( + WanNumericalToleranceV1, + WanTensorEnvelopeV2, +) +from video_lab.training_models import Sha256Digest + +WanProductionAuthorityName = Literal[ + "VIDEO_LAB_WAN_ADAPTER_PATHS_JSON", + "VIDEO_LAB_WAN_ARTIFACT_DATABASE", + "VIDEO_LAB_WAN_ARTIFACT_ROOT", + "VIDEO_LAB_WAN_CACHE_REVISION", + "VIDEO_LAB_WAN_CACHE_ROOT", + "VIDEO_LAB_WAN_COMPILATION_OPERATION_ID", + "VIDEO_LAB_WAN_COMPILATION_RECEIPT_ID", + "VIDEO_LAB_WAN_DEPENDENCY_VERSIONS_JSON", + "VIDEO_LAB_WAN_MEMORY_RELEASE_BOUND_BYTES", + "VIDEO_LAB_WAN_MODEL_REVISION", + "VIDEO_LAB_WAN_MODEL_ROOT", + "VIDEO_LAB_WAN_REQUIRED_ARTIFACT_BYTES", + "VIDEO_LAB_WAN_TENSOR_ENVELOPE_JSON", + "VIDEO_LAB_WAN_TOLERANCE_JSON", +] + +WAN_PRODUCTION_AUTHORITY_NAMES: tuple[WanProductionAuthorityName, ...] = ( + "VIDEO_LAB_WAN_ADAPTER_PATHS_JSON", + "VIDEO_LAB_WAN_ARTIFACT_DATABASE", + "VIDEO_LAB_WAN_ARTIFACT_ROOT", + "VIDEO_LAB_WAN_CACHE_REVISION", + "VIDEO_LAB_WAN_CACHE_ROOT", + "VIDEO_LAB_WAN_COMPILATION_OPERATION_ID", + "VIDEO_LAB_WAN_COMPILATION_RECEIPT_ID", + "VIDEO_LAB_WAN_DEPENDENCY_VERSIONS_JSON", + "VIDEO_LAB_WAN_MEMORY_RELEASE_BOUND_BYTES", + "VIDEO_LAB_WAN_MODEL_REVISION", + "VIDEO_LAB_WAN_MODEL_ROOT", + "VIDEO_LAB_WAN_REQUIRED_ARTIFACT_BYTES", + "VIDEO_LAB_WAN_TENSOR_ENVELOPE_JSON", + "VIDEO_LAB_WAN_TOLERANCE_JSON", +) + +_AUTHORITY_NAMES = frozenset(WAN_PRODUCTION_AUTHORITY_NAMES) +_RECIPE_EVIDENCE_NAMES: tuple[WanProductionAuthorityName, ...] = ( + "VIDEO_LAB_WAN_ADAPTER_PATHS_JSON", + "VIDEO_LAB_WAN_CACHE_REVISION", + "VIDEO_LAB_WAN_DEPENDENCY_VERSIONS_JSON", + "VIDEO_LAB_WAN_MEMORY_RELEASE_BOUND_BYTES", + "VIDEO_LAB_WAN_MODEL_REVISION", + "VIDEO_LAB_WAN_REQUIRED_ARTIFACT_BYTES", + "VIDEO_LAB_WAN_TENSOR_ENVELOPE_JSON", + "VIDEO_LAB_WAN_TOLERANCE_JSON", +) +_SCIENTIFIC_EVIDENCE_NAMES: tuple[WanProductionAuthorityName, ...] = tuple( + sorted( + ( + *_RECIPE_EVIDENCE_NAMES, + "VIDEO_LAB_WAN_COMPILATION_OPERATION_ID", + "VIDEO_LAB_WAN_COMPILATION_RECEIPT_ID", + ) + ) +) +_RECIPE_EVIDENCE_SET = frozenset(_RECIPE_EVIDENCE_NAMES) +_SCIENTIFIC_EVIDENCE_SET = frozenset(_SCIENTIFIC_EVIDENCE_NAMES) +_CANDIDATE_FILENAME = "wan-production-qualification-candidate.v1.json" +_MAXIMUM_CANDIDATE_BYTES = 16 * 1024 * 1024 +_MAXIMUM_AUTHORITY_BYTES = 64 * 1024 * 1024 * 1024 +_MAXIMUM_AUTHORITY_DEPTH = 32 +_MAXIMUM_SEARCH_DIRECTORIES = 250_000 +_MAXIMUM_SEARCH_FILES = 2_000_000 +_MAXIMUM_ROOT_INVENTORY_DIRECTORIES = 25_000 +_MAXIMUM_ROOT_INVENTORY_FILES = 100_000 +_REVISION = re.compile(r"^[0-9a-f]{40}$") +_DEPENDENCIES = ("diffusers", "peft", "safetensors", "torch", "transformers") +_ADAPTER_ROLES = ( + "high_transformer", + "low_transformer", + "scheduler", + "text_encoder", + "tokenizer", + "transformer_config", + "vae", + "wan_snapshot", +) + + +class _StrictQualificationModel(FrozenModel): + model_config = ConfigDict( + frozen=True, + extra="forbid", + strict=True, + revalidate_instances="always", + hide_input_in_errors=True, + ) + + def model_copy( + self, *, update: Mapping[str, Any] | None = None, deep: bool = False + ) -> Self: + values = {name: getattr(self, name) for name in type(self).model_fields} + if deep: + values = deepcopy(values) + if update is not None: + values.update(update) + return type(self).model_validate(values, strict=True) + + +def _exact_pairs(value: object, *, label: str) -> object: + if type(value) is not tuple: + raise ValueError(f"{label} must be an exact tuple") + for entry in cast(tuple[object, ...], value): + if type(entry) is not tuple or len(cast(tuple[object, ...], entry)) != 2: + raise ValueError(f"{label} entries must be exact pairs") + return value + + +def _validate_authority_pairs( + value: tuple[tuple[WanProductionAuthorityName, str], ...], + *, + allowed: frozenset[str] = _AUTHORITY_NAMES, +) -> tuple[tuple[WanProductionAuthorityName, str], ...]: + names = tuple(name for name, _binding in value) + if names != tuple(sorted(set(names))) or any(name not in allowed for name in names): + raise ValueError("authority bindings must be canonical, unique, and closed") + if any(type(binding) is not str or not binding or "\x00" in binding for _, binding in value): + raise ValueError("authority bindings must be non-empty exact strings") + return value + + +class WanQualificationSearchV1(_StrictQualificationModel): + schema_version: Literal["wan-production-qualification-search.v1"] + search_roots: tuple[Path, ...] + maximum_directories: Annotated[ + StrictInt, Field(gt=0, le=_MAXIMUM_SEARCH_DIRECTORIES) + ] + maximum_files: Annotated[StrictInt, Field(gt=0, le=_MAXIMUM_SEARCH_FILES)] + + @field_validator("search_roots", mode="before") + @classmethod + def validate_exact_roots(cls, value: object) -> object: + if type(value) is not tuple: + raise ValueError("search roots must be an exact tuple") + return value + + @field_validator("search_roots") + @classmethod + def validate_roots(cls, value: tuple[Path, ...]) -> tuple[Path, ...]: + if not value or any(not isinstance(root, Path) or not root.is_absolute() for root in value): + raise ValueError("search roots must be explicit absolute paths") + texts = tuple(os.fspath(root) for root in value) + if texts != tuple(sorted(set(texts))) or any( + text.startswith("//") or os.path.normpath(text) != text for text in texts + ): + raise ValueError("search roots must be sorted and unique") + return value + + @field_validator("maximum_directories", "maximum_files", mode="before") + @classmethod + def validate_exact_limits(cls, value: object) -> int: + if type(value) is not int: + raise ValueError("search limits must be exact built-in integers") + return value + + +class WanQualificationCandidateV1(_StrictQualificationModel): + schema_version: Literal["wan-production-qualification-candidate.v1"] + bindings: tuple[tuple[WanProductionAuthorityName, str], ...] + accepted_compilation_evidence: tuple[ + tuple[WanProductionAuthorityName, str], ... + ] + + @field_validator("bindings", "accepted_compilation_evidence", mode="before") + @classmethod + def validate_exact_containers(cls, value: object) -> object: + return _exact_pairs(value, label="candidate authority") + + @field_validator("bindings") + @classmethod + def validate_bindings( + cls, value: tuple[tuple[WanProductionAuthorityName, str], ...] + ) -> tuple[tuple[WanProductionAuthorityName, str], ...]: + return _validate_authority_pairs(value) + + @field_validator("accepted_compilation_evidence") + @classmethod + def validate_evidence( + cls, value: tuple[tuple[WanProductionAuthorityName, str], ...] + ) -> tuple[tuple[WanProductionAuthorityName, str], ...]: + return _validate_authority_pairs(value, allowed=_SCIENTIFIC_EVIDENCE_SET) + + @model_validator(mode="after") + def validate_evidence_bindings(self) -> WanQualificationCandidateV1: + bindings = dict(self.bindings) + if any(bindings.get(name) != value for name, value in self.accepted_compilation_evidence): + raise ValueError("accepted compilation evidence must bind candidate values exactly") + return self + + +class _WanQualificationRecipeV1(_StrictQualificationModel): + schema_version: Literal["wan-production-qualification-recipe.v1"] + authority_bindings: tuple[tuple[WanProductionAuthorityName, str], ...] + + @field_validator("authority_bindings", mode="before") + @classmethod + def validate_exact_container(cls, value: object) -> object: + return _exact_pairs(value, label="qualification recipe authority") + + @field_validator("authority_bindings") + @classmethod + def validate_bindings( + cls, value: tuple[tuple[WanProductionAuthorityName, str], ...] + ) -> tuple[tuple[WanProductionAuthorityName, str], ...]: + return _validate_authority_pairs(value, allowed=_RECIPE_EVIDENCE_SET) + + +class WanQualificationRejectionV1(_StrictQualificationModel): + candidate_digest: Sha256Digest + reason: Literal["malformed", "unsupported", "incomplete"] + + +class WanQualificationManifestV1(_StrictQualificationModel): + schema_version: Literal["wan-production-qualification-manifest.v1"] + status: Literal["qualified", "missing", "ambiguous"] + bindings: tuple[tuple[WanProductionAuthorityName, str], ...] + missing_roles: tuple[WanProductionAuthorityName, ...] + ambiguous_roles: tuple[WanProductionAuthorityName, ...] + candidate_digests: tuple[tuple[WanProductionAuthorityName, Sha256Digest], ...] + rejections: tuple[WanQualificationRejectionV1, ...] = () + + @field_validator("bindings", "candidate_digests", mode="before") + @classmethod + def validate_exact_pair_containers(cls, value: object) -> object: + return _exact_pairs(value, label="manifest authority") + + @field_validator("missing_roles", "ambiguous_roles", mode="before") + @classmethod + def validate_exact_role_containers(cls, value: object) -> object: + if type(value) is not tuple: + raise ValueError("manifest roles must be exact tuples") + return value + + @field_validator("rejections", mode="before") + @classmethod + def validate_exact_rejection_container(cls, value: object) -> object: + if type(value) is not tuple: + raise ValueError("manifest rejections must be an exact tuple") + return value + + @field_validator("rejections") + @classmethod + def validate_rejections( + cls, value: tuple[WanQualificationRejectionV1, ...] + ) -> tuple[WanQualificationRejectionV1, ...]: + keys = tuple((item.candidate_digest, item.reason) for item in value) + if keys != tuple(sorted(set(keys))): + raise ValueError("manifest rejections must be canonical and unique") + return value + + @field_validator("bindings") + @classmethod + def validate_bindings( + cls, value: tuple[tuple[WanProductionAuthorityName, str], ...] + ) -> tuple[tuple[WanProductionAuthorityName, str], ...]: + return _validate_authority_pairs(value) + + @field_validator("candidate_digests") + @classmethod + def validate_candidate_digests( + cls, value: tuple[tuple[WanProductionAuthorityName, Sha256Digest], ...] + ) -> tuple[tuple[WanProductionAuthorityName, Sha256Digest], ...]: + names = tuple(name for name, _digest in value) + if names != tuple(sorted(set(names))) or any( + name not in _AUTHORITY_NAMES for name in names + ): + raise ValueError("candidate digests must be canonical and closed") + return value + + @field_validator("missing_roles", "ambiguous_roles") + @classmethod + def validate_roles( + cls, value: tuple[WanProductionAuthorityName, ...] + ) -> tuple[WanProductionAuthorityName, ...]: + if value != tuple(sorted(set(value))) or any( + name not in _AUTHORITY_NAMES for name in value + ): + raise ValueError("manifest roles must be canonical, unique, and closed") + return value + + @model_validator(mode="after") + def validate_status(self) -> WanQualificationManifestV1: + binding_names = tuple(name for name, _ in self.bindings) + digest_names = tuple(name for name, _ in self.candidate_digests) + if self.status == "qualified": + if ( + binding_names != WAN_PRODUCTION_AUTHORITY_NAMES + or digest_names != WAN_PRODUCTION_AUTHORITY_NAMES + or self.missing_roles + or self.ambiguous_roles + ): + raise ValueError("qualified manifest must bind one complete authority set") + elif self.status == "missing": + if ( + self.bindings + or self.candidate_digests + or self.ambiguous_roles + or (not self.missing_roles and not self.rejections) + ): + raise ValueError("missing manifest fields are mutually exclusive") + elif ( + self.bindings + or self.candidate_digests + or self.missing_roles + or not self.ambiguous_roles + ): + raise ValueError("ambiguous manifest fields are mutually exclusive") + return self + + +def load_wan_qualification_manifest(path: Path) -> WanQualificationManifestV1: + """Load one strict manifest through a retained, read-only local authority.""" + + unavailable = "wan_mps_production_canary_inventory_manifest_unavailable" + invalid = "wan_mps_production_canary_inventory_manifest_invalid" + if ( + not isinstance(path, Path) + or not path.is_absolute() + or path.name in {"", ".", ".."} + or "\x00" in path.name + ): + raise ValueError(invalid) + try: + authority = _capture_existing_directory_authority( + path.parent, error_code=unavailable + ) + except ValueError: + raise ValueError(unavailable) from None + try: + try: + expected = os.stat( + path.name, dir_fd=authority.descriptor, follow_symlinks=False + ) + except FileNotFoundError: + raise ValueError(unavailable) from None + payload = _read_regular_file_at(authority.descriptor, path.name, expected) + authority.reverify() + if payload is None: + raise ValueError(invalid) + decoded = _load_strict_json(payload) + return WanQualificationManifestV1.model_validate( + _json_arrays_to_tuples(decoded), strict=True + ) + except ValueError as error: + if str(error) == unavailable: + raise + raise ValueError(invalid) from None + except (OSError, TypeError): + raise ValueError(invalid) from None + finally: + authority.close() + + +def discover_wan_production_authorities( + request: WanQualificationSearchV1, +) -> WanQualificationManifestV1: + """Resolve a unique complete set from explicit local roots without mutation.""" + + request = WanQualificationSearchV1.model_validate(request, strict=True) + authorities = _capture_search_roots(request.search_roots) + candidates: list[tuple[dict[WanProductionAuthorityName, str], str]] = [] + rejected: set[tuple[str, Literal["malformed", "unsupported", "incomplete"]]] = set() + try: + for payload, digest in _walk_candidate_payloads(request, authorities): + try: + decoded = _load_strict_json(payload) + except (TypeError, ValueError): + rejected.add((digest, "malformed")) + continue + try: + candidate = WanQualificationCandidateV1.model_validate( + _json_arrays_to_tuples(decoded), strict=True + ) + except (TypeError, ValueError): + rejected.add((digest, "unsupported")) + continue + accepted = _qualify_candidate(candidate, authorities, request) + candidates.append((accepted, digest)) + if tuple(sorted(accepted)) != WAN_PRODUCTION_AUTHORITY_NAMES: + rejected.add((digest, "incomplete")) + finally: + for authority in reversed(authorities): + authority.close() + + complete: dict[ + tuple[tuple[WanProductionAuthorityName, str], ...], + tuple[str, ...], + ] = {} + for accepted, digest in candidates: + if tuple(sorted(accepted)) != WAN_PRODUCTION_AUTHORITY_NAMES: + continue + bindings = tuple(sorted(accepted.items())) + complete[bindings] = tuple(sorted(set((*complete.get(bindings, ()), digest)))) + if len(complete) == 1: + bindings, digests = next(iter(complete.items())) + selected_digest = digests[0] + return WanQualificationManifestV1( + schema_version="wan-production-qualification-manifest.v1", + status="qualified", + bindings=bindings, + missing_roles=(), + ambiguous_roles=(), + candidate_digests=tuple((name, selected_digest) for name, _ in bindings), + rejections=_qualification_rejections(rejected), + ) + if len(complete) > 1: + all_bindings = tuple(complete) + ambiguous = tuple( + name + for name in WAN_PRODUCTION_AUTHORITY_NAMES + if len({dict(bindings)[name] for bindings in all_bindings}) > 1 + ) + return _unqualified_manifest( + status="ambiguous", roles=ambiguous, rejected=rejected + ) + + role_values: dict[WanProductionAuthorityName, set[str]] = { + name: set() for name in WAN_PRODUCTION_AUTHORITY_NAMES + } + for accepted, _digest in candidates: + for name, value in accepted.items(): + role_values[name].add(value) + ambiguous = tuple(name for name, values in role_values.items() if len(values) > 1) + if ambiguous: + return _unqualified_manifest( + status="ambiguous", roles=ambiguous, rejected=rejected + ) + missing = tuple(name for name, values in role_values.items() if not values) + return _unqualified_manifest( + status="missing", + roles=missing if candidates else WAN_PRODUCTION_AUTHORITY_NAMES, + rejected=rejected, + ) + + +def _qualification_rejections( + rejected: set[tuple[str, Literal["malformed", "unsupported", "incomplete"]]], +) -> tuple[WanQualificationRejectionV1, ...]: + return tuple( + WanQualificationRejectionV1(candidate_digest=digest, reason=reason) + for digest, reason in sorted(rejected) + ) + + +def _unqualified_manifest( + *, + status: Literal["missing", "ambiguous"], + roles: tuple[WanProductionAuthorityName, ...], + rejected: set[ + tuple[str, Literal["malformed", "unsupported", "incomplete"]] + ] | None = None, +) -> WanQualificationManifestV1: + return WanQualificationManifestV1( + schema_version="wan-production-qualification-manifest.v1", + status=status, + bindings=(), + missing_roles=roles if status == "missing" else (), + ambiguous_roles=roles if status == "ambiguous" else (), + candidate_digests=(), + rejections=_qualification_rejections(rejected or set()), + ) + + +def _capture_search_roots( + roots: tuple[Path, ...], +) -> tuple[RetainedDirectoryAuthority, ...]: + for index, root in enumerate(roots): + if any( + root != other and root.is_relative_to(other) + for other in roots[:index] + ): + raise ValueError("wan_qualification_search_root_invalid") + authorities: list[RetainedDirectoryAuthority] = [] + identities: set[tuple[int, int]] = set() + try: + for root in roots: + authority = _capture_existing_directory_authority( + root, error_code="wan_qualification_search_root_invalid" + ) + identity = ( + os.fstat(authority.descriptor).st_dev, + os.fstat(authority.descriptor).st_ino, + ) + if identity in identities: + authority.close() + raise ValueError + identities.add(identity) + authorities.append(authority) + return tuple(authorities) + except (OSError, ValueError): + for authority in reversed(authorities): + authority.close() + raise ValueError("wan_qualification_search_root_invalid") from None + + +def _capture_existing_directory_authority( + path: Path, *, error_code: str +) -> RetainedDirectoryAuthority: + descriptors: list[int] = [] + flags = ( + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0) + ) + try: + if not isinstance(path, Path) or not path.is_absolute() or not error_code: + raise ValueError + descriptors.append(os.open(path.anchor, flags)) + for component in path.parts[1:]: + if component in {"", ".", ".."}: + raise ValueError + expected = os.stat( + component, dir_fd=descriptors[-1], follow_symlinks=False + ) + if not stat.S_ISDIR(expected.st_mode): + raise ValueError + child = os.open(component, flags, dir_fd=descriptors[-1]) + descriptors.append(child) + if not os.path.samestat(expected, os.fstat(child)): + raise ValueError + identities = tuple( + (os.fstat(descriptor).st_dev, os.fstat(descriptor).st_ino) + for descriptor in descriptors + ) + authority = RetainedDirectoryAuthority( + path=path, + descriptor=descriptors[-1], + descriptors=tuple(descriptors), + identities=identities, + error_code=error_code, + ) + if not _retained_search_root_is_current(authority): + raise ValueError + return authority + except (OSError, ValueError): + for descriptor in reversed(descriptors): + try: + os.close(descriptor) + except OSError: + pass + raise ValueError(error_code) from None + + +def _retained_search_root_is_current( + authority: RetainedDirectoryAuthority, +) -> bool: + try: + if not _retained_directory_is_stable(authority): + return False + for parent, component, child in zip( + authority.descriptors[:-1], + authority.path.parts[1:], + authority.descriptors[1:], + strict=True, + ): + linked = os.stat(component, dir_fd=parent, follow_symlinks=False) + observed = os.fstat(child) + if not stat.S_ISDIR(linked.st_mode) or not os.path.samestat(linked, observed): + return False + return True + except (OSError, ValueError): + return False + + +def _walk_candidate_payloads( + request: WanQualificationSearchV1, + authorities: tuple[RetainedDirectoryAuthority, ...], +) -> Iterator[tuple[bytes, Sha256Digest]]: + directory_count = 0 + file_count = 0 + + def walk(descriptor: int) -> Iterator[tuple[bytes, Sha256Digest]]: + nonlocal directory_count, file_count + directory_count += 1 + if directory_count > request.maximum_directories: + raise ValueError("wan_qualification_search_limit_exceeded") + with os.scandir(descriptor) as entries: + for entry in entries: + try: + info = entry.stat(follow_symlinks=False) + except OSError: + continue + if stat.S_ISDIR(info.st_mode): + child: int | None = None + try: + child = os.open( + entry.name, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=descriptor, + ) + if not os.path.samestat(info, os.fstat(child)): + continue + yield from walk(child) + except OSError: + continue + finally: + if child is not None: + os.close(child) + continue + file_count += 1 + if file_count > request.maximum_files: + raise ValueError("wan_qualification_search_limit_exceeded") + if not stat.S_ISREG(info.st_mode) or entry.name != _CANDIDATE_FILENAME: + continue + payload = _read_regular_file_at(descriptor, entry.name, info) + if payload is not None: + yield payload, "sha256:" + hashlib.sha256(payload).hexdigest() + + for authority in authorities: + root_descriptor = os.dup(authority.descriptor) + try: + yield from walk(root_descriptor) + finally: + os.close(root_descriptor) + + +def _read_regular_file_at( + parent_descriptor: int, name: str, expected: os.stat_result +) -> bytes | None: + descriptor: int | None = None + try: + expected_identity = _stable_regular_identity(expected) + if expected.st_size > _MAXIMUM_CANDIDATE_BYTES: + return None + descriptor = os.open( + name, + os.O_RDONLY + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=parent_descriptor, + ) + before = os.fstat(descriptor) + identity = _stable_regular_identity(before) + if identity != expected_identity or before.st_size > _MAXIMUM_CANDIDATE_BYTES: + return None + payload = bytearray() + digest = hashlib.sha256() + offset = 0 + while offset < before.st_size: + chunk = os.pread( + descriptor, min(1024 * 1024, before.st_size - offset), offset + ) + if not chunk: + return None + payload.extend(chunk) + digest.update(chunk) + offset += len(chunk) + if os.pread(descriptor, 1, before.st_size): + return None + if _stable_regular_identity(os.fstat(descriptor)) != identity: + return None + verification = hashlib.sha256() + offset = 0 + while offset < before.st_size: + chunk = os.pread( + descriptor, min(1024 * 1024, before.st_size - offset), offset + ) + if not chunk: + return None + verification.update(chunk) + offset += len(chunk) + if ( + os.pread(descriptor, 1, before.st_size) + or verification.digest() != digest.digest() + or _stable_regular_identity(os.fstat(descriptor)) != identity + ): + return None + return bytes(payload) + except (OSError, ValueError): + return None + finally: + if descriptor is not None: + os.close(descriptor) + + +def _json_arrays_to_tuples(value: object) -> object: + if type(value) is list: + return tuple(_json_arrays_to_tuples(item) for item in cast(list[object], value)) + if type(value) is dict: + return { + key: _json_arrays_to_tuples(item) + for key, item in cast(dict[object, object], value).items() + } + return value + + +def _strict_json_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON object key") + result[key] = value + return result + + +def _reject_json_constant(_value: str) -> NoReturn: + raise ValueError("non-finite JSON number") + + +def _load_strict_json(value: str | bytes) -> object: + return json.loads( + value, + object_pairs_hook=_strict_json_object, + parse_constant=_reject_json_constant, + ) + + +def _qualify_candidate( + candidate: WanQualificationCandidateV1, + authorities: tuple[RetainedDirectoryAuthority, ...], + request: WanQualificationSearchV1, +) -> dict[WanProductionAuthorityName, str]: + supplied = dict(candidate.bindings) + evidence = dict(candidate.accepted_compilation_evidence) + accepted: dict[WanProductionAuthorityName, str] = {} + if not all(_retained_search_root_is_current(item) for item in authorities): + return accepted + + artifact_root = _safe_directory_binding( + supplied.get("VIDEO_LAB_WAN_ARTIFACT_ROOT"), authorities + ) + database = _safe_file_binding( + supplied.get("VIDEO_LAB_WAN_ARTIFACT_DATABASE"), authorities + ) + receipt_id = supplied.get("VIDEO_LAB_WAN_COMPILATION_RECEIPT_ID") + operation_id = supplied.get("VIDEO_LAB_WAN_COMPILATION_OPERATION_ID") + provenance = ( + _load_compilation_authorities( + database, artifact_root, receipt_id=receipt_id, operation_id=operation_id + ) + if database is not None and artifact_root is not None + else None + ) + if provenance is not None: + assert database is not None and artifact_root is not None + accepted["VIDEO_LAB_WAN_ARTIFACT_DATABASE"] = os.fspath(database) + accepted["VIDEO_LAB_WAN_ARTIFACT_ROOT"] = os.fspath(artifact_root) + for name, authoritative_value in provenance.items(): + supplied_value = supplied.get(name) + if supplied_value is None or evidence.get(name) != supplied_value: + continue + normalized = _normalize_scientific_binding(name, supplied_value) + if normalized == authoritative_value: + accepted[name] = normalized + + model_root = _safe_directory_binding( + supplied.get("VIDEO_LAB_WAN_MODEL_ROOT"), authorities + ) + model_revision = accepted.get("VIDEO_LAB_WAN_MODEL_REVISION") + adapter_value = accepted.get("VIDEO_LAB_WAN_ADAPTER_PATHS_JSON") + if ( + model_root is not None + and model_revision is not None + and _REVISION.fullmatch(model_revision) + and adapter_value is not None + and _validate_model_inventory(model_root, model_revision, adapter_value, request) + ): + accepted["VIDEO_LAB_WAN_MODEL_ROOT"] = os.fspath(model_root) + accepted["VIDEO_LAB_WAN_MODEL_REVISION"] = model_revision + + cache_root = _safe_directory_binding( + supplied.get("VIDEO_LAB_WAN_CACHE_ROOT"), authorities + ) + cache_revision = accepted.get("VIDEO_LAB_WAN_CACHE_REVISION") + if ( + cache_root is not None + and cache_revision is not None + and _REVISION.fullmatch(cache_revision) + and _validate_cache_revision(cache_root, cache_revision, request) + ): + accepted["VIDEO_LAB_WAN_CACHE_ROOT"] = os.fspath(cache_root) + accepted["VIDEO_LAB_WAN_CACHE_REVISION"] = cache_revision + + if not all(_retained_search_root_is_current(item) for item in authorities): + return {} + return accepted + + +def _normalize_scientific_binding( + name: WanProductionAuthorityName, value: str +) -> str | None: + try: + if name == "VIDEO_LAB_WAN_ADAPTER_PATHS_JSON": + parsed = _load_strict_json(value) + if type(parsed) is not dict or tuple(sorted(parsed)) != _ADAPTER_ROLES: + return None + for role in _ADAPTER_ROLES: + _safe_relative_path(parsed[role], allow_dot=role == "wan_snapshot") + return canonical_json(parsed) + if name == "VIDEO_LAB_WAN_DEPENDENCY_VERSIONS_JSON": + parsed = _load_strict_json(value) + if type(parsed) is not list: + return None + pairs = tuple(tuple(item) if type(item) is list else item for item in parsed) + if any( + type(item) is not tuple + or len(item) != 2 + or type(item[0]) is not str + or type(item[1]) is not str + for item in pairs + ): + return None + expected = tuple((name, importlib.metadata.version(name)) for name in _DEPENDENCIES) + if pairs != expected: + return None + return _canonical_json_value(pairs) + if name in { + "VIDEO_LAB_WAN_REQUIRED_ARTIFACT_BYTES", + "VIDEO_LAB_WAN_MEMORY_RELEASE_BOUND_BYTES", + }: + if not value.isascii() or not value.isdigit() or (len(value) > 1 and value[0] == "0"): + return None + amount = int(value) + if amount <= 0: + return None + return str(amount) + if name == "VIDEO_LAB_WAN_TOLERANCE_JSON": + parsed = WanNumericalToleranceV1.model_validate( + _load_strict_json(value), strict=True + ) + return canonical_json(parsed.model_dump(mode="json", exclude={"schema_version"})) + if name == "VIDEO_LAB_WAN_TENSOR_ENVELOPE_JSON": + raw = _load_strict_json(value) + if type(raw) is not list: + return None + envelope = tuple( + WanTensorEnvelopeV2.model_validate(_json_arrays_to_tuples(item), strict=True) + for item in raw + ) + names = tuple(item.name for item in envelope) + if ( + not names + or names != tuple(sorted(set(names))) + or {item.role for item in envelope} != {"activation", "optimizer", "parameter"} + ): + return None + return _canonical_json_value( + tuple(item.model_dump(mode="json") for item in envelope) + ) + if name in { + "VIDEO_LAB_WAN_COMPILATION_OPERATION_ID", + "VIDEO_LAB_WAN_COMPILATION_RECEIPT_ID", + }: + if not value.isascii() or not re.fullmatch(r"[a-z][a-z0-9_]{2,127}", value): + return None + return value + if name in { + "VIDEO_LAB_WAN_CACHE_REVISION", + "VIDEO_LAB_WAN_MODEL_REVISION", + }: + if _REVISION.fullmatch(value) is None: + return None + return value + except (ImportError, LookupError, TypeError, ValueError): + return None + return None + + +def _safe_relative_path(value: object, *, allow_dot: bool = False) -> Path: + if type(value) is not str: + raise ValueError + path = PurePath(value) + windows = PureWindowsPath(value) + if allow_dot and value == ".": + return Path(value) + if ( + not value + or value == "." + or path.is_absolute() + or windows.anchor + or windows.drive + or "\\" in value + or any(part in {"", ".", ".."} for part in path.parts) + or any(PureWindowsPath(part).is_reserved() for part in path.parts) + ): + raise ValueError + return Path(value) + + +def _canonical_json_value(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def _safe_directory_binding( + value: str | None, authorities: tuple[RetainedDirectoryAuthority, ...] +) -> Path | None: + return _safe_path_binding(value, authorities, directory=True) + + +def _safe_file_binding( + value: str | None, authorities: tuple[RetainedDirectoryAuthority, ...] +) -> Path | None: + return _safe_path_binding(value, authorities, directory=False) + + +def _safe_path_binding( + value: str | None, + authorities: tuple[RetainedDirectoryAuthority, ...], + *, + directory: bool, +) -> Path | None: + if type(value) is not str or not value or "\x00" in value: + return None + if "\\" in value or value.startswith("//") or os.path.normpath(value) != value: + return None + path = Path(value) + containing_roots = tuple( + authority + for authority in authorities + if _lexically_contained(path, authority.path) + ) + if not path.is_absolute() or not containing_roots: + return None + if not any( + _validate_contained_path(path, authority, directory=directory) + for authority in containing_roots + ): + return None + return path + + +def _lexically_contained(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + except ValueError: + return False + return True + + +def _validate_contained_path( + path: Path, authority: RetainedDirectoryAuthority, *, directory: bool +) -> bool: + descriptors: list[int] = [] + identities: list[tuple[int, int, int, int, int, int, int]] = [] + try: + relative = path.relative_to(authority.path) + if not _retained_search_root_is_current(authority): + return False + if not relative.parts: + return directory and _retained_search_root_is_current(authority) + parent = authority.descriptor + for index, component in enumerate(relative.parts): + last = index == len(relative.parts) - 1 + info = os.stat(component, dir_fd=parent, follow_symlinks=False) + expected_directory = directory if last else True + if (expected_directory and not stat.S_ISDIR(info.st_mode)) or ( + not expected_directory and not stat.S_ISREG(info.st_mode) + ): + return False + flags = ( + os.O_RDONLY + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0) + ) + if expected_directory: + flags |= os.O_DIRECTORY + child = os.open(component, flags, dir_fd=parent) + descriptors.append(child) + expected_identity = ( + _directory_inventory_identity(info) + if expected_directory + else _stable_regular_identity(info) + ) + observed_identity = ( + _directory_inventory_identity(os.fstat(child)) + if expected_directory + else _stable_regular_identity(os.fstat(child)) + ) + if expected_identity != observed_identity: + return False + identities.append(expected_identity) + parent = child + if any( + identity + != ( + _directory_inventory_identity(os.fstat(descriptor)) + if stat.S_ISDIR(os.fstat(descriptor).st_mode) + else _stable_regular_identity(os.fstat(descriptor)) + ) + for descriptor, identity in zip(descriptors, identities, strict=True) + ): + return False + return _retained_search_root_is_current(authority) + except (OSError, ValueError): + return False + finally: + for descriptor in reversed(descriptors): + os.close(descriptor) + + +def _validate_model_inventory( + root: Path, + revision: str, + adapter_value: str, + request: WanQualificationSearchV1, +) -> bool: + try: + adapter = _load_strict_json(adapter_value) + if type(adapter) is not dict: + return False + paths = { + name: _safe_relative_path(value, allow_dot=name == "wan_snapshot") + for name, value in cast(dict[str, object], adapter).items() + } + if paths["wan_snapshot"] != Path("."): + return False + inventory = _bounded_root_inventory(root, request) + if not _matches_local_revision(root, revision, inventory.manifest_digest): + return False + for role in ("scheduler", "text_encoder", "tokenizer", "vae"): + location = paths[role] + if not any( + item == location or item.is_relative_to(location) + for item in inventory.files + ): + return False + for role in ("transformer_config", "high_transformer", "low_transformer"): + if paths[role] not in inventory.files: + return False + return True + except (OSError, TypeError, ValueError): + return False + + +def _matches_local_revision(root: Path, revision: str, manifest_digest: str) -> bool: + if root.name == revision and root.parent.name == "snapshots": + return True + component_revision = hashlib.sha256(manifest_digest.encode("ascii")).hexdigest()[:40] + return revision == component_revision + + +def _validate_cache_revision( + root: Path, revision: str, request: WanQualificationSearchV1 +) -> bool: + authority: RetainedDirectoryAuthority | None = None + try: + authority, inventory = _capture_bounded_root_inventory(root, request) + if Path("snapshots", revision) in inventory.directories: + return True + for relative in sorted( + item + for item in inventory.files + if item.parent == Path("refs") + ): + payload = _read_bounded_relative_file( + authority.descriptor, relative, maximum_bytes=4096 + ) + if payload is not None and payload.decode("utf-8").strip() == revision: + return True + return _matches_local_revision(root, revision, inventory.manifest_digest) + except (OSError, UnicodeError, ValueError): + return False + finally: + if authority is not None: + authority.close() + + +@dataclass(frozen=True, slots=True) +class _BoundedRootInventory: + manifest_digest: str + files: frozenset[Path] + directories: frozenset[Path] + + +def _bounded_root_inventory( + root: Path, request: WanQualificationSearchV1 +) -> _BoundedRootInventory: + authority: RetainedDirectoryAuthority | None = None + try: + authority, inventory = _capture_bounded_root_inventory(root, request) + return inventory + finally: + if authority is not None: + authority.close() + + +def _capture_bounded_root_inventory( + root: Path, request: WanQualificationSearchV1 +) -> tuple[RetainedDirectoryAuthority, _BoundedRootInventory]: + authority = _capture_existing_directory_authority( + root, error_code="wan_qualification_inventory_root_invalid" + ) + try: + records: list[bytes] = [] + files: set[Path] = set() + directories: set[Path] = set() + observed_bytes = 0 + observed_directories = 0 + maximum_files = min(request.maximum_files, _MAXIMUM_ROOT_INVENTORY_FILES) + maximum_directories = min( + request.maximum_directories, _MAXIMUM_ROOT_INVENTORY_DIRECTORIES + ) + + def walk(descriptor: int, relative: Path) -> None: + nonlocal observed_bytes, observed_directories + before = _directory_inventory_identity(os.fstat(descriptor)) + with os.scandir(descriptor) as scan: + for entry in scan: + child_relative = relative / entry.name + if len(child_relative.parts) > _MAXIMUM_AUTHORITY_DEPTH: + raise ValueError("wan_qualification_inventory_limit_exceeded") + info = entry.stat(follow_symlinks=False) + if stat.S_ISDIR(info.st_mode): + observed_directories += 1 + if observed_directories > maximum_directories: + raise ValueError("wan_qualification_inventory_limit_exceeded") + child = os.open( + entry.name, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=descriptor, + ) + try: + child_identity = _directory_inventory_identity(info) + if child_identity != _directory_inventory_identity( + os.fstat(child) + ): + raise ValueError("wan_qualification_inventory_changed") + directories.add(child_relative) + walk(child, child_relative) + if child_identity != _directory_inventory_identity( + os.fstat(child) + ): + raise ValueError("wan_qualification_inventory_changed") + finally: + os.close(child) + continue + if not stat.S_ISREG(info.st_mode): + raise ValueError("wan_qualification_inventory_special_file") + if len(files) >= maximum_files: + raise ValueError("wan_qualification_inventory_limit_exceeded") + observed_bytes += info.st_size + if observed_bytes > _MAXIMUM_AUTHORITY_BYTES: + raise ValueError("wan_qualification_inventory_limit_exceeded") + record = _hash_regular_file_at( + descriptor, entry.name, info, child_relative + ) + files.add(child_relative) + records.append(record) + if before != _directory_inventory_identity(os.fstat(descriptor)): + raise ValueError("wan_qualification_inventory_changed") + + walk(authority.descriptor, Path()) + if not _retained_directory_is_stable(authority): + raise ValueError("wan_qualification_inventory_changed") + inventory = _BoundedRootInventory( + manifest_digest="sha256:" + + hashlib.sha256(b"\n".join(sorted(records))).hexdigest(), + files=frozenset(files), + directories=frozenset(directories), + ) + return authority, inventory + except BaseException: + authority.close() + raise + + +def _directory_inventory_identity( + observed: os.stat_result, +) -> tuple[int, int, int, int, int, int, int]: + if not stat.S_ISDIR(observed.st_mode): + raise ValueError("wan_qualification_inventory_changed") + return ( + observed.st_dev, + observed.st_ino, + observed.st_mode, + observed.st_nlink, + observed.st_size, + observed.st_mtime_ns, + observed.st_ctime_ns, + ) + + +def _hash_regular_file_at( + directory_fd: int, + name: str, + expected: os.stat_result, + relative: Path, +) -> bytes: + descriptor = os.open( + name, + os.O_RDONLY + | os.O_NONBLOCK + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=directory_fd, + ) + try: + before = os.fstat(descriptor) + if _stable_regular_identity(expected) != _stable_regular_identity(before): + raise ValueError("wan_qualification_inventory_changed") + digest = hashlib.sha256() + offset = 0 + while offset < before.st_size: + chunk = os.pread(descriptor, min(1 << 20, before.st_size - offset), offset) + if not chunk: + raise ValueError("wan_qualification_inventory_changed") + digest.update(chunk) + offset += len(chunk) + if os.pread(descriptor, 1, offset) or _stable_regular_identity( + before + ) != _stable_regular_identity(os.fstat(descriptor)): + raise ValueError("wan_qualification_inventory_changed") + return ( + relative.as_posix().encode() + + b"\0" + + str(offset).encode() + + b"\0" + + digest.digest() + ) + finally: + os.close(descriptor) + + +def _read_bounded_relative_file( + root_descriptor: int, relative: Path, *, maximum_bytes: int +) -> bytes | None: + opened: list[int] = [] + parent = root_descriptor + try: + for component in relative.parts[:-1]: + parent = os.open( + component, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=parent, + ) + opened.append(parent) + info = os.stat(relative.name, dir_fd=parent, follow_symlinks=False) + if not stat.S_ISREG(info.st_mode) or info.st_size > maximum_bytes: + return None + return _read_regular_file_at(parent, relative.name, info) + except (OSError, ValueError): + return None + finally: + for descriptor in reversed(opened): + os.close(descriptor) + return False + + +def _retained_directory_is_stable(authority: RetainedDirectoryAuthority) -> bool: + try: + return tuple( + (os.fstat(descriptor).st_dev, os.fstat(descriptor).st_ino) + for descriptor in authority.descriptors + ) == authority.identities + except OSError: + return False + + +def _stable_regular_identity( + observed: os.stat_result, +) -> tuple[int, int, int, int, int, int, int]: + if not stat.S_ISREG(observed.st_mode) or observed.st_nlink != 1: + raise ValueError("qualification file authority invalid") + return ( + observed.st_dev, + observed.st_ino, + observed.st_mode, + observed.st_nlink, + observed.st_size, + observed.st_mtime_ns, + observed.st_ctime_ns, + ) + + +def _read_retained_artifact( + authority: RetainedDirectoryAuthority, + relative_value: str, + *, + expected_size: int, + expected_digest: str, + capture_payload: bool, +) -> bytes | None: + descriptors: list[int] = [] + try: + relative = _safe_relative_path(relative_value) + parent = authority.descriptor + for component in relative.parts[:-1]: + child = os.open( + component, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=parent, + ) + descriptors.append(child) + parent = child + descriptor = os.open( + relative.name, + os.O_RDONLY + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=parent, + ) + descriptors.append(descriptor) + identity = _stable_regular_identity(os.fstat(descriptor)) + if ( + expected_size < 0 + or expected_size > _MAXIMUM_AUTHORITY_BYTES + or identity[4] != expected_size + ): + return None + digest = hashlib.sha256() + payload = bytearray() + offset = 0 + while offset < expected_size: + chunk = os.pread(descriptor, min(1024 * 1024, expected_size - offset), offset) + if not chunk: + return None + digest.update(chunk) + if capture_payload: + payload.extend(chunk) + offset += len(chunk) + if os.pread(descriptor, 1, expected_size): + return None + if _stable_regular_identity(os.fstat(descriptor)) != identity: + return None + if digest.hexdigest() != expected_digest or not _retained_directory_is_stable(authority): + return None + return bytes(payload) + except (OSError, TypeError, ValueError): + return None + finally: + for descriptor in reversed(descriptors): + os.close(descriptor) + + +def _input_from_row(row: sqlite3.Row) -> InputBinding: + return InputBinding( + role=cast(Literal["source_manifest", "transform_recipe"], str(row["role"])), + artifact_id=str(row["artifact_id"]), + sha256=str(row["sha256"]), + byte_size=int(row["byte_size"]), + media_type=str(row["media_type"]), + ) + + +def _output_from_row(row: sqlite3.Row) -> CompilationOutput: + return CompilationOutput( + role=str(row["role"]), + artifact_id=str(row["artifact_id"]), + sha256=str(row["sha256"]), + byte_size=int(row["byte_size"]), + media_type=str(row["media_type"]), + ) + + +def _load_compilation_authorities( + database: Path, + artifact_root: Path, + *, + receipt_id: str | None, + operation_id: str | None, +) -> dict[WanProductionAuthorityName, str] | None: + if receipt_id is None or operation_id is None: + return None + database_authority: RetainedSqliteAuthority | None = None + artifact_authority: RetainedDirectoryAuthority | None = None + try: + database_authority = RetainedSqliteAuthority.capture(database) + if os.fstat(database_authority.descriptor).st_nlink != 1: + return None + artifact_authority = _capture_existing_directory_authority( + artifact_root, error_code="wan_qualification_artifact_root_invalid" + ) + with database_authority.connect_readonly() as connection: + operation = connection.execute( + "SELECT id,operation_key,compiler_id,request_digest,request_json,status,attempt," + "receipt_id,receipt_digest,receipt_json FROM dataset_compilation_operations " + "WHERE id=? AND receipt_id=?", + (operation_id, receipt_id), + ).fetchone() + if operation is None: + return None + request = DatasetCompilationRequest.model_validate( + _json_arrays_to_tuples(_load_strict_json(str(operation["request_json"]))), + strict=True, + ) + receipt = DatasetCompilationReceipt.model_validate( + _json_arrays_to_tuples(_load_strict_json(str(operation["receipt_json"]))), + strict=True, + ) + canonical_request = canonical_json(request.model_dump(mode="json")) + canonical_receipt = canonical_json(receipt.model_dump(mode="json")) + receipt_digest = hashlib.sha256(canonical_receipt.encode()).hexdigest() + if ( + str(operation["id"]) != operation_id + or str(operation["status"]) != "completed" + or str(operation["operation_key"]) != request.operation_key + or request.operation_key != receipt.operation_key + or request.operation_key != "wan_production_qualification" + or str(operation["compiler_id"]) != request.compiler_id + or request.compiler_id != receipt.compiler_id + or request.compiler_id != "wan_curated_v1" + or str(operation["request_json"]) != canonical_request + or str(operation["request_digest"]) != request.digest + or str(operation["request_digest"]) != receipt.request_digest + or str(operation["receipt_id"]) != receipt_id + or str(operation["receipt_json"]) != canonical_receipt + or str(operation["receipt_digest"]) != receipt_digest + or receipt.receipt_id != receipt_id + or receipt.operation_id != operation_id + or receipt.attempt != int(operation["attempt"]) + or receipt.source != request.source + or receipt.recipe != request.recipe + or receipt.reproduces_receipt_id != request.reproduces_receipt_id + or request.output_roles != ("batch", "manifest", "provenance") + or tuple(output.role for output in receipt.outputs) != request.output_roles + ): + return None + input_rows = connection.execute( + "SELECT operation_id,role,artifact_id,sha256,byte_size,media_type " + "FROM dataset_compilation_inputs WHERE operation_id=? ORDER BY role", + (operation_id,), + ).fetchall() + output_rows = connection.execute( + "SELECT receipt_id,operation_id,role,artifact_id,sha256,byte_size," + "media_type,attempt " + "FROM dataset_compilation_outputs WHERE operation_id=? ORDER BY role", + (operation_id,), + ).fetchall() + if ( + any(str(row["operation_id"]) != operation_id for row in input_rows) + or any( + str(row["receipt_id"]) != receipt.receipt_id + or str(row["receipt_id"]) != str(operation["receipt_id"]) + or str(row["operation_id"]) != receipt.operation_id + or int(row["attempt"]) != receipt.attempt + or int(row["attempt"]) != int(operation["attempt"]) + for row in output_rows + ) + or tuple(_input_from_row(row) for row in input_rows) + != (request.source, request.recipe) + or tuple(_output_from_row(row) for row in output_rows) != receipt.outputs + ): + return None + completed = connection.execute( + "SELECT operation_id,attempt,evidence_digest " + "FROM dataset_compilation_events " + "WHERE operation_id=? AND event_kind='completed' ORDER BY sequence", + (operation_id,), + ).fetchall() + if len(completed) != 1 or tuple(completed[0]) != ( + operation_id, + receipt.attempt, + receipt_digest, + ): + return None + bindings: tuple[InputBinding | CompilationOutput, ...] = ( + request.source, + request.recipe, + *receipt.outputs, + ) + recipe_payload: bytes | None = None + for binding in bindings: + artifact = connection.execute( + "SELECT id,sha256,byte_size,media_type,relative_path FROM artifacts WHERE id=?", + (binding.artifact_id,), + ).fetchone() + if artifact is None or tuple(artifact)[:4] != ( + binding.artifact_id, + binding.sha256, + binding.byte_size, + binding.media_type, + ): + return None + payload = _read_retained_artifact( + artifact_authority, + str(artifact["relative_path"]), + expected_size=binding.byte_size, + expected_digest=binding.sha256, + capture_payload=binding.role == "transform_recipe", + ) + if payload is None: + return None + if binding.role == "transform_recipe": + recipe_payload = payload + database_authority.reverify() + if recipe_payload is None or not _retained_directory_is_stable(artifact_authority): + return None + recipe = _WanQualificationRecipeV1.model_validate( + _json_arrays_to_tuples(_load_strict_json(recipe_payload)), strict=True + ) + authorities: dict[WanProductionAuthorityName, str] = { + "VIDEO_LAB_WAN_COMPILATION_OPERATION_ID": operation_id, + "VIDEO_LAB_WAN_COMPILATION_RECEIPT_ID": receipt_id, + } + for name, value in recipe.authority_bindings: + normalized = _normalize_scientific_binding(name, value) + if normalized is not None: + authorities[name] = normalized + return authorities + except (OSError, sqlite3.Error, TypeError, ValueError): + return None + finally: + if artifact_authority is not None: + artifact_authority.close() + if database_authority is not None: + database_authority.close() + + +__all__ = ( + "WAN_PRODUCTION_AUTHORITY_NAMES", + "WanProductionAuthorityName", + "WanQualificationCandidateV1", + "WanQualificationManifestV1", + "WanQualificationRejectionV1", + "WanQualificationSearchV1", + "discover_wan_production_authorities", +) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/launcher.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/launcher.py new file mode 100644 index 0000000..47eb9a3 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/launcher.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +_import = __import__ + + +def main() -> None: + """Load the CLI without importing optional inference dependencies.""" + cli = _import("video_lab.cli", fromlist=("app",)) + cli.app() + + +if __name__ == "__main__": + main() diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/local_authority.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/local_authority.py new file mode 100644 index 0000000..c384df7 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/local_authority.py @@ -0,0 +1,689 @@ +"""Retained, pathname-aware authorities for local filesystems and SQLite.""" + +from __future__ import annotations + +import ctypes +import errno +import hashlib +import os +import re +import sqlite3 +import stat +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from threading import RLock + +MAXIMUM_SQLITE_BYTES = 1024 * 1024 * 1024 + +_DIRECTORY_AUTHORITY_CHANGED = "wan_canary_directory_authority_changed" +_DATABASE_AUTHORITY_CHANGED = "wan_canary_database_authority_changed" + +_DIRECTORY_FLAGS = ( + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0) +) +_FILE_SAFETY_FLAGS = getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0) +_UNKNOWN_SQLITE_FUNCTION = re.compile(r"^unknown function: ([A-Za-z_][A-Za-z0-9_]*)\(\)$") +_SQLITE_SIDECAR_SUFFIXES = ("-wal", "-shm", "-journal") +_LINUX_AT_FDCWD = -100 +_LINUX_AT_SYMLINK_FOLLOW = 0x400 +_LINUX_AT_EMPTY_PATH = 0x1000 +_LINKAT_CAPABILITY_ERRNOS = frozenset((errno.EACCES, errno.ENOENT, errno.EPERM)) + + +@dataclass(slots=True) +class _AuthorityState: + lock: RLock = field(default_factory=RLock) + closed: bool = False + + +def _safe_child_name(name: str, *, error_code: str) -> str: + if ( + type(name) is not str + or name in {"", ".", ".."} + or "\x00" in name + or Path(name).name != name + or os.path.isabs(name) + ): + raise ValueError(error_code) + return name + + +def _directory_identity(descriptor: int) -> tuple[int, int]: + observed = os.fstat(descriptor) + if not stat.S_ISDIR(observed.st_mode): + raise OSError(errno.ENOTDIR, "retained authority is not a directory") + return observed.st_dev, observed.st_ino + + +def _open_directory_chain(path: Path) -> list[int]: + descriptors: list[int] = [] + try: + current = os.open(path.anchor, _DIRECTORY_FLAGS) + descriptors.append(current) + for component in path.parts[1:]: + if component in {"", ".", ".."}: + raise OSError(errno.EINVAL, "invalid directory authority") + current = os.open(component, _DIRECTORY_FLAGS, dir_fd=current) + descriptors.append(current) + return descriptors + except BaseException: + for descriptor in reversed(descriptors): + os.close(descriptor) + raise + + +def _create_directory_leaf(path: Path) -> None: + parents = _open_directory_chain(path.parent) + try: + os.mkdir(path.name, mode=0o700, dir_fd=parents[-1]) + os.fsync(parents[-1]) + finally: + for descriptor in reversed(parents): + os.close(descriptor) + + +@dataclass(frozen=True, slots=True) +class RetainedDirectoryAuthority: + path: Path + descriptor: int + descriptors: tuple[int, ...] + identities: tuple[tuple[int, int], ...] + error_code: str = _DIRECTORY_AUTHORITY_CHANGED + _state: _AuthorityState = field( + default_factory=_AuthorityState, compare=False, repr=False + ) + + @classmethod + def capture( + cls, + path: Path, + *, + error_code: str = _DIRECTORY_AUTHORITY_CHANGED, + ) -> RetainedDirectoryAuthority: + if not isinstance(path, Path) or not path.is_absolute() or not error_code: + raise ValueError(error_code or _DIRECTORY_AUTHORITY_CHANGED) + descriptors: list[int] = [] + try: + try: + descriptors = _open_directory_chain(path) + except FileNotFoundError: + _create_directory_leaf(path) + descriptors = _open_directory_chain(path) + identities = tuple(_directory_identity(item) for item in descriptors) + authority = cls( + path=path, + descriptor=descriptors[-1], + descriptors=tuple(descriptors), + identities=identities, + error_code=error_code, + ) + authority.reverify() + return authority + except (OSError, ValueError) as error: + for descriptor in reversed(descriptors): + try: + os.close(descriptor) + except OSError: + pass + if isinstance(error, ValueError) and str(error) == error_code: + raise + raise ValueError(error_code) from None + except BaseException: + for descriptor in reversed(descriptors): + try: + os.close(descriptor) + except OSError: + pass + raise + + def _reverify_locked(self) -> None: + if self._state.closed: + raise ValueError(self.error_code) + try: + retained = tuple(_directory_identity(item) for item in self.descriptors) + current = _open_directory_chain(self.path) + try: + observed = tuple(_directory_identity(item) for item in current) + finally: + for descriptor in reversed(current): + os.close(descriptor) + if retained != self.identities or observed != self.identities: + raise OSError(errno.ESTALE, "directory authority changed") + except OSError: + raise ValueError(self.error_code) from None + + def reverify(self) -> None: + with self._state.lock: + self._reverify_locked() + + def open_file(self, name: str, flags: int, *, mode: int = 0o600) -> int: + name = _safe_child_name(name, error_code=self.error_code) + with self._state.lock: + self._reverify_locked() + descriptor = os.open( + name, + flags | _FILE_SAFETY_FLAGS, + mode, + dir_fd=self.descriptor, + ) + try: + self._reverify_locked() + except BaseException: + os.close(descriptor) + raise + return descriptor + + def open_exclusive(self, name: str, *, mode: int = 0o600) -> int: + return self.open_file(name, os.O_RDWR | os.O_CREAT | os.O_EXCL, mode=mode) + + def contains(self, name: str) -> bool: + name = _safe_child_name(name, error_code=self.error_code) + with self._state.lock: + self._reverify_locked() + try: + os.stat(name, dir_fd=self.descriptor, follow_symlinks=False) + except FileNotFoundError: + present = False + else: + present = True + self._reverify_locked() + return present + + def unlink(self, name: str, *, missing_ok: bool = False) -> None: + name = _safe_child_name(name, error_code=self.error_code) + with self._state.lock: + self._reverify_locked() + try: + os.unlink(name, dir_fd=self.descriptor) + except FileNotFoundError: + if not missing_ok: + raise + os.fsync(self.descriptor) + self._reverify_locked() + + def close(self) -> None: + with self._state.lock: + if self._state.closed: + return + self._state.closed = True + first_error: OSError | None = None + for descriptor in reversed(self.descriptors): + try: + os.close(descriptor) + except OSError as error: + if error.errno != errno.EBADF and first_error is None: + first_error = error + if first_error is not None: + raise ValueError(self.error_code) from None + + +def _file_identity(observed: os.stat_result) -> tuple[int, int, int, int]: + if not stat.S_ISREG(observed.st_mode): + raise OSError(errno.EINVAL, "retained authority is not a regular file") + return ( + observed.st_dev, + observed.st_ino, + observed.st_size, + observed.st_mtime_ns, + ) + + +def _pread_bounded(descriptor: int, *, maximum_bytes: int) -> bytes: + before = os.fstat(descriptor) + identity = _file_identity(before) + size = identity[2] + if size < 0 or size > maximum_bytes: + raise ValueError(_DATABASE_AUTHORITY_CHANGED) + payload = bytearray() + offset = 0 + while offset < size: + chunk = os.pread(descriptor, min(1024 * 1024, size - offset), offset) + if not chunk: + raise ValueError(_DATABASE_AUTHORITY_CHANGED) + payload.extend(chunk) + offset += len(chunk) + if os.pread(descriptor, 1, size): + raise ValueError(_DATABASE_AUTHORITY_CHANGED) + if _file_identity(os.fstat(descriptor)) != identity: + raise ValueError(_DATABASE_AUTHORITY_CHANGED) + return bytes(payload) + + +def _assert_no_sqlite_sidecars(directory_descriptor: int, name: str) -> None: + for suffix in _SQLITE_SIDECAR_SUFFIXES: + try: + os.stat( + name + suffix, + dir_fd=directory_descriptor, + follow_symlinks=False, + ) + except FileNotFoundError: + continue + raise OSError(errno.EBUSY, "SQLite sidecar state is active") + + +def _call_fclonefileat( + source_descriptor: int, + destination_descriptor: int, + name: str, +) -> int: + """Atomically clone, then immediately retain the published inode readonly. + + Darwin's public clone primitive returns status rather than a descriptor, so + clone creation and descriptor capture are necessarily separate syscalls. + """ + + library = ctypes.CDLL(None, use_errno=True) + clone = library.fclonefileat + clone.argtypes = ( + ctypes.c_int, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint32, + ) + clone.restype = ctypes.c_int + result = clone( + source_descriptor, + destination_descriptor, + os.fsencode(name), + 0, + ) + if result != 0: + observed_errno = ctypes.get_errno() + raise OSError(observed_errno, os.strerror(observed_errno)) + return os.open( + name, + os.O_RDONLY | _FILE_SAFETY_FLAGS, + dir_fd=destination_descriptor, + ) + + +def _call_linkat( + source_descriptor: int, + source_name: bytes, + destination_descriptor: int, + destination_name: str, + flags: int, +) -> None: + library = ctypes.CDLL(None, use_errno=True) + link = library.linkat + link.argtypes = ( + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ) + link.restype = ctypes.c_int + result = link( + source_descriptor, + source_name, + destination_descriptor, + os.fsencode(destination_name), + flags, + ) + if result != 0: + observed_errno = ctypes.get_errno() + raise OSError(observed_errno, os.strerror(observed_errno)) + + +def _link_anonymous_file( + descriptor: int, + destination_descriptor: int, + name: str, +) -> None: + """Link an ``O_TMPFILE`` descriptor without requiring a Linux capability.""" + + try: + _call_linkat( + descriptor, + b"", + destination_descriptor, + name, + _LINUX_AT_EMPTY_PATH, + ) + return + except OSError as error: + if error.errno not in _LINKAT_CAPABILITY_ERRNOS: + raise + _call_linkat( + _LINUX_AT_FDCWD, + os.fsencode(f"/proc/self/fd/{descriptor}"), + destination_descriptor, + name, + _LINUX_AT_SYMLINK_FOLLOW, + ) + + +def _publish_atomic_clone( + *, + source_descriptor: int, + destination_descriptor: int, + name: str, + payload: bytes, +) -> int: + """Publish exact bytes without any cleanup-requiring visible stage.""" + + if sys.platform == "darwin": + return _call_fclonefileat( + source_descriptor, + destination_descriptor, + name, + ) + if sys.platform.startswith("linux") and hasattr(os, "O_TMPFILE"): + descriptor = os.open( + ".", + os.O_RDWR | os.O_TMPFILE | getattr(os, "O_CLOEXEC", 0), + 0o600, + dir_fd=destination_descriptor, + ) + try: + remaining = memoryview(payload) + while remaining: + written = os.write(descriptor, remaining) + if written <= 0: + raise OSError(errno.EIO, "database clone write failed") + remaining = remaining[written:] + os.fsync(descriptor) + if _pread_bounded(descriptor, maximum_bytes=MAXIMUM_SQLITE_BYTES) != payload: + raise OSError(errno.EIO, "database clone verification failed") + _link_anonymous_file(descriptor, destination_descriptor, name) + return descriptor + except BaseException: + os.close(descriptor) + raise + raise OSError(errno.ENOTSUP, "atomic retained clone publication is unavailable") + + +def _private_pager_payload(payload: bytes) -> bytes: + """Make a closed WAL image readable by SQLite's anonymous private pager. + + SQLite persists WAL mode in header bytes 18 and 19 even after the final + connection checkpoints and removes the WAL. ``deserialize`` cannot attach + a WAL file to ``:memory:``, so normalize only those pager-mode bytes in the + ephemeral image. The authority verifies that no WAL, shared-memory, or + rollback-journal sidecar exists before this normalization. Authority + digests and clone writes continue to use the exact retained payload. + """ + + if len(payload) >= 20 and payload[:16] == b"SQLite format 3\x00": + read_version, write_version = payload[18], payload[19] + if (read_version, write_version) == (2, 2): + private = bytearray(payload) + private[18:20] = b"\x01\x01" + return bytes(private) + return payload + + +def _validate_sqlite_integrity(connection: sqlite3.Connection) -> None: + registered: set[str] = set() + while True: + try: + integrity = connection.execute("PRAGMA integrity_check").fetchone() + except sqlite3.OperationalError as error: + match = _UNKNOWN_SQLITE_FUNCTION.fullmatch(str(error)) + if match is None or match.group(1) in registered or len(registered) >= 128: + raise + name = match.group(1) + registered.add(name) + connection.create_function(name, -1, lambda *_arguments: 1, deterministic=True) + continue + if integrity is None or integrity[0] != "ok": + raise sqlite3.DatabaseError("retained SQLite clone failed integrity check") + return + + +@dataclass(frozen=True, slots=True) +class RetainedSqliteAuthority: + path: Path + descriptor: int + parent_descriptor: int + parent_device: int + parent_inode: int + device: int + inode: int + size: int + mtime_ns: int + digest: str + _state: _AuthorityState = field( + default_factory=_AuthorityState, compare=False, repr=False + ) + + @classmethod + def capture(cls, path: Path) -> RetainedSqliteAuthority: + if not isinstance(path, Path) or not path.is_absolute(): + raise ValueError(_DATABASE_AUTHORITY_CHANGED) + descriptor = -1 + parent_descriptor = -1 + try: + parent_descriptors = _open_directory_chain(path.parent) + parent_descriptor = parent_descriptors.pop() + for retained_parent in reversed(parent_descriptors): + os.close(retained_parent) + descriptor = os.open( + path.name, + os.O_RDONLY | _FILE_SAFETY_FLAGS, + dir_fd=parent_descriptor, + ) + _assert_no_sqlite_sidecars(parent_descriptor, path.name) + payload = _pread_bounded( + descriptor, maximum_bytes=MAXIMUM_SQLITE_BYTES + ) + _assert_no_sqlite_sidecars(parent_descriptor, path.name) + observed = os.fstat(descriptor) + parent_observed = os.fstat(parent_descriptor) + authority = cls( + path=path, + descriptor=descriptor, + parent_descriptor=parent_descriptor, + parent_device=parent_observed.st_dev, + parent_inode=parent_observed.st_ino, + device=observed.st_dev, + inode=observed.st_ino, + size=observed.st_size, + mtime_ns=observed.st_mtime_ns, + digest="sha256:" + hashlib.sha256(payload).hexdigest(), + ) + authority.reverify() + return authority + except (OSError, ValueError) as error: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + pass + if parent_descriptor >= 0: + try: + os.close(parent_descriptor) + except OSError: + pass + if isinstance(error, ValueError) and str(error) == _DATABASE_AUTHORITY_CHANGED: + raise + raise ValueError(_DATABASE_AUTHORITY_CHANGED) from None + except BaseException: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + pass + if parent_descriptor >= 0: + try: + os.close(parent_descriptor) + except OSError: + pass + raise + + def _expected_identity(self) -> tuple[int, int, int, int]: + return self.device, self.inode, self.size, self.mtime_ns + + def _verified_payload_locked(self) -> bytes: + if self._state.closed: + raise ValueError(_DATABASE_AUTHORITY_CHANGED) + try: + retained = _file_identity(os.fstat(self.descriptor)) + retained_parent = _directory_identity(self.parent_descriptor) + current_parents = _open_directory_chain(self.path.parent) + try: + current_parent = _directory_identity(current_parents[-1]) + finally: + for current_descriptor in reversed(current_parents): + os.close(current_descriptor) + expected_parent = self.parent_device, self.parent_inode + current = _file_identity( + os.stat( + self.path.name, + dir_fd=self.parent_descriptor, + follow_symlinks=False, + ) + ) + if retained != self._expected_identity() or current != self._expected_identity(): + raise OSError(errno.ESTALE, "database authority changed") + if retained_parent != expected_parent or current_parent != expected_parent: + raise OSError(errno.ESTALE, "database parent authority changed") + _assert_no_sqlite_sidecars(self.parent_descriptor, self.path.name) + payload = _pread_bounded( + self.descriptor, maximum_bytes=MAXIMUM_SQLITE_BYTES + ) + if "sha256:" + hashlib.sha256(payload).hexdigest() != self.digest: + raise OSError(errno.ESTALE, "database authority changed") + if _file_identity(os.fstat(self.descriptor)) != self._expected_identity(): + raise OSError(errno.ESTALE, "database authority changed") + _assert_no_sqlite_sidecars(self.parent_descriptor, self.path.name) + return payload + except (OSError, ValueError): + raise ValueError(_DATABASE_AUTHORITY_CHANGED) from None + + def reverify(self) -> None: + with self._state.lock: + self._verified_payload_locked() + + @contextmanager + def connect_readonly(self) -> Iterator[sqlite3.Connection]: + with self._state.lock: + payload = self._verified_payload_locked() + connection = sqlite3.connect(":memory:", isolation_level=None) + try: + connection.deserialize(_private_pager_payload(payload)) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA query_only = ON") + self._verified_payload_locked() + yield connection + self._verified_payload_locked() + finally: + connection.close() + + def clone_to( + self, destination: RetainedDirectoryAuthority, name: str + ) -> RetainedSqliteAuthority: + """Clone into an app-owned, mutation-exclusive destination namespace. + + Darwin exact publication relies on that ownership contract: public Darwin + APIs cannot defend arbitrary, uncoordinated same-UID child-entry mutation + between clone installation and retaining the published descriptor. + """ + + name = _safe_child_name(name, error_code=destination.error_code) + source_descriptor = -1 + destination_descriptor = -1 + clone_descriptor = -1 + with self._state.lock: + payload = self._verified_payload_locked() + source_descriptor = os.dup(self.descriptor) + try: + with destination._state.lock: + destination._reverify_locked() + destination_descriptor = os.dup(destination.descriptor) + connection = sqlite3.connect(":memory:", isolation_level=None) + try: + connection.deserialize(_private_pager_payload(payload)) + connection.execute("PRAGMA query_only = ON") + _validate_sqlite_integrity(connection) + finally: + connection.close() + if _file_identity(os.fstat(source_descriptor)) != self._expected_identity(): + raise OSError(errno.ESTALE, "database authority changed") + if _pread_bounded( + source_descriptor, + maximum_bytes=MAXIMUM_SQLITE_BYTES, + ) != payload: + raise OSError(errno.ESTALE, "database authority changed") + if _directory_identity(destination_descriptor) != destination.identities[-1]: + raise OSError(errno.ESTALE, "destination authority changed") + current_directories = _open_directory_chain(destination.path) + try: + current_identities = tuple( + _directory_identity(item) for item in current_directories + ) + finally: + for current_descriptor in reversed(current_directories): + os.close(current_descriptor) + if current_identities != destination.identities: + raise OSError(errno.ESTALE, "destination authority changed") + clone_descriptor = _publish_atomic_clone( + source_descriptor=source_descriptor, + destination_descriptor=destination_descriptor, + name=name, + payload=payload, + ) + observed = os.fstat(clone_descriptor) + if _pread_bounded( + clone_descriptor, + maximum_bytes=MAXIMUM_SQLITE_BYTES, + ) != payload: + raise OSError(errno.EIO, "database clone verification failed") + current_clone = _file_identity( + os.stat( + name, + dir_fd=destination_descriptor, + follow_symlinks=False, + ) + ) + if current_clone != _file_identity(observed): + raise OSError(errno.ESTALE, "database clone publication changed") + parent_observed = os.fstat(destination_descriptor) + clone = RetainedSqliteAuthority( + path=destination.path / name, + descriptor=clone_descriptor, + parent_descriptor=destination_descriptor, + parent_device=parent_observed.st_dev, + parent_inode=parent_observed.st_ino, + device=observed.st_dev, + inode=observed.st_ino, + size=observed.st_size, + mtime_ns=observed.st_mtime_ns, + digest="sha256:" + hashlib.sha256(payload).hexdigest(), + ) + clone_descriptor = -1 + destination_descriptor = -1 + return clone + except BaseException: + if clone_descriptor >= 0: + os.close(clone_descriptor) + if destination_descriptor >= 0: + os.close(destination_descriptor) + raise + finally: + if source_descriptor >= 0: + os.close(source_descriptor) + + def close(self) -> None: + with self._state.lock: + if self._state.closed: + return + self._state.closed = True + try: + os.close(self.descriptor) + except OSError as error: + if error.errno != errno.EBADF: + raise ValueError(_DATABASE_AUTHORITY_CHANGED) from None + try: + os.close(self.parent_descriptor) + except OSError as error: + if error.errno != errno.EBADF: + raise ValueError(_DATABASE_AUTHORITY_CHANGED) from None diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/normal_generation.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/normal_generation.py new file mode 100644 index 0000000..9b98540 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/normal_generation.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import cast + +from video_lab.artifacts import ArtifactStore +from video_lab.comparison import build_controlled_pair +from video_lab.domain import ExperimentDraft, JobState, Manifest +from video_lab.repositories import ExperimentRepository, JobRepository +from video_lab.runtime_identity import runtime_fingerprint +from video_lab.worker import Worker + + +@dataclass(frozen=True) +class CommittedGenerationEvidence: + job_id: str + artifact_sha256: str + artifact: dict[str, object] + source_fingerprint: str + runtime_fingerprint: str + placement_fingerprint: str + placement: str + fallback_violations: tuple[str, ...] = () + experiment_id: str | None = None + + def stage_details(self) -> dict[str, object]: + return { + "job_id": self.job_id, + "artifact_sha256": self.artifact_sha256, + "artifact": self.artifact, + "source_fingerprint": self.source_fingerprint, + "runtime_fingerprint": self.runtime_fingerprint, + "placement_fingerprint": self.placement_fingerprint, + "placement": self.placement, + "fallback_violations": self.fallback_violations, + } + + +class RepositoryWorkerGenerationRunner: + """Runs manifests through persisted experiment/jobs, Worker, and ArtifactStore.""" + + def __init__( + self, + *, + experiments: ExperimentRepository, + jobs: JobRepository, + artifacts: ArtifactStore, + worker: Worker, + ) -> None: + self._experiments = experiments + self._jobs = jobs + self._artifacts = artifacts + self._worker = worker + + def run_one(self, manifest: Manifest) -> CommittedGenerationEvidence: + job = self._experiments.create_single_generation(manifest) + outcome = self._worker.run_once(job.id) + if outcome.status != "completed" or outcome.job_id != job.id: + raise RuntimeError( + f"normal generation job failed: {outcome.error_code}: " + f"{self._failure_detail(job.id)}" + ) + return self._load(job.id, manifest) + + def run_pair( + self, + baseline: Manifest, + candidate: Manifest, + *, + variable_path: str, + ) -> tuple[CommittedGenerationEvidence, CommittedGenerationEvidence]: + pair = build_controlled_pair( + ExperimentDraft( + baseline=baseline, + variable_path=variable_path, + alternate_value=_value_at_path(candidate, variable_path), + ) + ) + if pair.variant_b != candidate: + raise ValueError(f"controlled pair differs outside {variable_path}") + experiment, jobs = self._experiments.create_with_jobs(pair) + for expected in jobs: + outcome = self._worker.run_once(expected.id) + if outcome.status != "completed" or outcome.job_id != expected.id: + raise RuntimeError( + f"normal generation job failed: {outcome.error_code}: " + f"{self._failure_detail(expected.id)}" + ) + return tuple( + self._load( + job.id, + self._jobs.manifest_for_job(job.id), + experiment_id=experiment.id, + ) + for job in jobs + ) # type: ignore[return-value] + + def _failure_detail(self, job_id: str) -> str: + events = self._jobs.events_for_job(job_id) + message = None if not events else events[-1].payload.get("message") + return str(message or "failure detail unavailable") + + def _load( + self, job_id: str, manifest: Manifest, *, experiment_id: str | None = None + ) -> CommittedGenerationEvidence: + if self._jobs.manifest_for_job(job_id) != manifest: + raise RuntimeError("committed job manifest differs from request") + if self._jobs.get(job_id).state is not JobState.COMPLETED: + raise RuntimeError("normal generation job did not complete") + artifacts = self._artifacts.list_for_job(job_id) + if len(artifacts) != 1 or artifacts[0].media_type != "video/mp4": + raise RuntimeError("normal generation job lacks one committed MP4") + completed = self._jobs.events_for_job(job_id)[-1] + if completed.kind != "job_completed": + raise RuntimeError("normal generation job lacks completion evidence") + payload = completed.payload + telemetry = payload.get("adapter_telemetry") + if not isinstance(telemetry, Mapping): + raise RuntimeError("normal generation job lacks adapter telemetry") + observed = telemetry.get("observed_configuration") + if not isinstance(observed, Mapping): + raise RuntimeError("normal generation job lacks observed configuration") + decoded = payload.get("decoded_frame_sha256") + if not isinstance(decoded, Sequence) or isinstance(decoded, str | bytes | bytearray): + raise RuntimeError("normal generation job lacks decoded frame evidence") + decoded_hashes = tuple(str(value) for value in decoded) + artifact = artifacts[0] + return CommittedGenerationEvidence( + job_id=job_id, + artifact_sha256=artifact.sha256, + artifact={ + "format": "mp4", + "valid": True, + "sha256": artifact.sha256, + "decoded_frame_sha256": decoded_hashes, + "codec": payload.get("codec", "h264"), + "duration_seconds": payload.get("duration_seconds", 0.0), + "width": payload["width"], + "height": payload["height"], + "num_frames": payload["frame_count"], + "fps": payload["fps"], + }, + source_fingerprint=str(observed["source_fingerprint"]), + runtime_fingerprint=runtime_fingerprint( + cast(Mapping[str, str], payload["runtime_versions"]) + ), + placement_fingerprint=str(observed["placement_fingerprint"]), + placement="local_hybrid", + fallback_violations=tuple(cast(list[str], telemetry.get("fallback_violations", []))), + experiment_id=experiment_id, + ) + + +def _value_at_path(manifest: Manifest, variable_path: str) -> object: + value: object = manifest.model_dump(mode="json") + for part in variable_path.split("."): + if not isinstance(value, Mapping) or part not in value: + raise ValueError(f"unknown variable path: {variable_path}") + value = value[part] + return value diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/operations.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/operations.py new file mode 100644 index 0000000..0afce21 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/operations.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from typing import Any, Literal + +from video_lab.adapter_registry import AdapterRegistry +from video_lab.capabilities import AdapterCapabilities +from video_lab.domain import ExperimentDraft, FrozenModel, Manifest +from video_lab.services import ExperimentService, ExperimentView + + +class ManifestFieldSpec(FrozenModel): + path: str + label: str + value_type: Literal["string", "integer", "number", "boolean"] + options: tuple[str, ...] = () + minimum: float | None = None + maximum: float | None = None + step: float | None = None + rule: Literal["16n+1"] | None = None + + +class AdapterCatalogEntry(FrozenModel): + adapter_id: str + display_name: str + capabilities: AdapterCapabilities + fields: tuple[ManifestFieldSpec, ...] + source_model: str | None = None + frame_rule: str | None = None + declared_profiles: tuple[str, ...] = () + generation_enabled: bool = True + + +class ExperimentPreview(FrozenModel): + variant_a: Manifest + variant_b: Manifest + variable_path: str + changed_paths: tuple[str, ...] + + +def _field_specs(capabilities: AdapterCapabilities) -> tuple[ManifestFieldSpec, ...]: + return ( + ManifestFieldSpec(path="prompt", label="Prompt", value_type="string"), + ManifestFieldSpec(path="negative_prompt", label="Negative prompt", value_type="string"), + ManifestFieldSpec(path="seed", label="Seed", value_type="integer"), + ManifestFieldSpec( + path="width", + label="Width", + value_type="integer", + minimum=capabilities.min_width, + maximum=capabilities.max_width, + step=capabilities.width_multiple, + ), + ManifestFieldSpec( + path="height", + label="Height", + value_type="integer", + minimum=capabilities.min_height, + maximum=capabilities.max_height, + step=capabilities.height_multiple, + ), + ManifestFieldSpec( + path="num_frames", + label="Frames", + value_type="integer", + minimum=capabilities.min_frames, + maximum=capabilities.max_frames, + rule=None if capabilities.frame_rule == "any" else capabilities.frame_rule, + ), + ManifestFieldSpec(path="fps", label="FPS", value_type="integer", minimum=1), + ManifestFieldSpec( + path="inference_steps", label="Inference steps", value_type="integer", minimum=1 + ), + ManifestFieldSpec( + path="guidance_scale", label="Guidance scale", value_type="number", minimum=0 + ), + ManifestFieldSpec( + path="scheduler", + label="Scheduler", + value_type="string", + options=capabilities.schedulers, + ), + ) + + +class LaboratoryOperations: + def __init__(self, registry: AdapterRegistry, service: ExperimentService) -> None: + self.registry = registry + self.service = service + + @staticmethod + def catalog_for(registry: AdapterRegistry) -> tuple[AdapterCatalogEntry, ...]: + return tuple( + AdapterCatalogEntry( + adapter_id=metadata.adapter_id, + display_name=metadata.display_name, + capabilities=capabilities, + fields=_field_specs(capabilities), + source_model=metadata.source_model, + frame_rule=metadata.frame_rule, + declared_profiles=metadata.declared_profiles, + generation_enabled=metadata.generation_enabled, + ) + for metadata, capabilities in registry.catalog_metadata() + ) + + def catalog(self) -> tuple[AdapterCatalogEntry, ...]: + return self.catalog_for(self.registry) + + def preview( + self, draft: ExperimentDraft | dict[str, Any], adapter_id: str + ) -> ExperimentPreview: + canonical = ExperimentDraft.model_validate(draft) + pair = self.service.preview(canonical, self.registry.capabilities(adapter_id)) + return ExperimentPreview.model_validate(pair.model_dump(mode="json")) + + def create(self, draft: ExperimentDraft | dict[str, Any], adapter_id: str) -> ExperimentView: + canonical = ExperimentDraft.model_validate(draft) + return self.service.create(canonical, self.registry.capabilities(adapter_id)) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/preference_optimizer.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/preference_optimizer.py new file mode 100644 index 0000000..e8b9d71 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/preference_optimizer.py @@ -0,0 +1,560 @@ +from __future__ import annotations + +import hashlib +import math +from enum import StrEnum +from typing import Literal, Self + +import numpy as np +import numpy.typing as npt +from pydantic import Field, model_validator + +from video_lab.domain import FrozenModel, canonical_json + +FloatArray = npt.NDArray[np.float64] + +OPTIMIZER_VERSION: Literal["preference-gp-laplace.v1"] = "preference-gp-laplace.v1" +OPTIMIZER_CONFIG_SCHEMA_VERSION: Literal["preference-optimizer-config.v1"] = ( + "preference-optimizer-config.v1" +) +_ADAPTER_SAFE_LOWER_BOUND = 1.0 +_ADAPTER_SAFE_UPPER_BOUND = 5.0 +# V1 is qualified on at most 17 points, bounding cubic GP work and pair enumeration. +_MAX_GUIDANCE_GRID_CARDINALITY = 17 + + +class PreferenceOutcome(StrEnum): + A = "a" + B = "b" + TIE = "no_perceptible_difference" + + +class StopReason(StrEnum): + CONTINUE = "continue" + SEARCH_CONVERGED = "search_converged" + SEARCH_BUDGET_EXHAUSTED = "search_budget_exhausted" + CANDIDATE_SPACE_EXHAUSTED = "candidate_space_exhausted" + + +class SearchDisposition(StrEnum): + INCONCLUSIVE_WITHIN_BUDGET = "inconclusive_within_budget" + + +class CandidatePair(FrozenModel): + value_a: float + value_b: float + + @model_validator(mode="after") + def distinct_values(self) -> Self: + if not math.isfinite(self.value_a) or not math.isfinite(self.value_b): + raise ValueError("candidate pair values must be finite") + if self.value_a == self.value_b: + raise ValueError("candidate pair values must differ") + return self + + +class PreferenceObservation(CandidatePair): + outcome: PreferenceOutcome + + +class OptimizerConfig(FrozenModel): + schema_version: Literal["preference-optimizer-config.v1"] = OPTIMIZER_CONFIG_SCHEMA_VERSION + optimizer_version: Literal["preference-gp-laplace.v1"] = OPTIMIZER_VERSION + lower_bound: float = 1.0 + upper_bound: float = 5.0 + resolution: float = Field(default=0.25, gt=0) + kernel_amplitude: float = Field(default=1.0, gt=0) + kernel_length_scale: float = Field(default=0.75, gt=0) + jitter: float = Field(default=1e-6, gt=0) + tie_threshold: float = Field(default=0.35, gt=0) + minimum_comparisons: Literal[4] = 4 + maximum_comparisons: Literal[10] = 10 + concentration_mass: float = Field(default=0.90, gt=0, lt=1) + concentration_width: float = Field(default=0.5, ge=0) + information_threshold_bits: float = Field(default=0.01, ge=0) + posterior_draws: int = Field(default=4096, ge=256) + + @model_validator(mode="after") + def valid_interval_and_budget(self) -> Self: + if not all( + math.isfinite(value) + for value in ( + self.lower_bound, + self.upper_bound, + self.resolution, + self.kernel_amplitude, + self.kernel_length_scale, + self.jitter, + self.tie_threshold, + self.concentration_mass, + self.concentration_width, + self.information_threshold_bits, + ) + ): + raise ValueError("optimizer floating-point inputs must be finite") + if ( + self.lower_bound < _ADAPTER_SAFE_LOWER_BOUND + or self.upper_bound > _ADAPTER_SAFE_UPPER_BOUND + ): + raise ValueError("optimizer bounds must stay within the adapter-safe range") + if self.upper_bound <= self.lower_bound: + raise ValueError("optimizer bounds must form an exact positive grid") + span_steps = (self.upper_bound - self.lower_bound) / self.resolution + if not math.isfinite(span_steps): + raise ValueError("optimizer grid cardinality must be finite and bounded") + if span_steps > (_MAX_GUIDANCE_GRID_CARDINALITY - 1) + 1e-9: + raise ValueError( + "optimizer grid cardinality exceeds the v1 maximum of " + f"{_MAX_GUIDANCE_GRID_CARDINALITY}" + ) + rounded_steps = round(span_steps) + if not math.isclose(span_steps, rounded_steps, rel_tol=0.0, abs_tol=1e-9): + raise ValueError("optimizer bounds must form an exact positive grid") + return self + + @property + def initial_pair(self) -> CandidatePair: + grid = build_guidance_grid(self) + lower_target = self.lower_bound + 0.25 * (self.upper_bound - self.lower_bound) + upper_target = self.lower_bound + 0.75 * (self.upper_bound - self.lower_bound) + return CandidatePair( + value_a=_nearest_grid_value(grid, lower_target), + value_b=_nearest_grid_value(grid, upper_target), + ) + + def validate_observations(self, observations: tuple[PreferenceObservation, ...]) -> None: + self.validate_observation_grid(observations) + if len(observations) > self.maximum_comparisons: + raise ValueError("preference observations exceed the comparison budget") + + def validate_observation_grid(self, observations: tuple[PreferenceObservation, ...]) -> None: + allowed = set(build_guidance_grid(self)) + if any(item.value_a not in allowed or item.value_b not in allowed for item in observations): + raise ValueError("preference observation value is outside the configured grid") + + +class PosteriorSummary(FrozenModel): + grid: tuple[float, ...] + mean: tuple[float, ...] + covariance: tuple[tuple[float, ...], ...] + argmax_probability: tuple[float, ...] + recommendation: float + concentration_interval: tuple[float, float] + concentration_probability: float + content_digest: str + + +class AcquisitionScore(FrozenModel): + pair: CandidatePair + expected_information_bits: float = Field(ge=0) + + +class StoppingDecision(FrozenModel): + stop: bool + reason: StopReason + disposition: SearchDisposition | None = None + comparison_count: int = Field(ge=0) + maximum_information_bits: float = Field(ge=0) + + +class OptimizerDecision(FrozenModel): + posterior: PosteriorSummary + candidate_scores: tuple[AcquisitionScore, ...] + selected_pair: CandidatePair | None + stopping: StoppingDecision + + +def build_guidance_grid(config: OptimizerConfig) -> tuple[float, ...]: + steps = round((config.upper_bound - config.lower_bound) / config.resolution) + return tuple( + round(config.lower_bound + index * config.resolution, 12) for index in range(steps + 1) + ) + + +def _nearest_grid_value(grid: tuple[float, ...], target: float) -> float: + return min(grid, key=lambda value: (abs(value - target), value)) + + +def _sigmoid(value: float) -> float: + if value >= 0: + inverse = math.exp(-value) + return 1.0 / (1.0 + inverse) + exponential = math.exp(value) + return exponential / (1.0 + exponential) + + +def outcome_probabilities( + config: OptimizerConfig, + *, + difference_mean: float, + difference_variance: float, +) -> dict[PreferenceOutcome, float]: + scale = math.sqrt(1.0 + math.pi * max(0.0, difference_variance) / 8.0) + mean = difference_mean / scale + threshold = config.tie_threshold / scale + probability_a = _sigmoid(mean - threshold) + probability_b = _sigmoid(-mean - threshold) + probability_tie = max(0.0, 1.0 - probability_a - probability_b) + total = probability_a + probability_b + probability_tie + return { + PreferenceOutcome.A: probability_a / total, + PreferenceOutcome.B: probability_b / total, + PreferenceOutcome.TIE: probability_tie / total, + } + + +def _observation_probability( + config: OptimizerConfig, difference: float, outcome: PreferenceOutcome +) -> float: + return outcome_probabilities(config, difference_mean=difference, difference_variance=0.0)[ + outcome + ] + + +def _outcome_log_likelihood( + config: OptimizerConfig, difference: float, outcome: PreferenceOutcome +) -> float: + return math.log(max(_observation_probability(config, difference, outcome), 1e-15)) + + +def _rbf_covariance(config: OptimizerConfig, grid: FloatArray) -> FloatArray: + distances = grid[:, None] - grid[None, :] + covariance = config.kernel_amplitude**2 * np.exp( + -0.5 * distances**2 / config.kernel_length_scale**2 + ) + covariance += np.eye(len(grid), dtype=np.float64) * config.jitter + return covariance + + +def _log_likelihood_derivatives( + config: OptimizerConfig, difference: float, outcome: PreferenceOutcome +) -> tuple[float, float]: + step = 1e-4 + center = _outcome_log_likelihood(config, difference, outcome) + lower = _outcome_log_likelihood(config, difference - step, outcome) + upper = _outcome_log_likelihood(config, difference + step, outcome) + first = (upper - lower) / (2.0 * step) + second = (upper - 2.0 * center + lower) / (step * step) + return first, min(second, -1e-10) + + +def _map_utility( + config: OptimizerConfig, + grid: tuple[float, ...], + observations: tuple[PreferenceObservation, ...], + prior_inverse: FloatArray, +) -> tuple[FloatArray, FloatArray]: + indexes = {value: index for index, value in enumerate(grid)} + utility = np.zeros(len(grid), dtype=np.float64) + precision = prior_inverse.copy() + for _iteration in range(80): + gradient = -(prior_inverse @ utility) + precision = prior_inverse.copy() + for observation in observations: + index_a = indexes[observation.value_a] + index_b = indexes[observation.value_b] + first, second = _log_likelihood_derivatives( + config, utility[index_a] - utility[index_b], observation.outcome + ) + direction = np.zeros(len(grid), dtype=np.float64) + direction[index_a] = 1.0 + direction[index_b] = -1.0 + gradient += first * direction + precision += -second * np.outer(direction, direction) + step = np.linalg.solve(precision, gradient) + candidate = utility + step + if float(np.max(np.abs(candidate - utility))) < 1e-9: + utility = candidate + break + utility = candidate + else: + raise RuntimeError("preference posterior MAP did not converge") + return utility, precision + + +def _posterior_covariance(precision: FloatArray) -> FloatArray: + return np.linalg.inv(precision) + + +def _sampling_seed(config: OptimizerConfig, observations: tuple[PreferenceObservation, ...]) -> int: + payload = canonical_json( + { + "optimizer_version": config.optimizer_version, + "config": config.model_dump(mode="json"), + "observations": [item.model_dump(mode="json") for item in observations], + } + ) + return int.from_bytes(hashlib.sha256(payload.encode()).digest()[:8], "big") + + +def _standard_normal_draws( + config: OptimizerConfig, + observations: tuple[PreferenceObservation, ...], +) -> FloatArray: + generator = np.random.default_rng(_sampling_seed(config, observations)) + return generator.standard_normal( + size=(config.posterior_draws, len(build_guidance_grid(config))) + ) + + +def shortest_mass_interval( + *, + grid: tuple[float, ...], + probability: tuple[float, ...], + target_mass: float, +) -> tuple[tuple[float, float], float]: + if len(grid) != len(probability) or not grid: + raise ValueError("grid and probability must have equal non-zero length") + candidates: list[tuple[float, float, float, float]] = [] + for lower in range(len(grid)): + mass = 0.0 + for upper in range(lower, len(grid)): + mass += probability[upper] + if mass + 1e-12 >= target_mass: + candidates.append((grid[upper] - grid[lower], grid[lower], grid[upper], mass)) + break + if not candidates: + return (grid[0], grid[-1]), float(sum(probability)) + _width, lower_value, upper_value, mass = min(candidates) + return (lower_value, upper_value), mass + + +def _argmax_projection( + config: OptimizerConfig, + grid: tuple[float, ...], + mean: FloatArray, + covariance: FloatArray, + standard_normal_draws: FloatArray, +) -> tuple[FloatArray, int, tuple[float, float], float]: + stabilized = covariance + np.eye(len(grid), dtype=np.float64) * config.jitter + factor = np.linalg.cholesky(stabilized) + samples = mean + standard_normal_draws @ factor.T + winners = np.argmax(samples, axis=1) + counts = np.bincount(winners, minlength=len(grid)).astype(np.float64) + probabilities = counts / float(config.posterior_draws) + recommendation_index = int(np.argmax(probabilities)) + interval, mass = shortest_mass_interval( + grid=grid, + probability=tuple(float(value) for value in probabilities), + target_mass=config.concentration_mass, + ) + return probabilities, recommendation_index, interval, mass + + +def fit_posterior( + config: OptimizerConfig, + observations: tuple[PreferenceObservation, ...], +) -> PosteriorSummary: + config.validate_observations(observations) + standard_normal_draws = _standard_normal_draws(config, observations) + return _fit_posterior_with_draws(config, observations, standard_normal_draws) + + +def _fit_posterior_with_draws( + config: OptimizerConfig, + observations: tuple[PreferenceObservation, ...], + standard_normal_draws: FloatArray, +) -> PosteriorSummary: + config.validate_observation_grid(observations) + grid = build_guidance_grid(config) + expected_shape = (config.posterior_draws, len(grid)) + if standard_normal_draws.shape != expected_shape: + raise ValueError("standard-normal draws do not match the optimizer grid") + try: + grid_array = np.asarray(grid, dtype=np.float64) + prior = _rbf_covariance(config, grid_array) + prior_inverse = np.linalg.inv(prior) + mean, precision = _map_utility(config, grid, observations, prior_inverse) + covariance = _posterior_covariance(precision) + probabilities, recommendation_index, interval, mass = _argmax_projection( + config, grid, mean, covariance, standard_normal_draws + ) + except np.linalg.LinAlgError as error: + raise RuntimeError("preference posterior numerical failure") from error + payload = { + "optimizer_version": config.optimizer_version, + "config": config.model_dump(mode="json"), + "observations": [item.model_dump(mode="json") for item in observations], + "grid": list(grid), + "mean": mean.tolist(), + "covariance": covariance.tolist(), + "argmax_probability": probabilities.tolist(), + "recommendation": grid[recommendation_index], + "concentration_interval": list(interval), + "concentration_probability": mass, + } + digest = hashlib.sha256(canonical_json(payload).encode()).hexdigest() + return PosteriorSummary( + grid=grid, + mean=tuple(float(value) for value in mean), + covariance=tuple(tuple(float(value) for value in row) for row in covariance), + argmax_probability=tuple(float(value) for value in probabilities), + recommendation=grid[recommendation_index], + concentration_interval=interval, + concentration_probability=mass, + content_digest=f"sha256:{digest}", + ) + + +def _entropy_bits(probability: tuple[float, ...]) -> float: + return -sum(value * math.log2(value) for value in probability if value > 0.0) + + +def _canonical_pair(left: float, right: float) -> CandidatePair: + return CandidatePair(value_a=min(left, right), value_b=max(left, right)) + + +def _eligible_pairs( + config: OptimizerConfig, + posterior: PosteriorSummary, +) -> tuple[CandidatePair, ...]: + lower, upper = posterior.concentration_interval + values = build_guidance_grid(config) + result: list[CandidatePair] = [] + for index, value_a in enumerate(values): + for value_b in values[index + 1 :]: + pair = CandidatePair(value_a=value_a, value_b=value_b) + if lower <= value_a <= upper or lower <= value_b <= upper: + result.append(pair) + return tuple(result) + + +def _difference_moments(posterior: PosteriorSummary, pair: CandidatePair) -> tuple[float, float]: + indexes = {value: index for index, value in enumerate(posterior.grid)} + index_a = indexes[pair.value_a] + index_b = indexes[pair.value_b] + covariance = posterior.covariance + mean = posterior.mean[index_a] - posterior.mean[index_b] + variance = ( + covariance[index_a][index_a] + + covariance[index_b][index_b] + - 2.0 * covariance[index_a][index_b] + ) + return mean, max(0.0, variance) + + +def _expected_information_estimate_bits( + config: OptimizerConfig, + observations: tuple[PreferenceObservation, ...], + posterior: PosteriorSummary, + pair: CandidatePair, + standard_normal_draws: FloatArray, +) -> float: + """Estimate EIG with version-pinned common random numbers for all outcomes.""" + mean, variance = _difference_moments(posterior, pair) + predictive = outcome_probabilities(config, difference_mean=mean, difference_variance=variance) + current_entropy = _entropy_bits(posterior.argmax_probability) + expected_entropy = 0.0 + for outcome in PreferenceOutcome: + hypothetical = observations + ( + PreferenceObservation(value_a=pair.value_a, value_b=pair.value_b, outcome=outcome), + ) + updated = _fit_posterior_with_draws(config, hypothetical, standard_normal_draws) + expected_entropy += predictive[outcome] * _entropy_bits(updated.argmax_probability) + return max(0.0, current_entropy - expected_entropy) + + +def _order_acquisition_scores( + scores: tuple[AcquisitionScore, ...], repeated: set[CandidatePair] +) -> tuple[AcquisitionScore, ...]: + return tuple( + sorted( + scores, + key=lambda score: ( + -score.expected_information_bits, + score.pair in repeated, + score.pair.value_a, + score.pair.value_b, + ), + ) + ) + + +def _candidate_scores( + config: OptimizerConfig, + observations: tuple[PreferenceObservation, ...], + posterior: PosteriorSummary, + standard_normal_draws: FloatArray, +) -> tuple[AcquisitionScore, ...]: + repeated = {_canonical_pair(item.value_a, item.value_b) for item in observations} + scores: list[AcquisitionScore] = [] + for pair in _eligible_pairs(config, posterior): + estimate = _expected_information_estimate_bits( + config, observations, posterior, pair, standard_normal_draws + ) + if pair in repeated and estimate <= 0.0: + continue + scores.append(AcquisitionScore(pair=pair, expected_information_bits=estimate)) + return _order_acquisition_scores(tuple(scores), repeated) + + +def _convergence_satisfied( + config: OptimizerConfig, + posterior: PosteriorSummary, + *, + comparison_count: int, + maximum_information_bits: float, +) -> bool: + interval_width = posterior.concentration_interval[1] - posterior.concentration_interval[0] + return ( + comparison_count >= config.minimum_comparisons + and posterior.concentration_probability >= config.concentration_mass + and interval_width <= config.concentration_width + and maximum_information_bits < config.information_threshold_bits + ) + + +def optimize_preference( + config: OptimizerConfig, + observations: tuple[PreferenceObservation, ...], +) -> OptimizerDecision: + config.validate_observations(observations) + standard_normal_draws = _standard_normal_draws(config, observations) + posterior = _fit_posterior_with_draws(config, observations, standard_normal_draws) + if not observations: + return OptimizerDecision( + posterior=posterior, + candidate_scores=(), + selected_pair=config.initial_pair, + stopping=StoppingDecision( + stop=False, + reason=StopReason.CONTINUE, + comparison_count=0, + maximum_information_bits=0.0, + ), + ) + ordered = _candidate_scores(config, observations, posterior, standard_normal_draws) + maximum_information = ordered[0].expected_information_bits if ordered else 0.0 + converged = _convergence_satisfied( + config, + posterior, + comparison_count=len(observations), + maximum_information_bits=maximum_information, + ) + at_budget = len(observations) >= config.maximum_comparisons + exhausted = not ordered + if converged: + reason = StopReason.SEARCH_CONVERGED + disposition = None + elif at_budget: + reason = StopReason.SEARCH_BUDGET_EXHAUSTED + disposition = SearchDisposition.INCONCLUSIVE_WITHIN_BUDGET + elif exhausted: + reason = StopReason.CANDIDATE_SPACE_EXHAUSTED + disposition = None + else: + reason = StopReason.CONTINUE + disposition = None + stop = converged or at_budget or exhausted + return OptimizerDecision( + posterior=posterior, + candidate_scores=ordered, + selected_pair=None if stop else ordered[0].pair, + stopping=StoppingDecision( + stop=stop, + reason=reason, + disposition=disposition, + comparison_count=len(observations), + maximum_information_bits=maximum_information, + ), + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/py.typed b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/queue_service.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/queue_service.py new file mode 100644 index 0000000..84946ff --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/queue_service.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from video_lab.artifacts import ArtifactStore +from video_lab.domain import FrozenModel, JobState, RunEvent, TelemetrySample +from video_lab.repositories import ExperimentRepository, JobRepository, StoredExperiment +from video_lab.review_service import ReviewService + +_EVIDENCE_LIMIT = 20 +_TEXT_LIMIT = 240 +_ACTIVE_STATES = { + state.value + for state in JobState + if state not in {JobState.QUEUED, JobState.COMPLETED, JobState.FAILED, JobState.CANCELLED} +} + + +class EvidenceView(FrozenModel): + sequence: int + kind: str + phase: str | None = None + message: str | None = None + + +class TelemetrySummary(FrozenModel): + elapsed_seconds: float + process_rss_bytes: int + system_used_bytes: int + system_available_bytes: int + swap_used_bytes: int + thermal_pressure: str | None + + +class ArtifactView(FrozenModel): + id: str + media_type: str + byte_size: int + + +class JobProgressView(FrozenModel): + id: str + variant_label: str + state: str + updated_at: str + attempt_count: int + recovery_required: bool + progress: dict[str, object] | None + evidence: tuple[EvidenceView, ...] + telemetry: TelemetrySummary | None + artifacts: tuple[ArtifactView, ...] + can_cancel: bool + + +class ExperimentQueueView(FrozenModel): + id: str + variable_label: str + reviewable: bool + reviewability_reasons: tuple[str, ...] + review_url: str | None + jobs: tuple[JobProgressView, ...] + + +class QueueSnapshot(FrozenModel): + schema_version: str = "queue.v1" + experiments: tuple[ExperimentQueueView, ...] + worker: dict[str, object] + + +class QueueService: + def __init__( + self, + experiments: ExperimentRepository, + jobs: JobRepository, + artifacts: ArtifactStore, + reviews: ReviewService, + *, + worker_status: Callable[[], dict[str, object]] | None = None, + ) -> None: + self.experiments = experiments + self.jobs = jobs + self.artifacts = artifacts + self.reviews = reviews + self.worker_status = worker_status + + def snapshot(self) -> QueueSnapshot: + eligibility = {item.experiment_id: item for item in self.reviews.queue()} + experiments = tuple( + self._experiment(experiment, eligibility[experiment.id].reason_codes) + for experiment in self.experiments.list() + ) + return QueueSnapshot(experiments=experiments, worker=self._worker()) + + def _worker(self) -> dict[str, object]: + if self.worker_status is None: + return {"state": "unavailable"} + status = self.worker_status() + return { + key: status[key] + for key in ("state", "worker_id", "job_ids", "exit_code") + if key in status + } + + def _experiment( + self, experiment: StoredExperiment, reasons: tuple[str, ...] + ) -> ExperimentQueueView: + baseline = experiment.variants[0].manifest + alternate = experiment.variants[1].manifest + left = _value_at(baseline.model_dump(mode="json"), experiment.variable_path) + right = _value_at(alternate.model_dump(mode="json"), experiment.variable_path) + labels = {variant.id: variant.label for variant in experiment.variants} + jobs = tuple( + self._job(job.id, labels[job.variant_id]) + for job in self.jobs.list_for_experiment(experiment.id) + ) + reviewable = not reasons + return ExperimentQueueView( + id=experiment.id, + variable_label=f"{experiment.variable_path}: {left} -> {right}", + reviewable=reviewable, + reviewability_reasons=reasons, + review_url="/api/reviews" if reviewable else None, + jobs=jobs, + ) + + def _job(self, job_id: str, variant_label: str) -> JobProgressView: + job = self.jobs.get(job_id) + attempts = self.jobs.attempts_for_job(job.id) + events = self.jobs.events_for_job(job.id) + latest_attempt = attempts[-1] if attempts else None + telemetry = ( + self.jobs.telemetry_for_attempt(latest_attempt.id) if latest_attempt is not None else () + ) + progress = _latest_progress(events) + return JobProgressView( + id=job.id, + variant_label=variant_label, + state=job.state.value, + updated_at=job.updated_at.isoformat(), + attempt_count=len(attempts), + recovery_required=bool( + latest_attempt + and latest_attempt.status.value == "active" + and job.state.value in _ACTIVE_STATES + ), + progress=progress, + evidence=tuple(_evidence(event) for event in events[-_EVIDENCE_LIMIT:]), + telemetry=_telemetry(telemetry[-1]) if telemetry else None, + artifacts=tuple( + ArtifactView(id=item.id, media_type=item.media_type, byte_size=item.byte_size) + for item in self.artifacts.list_for_job(job.id) + ), + can_cancel=job.state.value == "queued" or job.state.value in _ACTIVE_STATES, + ) + + +def _value_at(payload: dict[str, Any], path: str) -> Any: + value: Any = payload + for component in path.split("."): + value = value[int(component)] if isinstance(value, list) else value[component] + return value + + +def _latest_progress(events: tuple[RunEvent, ...]) -> dict[str, object] | None: + for event in reversed(events): + if event.kind == "progress": + return { + key: event.payload[key] + for key in ("phase", "step", "total") + if key in event.payload + } + return None + + +def _evidence(event: RunEvent) -> EvidenceView: + phase = event.payload.get("phase") + message = event.payload.get("message") + safe_message = None + if isinstance(message, str): + safe_message = message.rsplit("/", maxsplit=1)[-1][:_TEXT_LIMIT] + return EvidenceView( + sequence=event.id, + kind=event.kind[:80], + phase=phase[:80] if isinstance(phase, str) else None, + message=safe_message, + ) + + +def _telemetry(sample: TelemetrySample) -> TelemetrySummary: + return TelemetrySummary( + elapsed_seconds=sample.elapsed_seconds, + process_rss_bytes=sample.process_rss_bytes, + system_used_bytes=sample.system_used_bytes, + system_available_bytes=sample.system_available_bytes, + swap_used_bytes=sample.swap_used_bytes, + thermal_pressure=sample.thermal_pressure, + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/repositories.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/repositories.py new file mode 100644 index 0000000..99bbcf1 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/repositories.py @@ -0,0 +1,1293 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from dataclasses import dataclass +from datetime import datetime + +from video_lab.comparison import ControlledPair +from video_lab.database import Database +from video_lab.database import validation_experiment_id as _validation_experiment_id +from video_lab.domain import ( + ClaimedJob, + FrozenModel, + Job, + JobState, + Manifest, + RunAttempt, + RunAttemptStatus, + RunEvent, + TelemetrySample, + Variant, + canonical_json, + new_id, + utc_now, +) +from video_lab.states import InvalidTransition, transition_job + + +class RecordNotFound(LookupError): + pass + + +def validation_experiment_id(campaign_id: str, trial_id: str) -> str: + return _validation_experiment_id(campaign_id, trial_id) + + +class AdaptiveExperimentConflict(RuntimeError): + pass + + +class ValidationExperimentConflict(RuntimeError): + pass + + +@dataclass(frozen=True) +class ActiveWorkerLease: + job_id: str + attempt_id: str + worker_id: str + + +class StoredExperiment(FrozenModel): + id: str + schema_version: str + variable_path: str + changed_paths: tuple[str, ...] + created_at: datetime + variants: tuple[Variant, Variant] + + +class ExperimentRepository: + def __init__(self, database: Database) -> None: + self.database = database + + def create(self, pair: ControlledPair) -> StoredExperiment: + stored = _new_stored_experiment(pair) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + _insert_stored_experiment(connection, stored) + connection.commit() + except Exception: + connection.rollback() + raise + return stored + + def create_with_jobs(self, pair: ControlledPair) -> tuple[StoredExperiment, tuple[Job, Job]]: + stored = _new_stored_experiment(pair) + now = utc_now() + jobs = ( + Job( + id=new_id("job"), + variant_id=stored.variants[0].id, + state=JobState.QUEUED, + created_at=now, + updated_at=now, + ), + Job( + id=new_id("job"), + variant_id=stored.variants[1].id, + state=JobState.QUEUED, + created_at=now, + updated_at=now, + ), + ) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + _insert_stored_experiment(connection, stored) + connection.executemany( + "INSERT INTO jobs(id, variant_id, state, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + [ + ( + job.id, + job.variant_id, + job.state.value, + job.created_at.isoformat(), + job.updated_at.isoformat(), + ) + for job in jobs + ], + ) + connection.commit() + except Exception: + connection.rollback() + raise + return stored, jobs + + def create_adaptive_with_jobs( + self, + pair: ControlledPair, + *, + study_id: str, + optimizer_output_digest: str, + ) -> tuple[StoredExperiment, tuple[Job, Job]]: + """Create or verify one deterministically identified adaptive aggregate.""" + stored, expected_jobs = _new_adaptive_experiment( + pair, + study_id=study_id, + optimizer_output_digest=optimizer_output_digest, + ) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + existing = connection.execute( + "SELECT * FROM experiments WHERE id = ?", (stored.id,) + ).fetchone() + if existing is None: + _insert_stored_experiment(connection, stored) + connection.executemany( + "INSERT INTO jobs(id, variant_id, state, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + [ + ( + job.id, + job.variant_id, + job.state.value, + job.created_at.isoformat(), + job.updated_at.isoformat(), + ) + for job in expected_jobs + ], + ) + persisted = stored + jobs = expected_jobs + else: + persisted, jobs = _verify_adaptive_experiment( + connection, existing, stored, expected_jobs + ) + connection.commit() + except sqlite3.IntegrityError as error: + connection.rollback() + raise AdaptiveExperimentConflict( + "adaptive experiment identity collided with persisted evidence" + ) from error + except Exception: + connection.rollback() + raise + return persisted, jobs + + def create_validation_with_jobs( + self, + pair: ControlledPair, + *, + campaign_id: str, + trial_id: str, + ) -> tuple[StoredExperiment, tuple[Job, Job]]: + """Create or verify one deterministic validation experiment and its two jobs.""" + stored, expected_jobs = _new_validation_experiment( + pair, campaign_id=campaign_id, trial_id=trial_id + ) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + existing = connection.execute( + "SELECT * FROM experiments WHERE id = ?", (stored.id,) + ).fetchone() + if existing is None: + _insert_stored_experiment(connection, stored) + connection.executemany( + "INSERT INTO jobs(id, variant_id, state, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + [ + ( + job.id, + job.variant_id, + job.state.value, + job.created_at.isoformat(), + job.updated_at.isoformat(), + ) + for job in expected_jobs + ], + ) + persisted, jobs = stored, expected_jobs + else: + try: + persisted, jobs = _verify_adaptive_experiment( + connection, existing, stored, expected_jobs + ) + except AdaptiveExperimentConflict as error: + raise ValidationExperimentConflict( + "validation experiment identity collided with persisted evidence" + ) from error + _ensure_validation_experiment_binding( + connection, + campaign_id=campaign_id, + trial_id=trial_id, + experiment_id=stored.id, + ) + connection.commit() + except sqlite3.IntegrityError as error: + connection.rollback() + raise ValidationExperimentConflict( + "validation experiment identity collided with persisted evidence" + ) from error + except Exception: + connection.rollback() + raise + return persisted, jobs + + def create_single_generation(self, manifest: Manifest) -> Job: + """Persist one qualification experiment, variant, and queued job atomically.""" + now = utc_now() + experiment_id = new_id("experiment") + variant_id = new_id("variant") + job = Job( + id=new_id("job"), + variant_id=variant_id, + state=JobState.QUEUED, + created_at=now, + updated_at=now, + ) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + connection.execute( + "INSERT INTO experiments VALUES (?, ?, ?, ?, ?)", + (experiment_id, "qualification.v1", "", "[]", now.isoformat()), + ) + connection.execute( + "INSERT INTO variants(id, experiment_id, label, manifest_json) " + "VALUES (?, ?, 'A', ?)", + (variant_id, experiment_id, canonical_json(manifest)), + ) + connection.execute( + "INSERT INTO jobs(id, variant_id, state, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (job.id, variant_id, job.state.value, now.isoformat(), now.isoformat()), + ) + connection.commit() + except Exception: + connection.rollback() + raise + return job + + def get(self, experiment_id: str) -> StoredExperiment: + return self._get(experiment_id, adaptive=False) + + def get_validation(self, experiment_id: str) -> StoredExperiment: + if not experiment_id.startswith("validation_exp_"): + raise RecordNotFound(f"validation experiment not found: {experiment_id}") + with self.database.connect() as connection: + row = connection.execute( + "SELECT * FROM experiments WHERE id = ? AND schema_version = ?", + (experiment_id, "validation-controlled.v1"), + ).fetchone() + if row is None: + raise RecordNotFound(f"validation experiment not found: {experiment_id}") + variants = connection.execute( + "SELECT * FROM variants WHERE experiment_id = ? ORDER BY label", (experiment_id,) + ).fetchall() + return _stored_experiment(row, variants) + + def get_adaptive(self, experiment_id: str) -> StoredExperiment: + return self._get(experiment_id, adaptive=True) + + def _get(self, experiment_id: str, *, adaptive: bool) -> StoredExperiment: + with self.database.connect() as connection: + row = connection.execute( + "SELECT experiments.* FROM experiments WHERE experiments.id = ? " + + ( + "AND EXISTS (SELECT 1 FROM adaptive_study_events AS event " + "WHERE json_extract(event.payload_json, '$.experiment_id') = experiments.id)" + if adaptive + else "AND experiments.schema_version <> 'validation-controlled.v1' " + "AND NOT EXISTS (SELECT 1 FROM adaptive_study_events AS event " + "WHERE json_extract(event.payload_json, '$.experiment_id') = experiments.id)" + ), + (experiment_id,), + ).fetchone() + if row is None: + raise RecordNotFound(f"experiment not found: {experiment_id}") + variants = connection.execute( + "SELECT * FROM variants WHERE experiment_id = ? ORDER BY label", (experiment_id,) + ).fetchall() + return _stored_experiment(row, variants) + + def list(self, limit: int = 50) -> tuple[StoredExperiment, ...]: + with self.database.connect() as connection: + rows = connection.execute( + "SELECT experiments.id FROM experiments " + "JOIN variants ON variants.experiment_id = experiments.id " + "WHERE NOT EXISTS (SELECT 1 FROM adaptive_study_events AS event " + "WHERE json_extract(event.payload_json, '$.experiment_id') = experiments.id) " + "AND experiments.schema_version <> 'validation-controlled.v1' " + "GROUP BY experiments.id HAVING COUNT(variants.id) = 2 " + "ORDER BY experiments.created_at DESC, experiments.id DESC LIMIT ?", + (limit,), + ).fetchall() + return tuple(self.get(str(row["id"])) for row in rows) + + def list_adaptive_for_study(self, study_id: str) -> tuple[StoredExperiment, ...]: + with self.database.connect() as connection: + rows = connection.execute( + "SELECT DISTINCT experiments.id, experiments.created_at FROM experiments " + "JOIN adaptive_study_events AS event " + "ON json_extract(event.payload_json, '$.experiment_id') = experiments.id " + "WHERE event.study_id = ? ORDER BY experiments.created_at, experiments.id", + (study_id,), + ).fetchall() + return tuple(self.get_adaptive(str(row["id"])) for row in rows) + + def is_generic_visible(self, experiment_id: str) -> bool: + with self.database.connect() as connection: + row = connection.execute( + "SELECT 1 FROM experiments WHERE id = ? " + "AND schema_version <> 'validation-controlled.v1' " + "AND NOT EXISTS (SELECT 1 FROM adaptive_study_events AS event " + "WHERE json_extract(event.payload_json, '$.experiment_id') = experiments.id)", + (experiment_id,), + ).fetchone() + return row is not None + + +class JobRepository: + def __init__(self, database: Database) -> None: + self.database = database + + def create_for_experiment(self, experiment_id: str) -> tuple[Job, Job]: + now = utc_now() + with self.database.connect() as connection: + variants = connection.execute( + "SELECT id FROM variants WHERE experiment_id = ? ORDER BY label", (experiment_id,) + ).fetchall() + if not variants: + raise RecordNotFound(f"experiment not found: {experiment_id}") + if len(variants) != 2: + raise ValueError(f"experiment must have exactly two variants: {experiment_id}") + jobs = ( + Job( + id=new_id("job"), + variant_id=str(variants[0]["id"]), + state=JobState.QUEUED, + created_at=now, + updated_at=now, + ), + Job( + id=new_id("job"), + variant_id=str(variants[1]["id"]), + state=JobState.QUEUED, + created_at=now, + updated_at=now, + ), + ) + connection.execute("BEGIN IMMEDIATE") + try: + connection.executemany( + "INSERT INTO jobs(id, variant_id, state, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + [ + (job.id, job.variant_id, job.state.value, now.isoformat(), now.isoformat()) + for job in jobs + ], + ) + connection.commit() + except Exception: + connection.rollback() + raise + return jobs + + def get(self, job_id: str) -> Job: + with self.database.connect() as connection: + row = connection.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone() + if row is None: + raise RecordNotFound(f"job not found: {job_id}") + return _job(row) + + def list_for_experiment(self, experiment_id: str) -> tuple[Job, ...]: + with self.database.connect() as connection: + exists = connection.execute( + "SELECT 1 FROM experiments WHERE id = ?", (experiment_id,) + ).fetchone() + if exists is None: + raise RecordNotFound(f"experiment not found: {experiment_id}") + rows = connection.execute( + "SELECT jobs.* FROM jobs JOIN variants ON variants.id = jobs.variant_id " + "WHERE variants.experiment_id = ? ORDER BY variants.label", + (experiment_id,), + ).fetchall() + return tuple(_job(row) for row in rows) + + def transition(self, job_id: str, target: JobState) -> Job: + current = self.get(job_id) + state = transition_job(current.state, target) + updated_at = utc_now() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + cursor = connection.execute( + "UPDATE jobs SET state = ?, updated_at = ? WHERE id = ? AND state = ?", + (state.value, updated_at.isoformat(), job_id, current.state.value), + ) + if cursor.rowcount != 1: + raise sqlite3.OperationalError("job changed during transition") + connection.commit() + except Exception: + connection.rollback() + raise + return current.model_copy(update={"state": state, "updated_at": updated_at}) + + def requeue_failed_adaptive( + self, + study_id: str, + experiment_id: str, + job_ids: tuple[str, ...], + ) -> tuple[Job, ...]: + now = utc_now() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + event_row = connection.execute( + "SELECT new_state, payload_json FROM adaptive_study_events " + "WHERE study_id = ? ORDER BY sequence DESC LIMIT 1", + (study_id,), + ).fetchone() + if event_row is None or str(event_row["new_state"]) != "paused": + raise AdaptiveExperimentConflict( + "adaptive jobs can only be requeued for the active paused pair" + ) + try: + payload = json.loads(str(event_row["payload_json"])) + output_digest = str(payload["optimizer_output_digest"]) + except (KeyError, TypeError, json.JSONDecodeError) as error: + raise AdaptiveExperimentConflict( + "paused adaptive event has invalid generation evidence" + ) from error + if ( + payload.get("experiment_id") != experiment_id + or payload.get("resume_state") != "generating_pair" + or adaptive_experiment_id(study_id, output_digest) != experiment_id + ): + raise AdaptiveExperimentConflict( + "adaptive jobs do not belong to the active paused pair" + ) + rows = connection.execute( + "SELECT jobs.*, variants.label FROM jobs " + "JOIN variants ON variants.id = jobs.variant_id " + "WHERE variants.experiment_id = ? ORDER BY variants.label", + (experiment_id,), + ).fetchall() + persisted_ids = tuple(str(row["id"]) for row in rows) + expected_ids = tuple( + _adaptive_record_id("job", study_id, output_digest, role=f"job:{row['label']}") + for row in rows + ) + if ( + len(rows) != 2 + or len(set(job_ids)) != 2 + or set(job_ids) != set(persisted_ids) + or persisted_ids != expected_ids + ): + raise AdaptiveExperimentConflict( + "adaptive retry jobs do not match the deterministic active pair jobs" + ) + retry_rows: list[tuple[sqlite3.Row, sqlite3.Row]] = [] + active_states = { + JobState.PREPARING, + JobState.DOWNLOADING, + JobState.LOADING, + JobState.GENERATING, + JobState.ENCODING, + } + for row in rows: + active_attempt = connection.execute( + "SELECT 1 FROM run_attempts WHERE job_id = ? AND status = ? LIMIT 1", + (str(row["id"]), RunAttemptStatus.ACTIVE.value), + ).fetchone() + if active_attempt is not None: + raise AdaptiveExperimentConflict( + "adaptive retry job still has an active attempt" + ) + latest_attempt = connection.execute( + "SELECT * FROM run_attempts WHERE job_id = ? ORDER BY number DESC LIMIT 1", + (str(row["id"]),), + ).fetchone() + state = JobState(str(row["state"])) + if state is JobState.FAILED: + if latest_attempt is None or str(latest_attempt["status"]) not in { + RunAttemptStatus.FAILED.value, + RunAttemptStatus.INTERRUPTED.value, + }: + raise AdaptiveExperimentConflict( + "failed adaptive job has no finalized failure evidence" + ) + retry_rows.append((row, latest_attempt)) + elif state in active_states: + if ( + latest_attempt is None + or str(latest_attempt["status"]) != RunAttemptStatus.INTERRUPTED.value + ): + raise AdaptiveExperimentConflict( + "adaptive generation job is active without interrupted evidence" + ) + retry_rows.append((row, latest_attempt)) + elif state not in {JobState.QUEUED, JobState.COMPLETED}: + raise AdaptiveExperimentConflict( + f"adaptive retry does not support job state {state.value}" + ) + for row, attempt_row in retry_rows: + cursor = connection.execute( + "UPDATE jobs SET state = ?, updated_at = ?, error_code = NULL, " + "error_message = NULL, claimed_by = NULL, claimed_at = NULL " + "WHERE id = ? AND state = ?", + ( + JobState.QUEUED.value, + now.isoformat(), + str(row["id"]), + str(row["state"]), + ), + ) + if cursor.rowcount != 1: + raise sqlite3.OperationalError("adaptive retry job changed during reset") + _insert_event( + connection, + str(row["id"]), + str(attempt_row["id"]), + "adaptive_retry_queued", + {"study_id": study_id, "experiment_id": experiment_id}, + now, + ) + persisted = tuple( + _job( + connection.execute( + "SELECT * FROM jobs WHERE id = ?", (str(row["id"]),) + ).fetchone() + ) + for row in rows + ) + connection.commit() + except Exception: + connection.rollback() + raise + return persisted + + def claim_next(self, worker_id: str, job_id: str | None = None) -> ClaimedJob | None: + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + row = connection.execute( + ( + "SELECT * FROM jobs WHERE state = ? ORDER BY created_at, id LIMIT 1" + if job_id is None + else "SELECT * FROM jobs WHERE state = ? AND id = ? LIMIT 1" + ), + (JobState.QUEUED.value,) if job_id is None else (JobState.QUEUED.value, job_id), + ).fetchone() + if row is None: + connection.commit() + return None + job = _job(row) + state = transition_job(job.state, JobState.PREPARING) + now = utc_now() + cursor = connection.execute( + "UPDATE jobs SET state = ?, updated_at = ?, claimed_by = ?, claimed_at = ? " + "WHERE id = ? AND state = ? AND claimed_by IS NULL", + ( + state.value, + now.isoformat(), + worker_id, + now.isoformat(), + job.id, + JobState.QUEUED.value, + ), + ) + if cursor.rowcount != 1: + raise sqlite3.OperationalError("job changed during claim") + next_number = int( + connection.execute( + "SELECT coalesce(max(number), 0) + 1 FROM run_attempts WHERE job_id = ?", + (job.id,), + ).fetchone()[0] + ) + attempt = RunAttempt( + id=new_id("attempt"), + job_id=job.id, + number=next_number, + worker_id=worker_id, + status=RunAttemptStatus.ACTIVE, + started_at=now, + ) + connection.execute( + "INSERT INTO run_attempts(" + "id, job_id, number, worker_id, status, started_at" + ") VALUES (?, ?, ?, ?, ?, ?)", + ( + attempt.id, + attempt.job_id, + attempt.number, + attempt.worker_id, + attempt.status.value, + attempt.started_at.isoformat(), + ), + ) + _insert_event( + connection, + job.id, + attempt.id, + "job_claimed", + {"worker_id": worker_id}, + now, + ) + connection.commit() + except Exception: + connection.rollback() + raise + claimed_job = job.model_copy(update={"state": state, "updated_at": now}) + return ClaimedJob(job=claimed_job, attempt=attempt) + + def append_event( + self, + job_id: str, + attempt_id: str, + kind: str, + payload: dict[str, object], + ) -> RunEvent: + now = utc_now() + with self.database.connect() as connection: + event_id = _insert_event(connection, job_id, attempt_id, kind, payload, now) + return RunEvent( + id=event_id, + job_id=job_id, + attempt_id=attempt_id, + kind=kind, + payload=payload, + created_at=now, + ) + + def events_for_job(self, job_id: str) -> tuple[RunEvent, ...]: + with self.database.connect() as connection: + rows = connection.execute( + "SELECT * FROM run_events WHERE job_id = ? ORDER BY id", (job_id,) + ).fetchall() + return tuple(_run_event(row) for row in rows) + + def append_telemetry(self, sample: TelemetrySample) -> TelemetrySample: + sample = TelemetrySample.model_validate(sample.model_dump(round_trip=True)) + with self.database.connect() as connection: + cursor = connection.execute( + "INSERT INTO telemetry_samples(" + "attempt_id, recorded_at, elapsed_seconds, process_rss_bytes, " + "system_used_bytes, system_available_bytes, swap_used_bytes, thermal_pressure" + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + sample.attempt_id, + sample.recorded_at.isoformat(), + sample.elapsed_seconds, + sample.process_rss_bytes, + sample.system_used_bytes, + sample.system_available_bytes, + sample.swap_used_bytes, + sample.thermal_pressure, + ), + ) + sample_id = cursor.lastrowid + if sample_id is None: + raise sqlite3.OperationalError("telemetry insert did not produce an id") + return sample.model_copy(update={"id": sample_id}) + + def telemetry_for_attempt(self, attempt_id: str) -> tuple[TelemetrySample, ...]: + with self.database.connect() as connection: + rows = connection.execute( + "SELECT * FROM telemetry_samples WHERE attempt_id = ? ORDER BY id", (attempt_id,) + ).fetchall() + return tuple(_telemetry_sample(row) for row in rows) + + def attempts_for_job(self, job_id: str) -> tuple[RunAttempt, ...]: + with self.database.connect() as connection: + rows = connection.execute( + "SELECT * FROM run_attempts WHERE job_id = ? ORDER BY number", (job_id,) + ).fetchall() + return tuple(_run_attempt(row) for row in rows) + + def active_worker_leases(self) -> tuple[ActiveWorkerLease, ...]: + with self.database.connect() as connection: + rows = connection.execute( + "SELECT jobs.id AS job_id, run_attempts.id AS attempt_id, " + "run_attempts.worker_id FROM jobs JOIN run_attempts " + "ON run_attempts.job_id = jobs.id WHERE run_attempts.status = ? " + "AND jobs.claimed_by = run_attempts.worker_id " + "ORDER BY jobs.created_at, jobs.id, run_attempts.number", + (RunAttemptStatus.ACTIVE.value,), + ).fetchall() + return tuple( + ActiveWorkerLease( + job_id=str(row["job_id"]), + attempt_id=str(row["attempt_id"]), + worker_id=str(row["worker_id"]), + ) + for row in rows + ) + + def manifest_for_job(self, job_id: str) -> Manifest: + with self.database.connect() as connection: + row = connection.execute( + "SELECT variants.manifest_json FROM jobs " + "JOIN variants ON variants.id = jobs.variant_id WHERE jobs.id = ?", + (job_id,), + ).fetchone() + if row is None: + raise RecordNotFound(f"job not found: {job_id}") + return Manifest.model_validate_json(str(row["manifest_json"])) + + def request_cancellation(self, job_id: str) -> Job: + now = utc_now() + active_states = { + JobState.PREPARING, + JobState.DOWNLOADING, + JobState.LOADING, + JobState.GENERATING, + JobState.ENCODING, + } + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + row = connection.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone() + if row is None: + raise RecordNotFound(f"job not found: {job_id}") + job = _job(row) + if job.state is JobState.CANCELLED: + connection.commit() + return job + if job.state in {JobState.COMPLETED, JobState.FAILED}: + raise InvalidTransition(f"cannot cancel terminal job in state {job.state}") + if job.state is JobState.QUEUED: + connection.execute( + "UPDATE jobs SET state = ?, updated_at = ?, claimed_by = NULL, " + "claimed_at = NULL WHERE id = ?", + (JobState.CANCELLED.value, now.isoformat(), job_id), + ) + elif job.state in active_states: + attempt_row = connection.execute( + "SELECT id FROM run_attempts WHERE job_id = ? AND status = ? " + "ORDER BY number DESC LIMIT 1", + (job_id, RunAttemptStatus.ACTIVE.value), + ).fetchone() + if attempt_row is None: + raise sqlite3.OperationalError("active job has no active attempt") + connection.execute( + "UPDATE jobs SET state = ?, updated_at = ? WHERE id = ?", + (JobState.CANCELLED.value, now.isoformat(), job_id), + ) + _insert_event( + connection, + job_id, + str(attempt_row["id"]), + "cancellation_requested", + {}, + now, + ) + else: # pragma: no cover - closed enum protects this branch + raise InvalidTransition(f"cannot cancel job in state {job.state}") + connection.commit() + except Exception: + connection.rollback() + raise + return job.model_copy(update={"state": JobState.CANCELLED, "updated_at": now}) + + def finalize_attempt( + self, + job_id: str, + attempt_id: str, + *, + job_state: JobState, + attempt_status: RunAttemptStatus, + artifact_id: str | None, + error_code: str | None, + error_message: str | None, + event_kind: str, + event_payload: dict[str, object], + ) -> tuple[Job, RunAttempt]: + expected = { + JobState.COMPLETED: RunAttemptStatus.COMPLETED, + JobState.FAILED: RunAttemptStatus.FAILED, + JobState.CANCELLED: RunAttemptStatus.CANCELLED, + } + if expected.get(job_state) is not attempt_status: + raise ValueError("job state and attempt status do not match") + if job_state is JobState.COMPLETED: + if artifact_id is None or error_code is not None or error_message is not None: + raise ValueError("completed attempt requires an artifact and no error") + elif artifact_id is not None or error_code is None or error_message is None: + raise ValueError("failed or cancelled attempt requires an error and no artifact") + if not event_kind: + raise ValueError("terminal event kind must not be empty") + + now = utc_now() + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + job_row = connection.execute( + "SELECT * FROM jobs WHERE id = ?", (job_id,) + ).fetchone() + if job_row is None: + raise RecordNotFound(f"job not found: {job_id}") + job = _job(job_row) + if job.state is JobState.CANCELLED: + if job_state is not JobState.CANCELLED: + raise InvalidTransition("cancelled job may only finalize as cancelled") + else: + transition_job(job.state, job_state) + attempt_row = connection.execute( + "SELECT * FROM run_attempts WHERE id = ? AND job_id = ?", + (attempt_id, job_id), + ).fetchone() + if attempt_row is None: + raise RecordNotFound(f"attempt not found: {attempt_id}") + attempt = _run_attempt(attempt_row) + if attempt.status is not RunAttemptStatus.ACTIVE: + raise ValueError("attempt is already finalized") + cursor = connection.execute( + "UPDATE run_attempts SET status = ?, finished_at = ?, error_code = ?, " + "error_message = ?, artifact_id = ? WHERE id = ? AND status = ?", + ( + attempt_status.value, + now.isoformat(), + error_code, + error_message, + artifact_id, + attempt_id, + RunAttemptStatus.ACTIVE.value, + ), + ) + if cursor.rowcount != 1: + raise sqlite3.OperationalError("attempt changed during finalization") + _insert_event(connection, job_id, attempt_id, event_kind, event_payload, now) + if artifact_id is not None: + connection.execute( + "INSERT OR IGNORE INTO job_artifacts(job_id, artifact_id, created_at) " + "VALUES (?, ?, ?)", + (job_id, artifact_id, now.isoformat()), + ) + connection.execute( + "UPDATE jobs SET state = ?, updated_at = ?, error_code = ?, " + "error_message = ?, claimed_by = NULL, claimed_at = NULL WHERE id = ?", + (job_state.value, now.isoformat(), error_code, error_message, job_id), + ) + connection.commit() + except Exception: + connection.rollback() + raise + return ( + job.model_copy(update={"state": job_state, "updated_at": now}), + attempt.model_copy( + update={ + "status": attempt_status, + "finished_at": now, + "error_code": error_code, + "error_message": error_message, + "artifact_id": artifact_id, + } + ), + ) + + def recover_interrupted(self, worker_id: str) -> tuple[Job, ...]: + active_states = ( + JobState.PREPARING, + JobState.DOWNLOADING, + JobState.LOADING, + JobState.GENERATING, + JobState.ENCODING, + ) + placeholders = ",".join("?" for _ in active_states) + now = utc_now() + recovered: list[Job] = [] + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + rows = connection.execute( + "SELECT jobs.*, run_attempts.id AS attempt_id FROM jobs " + "JOIN run_attempts ON run_attempts.job_id = jobs.id " + f"WHERE jobs.state IN ({placeholders}) AND run_attempts.status = ? " + "AND jobs.claimed_by = ? AND run_attempts.worker_id = ? " + "ORDER BY jobs.created_at, jobs.id, run_attempts.number", + ( + *tuple(state.value for state in active_states), + RunAttemptStatus.ACTIVE.value, + worker_id, + worker_id, + ), + ).fetchall() + for row in rows: + attempt_cursor = connection.execute( + "UPDATE run_attempts SET status = ?, finished_at = ?, " + "error_code = ?, error_message = ? WHERE id = ? AND status = ? " + "AND worker_id = ?", + ( + RunAttemptStatus.INTERRUPTED.value, + now.isoformat(), + "worker_interrupted", + "active attempt interrupted before completion", + str(row["attempt_id"]), + RunAttemptStatus.ACTIVE.value, + worker_id, + ), + ) + if attempt_cursor.rowcount != 1: + raise sqlite3.OperationalError("attempt lease changed during recovery") + _insert_event( + connection, + str(row["id"]), + str(row["attempt_id"]), + "worker_interrupted", + {"recovered_by": worker_id}, + now, + ) + job_cursor = connection.execute( + "UPDATE jobs SET state = ?, updated_at = ?, claimed_by = NULL, " + "claimed_at = NULL WHERE id = ? AND claimed_by = ?", + ( + JobState.QUEUED.value, + now.isoformat(), + str(row["id"]), + worker_id, + ), + ) + if job_cursor.rowcount != 1: + raise sqlite3.OperationalError("job lease changed during recovery") + recovered.append( + _job(row).model_copy(update={"state": JobState.QUEUED, "updated_at": now}) + ) + connection.commit() + except Exception: + connection.rollback() + raise + return tuple(recovered) + + +def _stored_experiment(row: sqlite3.Row, variant_rows: list[sqlite3.Row]) -> StoredExperiment: + variants = tuple( + Variant( + id=str(item["id"]), + experiment_id=str(item["experiment_id"]), + label=str(item["label"]), + manifest=Manifest.model_validate_json(str(item["manifest_json"])), + ) + for item in variant_rows + ) + if len(variants) != 2: + raise ValueError(f"experiment must have exactly two variants: {row['id']}") + return StoredExperiment( + id=str(row["id"]), + schema_version=str(row["schema_version"]), + variable_path=str(row["variable_path"]), + changed_paths=tuple(json.loads(str(row["changed_paths_json"]))), + created_at=datetime.fromisoformat(str(row["created_at"])), + variants=(variants[0], variants[1]), + ) + + +def _new_stored_experiment(pair: ControlledPair) -> StoredExperiment: + experiment_id = new_id("exp") + variants = ( + Variant( + id=new_id("var"), + experiment_id=experiment_id, + label="A", + manifest=pair.variant_a, + ), + Variant( + id=new_id("var"), + experiment_id=experiment_id, + label="B", + manifest=pair.variant_b, + ), + ) + return StoredExperiment( + id=experiment_id, + schema_version=pair.variant_a.schema_version, + variable_path=pair.variable_path, + changed_paths=pair.changed_paths, + created_at=utc_now(), + variants=variants, + ) + + +def adaptive_experiment_id(study_id: str, optimizer_output_digest: str) -> str: + return _adaptive_record_id("exp", study_id, optimizer_output_digest, role="experiment") + + +def _adaptive_record_id( + record_kind: str, + study_id: str, + optimizer_output_digest: str, + *, + role: str, +) -> str: + if not study_id: + raise ValueError("adaptive study id must not be empty") + if len(optimizer_output_digest) != 64 or set(optimizer_output_digest) - set("0123456789abcdef"): + raise ValueError("optimizer output digest must be lowercase sha256 hex") + payload = canonical_json( + { + "schema_version": "adaptive-experiment-identity.v1", + "domain": f"video-lab.adaptive.{record_kind}", + "study_id": study_id, + "optimizer_output_digest": optimizer_output_digest, + "role": role, + } + ) + digest = hashlib.sha256(payload.encode()).hexdigest() + return f"adaptive_{record_kind}_{digest}" + + +def _validation_record_id(record_kind: str, campaign_id: str, trial_id: str, *, role: str) -> str: + if not campaign_id or not trial_id: + raise ValueError("validation campaign and trial ids must not be empty") + payload = canonical_json( + { + "schema_version": "validation-experiment-identity.v1", + "domain": f"video-lab.validation.{record_kind}", + "campaign_id": campaign_id, + "trial_id": trial_id, + "role": role, + } + ) + return f"validation_{record_kind}_{hashlib.sha256(payload.encode()).hexdigest()}" + + +def _new_validation_experiment( + pair: ControlledPair, *, campaign_id: str, trial_id: str +) -> tuple[StoredExperiment, tuple[Job, Job]]: + experiment_id = validation_experiment_id(campaign_id, trial_id) + now = utc_now() + variants = tuple( + Variant( + id=_validation_record_id("var", campaign_id, trial_id, role=f"variant:{label}"), + experiment_id=experiment_id, + label=label, + manifest=manifest, + ) + for label, manifest in (("A", pair.variant_a), ("B", pair.variant_b)) + ) + stored = StoredExperiment( + id=experiment_id, + schema_version="validation-controlled.v1", + variable_path=pair.variable_path, + changed_paths=pair.changed_paths, + created_at=now, + variants=(variants[0], variants[1]), + ) + jobs = tuple( + Job( + id=_validation_record_id("job", campaign_id, trial_id, role=f"job:{variant.label}"), + variant_id=variant.id, + state=JobState.QUEUED, + created_at=now, + updated_at=now, + ) + for variant in variants + ) + return stored, (jobs[0], jobs[1]) + + +def _ensure_validation_experiment_binding( + connection: sqlite3.Connection, + *, + campaign_id: str, + trial_id: str, + experiment_id: str, +) -> None: + row = connection.execute( + "SELECT experiment_id FROM validation_experiment_bindings " + "WHERE campaign_id=? AND trial_id=?", + (campaign_id, trial_id), + ).fetchone() + if row is None: + connection.execute( + "INSERT INTO validation_experiment_bindings(campaign_id, trial_id, experiment_id) " + "VALUES (?, ?, ?)", + (campaign_id, trial_id, experiment_id), + ) + elif str(row["experiment_id"]) != experiment_id: + raise ValidationExperimentConflict( + "validation trial is bound to a different controlled experiment" + ) + + +def _new_adaptive_experiment( + pair: ControlledPair, + *, + study_id: str, + optimizer_output_digest: str, +) -> tuple[StoredExperiment, tuple[Job, Job]]: + experiment_id = adaptive_experiment_id(study_id, optimizer_output_digest) + now = utc_now() + variants = ( + Variant( + id=_adaptive_record_id("var", study_id, optimizer_output_digest, role="variant:A"), + experiment_id=experiment_id, + label="A", + manifest=pair.variant_a, + ), + Variant( + id=_adaptive_record_id("var", study_id, optimizer_output_digest, role="variant:B"), + experiment_id=experiment_id, + label="B", + manifest=pair.variant_b, + ), + ) + stored = StoredExperiment( + id=experiment_id, + schema_version=pair.variant_a.schema_version, + variable_path=pair.variable_path, + changed_paths=pair.changed_paths, + created_at=now, + variants=variants, + ) + jobs = tuple( + Job( + id=_adaptive_record_id( + "job", study_id, optimizer_output_digest, role=f"job:{variant.label}" + ), + variant_id=variant.id, + state=JobState.QUEUED, + created_at=now, + updated_at=now, + ) + for variant in variants + ) + return stored, (jobs[0], jobs[1]) + + +def _verify_adaptive_experiment( + connection: sqlite3.Connection, + experiment_row: sqlite3.Row, + expected: StoredExperiment, + expected_jobs: tuple[Job, Job], +) -> tuple[StoredExperiment, tuple[Job, Job]]: + variant_rows = connection.execute( + "SELECT * FROM variants WHERE experiment_id = ? ORDER BY label", (expected.id,) + ).fetchall() + try: + persisted = _stored_experiment(experiment_row, variant_rows) + except (TypeError, ValueError) as error: + raise AdaptiveExperimentConflict( + "adaptive experiment aggregate is partial or invalid" + ) from error + if ( + persisted.schema_version != expected.schema_version + or persisted.variable_path != expected.variable_path + or persisted.changed_paths != expected.changed_paths + or tuple( + (variant.id, variant.experiment_id, variant.label, variant.manifest) + for variant in persisted.variants + ) + != tuple( + (variant.id, variant.experiment_id, variant.label, variant.manifest) + for variant in expected.variants + ) + ): + raise AdaptiveExperimentConflict( + "adaptive experiment identity does not match the controlled pair" + ) + job_rows = connection.execute( + "SELECT jobs.* FROM jobs JOIN variants ON variants.id = jobs.variant_id " + "WHERE variants.experiment_id = ? ORDER BY variants.label, jobs.id", + (expected.id,), + ).fetchall() + jobs = tuple(_job(row) for row in job_rows) + if tuple((job.id, job.variant_id) for job in jobs) != tuple( + (job.id, job.variant_id) for job in expected_jobs + ): + raise AdaptiveExperimentConflict( + "adaptive experiment jobs are partial or do not match deterministic identities" + ) + return persisted, (jobs[0], jobs[1]) + + +def _insert_stored_experiment(connection: sqlite3.Connection, stored: StoredExperiment) -> None: + connection.execute( + "INSERT INTO experiments VALUES (?, ?, ?, ?, ?)", + ( + stored.id, + stored.schema_version, + stored.variable_path, + json.dumps(list(stored.changed_paths), separators=(",", ":")), + stored.created_at.isoformat(), + ), + ) + connection.executemany( + "INSERT INTO variants(id, experiment_id, label, manifest_json) VALUES (?, ?, ?, ?)", + [ + (item.id, item.experiment_id, item.label, canonical_json(item.manifest)) + for item in stored.variants + ], + ) + + +def _job(row: sqlite3.Row) -> Job: + return Job( + id=str(row["id"]), + variant_id=str(row["variant_id"]), + state=JobState(str(row["state"])), + created_at=datetime.fromisoformat(str(row["created_at"])), + updated_at=datetime.fromisoformat(str(row["updated_at"])), + ) + + +def _run_attempt(row: sqlite3.Row) -> RunAttempt: + finished_at = row["finished_at"] + return RunAttempt( + id=str(row["id"]), + job_id=str(row["job_id"]), + number=int(row["number"]), + worker_id=str(row["worker_id"]), + status=RunAttemptStatus(str(row["status"])), + started_at=datetime.fromisoformat(str(row["started_at"])), + finished_at=None if finished_at is None else datetime.fromisoformat(str(finished_at)), + error_code=None if row["error_code"] is None else str(row["error_code"]), + error_message=None if row["error_message"] is None else str(row["error_message"]), + artifact_id=None if row["artifact_id"] is None else str(row["artifact_id"]), + ) + + +def _run_event(row: sqlite3.Row) -> RunEvent: + payload = json.loads(str(row["payload_json"])) + if not isinstance(payload, dict): + raise ValueError(f"event payload must be an object: {row['id']}") + return RunEvent( + id=int(row["id"]), + job_id=str(row["job_id"]), + attempt_id=str(row["attempt_id"]), + kind=str(row["kind"]), + payload=payload, + created_at=datetime.fromisoformat(str(row["created_at"])), + ) + + +def _telemetry_sample(row: sqlite3.Row) -> TelemetrySample: + return TelemetrySample( + id=int(row["id"]), + attempt_id=str(row["attempt_id"]), + recorded_at=datetime.fromisoformat(str(row["recorded_at"])), + elapsed_seconds=float(row["elapsed_seconds"]), + process_rss_bytes=int(row["process_rss_bytes"]), + system_used_bytes=int(row["system_used_bytes"]), + system_available_bytes=int(row["system_available_bytes"]), + swap_used_bytes=int(row["swap_used_bytes"]), + thermal_pressure=( + None if row["thermal_pressure"] is None else str(row["thermal_pressure"]) + ), + ) + + +def _insert_event( + connection: sqlite3.Connection, + job_id: str, + attempt_id: str, + kind: str, + payload: dict[str, object], + created_at: datetime, +) -> int: + cursor = connection.execute( + "INSERT INTO run_events(job_id, attempt_id, kind, payload_json, created_at) " + "VALUES (?, ?, ?, ?, ?)", + (job_id, attempt_id, kind, canonical_json(payload), created_at.isoformat()), + ) + event_id = cursor.lastrowid + if event_id is None: + raise sqlite3.OperationalError("event insert did not produce an id") + return event_id diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/review_service.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/review_service.py new file mode 100644 index 0000000..1a71f98 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/review_service.py @@ -0,0 +1,545 @@ +from __future__ import annotations + +import hashlib +import sqlite3 +from collections.abc import Sequence +from dataclasses import replace +from typing import Any, Literal, cast + +from video_lab.artifacts import Artifact, ArtifactError, ArtifactStore +from video_lab.domain import ( + DisplayPreference, + FrozenModel, + Manifest, + ReviewRevision, + RubricScore, +) +from video_lab.repositories import ( + ExperimentRepository, + JobRepository, + StoredExperiment, + validation_experiment_id, +) +from video_lab.reviews import ReviewRepository + +DisplayPosition = Literal["left", "right"] +_POSITIONS: tuple[DisplayPosition, DisplayPosition] = ("left", "right") + + +class ReviewQueueItem(FrozenModel): + experiment_id: str + reviewable: bool + reason_codes: tuple[str, ...] + + +class BlindMedia(FrozenModel): + display_position: DisplayPosition + url: str + fps: float = 1.0 + + +class BlindReviewView(FrozenModel): + session_id: str + media: tuple[BlindMedia, BlindMedia] + prompt: str | None = None + + +class ReviewSubmission(FrozenModel): + review_id: str + revision: int + status: Literal["submitted", "corrected"] + + +class ReviewRevealView(FrozenModel): + review_id: str + revision: int + canonical_by_display: dict[DisplayPosition, Literal["A", "B"]] + variable_path: str + baseline_value: Any + alternate_value: Any + structural_diff: tuple[str, ...] + manifests: dict[Literal["A", "B"], Manifest] + artifacts: dict[Literal["A", "B"], Artifact] + preferred_canonical_variant: Literal["A", "B"] | None + + +class ReviewService: + def __init__( + self, + experiments: ExperimentRepository, + jobs: JobRepository, + artifacts: ArtifactStore, + reviews: ReviewRepository, + ) -> None: + self.experiments = experiments + self.jobs = jobs + self.artifacts = artifacts + self.reviews = reviews + + def queue(self) -> tuple[ReviewQueueItem, ...]: + return tuple(self._queue_item(experiment) for experiment in self.experiments.list()) + + def start(self, experiment_id: str, reviewer_id: str) -> BlindReviewView: + return self._start(self.experiments.get(experiment_id), reviewer_id) + + def start_adaptive(self, experiment_id: str, reviewer_id: str) -> BlindReviewView: + return self._start(self.experiments.get_adaptive(experiment_id), reviewer_id) + + def start_validation( + self, + *, + campaign_id: str, + trial_id: str, + reviewer_id: str, + ) -> BlindReviewView: + experiment_id, winner_position, hidden_repeat = self._validation_trial_authorization( + campaign_id, trial_id, reviewer_id + ) + effective_reviewer = _validation_session_reviewer( + campaign_id, trial_id, reviewer_id, hidden_repeat=hidden_repeat + ) + view = self._start( + self.experiments.get_validation(experiment_id), + effective_reviewer, + validation=True, + ) + row = self._session_row(view.session_id) + with self.reviews.database.connect() as connection: + baseline = connection.execute( + "SELECT id FROM variants WHERE experiment_id = ? AND label = 'A'", + (experiment_id,), + ).fetchone() + if baseline is None: + raise ValueError("validation experiment has no winner variant") + actual_winner = "left" if str(row["left_variant_id"]) == str(baseline["id"]) else "right" + swap = actual_winner != winner_position + return BlindReviewView( + session_id=view.session_id, + media=( + BlindMedia( + display_position="left", + url=f"/validation/reviews/{view.session_id}/artifacts/" + f"{'right' if swap else 'left'}", + ), + BlindMedia( + display_position="right", + url=f"/validation/reviews/{view.session_id}/artifacts/" + f"{'left' if swap else 'right'}", + ), + ), + prompt=view.prompt, + ) + + def submit_validation( + self, + *, + campaign_id: str, + trial_id: str, + reviewer_id: str, + preference: DisplayPreference, + ) -> ReviewRevision: + experiment_id, winner_position, hidden_repeat = self._validation_trial_authorization( + campaign_id, trial_id, reviewer_id + ) + effective_reviewer = _validation_session_reviewer( + campaign_id, trial_id, reviewer_id, hidden_repeat=hidden_repeat + ) + session = self.reviews.start_session( + experiment_id, + effective_reviewer, + self._current_eligibility(experiment_id, propagate_artifact_errors=True)[1], + ) + row = self._session_row(session.id) + with self.reviews.database.connect() as connection: + baseline = connection.execute( + "SELECT id FROM variants WHERE experiment_id = ? AND label = 'A'", + (experiment_id,), + ).fetchone() + if baseline is None: + raise ValueError("validation experiment has no winner variant") + actual_winner = "left" if str(row["left_variant_id"]) == str(baseline["id"]) else "right" + actual_preference = preference + if preference is not DisplayPreference.TIE and actual_winner != winner_position: + actual_preference = ( + DisplayPreference.RIGHT + if preference is DisplayPreference.LEFT + else DisplayPreference.LEFT + ) + return self.reviews.submit_adaptive(session.id, actual_preference) + + def validation_trial_for_session( + self, *, campaign_id: str, session_id: str, reviewer_id: str + ) -> str: + session = self._session_row(session_id) + with self.reviews.database.connect() as connection: + rows = connection.execute( + "SELECT trial.trial_id, trial.reuses_trial_id FROM validation_trials AS trial " + "JOIN validation_campaigns AS campaign ON campaign.id = trial.campaign_id " + "WHERE trial.campaign_id = ? AND campaign.reviewer_id = ?", + (campaign_id, reviewer_id), + ).fetchall() + for row in rows: + trial_id = str(row["trial_id"]) + source_trial_id = str(row["reuses_trial_id"] or trial_id) + if str(session["reviewer_id"]) == _validation_session_reviewer( + campaign_id, + trial_id, + reviewer_id, + hidden_repeat=row["reuses_trial_id"] is not None, + ) and str(session["experiment_id"]) == validation_experiment_id( + campaign_id, source_trial_id + ): + return trial_id + raise LookupError(f"validation review not found: {session_id}") + + def artifact_for_validation_display( + self, + *, + campaign_id: str, + trial_id: str, + session_id: str, + position: str, + reviewer_id: str, + ) -> Artifact: + normalized = self._position(position) + bound_trial_id = self.validation_trial_for_session( + campaign_id=campaign_id, + session_id=session_id, + reviewer_id=reviewer_id, + ) + if bound_trial_id != trial_id: + raise LookupError(f"validation review not found: {session_id}") + row = self._session_row(session_id) + _, winner_position, _ = self._validation_trial_authorization( + campaign_id, trial_id, reviewer_id + ) + with self.reviews.database.connect() as connection: + baseline = connection.execute( + "SELECT id FROM variants WHERE experiment_id=? AND label='A'", + (row["experiment_id"],), + ).fetchone() + if baseline is None: + raise ValueError("validation experiment has no winner variant") + actual_winner = "left" if str(row["left_variant_id"]) == str(baseline["id"]) else "right" + underlying = normalized + if actual_winner != winner_position: + underlying = "right" if normalized == "left" else "left" + return self._pinned_artifact(row, self._position(underlying)) + + def _start( + self, experiment: StoredExperiment, reviewer_id: str, *, validation: bool = False + ) -> BlindReviewView: + reasons, artifacts_by_variant = self._current_eligibility(experiment.id) + ordered = tuple(dict.fromkeys(reasons)) + if ordered: + raise ValueError("experiment is not reviewable: " + ",".join(ordered)) + session = self.reviews.start_session(experiment.id, reviewer_id, artifacts_by_variant) + # Resolve through the store at the authorization boundary, even after queue validation. + for position in session.display_positions: + if validation: + self._pinned_artifact(self._session_row(session.id), position) + else: + self.artifact_for_display(session.id, position, reviewer_id) + return BlindReviewView( + session_id=session.id, + media=( + BlindMedia( + display_position="left", + url=f"/reviews/{session.id}/artifacts/left", + fps=1.0, + ), + BlindMedia( + display_position="right", + url=f"/reviews/{session.id}/artifacts/right", + fps=1.0, + ), + ), + prompt=self._shared_prompt(experiment), + ) + + def prompt_for_session(self, session_id: str, reviewer_id: str) -> str | None: + self._require_generic_session(session_id) + row = self._session_row(session_id) + if str(row["reviewer_id"]) != reviewer_id: + raise PermissionError("review session belongs to another reviewer") + return self._shared_prompt(self.experiments.get(str(row["experiment_id"]))) + + def artifact_for_display(self, session_id: str, position: str, reviewer_id: str) -> Artifact: + self._require_generic_session(session_id) + normalized = self._position(position) + row = self._session_row(session_id) + if str(row["reviewer_id"]) != reviewer_id: + raise PermissionError("review session belongs to another reviewer") + return self._pinned_artifact(row, normalized) + + def submit( + self, + session_id: str, + reviewer_id: str, + preference: DisplayPreference | None, + scores: Sequence[RubricScore], + notes: str | None, + ) -> ReviewSubmission: + self._require_generic_session(session_id) + if preference is None: + raise ValueError("display preference is required") + self._validate_owner_and_artifacts(session_id, reviewer_id) + if self.reviews.is_adaptive_session(session_id): + raise ValueError("adaptive review must use immutable preference submission") + revision = self.reviews.submit(session_id, preference, scores, notes) + return self._submission(revision, "submitted") + + def submit_adaptive( + self, + session_id: str, + reviewer_id: str, + preference: DisplayPreference, + ) -> ReviewRevision: + self._validate_owner_and_artifacts(session_id, reviewer_id) + if not self.reviews.is_adaptive_session(session_id): + raise ValueError("review session is not bound to an adaptive study") + return self.reviews.submit_adaptive(session_id, preference) + + def latest_adaptive(self, session_id: str, reviewer_id: str) -> ReviewRevision | None: + row = self._session_row(session_id) + if str(row["reviewer_id"]) != reviewer_id: + raise PermissionError("review session belongs to another reviewer") + if not self.reviews.is_adaptive_session(session_id): + raise ValueError("review session is not bound to an adaptive study") + return self.reviews.latest_for_session(session_id) + + def adaptive_binding_digest( + self, + session_id: str, + reviewer_id: str, + *, + study_id: str, + optimizer_step_ordinal: int, + optimizer_output_digest: str, + ) -> str: + return self.reviews.adaptive_binding_digest( + session_id, + reviewer_id, + study_id=study_id, + optimizer_step_ordinal=optimizer_step_ordinal, + optimizer_output_digest=optimizer_output_digest, + ) + + def correct( + self, + session_id: str, + reviewer_id: str, + supersedes_review_id: str, + preference: DisplayPreference | None, + scores: Sequence[RubricScore], + notes: str | None, + ) -> ReviewSubmission: + self._require_generic_session(session_id) + if preference is None: + raise ValueError("display preference is required") + row = self._session_row(session_id) + if str(row["reviewer_id"]) != reviewer_id: + raise PermissionError("review session belongs to another reviewer") + if self.reviews.is_adaptive_session(session_id): + raise ValueError("adaptive v1 review is immutable") + self._validate_owner_and_artifacts(session_id, reviewer_id) + revision = self.reviews.submit( + session_id, preference, scores, notes, supersedes_id=supersedes_review_id + ) + return self._submission(revision, "corrected") + + def reveal(self, session_id: str, reviewer_id: str) -> ReviewRevealView: + self._require_generic_session(session_id) + row = self._session_row(session_id) + if str(row["reviewer_id"]) != reviewer_id: + raise PermissionError("review session belongs to another reviewer") + revision = self.reviews.latest_for_session(session_id) + if revision is None: + raise PermissionError("review must be submitted before reveal") + experiment = self.experiments.get(str(row["experiment_id"])) + variants = {variant.id: variant for variant in experiment.variants} + canonical_by_id = { + variant.id: cast(Literal["A", "B"], variant.label) for variant in experiment.variants + } + mapping: dict[DisplayPosition, Literal["A", "B"]] = { + "left": canonical_by_id[str(row["left_variant_id"])], + "right": canonical_by_id[str(row["right_variant_id"])], + } + artifacts = { + mapping[position]: self._pinned_artifact(row, position) for position in _POSITIONS + } + baseline = experiment.variants[0].manifest + alternate = experiment.variants[1].manifest + return ReviewRevealView( + review_id=revision.id, + revision=revision.revision, + canonical_by_display=mapping, + variable_path=experiment.variable_path, + baseline_value=_value_at(baseline, experiment.variable_path), + alternate_value=_value_at(alternate, experiment.variable_path), + structural_diff=experiment.changed_paths, + manifests={ + "A": variants[experiment.variants[0].id].manifest, + "B": variants[experiment.variants[1].id].manifest, + }, + artifacts=artifacts, + preferred_canonical_variant=( + None + if revision.preferred_variant_id is None + else canonical_by_id[revision.preferred_variant_id] + ), + ) + + def _queue_item(self, experiment: StoredExperiment) -> ReviewQueueItem: + reasons, _ = self._current_eligibility(experiment.id) + ordered = tuple(dict.fromkeys(reasons)) + return ReviewQueueItem( + experiment_id=experiment.id, reviewable=not ordered, reason_codes=ordered + ) + + @staticmethod + def _shared_prompt(experiment: StoredExperiment) -> str | None: + prompts = {variant.manifest.prompt for variant in experiment.variants} + return prompts.pop() if len(prompts) == 1 else None + + def _current_eligibility( + self, experiment_id: str, *, propagate_artifact_errors: bool = False + ) -> tuple[list[str], dict[str, Artifact]]: + jobs = self.jobs.list_for_experiment(experiment_id) + reasons: list[str] = [] + artifacts_by_variant: dict[str, Artifact] = {} + if len(jobs) != 2 or any(job.state.value != "completed" for job in jobs): + reasons.append("jobs_not_completed") + for job in jobs: + registered = self.artifacts.list_for_job(job.id) + videos = tuple(item for item in registered if item.media_type == "video/mp4") + if not registered: + reasons.append("missing_artifact") + elif not videos: + reasons.append("non_video_artifact") + elif len(registered) != 1 or len(videos) != 1: + reasons.append("duplicate_artifact") + else: + artifact = videos[0] + try: + self.artifacts.resolve(artifact.id) + except ArtifactError: + if propagate_artifact_errors: + raise + reasons.append("corrupt_artifact") + else: + artifacts_by_variant[job.variant_id] = artifact + eligible_artifacts = tuple(artifacts_by_variant.values()) + if len(eligible_artifacts) == 2 and ( + eligible_artifacts[0].id == eligible_artifacts[1].id + or eligible_artifacts[0].sha256 == eligible_artifacts[1].sha256 + ): + reasons.append("duplicate_variant_artifact") + return reasons, artifacts_by_variant + + def _validate_owner_and_artifacts(self, session_id: str, reviewer_id: str) -> None: + row = self._session_row(session_id) + if str(row["reviewer_id"]) != reviewer_id: + raise PermissionError("review session belongs to another reviewer") + reasons, artifacts_by_variant = self._current_eligibility( + str(row["experiment_id"]), propagate_artifact_errors=True + ) + ordered = tuple(dict.fromkeys(reasons)) + if ordered: + raise ValueError("experiment is not reviewable: " + ",".join(ordered)) + for position in _POSITIONS: + variant_id = str(row[f"{position}_variant_id"]) + current = artifacts_by_variant.get(variant_id) + if current is None or ( + current.id, + current.sha256, + ) != ( + str(row[f"{position}_artifact_id"]), + str(row[f"{position}_artifact_sha256"]), + ): + raise ValueError("current artifacts do not match session-pinned artifacts") + + def _pinned_artifact(self, row: sqlite3.Row, position: DisplayPosition) -> Artifact: + artifact = self.artifacts.get(str(row[f"{position}_artifact_id"])) + if artifact.sha256 != str(row[f"{position}_artifact_sha256"]): + raise ArtifactError("session-pinned artifact identity does not match registry") + job_id = str(row[f"{position}_job_id"]) + with self.reviews.database.connect() as connection: + job = connection.execute("SELECT id FROM jobs WHERE id = ?", (job_id,)).fetchone() + if job is None: + raise ValueError("session-pinned variant job no longer exists") + return replace(artifact, job_id=job_id) + + def _session_row(self, session_id: str) -> sqlite3.Row: + with self.reviews.database.connect() as connection: + row = connection.execute( + "SELECT * FROM review_sessions WHERE id = ?", (session_id,) + ).fetchone() + if row is None: + raise LookupError(f"review session not found: {session_id}") + return cast(sqlite3.Row, row) + + def _require_generic_session(self, session_id: str) -> None: + row = self._session_row(session_id) + with self.reviews.database.connect() as connection: + experiment = connection.execute( + "SELECT schema_version FROM experiments WHERE id = ?", + (row["experiment_id"],), + ).fetchone() + if ( + experiment is not None + and str(experiment["schema_version"]) == "validation-controlled.v1" + ): + raise LookupError(f"review session not found: {session_id}") + + def _validation_trial_authorization( + self, campaign_id: str, trial_id: str, reviewer_id: str + ) -> tuple[str, str, bool]: + with self.reviews.database.connect() as connection: + row = connection.execute( + "SELECT campaign.reviewer_id, trial.winner_display_position, " + "trial.reuses_trial_id FROM validation_trials AS trial " + "JOIN validation_campaigns AS campaign ON campaign.id = trial.campaign_id " + "WHERE trial.campaign_id = ? AND trial.trial_id = ?", + (campaign_id, trial_id), + ).fetchone() + if row is None or str(row["reviewer_id"]) != reviewer_id: + raise LookupError(f"validation review not found: {trial_id}") + source_trial_id = str(row["reuses_trial_id"] or trial_id) + return ( + validation_experiment_id(campaign_id, source_trial_id), + str(row["winner_display_position"]), + row["reuses_trial_id"] is not None, + ) + + @staticmethod + def _position(position: str) -> DisplayPosition: + if position not in ("left", "right"): + raise ValueError("unknown display position") + return cast(DisplayPosition, position) + + @staticmethod + def _submission( + revision: ReviewRevision, status: Literal["submitted", "corrected"] + ) -> ReviewSubmission: + return ReviewSubmission(review_id=revision.id, revision=revision.revision, status=status) + + +def _value_at(manifest: Manifest, path: str) -> Any: + value: Any = manifest.model_dump(mode="json") + for part in path.split("."): + value = value[int(part)] if isinstance(value, list) else value[part] + return value + + +def _validation_session_reviewer( + campaign_id: str, + trial_id: str, + reviewer_id: str, + *, + hidden_repeat: bool, +) -> str: + if not hidden_repeat: + return reviewer_id + payload = f"validation-review-session.v1\x00{campaign_id}\x00{trial_id}\x00{reviewer_id}" + return "validation_reviewer_" + hashlib.sha256(payload.encode()).hexdigest() diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/reviews.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/reviews.py new file mode 100644 index 0000000..f8e03c8 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/reviews.py @@ -0,0 +1,452 @@ +from __future__ import annotations + +import hashlib +import secrets +import sqlite3 +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime +from typing import Literal, cast + +from video_lab.artifacts import Artifact +from video_lab.database import Database +from video_lab.domain import ( + DisplayPreference, + ReviewDimension, + ReviewRevision, + ReviewSession, + RubricScore, + canonical_json, + new_id, + utc_now, +) + +_POSITIONS: tuple[Literal["left", "right"], ...] = ("left", "right") +_SCORE_KEYS = {(position, dimension) for position in _POSITIONS for dimension in ReviewDimension} +ADAPTIVE_REVIEW_BINDING_SCHEMA_VERSION = "adaptive-review-binding.v1" + + +@dataclass(frozen=True) +class _ReviewSessionRecord: + id: str + experiment_id: str + reviewer_id: str + left_variant_id: str + right_variant_id: str + left_job_id: str + left_artifact_id: str + left_artifact_sha256: str + right_job_id: str + right_artifact_id: str + right_artifact_sha256: str + created_at: datetime + + +class ReviewRepository: + def __init__(self, database: Database) -> None: + self.database = database + + def start_session( + self, + experiment_id: str, + reviewer_id: str, + artifacts_by_variant: Mapping[str, Artifact], + ) -> ReviewSession: + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + existing = connection.execute( + "SELECT * FROM review_sessions WHERE experiment_id = ? AND reviewer_id = ?", + (experiment_id, reviewer_id), + ).fetchone() + if existing is not None: + connection.commit() + return _public_session(existing) + variants = connection.execute( + "SELECT id FROM variants WHERE experiment_id = ? ORDER BY label", + (experiment_id,), + ).fetchall() + if not variants: + raise LookupError(f"experiment not found: {experiment_id}") + if len(variants) != 2: + raise ValueError(f"experiment must have exactly two variants: {experiment_id}") + variant_ids = [str(row["id"]) for row in variants] + if set(artifacts_by_variant) != set(variant_ids): + raise ValueError("artifact pins must cover the experiment variants") + current = connection.execute( + "SELECT jobs.id AS job_id, jobs.variant_id, jobs.state, " + "artifacts.id AS artifact_id, " + "artifacts.sha256, artifacts.media_type FROM jobs " + "JOIN variants ON variants.id = jobs.variant_id " + "LEFT JOIN job_artifacts ON job_artifacts.job_id = jobs.id " + "LEFT JOIN artifacts ON artifacts.id = job_artifacts.artifact_id " + "WHERE variants.experiment_id = ?", + (experiment_id,), + ).fetchall() + current_pins = { + str(row["variant_id"]): ( + str(row["job_id"]), + str(row["artifact_id"]), + str(row["sha256"]), + ) + for row in current + if row["state"] == "completed" + and row["artifact_id"] is not None + and row["media_type"] == "video/mp4" + } + requested_pins = { + variant_id: (artifact.id, artifact.sha256) + for variant_id, artifact in artifacts_by_variant.items() + } + current_artifacts = { + variant_id: (pin[1], pin[2]) for variant_id, pin in current_pins.items() + } + if len(current) != 2 or current_artifacts != requested_pins: + raise ValueError("artifact pins no longer match current experiment artifacts") + shuffled = secrets.SystemRandom().sample(variant_ids, k=2) + left_artifact = artifacts_by_variant[shuffled[0]] + right_artifact = artifacts_by_variant[shuffled[1]] + record = _ReviewSessionRecord( + id=new_id("review_session"), + experiment_id=experiment_id, + reviewer_id=reviewer_id, + left_variant_id=shuffled[0], + right_variant_id=shuffled[1], + left_job_id=current_pins[shuffled[0]][0], + left_artifact_id=left_artifact.id, + left_artifact_sha256=left_artifact.sha256, + right_job_id=current_pins[shuffled[1]][0], + right_artifact_id=right_artifact.id, + right_artifact_sha256=right_artifact.sha256, + created_at=utc_now(), + ) + connection.execute( + "INSERT INTO review_sessions(id, experiment_id, reviewer_id, " + "left_variant_id, right_variant_id, left_artifact_id, " + "left_job_id, left_artifact_sha256, right_job_id, right_artifact_id, " + "right_artifact_sha256, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + record.id, + record.experiment_id, + record.reviewer_id, + record.left_variant_id, + record.right_variant_id, + record.left_artifact_id, + record.left_job_id, + record.left_artifact_sha256, + record.right_job_id, + record.right_artifact_id, + record.right_artifact_sha256, + record.created_at.isoformat(), + ), + ) + connection.commit() + return ReviewSession( + id=record.id, + experiment_id=record.experiment_id, + created_at=record.created_at, + ) + except Exception: + connection.rollback() + raise + + def submit( + self, + session_id: str, + preference: DisplayPreference, + scores: Sequence[RubricScore], + notes: str | None, + supersedes_id: str | None = None, + ) -> ReviewRevision: + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + session_row = connection.execute( + "SELECT * FROM review_sessions WHERE id = ?", (session_id,) + ).fetchone() + if session_row is None: + raise LookupError(f"review session not found: {session_id}") + if _is_adaptive_experiment(connection, str(session_row["experiment_id"])): + raise ValueError("adaptive review must use immutable preference submission") + ordered_scores = _validate_and_order_scores(scores) + normalized_notes = notes.strip() or None if notes is not None else None + latest_row = connection.execute( + "SELECT * FROM review_revisions WHERE session_id = ? " + "ORDER BY revision DESC LIMIT 1", + (session_id,), + ).fetchone() + superseded_row = None + if supersedes_id is not None: + superseded_row = connection.execute( + "SELECT session_id FROM review_revisions WHERE id = ?", (supersedes_id,) + ).fetchone() + if superseded_row is None: + raise ValueError("superseded review does not exist") + if str(superseded_row["session_id"]) != session_id: + raise ValueError("superseded review belongs to another session") + if latest_row is None: + if supersedes_id is not None: + raise ValueError("first review cannot supersede another review") + revision_number = 1 + else: + if supersedes_id is None: + raise ValueError("correction must specify the review it supersedes") + if supersedes_id != str(latest_row["id"]): + raise ValueError( + "correction must supersede the latest review for this session" + ) + revision_number = int(latest_row["revision"]) + 1 + + preferred_variant_id = None + if preference is DisplayPreference.LEFT: + preferred_variant_id = str(session_row["left_variant_id"]) + elif preference is DisplayPreference.RIGHT: + preferred_variant_id = str(session_row["right_variant_id"]) + revision = ReviewRevision( + id=new_id("review"), + session_id=session_id, + preferred_variant_id=preferred_variant_id, + display_preference=preference, + scores=ordered_scores, + notes=normalized_notes, + created_at=utc_now(), + revision=revision_number, + supersedes_review_id=supersedes_id, + ) + connection.execute( + "INSERT INTO review_revisions(id, session_id, preferred_variant_id, " + "display_preference, notes, created_at, revision, supersedes_review_id) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + revision.id, + revision.session_id, + revision.preferred_variant_id, + revision.display_preference.value, + revision.notes, + revision.created_at.isoformat(), + revision.revision, + revision.supersedes_review_id, + ), + ) + connection.executemany( + "INSERT INTO rubric_scores(review_id, display_position, dimension, score) " + "VALUES (?, ?, ?, ?)", + [ + (revision.id, score.display_position, score.dimension.value, score.score) + for score in ordered_scores + ], + ) + connection.commit() + return revision + except sqlite3.IntegrityError as error: + connection.rollback() + raise ValueError("review violates persistence constraints") from error + except Exception: + connection.rollback() + raise + + def submit_adaptive( + self, + session_id: str, + preference: DisplayPreference, + ) -> ReviewRevision: + """Persist one immutable adaptive preference, or return the identical retry.""" + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + session_row = connection.execute( + "SELECT * FROM review_sessions WHERE id = ?", (session_id,) + ).fetchone() + if session_row is None: + raise LookupError(f"review session not found: {session_id}") + latest_row = connection.execute( + "SELECT * FROM review_revisions WHERE session_id = ? " + "ORDER BY revision DESC LIMIT 1", + (session_id,), + ).fetchone() + if latest_row is not None: + score_rows = connection.execute( + "SELECT display_position, dimension, score FROM rubric_scores " + "WHERE review_id = ?", + (latest_row["id"],), + ).fetchall() + if ( + int(latest_row["revision"]) != 1 + or latest_row["supersedes_review_id"] is not None + or latest_row["notes"] is not None + or str(latest_row["display_preference"]) != preference.value + or score_rows + ): + raise ValueError("adaptive v1 review is immutable") + connection.commit() + return _revision(latest_row, score_rows) + + preferred_variant_id = None + if preference is DisplayPreference.LEFT: + preferred_variant_id = str(session_row["left_variant_id"]) + elif preference is DisplayPreference.RIGHT: + preferred_variant_id = str(session_row["right_variant_id"]) + revision = ReviewRevision( + id=new_id("review"), + session_id=session_id, + preferred_variant_id=preferred_variant_id, + display_preference=preference, + scores=(), + notes=None, + created_at=utc_now(), + revision=1, + supersedes_review_id=None, + ) + connection.execute( + "INSERT INTO review_revisions(id, session_id, preferred_variant_id, " + "display_preference, notes, created_at, revision, supersedes_review_id) " + "VALUES (?, ?, ?, ?, NULL, ?, 1, NULL)", + ( + revision.id, + revision.session_id, + revision.preferred_variant_id, + revision.display_preference.value, + revision.created_at.isoformat(), + ), + ) + connection.commit() + return revision + except Exception: + connection.rollback() + raise + + def is_adaptive_session(self, session_id: str) -> bool: + with self.database.connect() as connection: + row = connection.execute( + "SELECT experiment_id FROM review_sessions WHERE id = ?", + (session_id,), + ).fetchone() + return row is not None and _is_adaptive_experiment( + connection, str(row["experiment_id"]) + ) + + def adaptive_binding_digest( + self, + session_id: str, + reviewer_id: str, + *, + study_id: str, + optimizer_step_ordinal: int, + optimizer_output_digest: str, + ) -> str: + with self.database.connect() as connection: + row = connection.execute( + "SELECT * FROM review_sessions WHERE id = ?", (session_id,) + ).fetchone() + if row is None: + raise LookupError(f"review session not found: {session_id}") + if str(row["reviewer_id"]) != reviewer_id: + raise PermissionError("review session belongs to another reviewer") + if not _is_adaptive_experiment(connection, str(row["experiment_id"])): + raise ValueError("review session is not bound to an adaptive study") + return adaptive_review_binding_digest( + row, + study_id=study_id, + optimizer_step_ordinal=optimizer_step_ordinal, + optimizer_output_digest=optimizer_output_digest, + ) + + def latest_for_session(self, session_id: str) -> ReviewRevision | None: + with self.database.connect() as connection: + row = connection.execute( + "SELECT * FROM review_revisions WHERE session_id = ? " + "ORDER BY revision DESC LIMIT 1", + (session_id,), + ).fetchone() + if row is None: + return None + scores = connection.execute( + "SELECT display_position, dimension, score FROM rubric_scores " + "WHERE review_id = ? ORDER BY " + "CASE display_position WHEN 'left' THEN 0 ELSE 1 END, " + "CASE dimension " + "WHEN 'prompt_adherence' THEN 0 WHEN 'visual_quality' THEN 1 " + "WHEN 'motion_quality' THEN 2 WHEN 'temporal_consistency' THEN 3 ELSE 4 END", + (row["id"],), + ).fetchall() + return _revision(row, scores) + + +def adaptive_review_binding_digest( + session_row: sqlite3.Row, + *, + study_id: str, + optimizer_step_ordinal: int, + optimizer_output_digest: str, +) -> str: + payload = { + "schema_version": ADAPTIVE_REVIEW_BINDING_SCHEMA_VERSION, + "domain": "video-lab.adaptive.review-binding", + "study_id": study_id, + "optimizer_step_ordinal": optimizer_step_ordinal, + "optimizer_output_digest": optimizer_output_digest, + "experiment_id": str(session_row["experiment_id"]), + "review_session_id": str(session_row["id"]), + "reviewer_id": str(session_row["reviewer_id"]), + "left_variant_id": str(session_row["left_variant_id"]), + "right_variant_id": str(session_row["right_variant_id"]), + "left_job_id": str(session_row["left_job_id"]), + "right_job_id": str(session_row["right_job_id"]), + "left_artifact_id": str(session_row["left_artifact_id"]), + "right_artifact_id": str(session_row["right_artifact_id"]), + "left_artifact_sha256": str(session_row["left_artifact_sha256"]), + "right_artifact_sha256": str(session_row["right_artifact_sha256"]), + } + return hashlib.sha256(canonical_json(payload).encode()).hexdigest() + + +def _is_adaptive_experiment(connection: sqlite3.Connection, experiment_id: str) -> bool: + row = connection.execute( + "SELECT 1 FROM adaptive_study_events " + "WHERE json_extract(payload_json, '$.experiment_id') = ? LIMIT 1", + (experiment_id,), + ).fetchone() + return row is not None + + +def _validate_and_order_scores(scores: Sequence[RubricScore]) -> tuple[RubricScore, ...]: + by_key = {(score.display_position, score.dimension): score for score in scores} + if len(scores) != 10 or set(by_key) != _SCORE_KEYS: + raise ValueError("review requires exactly ten unique rubric scores") + return tuple( + by_key[(position, dimension)] for position in _POSITIONS for dimension in ReviewDimension + ) + + +def _public_session(row: sqlite3.Row) -> ReviewSession: + return ReviewSession( + id=str(row["id"]), + experiment_id=str(row["experiment_id"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + ) + + +def _revision(row: sqlite3.Row, score_rows: Sequence[sqlite3.Row]) -> ReviewRevision: + return ReviewRevision( + id=str(row["id"]), + session_id=str(row["session_id"]), + preferred_variant_id=( + None if row["preferred_variant_id"] is None else str(row["preferred_variant_id"]) + ), + display_preference=DisplayPreference(str(row["display_preference"])), + scores=tuple( + RubricScore( + display_position=cast(Literal["left", "right"], str(score["display_position"])), + dimension=ReviewDimension(str(score["dimension"])), + score=int(score["score"]), + ) + for score in score_rows + ), + notes=None if row["notes"] is None else str(row["notes"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + revision=int(row["revision"]), + supersedes_review_id=( + None if row["supersedes_review_id"] is None else str(row["supersedes_review_id"]) + ), + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/runtime_identity.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/runtime_identity.py new file mode 100644 index 0000000..959f91b --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/runtime_identity.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import hashlib +import json +import platform +import re +from collections.abc import Iterable, Mapping +from importlib.metadata import distributions +from pathlib import Path + +_NORMALIZE_DISTRIBUTION_NAME = re.compile(r"[-_.]+") +_VALID_NORMALIZED_DISTRIBUTION_NAME = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") + + +class RuntimeIdentityError(ValueError): + pass + + +def source_tree_digest(package_root: Path | None = None) -> str: + root = Path(__file__).resolve().parent if package_root is None else Path(package_root).resolve() + digest = hashlib.sha256(b"video-lab-source-tree.v1\0") + paths = sorted( + path + for path in root.rglob("*") + if path.is_file() + and "__pycache__" not in path.parts + and path.suffix not in {".pyc", ".pyo"} + and path.name != ".DS_Store" + ) + for path in paths: + relative = path.relative_to(root).as_posix().encode() + content = path.read_bytes() + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(len(content).to_bytes(8, "big")) + digest.update(content) + return f"source-sha256:{digest.hexdigest()}" + + +def animegen_runtime_versions( + *, + package_root: Path | None = None, + distribution_entries: Iterable[tuple[str, str]] | None = None, + python_version: str | None = None, +) -> dict[str, str]: + entries = ( + _installed_distribution_entries() if distribution_entries is None else distribution_entries + ) + return { + "python": platform.python_version() if python_version is None else python_version, + **_normalized_distribution_versions(entries), + "video-lab": source_tree_digest(package_root), + } + + +def _installed_distribution_entries() -> Iterable[tuple[str, str]]: + for distribution in distributions(): + name = distribution.metadata.get("Name") + version = distribution.version + if not isinstance(name, str) or not isinstance(version, str): + raise RuntimeIdentityError("installed distribution metadata is incomplete") + yield name, version + + +def _normalized_distribution_versions( + entries: Iterable[tuple[str, str]], +) -> dict[str, str]: + normalized: dict[str, str] = {} + for raw_name, raw_version in entries: + name = _NORMALIZE_DISTRIBUTION_NAME.sub("-", raw_name.strip()).lower() + version = raw_version.strip() + if not _VALID_NORMALIZED_DISTRIBUTION_NAME.fullmatch(name) or not version: + raise RuntimeIdentityError("installed distribution metadata is invalid") + existing = normalized.get(name) + if existing is not None and existing != version: + raise RuntimeIdentityError( + f"conflicting installed versions for normalized distribution {name}" + ) + normalized[name] = version + return {f"dist:{name}": normalized[name] for name in sorted(normalized)} + + +def runtime_fingerprint(runtime_versions: Mapping[str, str]) -> str: + canonical = json.dumps( + dict(runtime_versions), sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode() + return hashlib.sha256(canonical).hexdigest() diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/services.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/services.py new file mode 100644 index 0000000..5479f4f --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/services.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from video_lab.artifacts import ArtifactStore +from video_lab.capabilities import ( + AdapterCapabilities, + CapabilityViolation, + validate_manifest_support, +) +from video_lab.comparison import ControlledPair, build_controlled_pair +from video_lab.database import Database +from video_lab.domain import ExperimentDraft, Job, JobState, Manifest +from video_lab.repositories import ExperimentRepository, JobRepository, StoredExperiment + + +class ValidationFailure(ValueError): + def __init__(self, violations: tuple[CapabilityViolation, ...]) -> None: + super().__init__("manifest is not supported by the selected adapter") + self.violations = violations + + +@dataclass(frozen=True) +class ExperimentView: + experiment: StoredExperiment + jobs: tuple[Job, ...] + + +class ExperimentService: + def __init__( + self, + experiments: ExperimentRepository, + jobs: JobRepository, + artifacts: ArtifactStore, + ) -> None: + self.experiments = experiments + self.jobs = jobs + self.artifacts = artifacts + + @classmethod + def from_paths(cls, database_path: Path, artifact_root: Path) -> ExperimentService: + database = Database(database_path) + database.migrate() + return cls( + ExperimentRepository(database), + JobRepository(database), + ArtifactStore(database, artifact_root), + ) + + def create(self, draft: ExperimentDraft, capabilities: AdapterCapabilities) -> ExperimentView: + pair = self.preview(draft, capabilities) + experiment, jobs = self.experiments.create_with_jobs(pair) + return ExperimentView(experiment=experiment, jobs=jobs) + + def create_adaptive( + self, + draft: ExperimentDraft, + capabilities: AdapterCapabilities, + *, + study_id: str, + optimizer_output_digest: str, + ) -> ExperimentView: + pair = self.preview(draft, capabilities) + experiment, jobs = self.experiments.create_adaptive_with_jobs( + pair, + study_id=study_id, + optimizer_output_digest=optimizer_output_digest, + ) + return ExperimentView(experiment=experiment, jobs=jobs) + + def create_validation( + self, + draft: ExperimentDraft, + capabilities: AdapterCapabilities, + *, + campaign_id: str, + trial_id: str, + ) -> ExperimentView: + pair = self.preview(draft, capabilities) + experiment, jobs = self.experiments.create_validation_with_jobs( + pair, campaign_id=campaign_id, trial_id=trial_id + ) + return ExperimentView(experiment=experiment, jobs=jobs) + + def preview(self, draft: ExperimentDraft, capabilities: AdapterCapabilities) -> ControlledPair: + self._validate(draft.baseline, capabilities) + pair = build_controlled_pair(draft) + self._validate(pair.variant_b, capabilities) + return pair + + def get(self, experiment_id: str) -> ExperimentView: + experiment = self.experiments.get(experiment_id) + return ExperimentView( + experiment=experiment, + jobs=self.jobs.list_for_experiment(experiment_id), + ) + + def get_adaptive(self, experiment_id: str) -> ExperimentView: + experiment = self.experiments.get_adaptive(experiment_id) + return ExperimentView( + experiment=experiment, + jobs=self.jobs.list_for_experiment(experiment_id), + ) + + def get_validation(self, experiment_id: str) -> ExperimentView: + experiment = self.experiments.get_validation(experiment_id) + return ExperimentView( + experiment=experiment, + jobs=self.jobs.list_for_experiment(experiment_id), + ) + + def list(self, limit: int = 50) -> tuple[ExperimentView, ...]: + return tuple(self.get(experiment.id) for experiment in self.experiments.list(limit)) + + def list_adaptive_for_study(self, study_id: str) -> tuple[ExperimentView, ...]: + return tuple( + self.get_adaptive(experiment.id) + for experiment in self.experiments.list_adaptive_for_study(study_id) + ) + + def requeue_adaptive_failures(self, study_id: str, experiment_id: str) -> tuple[Job, ...]: + experiment = self.get_adaptive(experiment_id) + return self.jobs.requeue_failed_adaptive( + study_id, + experiment_id, + tuple(job.id for job in experiment.jobs), + ) + + def transition_job(self, job_id: str, target: JobState) -> Job: + return self.jobs.transition(job_id, target) + + @staticmethod + def _validate(manifest: Manifest, capabilities: AdapterCapabilities) -> None: + violations = validate_manifest_support(manifest, capabilities) + if violations: + raise ValidationFailure(violations) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/source_inventory.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/source_inventory.py new file mode 100644 index 0000000..7be8c24 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/source_inventory.py @@ -0,0 +1,556 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass, field, replace +from pathlib import Path, PurePosixPath +from types import MappingProxyType +from typing import Protocol + +from video_lab.feasibility_models import FeasibilityEvidenceError + +GIB = 1024**3 +COMMIT_LENGTH = 40 +HASHED_EVIDENCE_SUFFIXES = (".json", ".safetensors", ".bin", ".pt", ".ckpt") + +ANIMEGEN_REQUIRED_FILES: Mapping[str, tuple[str, ...]] = MappingProxyType( + { + "animegen": ("high_noise.safetensors", "low_noise.safetensors", "LICENSE"), + "wan-base": ( + "model_index.json", + "transformer/config.json", + "vae/config.json", + "vae/diffusion_pytorch_model.safetensors", + "scheduler/scheduler_config.json", + "tokenizer/tokenizer_config.json", + "tokenizer/spiece.model", + "text_encoder/config.json", + "text_encoder/model-00001-of-00003.safetensors", + "text_encoder/model-00002-of-00003.safetensors", + "text_encoder/model-00003-of-00003.safetensors", + "text_encoder/model.safetensors.index.json", + ), + "lightning-high": ("Wan2.2-T2V-A14B-4steps-lora-250928/high_noise_model.safetensors",), + "lightning-low": ("Wan2.2-T2V-A14B-4steps-lora-250928/low_noise_model.safetensors",), + } +) +ANIMEGEN_SOURCE_REPOS: Mapping[str, str] = MappingProxyType( + { + "animegen": "aidealab/AnimeGen-T2V", + "wan-base": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", + "lightning-high": "lightx2v/Wan2.2-Lightning", + "lightning-low": "lightx2v/Wan2.2-Lightning", + } +) + + +class SourceInventoryError(FeasibilityEvidenceError): + """Raised when immutable source evidence cannot be established.""" + + +class ModelInfo(Protocol): + def __call__(self, *, repo_id: str, revision: str, files_metadata: bool) -> object: ... + + +class SnapshotDownload(Protocol): + def __call__(self, **kwargs: object) -> str: ... + + +class DiskUsage(Protocol): + @property + def free(self) -> int: ... + + +@dataclass(frozen=True) +class SourceSpec: + component: str + repo_id: str + revision: str + required_files: tuple[str, ...] + allow_custom_code: bool = False + + def __init__( + self, + component: str, + repo_id: str, + revision: str, + required_files: Sequence[str], + allow_custom_code: bool = False, + ) -> None: + object.__setattr__(self, "component", _required(component, "component")) + object.__setattr__(self, "repo_id", _required(repo_id, "repo_id")) + object.__setattr__(self, "revision", _required(revision, "revision")) + object.__setattr__(self, "required_files", tuple(required_files)) + object.__setattr__(self, "allow_custom_code", allow_custom_code) + + +@dataclass(frozen=True) +class SourceFile: + relative_path: str + size: int + upstream_sha256: str | None = None + local_sha256: str | None = None + + def export_evidence(self) -> Mapping[str, object]: + evidence: dict[str, object] = { + "relative_path": self.relative_path, + "size": self.size, + } + if self.upstream_sha256: + evidence["upstream_sha256"] = self.upstream_sha256 + if self.local_sha256: + evidence["local_sha256"] = self.local_sha256 + return evidence + + +@dataclass(frozen=True) +class ResolvedSource: + component: str + repo_id: str + requested_revision: str + revision: str + license: str + files: tuple[SourceFile, ...] + + def export_evidence(self) -> Mapping[str, object]: + return { + "component": self.component, + "repo_id": self.repo_id, + "requested_revision": self.requested_revision, + "revision": self.revision, + "license": self.license, + "files": tuple(item.export_evidence() for item in self.files), + } + + +@dataclass(frozen=True) +class StoragePreflight: + expected_bytes: int + safety_margin_bytes: int + required_bytes: int + free_bytes: int + + def export_evidence(self) -> Mapping[str, int]: + return { + "expected_bytes": self.expected_bytes, + "safety_margin_bytes": self.safety_margin_bytes, + "required_bytes": self.required_bytes, + "free_bytes": self.free_bytes, + } + + +@dataclass(frozen=True) +class SourceInventory: + sources: tuple[ResolvedSource, ...] + expected_bytes: int + _client: SourceInventoryClient = field(repr=False, compare=False) + + def export_evidence(self) -> Mapping[str, object]: + return { + "expected_bytes": self.expected_bytes, + "sources": tuple(source.export_evidence() for source in self.sources), + } + + def preflight(self, disk_usage: Callable[[Path], DiskUsage] | None = None) -> StoragePreflight: + return self._client.preflight(self, disk_usage) + + def download(self) -> SourceInventory: + return self._client.download(self) + + +def canonical_source_digest(source: ResolvedSource) -> str: + files: list[dict[str, object]] = [] + for item in sorted(source.files, key=lambda value: value.relative_path): + checksum = item.local_sha256 or item.upstream_sha256 + if checksum is None: + raise SourceInventoryError( + f"source file lacks a verified checksum: {item.relative_path}" + ) + files.append({"path": item.relative_path, "size": item.size, "sha256": checksum}) + payload = { + "repo_id": source.repo_id, + "revision": source.revision, + "files": files, + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +class SourceInventoryClient: + def __init__( + self, + cache_root: Path, + model_info: ModelInfo, + snapshot_download: SnapshotDownload, + ) -> None: + self._cache_root = Path(cache_root) + self._model_info = model_info + self._snapshot_download = snapshot_download + + def resolve(self, specs: Iterable[SourceSpec]) -> SourceInventory: + requested = tuple(specs) + components = [spec.component for spec in requested] + if len(components) != len(set(components)): + raise SourceInventoryError("duplicate logical component") + + sources = tuple(self._resolve_one(spec) for spec in requested) + return SourceInventory( + sources=sources, + expected_bytes=sum(item.size for source in sources for item in source.files), + _client=self, + ) + + def preflight( + self, + inventory: SourceInventory, + disk_usage: Callable[[Path], DiskUsage] | None = None, + ) -> StoragePreflight: + self._prepare_cache_root() + try: + usage = (disk_usage or _disk_usage)(self._cache_root) + except OSError as exc: + raise SourceInventoryError("storage preflight filesystem access failed") from exc + free = usage.free + margin = max(20 * GIB, (inventory.expected_bytes + 4) // 5) + required = inventory.expected_bytes + margin + result = StoragePreflight(inventory.expected_bytes, margin, required, free) + if free < required: + raise SourceInventoryError( + f"insufficient free space: requires {required} bytes, found {free} bytes" + ) + return result + + def _prepare_cache_root(self) -> None: + root = self._cache_root + try: + if root.is_symlink(): + raise SourceInventoryError("cache root must not be a symlink") + if root.exists(): + if not root.is_dir(): + raise SourceInventoryError("cache root must be a directory") + return + for parent in root.parents: + if parent.exists(): + if parent.is_symlink(): + raise SourceInventoryError("cache parent must not be a symlink") + if not parent.is_dir(): + raise SourceInventoryError("cache parent must be a directory") + break + root.mkdir(parents=True, exist_ok=False) + if root.is_symlink() or not root.is_dir(): + raise SourceInventoryError("cache root creation was unsafe") + except SourceInventoryError: + raise + except (FileExistsError, OSError) as exc: + raise SourceInventoryError("cache root setup failed safely") from exc + + def download(self, inventory: SourceInventory) -> SourceInventory: + for source in inventory.sources: + self._assert_frozen_revision(source) + completed: list[ResolvedSource] = [] + for source in inventory.sources: + try: + target = Path( + self._snapshot_download( + repo_id=source.repo_id, + revision=source.revision, + cache_dir=str(self._cache_root), + allow_patterns=tuple(item.relative_path for item in source.files), + local_files_only=False, + resume_download=True, + ) + ) + except OSError as exc: + raise SourceInventoryError( + f"download filesystem access failed for {source.repo_id}" + ) from exc + self._assert_frozen_revision(source) + completed.append(replace(source, files=self._verify_download(target, source.files))) + sources = tuple(completed) + return SourceInventory(sources, inventory.expected_bytes, self) + + def verify_cached(self, inventory: SourceInventory) -> SourceInventory: + """Reverify a frozen imported inventory without metadata or acquisition calls.""" + completed: list[ResolvedSource] = [] + for source in inventory.sources: + try: + target = Path( + self._snapshot_download( + repo_id=source.repo_id, + revision=source.revision, + cache_dir=str(self._cache_root), + allow_patterns=tuple(item.relative_path for item in source.files), + local_files_only=True, + ) + ) + except OSError as exc: + raise SourceInventoryError( + f"cached snapshot unavailable for {source.repo_id}" + ) from exc + completed.append(replace(source, files=self._verify_download(target, source.files))) + sources = tuple(completed) + return SourceInventory(sources, inventory.expected_bytes, self) + + def refresh_cached( + self, inventory: SourceInventory, specs: Sequence[SourceSpec] + ) -> SourceInventory: + """Add newly required files from the same frozen local snapshots, then reverify.""" + if len(inventory.sources) != len(specs): + raise SourceInventoryError("cached inventory source set does not match specification") + if all( + tuple(item.relative_path for item in source.files) == spec.required_files + for source, spec in zip(inventory.sources, specs, strict=True) + ): + return inventory + refreshed: list[ResolvedSource] = [] + cache_root = self._cache_root.resolve() + for source, spec in zip(inventory.sources, specs, strict=True): + if ( + source.component != spec.component + or source.repo_id != spec.repo_id + or source.requested_revision != spec.revision + or source.revision != spec.revision + ): + raise SourceInventoryError(f"cached inventory identity mismatch: {spec.component}") + target = Path( + self._snapshot_download( + repo_id=source.repo_id, + revision=source.revision, + cache_dir=str(self._cache_root), + allow_patterns=spec.required_files, + local_files_only=True, + ) + ) + existing = {item.relative_path: item for item in source.files} + files: list[SourceFile] = [] + for name in spec.required_files: + item = existing.get(name) + if item is None: + path = (target / name).resolve(strict=True) + if not path.is_relative_to(cache_root) or not path.is_file(): + raise SourceInventoryError(f"cached file escapes cache: {name}") + item = SourceFile(name, path.stat().st_size, local_sha256=_sha256(path)) + files.append(item) + verified = self._verify_download(target, tuple(files)) + refreshed.append(replace(source, files=verified)) + sources = tuple(refreshed) + expected_bytes = sum(item.size for source in sources for item in source.files) + return SourceInventory(sources, expected_bytes, self) + + def _resolve_one(self, spec: SourceSpec) -> ResolvedSource: + if _is_mutable_revision(spec.revision): + raise SourceInventoryError(f"source {spec.repo_id} requires an immutable revision") + response = self._model_info( + repo_id=spec.repo_id, revision=spec.revision, files_metadata=True + ) + revision = str(getattr(response, "sha", "")) + if not _is_commit(revision): + raise SourceInventoryError( + f"source {spec.repo_id} did not resolve to a 40-character commit" + ) + license_name = _license_from(response) + if not license_name: + raise SourceInventoryError(f"source {spec.repo_id} is missing license metadata") + + siblings = tuple(getattr(response, "siblings", ()) or ()) + available: dict[str, object] = {} + for sibling in siblings: + name = str(getattr(sibling, "rfilename", "")) + _validate_relative_filename(name) + if name.lower().endswith(".py") and not spec.allow_custom_code: + raise SourceInventoryError(f"unexpected custom code in {spec.repo_id}: {name}") + if name in available: + raise SourceInventoryError(f"duplicate filename in {spec.repo_id}: {name}") + available[name] = sibling + + missing = [name for name in spec.required_files if name not in available] + if missing: + raise SourceInventoryError( + f"missing required file in {spec.repo_id}: {', '.join(missing)}" + ) + files = tuple(_source_file(name, available[name]) for name in spec.required_files) + return ResolvedSource( + spec.component, spec.repo_id, spec.revision, revision, license_name, files + ) + + def _verify_download( + self, target: Path, expected: tuple[SourceFile, ...] + ) -> tuple[SourceFile, ...]: + try: + cache_root = self._cache_root.resolve() + except OSError as exc: + raise SourceInventoryError("filesystem verification failed for cache root") from exc + verified: list[SourceFile] = [] + for item in expected: + path = target / item.relative_path + try: + is_symlink = path.is_symlink() + resolved = path.resolve(strict=True) + except FileNotFoundError as exc: + if is_symlink: + raise SourceInventoryError( + f"download symlink escapes cache: {item.relative_path}" + ) from exc + raise SourceInventoryError(f"download missing file: {item.relative_path}") from exc + except RuntimeError as exc: + raise SourceInventoryError( + f"download symlink resolution failed: {item.relative_path}" + ) from exc + except OSError as exc: + raise SourceInventoryError( + f"filesystem verification failed for {item.relative_path}" + ) from exc + try: + if not resolved.is_relative_to(cache_root) or not resolved.is_file(): + raise SourceInventoryError( + f"download symlink escapes cache: {item.relative_path}" + ) + if resolved.stat().st_size != item.size: + raise SourceInventoryError(f"download size mismatch: {item.relative_path}") + digest = _sha256(resolved) + except SourceInventoryError: + raise + except OSError as exc: + raise SourceInventoryError( + f"filesystem verification failed for {item.relative_path}" + ) from exc + if item.upstream_sha256 and digest != item.upstream_sha256: + raise SourceInventoryError(f"download hash mismatch: {item.relative_path}") + if item.local_sha256 and digest != item.local_sha256: + raise SourceInventoryError(f"cached hash mismatch: {item.relative_path}") + local_hash = digest if _needs_local_hash(item.relative_path) else None + verified.append(replace(item, local_sha256=local_hash)) + return tuple(verified) + + def _assert_frozen_revision(self, source: ResolvedSource) -> None: + current = self._model_info( + repo_id=source.repo_id, + revision=source.requested_revision, + files_metadata=True, + ) + if getattr(current, "sha", None) != source.revision: + raise SourceInventoryError( + f"resolved commit changed for {source.repo_id} between inventory and download" + ) + + +def inventory_from_evidence( + evidence: Mapping[str, object], client: SourceInventoryClient +) -> SourceInventory: + """Reconstruct a frozen inventory while keeping cache/client state operational-only.""" + raw_sources = evidence.get("sources") + expected_bytes = evidence.get("expected_bytes") + if ( + not isinstance(raw_sources, Sequence) + or isinstance(raw_sources, str | bytes | bytearray) + or not isinstance(expected_bytes, int) + or expected_bytes < 0 + ): + raise SourceInventoryError("invalid persisted source inventory") + sources: list[ResolvedSource] = [] + components: set[str] = set() + for raw_source in raw_sources: + if not isinstance(raw_source, Mapping): + raise SourceInventoryError("invalid persisted source identity") + component = _required(str(raw_source.get("component", "")), "component") + repo_id = _required(str(raw_source.get("repo_id", "")), "repo_id") + requested = str(raw_source.get("requested_revision", "")) + revision = str(raw_source.get("revision", "")) + license_name = _required(str(raw_source.get("license", "")), "license") + if component in components or not _is_commit(requested) or not _is_commit(revision): + raise SourceInventoryError("persisted source identity is not immutable") + components.add(component) + raw_files = raw_source.get("files") + if not isinstance(raw_files, Sequence) or isinstance(raw_files, str | bytes | bytearray): + raise SourceInventoryError("invalid persisted source files") + files: list[SourceFile] = [] + names: set[str] = set() + for raw_file in raw_files: + if not isinstance(raw_file, Mapping): + raise SourceInventoryError("invalid persisted source file") + name = str(raw_file.get("relative_path", "")) + _validate_relative_filename(name) + size = raw_file.get("size") + upstream = raw_file.get("upstream_sha256") + local = raw_file.get("local_sha256") + if name in names or not isinstance(size, int) or size < 0: + raise SourceInventoryError("invalid persisted source file") + for digest in (upstream, local): + if digest is not None and ( + not isinstance(digest, str) + or len(digest) != 64 + or any(char not in "0123456789abcdef" for char in digest) + ): + raise SourceInventoryError("invalid persisted source checksum") + names.add(name) + files.append(SourceFile(name, size, upstream, local)) + sources.append( + ResolvedSource(component, repo_id, requested, revision, license_name, tuple(files)) + ) + if sum(item.size for source in sources for item in source.files) != expected_bytes: + raise SourceInventoryError("persisted source inventory byte total does not match") + return SourceInventory(tuple(sources), expected_bytes, client) + + +def _required(value: str, label: str) -> str: + stripped = value.strip() + if not stripped: + raise SourceInventoryError(f"{label} must not be blank") + return stripped + + +def _is_mutable_revision(revision: str) -> bool: + return revision.lower() in {"main", "master", "latest", "refs/heads/main"} + + +def _is_commit(value: str) -> bool: + return len(value) == COMMIT_LENGTH and all(char in "0123456789abcdef" for char in value) + + +def _validate_relative_filename(name: str) -> None: + path = PurePosixPath(name) + if not name or path.is_absolute() or ".." in path.parts or "\\" in name: + raise SourceInventoryError(f"unsafe filename: {name}") + + +def _license_from(response: object) -> str | None: + card_data = getattr(response, "card_data", None) + if isinstance(card_data, Mapping): + value = card_data.get("license") + else: + value = getattr(card_data, "license", None) + return value.strip() if isinstance(value, str) and value.strip() else None + + +def _source_file(name: str, sibling: object) -> SourceFile: + size = getattr(sibling, "size", None) + if not isinstance(size, int) or size < 0: + raise SourceInventoryError(f"missing byte size for {name}") + lfs = getattr(sibling, "lfs", None) + sha = getattr(lfs, "sha256", None) if lfs is not None else None + if sha is not None and ( + not isinstance(sha, str) + or len(sha) != 64 + or any(character not in "0123456789abcdef" for character in sha) + ): + raise SourceInventoryError(f"invalid upstream hash for {name}") + return SourceFile(name, size, sha) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _needs_local_hash(name: str) -> bool: + lowered = name.lower() + return lowered.endswith(HASHED_EVIDENCE_SUFFIXES) or Path(lowered).name.startswith("license") + + +def _disk_usage(path: Path) -> DiskUsage: + return shutil.disk_usage(path) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/states.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/states.py new file mode 100644 index 0000000..61d2b08 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/states.py @@ -0,0 +1,26 @@ +from video_lab.domain import JobState + + +class InvalidTransition(ValueError): + pass + + +ALLOWED: dict[JobState, frozenset[JobState]] = { + JobState.QUEUED: frozenset({JobState.PREPARING, JobState.CANCELLED, JobState.FAILED}), + JobState.PREPARING: frozenset( + {JobState.DOWNLOADING, JobState.LOADING, JobState.CANCELLED, JobState.FAILED} + ), + JobState.DOWNLOADING: frozenset({JobState.LOADING, JobState.CANCELLED, JobState.FAILED}), + JobState.LOADING: frozenset({JobState.GENERATING, JobState.CANCELLED, JobState.FAILED}), + JobState.GENERATING: frozenset({JobState.ENCODING, JobState.CANCELLED, JobState.FAILED}), + JobState.ENCODING: frozenset({JobState.COMPLETED, JobState.CANCELLED, JobState.FAILED}), + JobState.COMPLETED: frozenset(), + JobState.FAILED: frozenset(), + JobState.CANCELLED: frozenset(), +} + + +def transition_job(current: JobState, target: JobState) -> JobState: + if target not in ALLOWED[current]: + raise InvalidTransition(f"cannot transition job from {current} to {target}") + return target diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/telemetry.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/telemetry.py new file mode 100644 index 0000000..25d7f7f --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/telemetry.py @@ -0,0 +1,54 @@ +from collections.abc import Callable + +import psutil # type: ignore[import-untyped] + +from video_lab.domain import TelemetrySample, utc_now + +ThermalProvider = Callable[[float], str | None] + + +class TelemetrySampler: + def __init__( + self, + *, + thermal_provider: ThermalProvider | None = None, + thermal_timeout_seconds: float = 0.1, + ) -> None: + if thermal_timeout_seconds <= 0: + raise ValueError("thermal timeout must be positive") + self._thermal_provider = thermal_provider + self._thermal_timeout_seconds = thermal_timeout_seconds + + def sample(self, attempt_id: str, elapsed_seconds: float) -> TelemetrySample: + try: + process_rss = int(psutil.Process().memory_info().rss) + except Exception: + process_rss = 0 + try: + virtual = psutil.virtual_memory() + system_used = int(virtual.used) + system_available = int(virtual.available) + except Exception: + system_used = 0 + system_available = 0 + try: + swap_used = int(psutil.swap_memory().used) + except Exception: + # Some sandboxed macOS processes cannot query sysctl swap data. + swap_used = 0 + thermal_pressure: str | None = None + if self._thermal_provider is not None: + try: + thermal_pressure = self._thermal_provider(self._thermal_timeout_seconds) + except Exception: + thermal_pressure = None + return TelemetrySample( + attempt_id=attempt_id, + recorded_at=utc_now(), + elapsed_seconds=elapsed_seconds, + process_rss_bytes=process_rss, + system_used_bytes=system_used, + system_available_bytes=system_available, + swap_used_bytes=swap_used, + thermal_pressure=thermal_pressure, + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainer_protocol.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainer_protocol.py new file mode 100644 index 0000000..3252c78 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainer_protocol.py @@ -0,0 +1,855 @@ +"""Dependency-free contracts for local character-training backends.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import posixpath +import stat +import threading +import unicodedata +from collections.abc import Callable, Mapping +from itertools import pairwise +from pathlib import Path +from typing import Annotated, Literal, Protocol, runtime_checkable + +from pydantic import Field, StrictFloat, StrictInt, field_validator, model_validator + +from video_lab.domain import FrozenModel, canonical_json +from video_lab.wan_pending import ( + trainable_parameter_specs_from_records, + validate_pending_gradients, + validate_trainable_parameter_specs, +) + +Sha256Digest = Annotated[str, Field(pattern=r"^sha256:[0-9a-f]{64}$")] + + +class TrainingMaterialHandles(FrozenModel): + recipe_digest: Sha256Digest + dataset_digest: Sha256Digest + base_model_digest: Sha256Digest + backend_digest: Sha256Digest + source_digest: Sha256Digest + runtime_digest: Sha256Digest + material_digest: Sha256Digest + + +class OptimizerParameter(FrozenModel): + name: str = Field(min_length=1) + value: float | int | bool | str + + @field_validator("value") + @classmethod + def finite_value(cls, value: float | int | bool | str) -> float | int | bool | str: + if isinstance(value, float) and not math.isfinite(value): + raise ValueError("optimizer parameter must be finite") + return value + + +class OptimizerConfiguration(FrozenModel): + algorithm: str = Field(min_length=1) + parameters: tuple[OptimizerParameter, ...] = Field(min_length=1) + + @model_validator(mode="after") + def canonical_parameters(self) -> OptimizerConfiguration: + names = tuple(item.name for item in self.parameters) + if names != tuple(sorted(set(names))): + raise ValueError("optimizer parameters must be sorted and unique") + return self + + +class ProbeTrainingRequest(FrozenModel): + request_id: str = Field(min_length=1) + backend_id: str = Field(min_length=1) + expected_backend_digest: Sha256Digest + materials: TrainingMaterialHandles + scratch_root: Path + + +class TrainerCapabilityReport(FrozenModel): + backend_id: str = Field(min_length=1) + backend_digest: Sha256Digest + runtime_digest: Sha256Digest + supported: bool + max_steps: int = Field(gt=0) + safe_boundaries: tuple[int, ...] + + @field_validator("safe_boundaries") + @classmethod + def validate_boundaries(cls, value: tuple[int, ...]) -> tuple[int, ...]: + if not value or tuple(sorted(set(value))) != value or value[0] <= 0: + raise ValueError("safe boundaries must be positive and strictly monotonic") + return value + + +class LoRAConfiguration(FrozenModel): + """Exact low-rank adapter configuration bound to one trainer request.""" + + rank: StrictInt = Field(gt=0) + alpha: StrictInt = Field(gt=0) + dropout: StrictFloat = Field(ge=0, lt=1) + + @field_validator("dropout") + @classmethod + def finite_dropout(cls, value: float) -> float: + if not math.isfinite(value): + raise ValueError("lora dropout must be finite") + return value + + +class CalibrationRequest(FrozenModel): + request_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + backend_id: str = Field(min_length=1) + materials: TrainingMaterialHandles + scratch_root: Path + steps: int = Field(ge=0) + trainable_targets: tuple[str, ...] = Field(min_length=1) + lora: LoRAConfiguration + seed: int + deadline_seconds: int = Field(gt=0) + + +class TrainingRequest(FrozenModel): + request_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + backend_id: str = Field(min_length=1) + materials: TrainingMaterialHandles + scratch_root: Path + total_steps: int = Field(ge=4) + checkpoint_steps: tuple[int, ...] = Field(min_length=1) + trainable_targets: tuple[str, ...] = Field(min_length=1) + lora: LoRAConfiguration + seed: int + deadline_seconds: int = Field(gt=0) + optimizer: OptimizerConfiguration = OptimizerConfiguration( + algorithm="adamw", + parameters=(OptimizerParameter(name="learning_rate", value=0.001),), + ) + start_step: int = Field(default=0, ge=0) + telemetry_sequence_start: int = Field(default=1, gt=0) + parent_checkpoint_id: str | None = None + parent_checkpoint_digest: Sha256Digest | None = None + optimizer_state_digest: Sha256Digest | None = None + + @model_validator(mode="after") + def validate_schedule(self) -> TrainingRequest: + if ( + tuple(sorted(set(self.checkpoint_steps))) != self.checkpoint_steps + or self.checkpoint_steps[-1] != self.total_steps + or self.checkpoint_steps[0] <= 0 + ): + raise ValueError("checkpoint schedule is invalid") + if not 0 <= self.start_step < self.checkpoint_steps[0]: + raise ValueError("resume start step is invalid") + resume_values = ( + self.parent_checkpoint_id, + self.parent_checkpoint_digest, + self.optimizer_state_digest, + ) + if any(value is None for value in resume_values) != all( + value is None for value in resume_values + ): + raise ValueError("resume identities must be complete") + return self + + +class TrainingProgress(FrozenModel): + step: int = Field(ge=0) + total_steps: int = Field(gt=0) + fraction: float = Field(ge=0, le=1) + loss: float + gradient_norm: float = Field(default=1.0, ge=0) + checkpoint_boundary: bool = False + + @model_validator(mode="after") + def validate_progress(self) -> TrainingProgress: + if self.step > self.total_steps or not all( + math.isfinite(value) for value in (self.fraction, self.loss, self.gradient_norm) + ): + raise ValueError("training progress must be bounded and finite") + expected = self.step / self.total_steps + if not math.isclose(self.fraction, expected, rel_tol=0, abs_tol=1e-12): + raise ValueError("training progress fraction does not match step") + return self + + +class CalibrationResult(FrozenModel): + schema_version: Literal["character-training-calibration-result.v1"] = ( + "character-training-calibration-result.v1" + ) + request_id: str + run_id: str + backend_digest: Sha256Digest + runtime_digest: Sha256Digest + calibration_digest: Sha256Digest + finite_gradients: bool + frozen_parameters_unchanged: bool + changed_targets: tuple[str, ...] + measurements: Mapping[str, float] + + @field_validator("measurements") + @classmethod + def finite_measurements(cls, value: Mapping[str, float]) -> Mapping[str, float]: + if not value or any(not math.isfinite(item) for item in value.values()): + raise ValueError("calibration measurements must be finite") + return value + + +class OutputFile(FrozenModel): + relative_path: str = Field(min_length=1) + digest: Sha256Digest + size_bytes: int = Field(gt=0) + role: str = Field(min_length=1) + + @field_validator("relative_path") + @classmethod + def contained_path(cls, value: str) -> str: + if value.startswith("/") or ".." in value.split("/"): + raise ValueError("output path escapes scratch root") + return value + + +class CheckpointResult(FrozenModel): + checkpoint_id: str + step: int = Field(gt=0) + fraction: float = Field(gt=0, le=1) + parent_checkpoint_id: str | None + files: tuple[OutputFile, ...] + telemetry_digest: Sha256Digest + frozen_parameters_digest: Sha256Digest + trainable_targets_digest: Sha256Digest + finite_gradient_proof_digest: Sha256Digest + + +class TrainingResult(FrozenModel): + request_id: str + run_id: str + backend_id: str + backend_digest: Sha256Digest + materials: TrainingMaterialHandles + output_root: Path + checkpoints: tuple[CheckpointResult, ...] + telemetry_digest: Sha256Digest + changed_targets: tuple[str, ...] + frozen_parameters_unchanged: bool + finite_gradients: bool + elapsed_seconds: float = Field(ge=0) + fault: str | None = None + + +class TrainingPublicationReceipt(FrozenModel): + schema_version: Literal["character-training-publication-receipt.v1"] = ( + "character-training-publication-receipt.v1" + ) + request_id: str + run_id: str + lifecycle_state: Literal["evaluating", "cancelled"] + checkpoint_ids: tuple[str, ...] + telemetry_digest: Sha256Digest + + +TrainingProgressCallback = Callable[[TrainingProgress], None] +CancellationCheck = Callable[[], bool] + + +class CancellationToken: + def __init__(self) -> None: + self._event = threading.Event() + + def request(self) -> None: + self._event.set() + + def cancelled(self) -> bool: + return self._event.is_set() + + +class TrainingCancelled(RuntimeError): + """Cooperative cancellation observed at one published checkpoint boundary.""" + + def __init__(self, boundary_step: int, result: TrainingResult | None = None) -> None: + self.boundary_step = boundary_step + self.result = result + super().__init__("training_cancelled") + + +@runtime_checkable +class TrainerBackend(Protocol): + backend_id: str + + def probe(self, request: ProbeTrainingRequest) -> TrainerCapabilityReport: ... + + def calibrate( + self, + request: CalibrationRequest, + progress: TrainingProgressCallback, + cancelled: CancellationCheck, + ) -> CalibrationResult: ... + + def train( + self, + request: TrainingRequest, + progress: TrainingProgressCallback, + cancelled: CancellationCheck, + ) -> TrainingResult: ... + + +class TrainerRegistry: + def __init__(self, *, reviewer_id: str) -> None: + self.reviewer_id = reviewer_id + self._backends: dict[str, TrainerBackend] = {} + + def register(self, backend: TrainerBackend, *, expected_id: str | None = None) -> None: + if expected_id is not None and backend.backend_id != expected_id: + raise ValueError("backend_id_mismatch") + if backend.backend_id in self._backends: + raise ValueError("backend_already_registered") + self._backends[backend.backend_id] = backend + + def resolve(self, reviewer_id: str, backend_id: str) -> TrainerBackend: + if reviewer_id != self.reviewer_id or backend_id not in self._backends: + raise LookupError("backend_not_registered") + backend = self._backends[backend_id] + if backend.backend_id != backend_id: + raise LookupError("backend_id_mismatch") + return backend + + +class TrainerResultInvalid(ValueError): + """A path-free rejection of untrusted backend output.""" + + +def validate_training_result(request: TrainingRequest, result: TrainingResult) -> None: + """Re-hash and structurally validate an untrusted backend result.""" + _validate_training_result(request, result, request.checkpoint_steps) + + +def validate_cancelled_training_result( + request: TrainingRequest, result: TrainingResult, boundary_step: int +) -> None: + """Validate a cooperative result as one exact non-empty checkpoint prefix.""" + if boundary_step not in request.checkpoint_steps: + raise TrainerResultInvalid("trainer_cancellation_boundary_invalid") + prefix = tuple(step for step in request.checkpoint_steps if step <= boundary_step) + if not prefix or prefix[-1] != boundary_step: + raise TrainerResultInvalid("trainer_cancellation_boundary_invalid") + _validate_training_result(request, result, prefix) + + +def _validate_training_result( + request: TrainingRequest, result: TrainingResult, expected_steps: tuple[int, ...] +) -> None: + if ( + result.request_id != request.request_id + or result.run_id != request.run_id + or result.backend_id != request.backend_id + or result.backend_digest != request.materials.backend_digest + or result.materials != request.materials + or result.output_root != request.scratch_root + or result.changed_targets != request.trainable_targets + or not result.frozen_parameters_unchanged + or not result.finite_gradients + or result.elapsed_seconds > request.deadline_seconds + or result.fault is not None + ): + raise TrainerResultInvalid("trainer_result_identity_invalid") + if len(result.checkpoints) != len(expected_steps): + raise TrainerResultInvalid("trainer_checkpoint_schedule_invalid") + try: + root_info = result.output_root.lstat() + except OSError: + raise TrainerResultInvalid("trainer_output_missing") from None + if stat.S_ISLNK(root_info.st_mode) or not stat.S_ISDIR(root_info.st_mode): + raise TrainerResultInvalid("trainer_output_unsafe") + root_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + root_flags |= os.O_NOFOLLOW + try: + root_fd = os.open(result.output_root, root_flags) + except OSError: + raise TrainerResultInvalid("trainer_output_unsafe") from None + try: + parent: str | None = request.parent_checkpoint_id + all_paths: set[str] = set() + for checkpoint, expected_step in zip(result.checkpoints, expected_steps, strict=True): + if ( + checkpoint.step != expected_step + or not math.isclose( + checkpoint.fraction, + expected_step / request.total_steps, + rel_tol=0, + abs_tol=1e-12, + ) + or checkpoint.parent_checkpoint_id != parent + ): + raise TrainerResultInvalid("trainer_checkpoint_lineage_invalid") + roles = {item.role for item in checkpoint.files} + if ( + roles + != { + "weights", + "optimizer", + "trainer_state", + "telemetry", + "trainable_targets", + "frozen_parameters", + "finite_gradient_proof", + } + or len(checkpoint.files) != 7 + ): + raise TrainerResultInvalid("trainer_checkpoint_inventory_invalid") + role_digests: dict[str, str] = {} + role_payloads: dict[str, bytes] = {} + for item in checkpoint.files: + normalized = unicodedata.normalize("NFC", item.relative_path) + canonical = posixpath.normpath(normalized) + if ( + normalized != item.relative_path + or canonical != item.relative_path + or canonical in {"", "."} + or canonical in all_paths + ): + raise TrainerResultInvalid("trainer_output_unsafe") + all_paths.add(canonical) + data = _read_output(root_fd, item.relative_path, item.size_bytes) + digest = "sha256:" + hashlib.sha256(data).hexdigest() + if digest != item.digest: + raise TrainerResultInvalid("trainer_output_digest_invalid") + if item.role == "weights": + _validate_safetensors(data) + elif item.role == "telemetry": + _validate_telemetry(data, request, checkpoint.step) + else: + role_payloads[item.role] = data + role_digests[item.role] = digest + physical_names = _validate_trainable_targets_payload( + role_payloads["trainable_targets"], request, checkpoint.step + ) + for role, data in role_payloads.items(): + if role != "trainable_targets": + _validate_evidence_payload( + data, + request, + checkpoint.step, + role, + physical_names=physical_names, + ) + if ( + checkpoint.telemetry_digest != role_digests["telemetry"] + or checkpoint.frozen_parameters_digest != role_digests["frozen_parameters"] + or checkpoint.trainable_targets_digest != role_digests["trainable_targets"] + or checkpoint.finite_gradient_proof_digest != role_digests["finite_gradient_proof"] + ): + raise TrainerResultInvalid("trainer_checkpoint_evidence_invalid") + parent = checkpoint.checkpoint_id + if result.telemetry_digest != result.checkpoints[-1].telemetry_digest: + raise TrainerResultInvalid("trainer_telemetry_digest_invalid") + final = os.fstat(root_fd) + current = result.output_root.lstat() + if (final.st_dev, final.st_ino) != (root_info.st_dev, root_info.st_ino) or ( + current.st_dev, + current.st_ino, + ) != (root_info.st_dev, root_info.st_ino): + raise TrainerResultInvalid("trainer_output_replaced") + except TrainerResultInvalid: + raise + except (OSError, ValueError, json.JSONDecodeError): + raise TrainerResultInvalid("trainer_output_invalid") from None + finally: + os.close(root_fd) + + +def _read_output(root_fd: int, relative_path: str, expected_size: int) -> bytes: + if relative_path.startswith("/") or ".." in relative_path.split("/"): + raise TrainerResultInvalid("trainer_output_unsafe") + flags = os.O_RDONLY | os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(relative_path, flags, dir_fd=root_fd) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_size != expected_size: + raise TrainerResultInvalid("trainer_output_size_invalid") + data = bytearray() + while len(data) <= expected_size: + chunk = os.read(descriptor, min(1024 * 1024, expected_size + 1 - len(data))) + if not chunk: + break + data.extend(chunk) + if len(data) != expected_size or os.fstat(descriptor).st_size != expected_size: + raise TrainerResultInvalid("trainer_output_size_invalid") + return bytes(data) + finally: + os.close(descriptor) + + +def _validate_safetensors(data: bytes) -> None: + if len(data) < 9: + raise TrainerResultInvalid("trainer_safetensors_invalid") + header_size = int.from_bytes(data[:8], "little") + if header_size <= 0 or header_size > len(data) - 8: + raise TrainerResultInvalid("trainer_safetensors_invalid") + raw_header = data[8 : 8 + header_size] + header = json.loads(raw_header) + if not isinstance(header, dict) or not header: + raise TrainerResultInvalid("trainer_safetensors_invalid") + canonical = json.dumps(header, sort_keys=True, separators=(",", ":")).encode() + if canonical + b" " * (header_size - len(canonical)) != raw_header: + raise TrainerResultInvalid("trainer_safetensors_invalid") + payload_size = len(data) - 8 - header_size + ranges: list[tuple[int, int]] = [] + widths = {"U8": 1, "I8": 1, "F16": 2, "BF16": 2, "F32": 4} + for value in header.values(): + if not isinstance(value, dict) or set(value) != {"dtype", "shape", "data_offsets"}: + raise TrainerResultInvalid("trainer_safetensors_invalid") + offsets = value["data_offsets"] + if ( + value["dtype"] not in {"U8", "I8", "F16", "BF16", "F32"} + or not isinstance(value["shape"], list) + or any(not isinstance(size, int) or size < 0 for size in value["shape"]) + or not isinstance(offsets, list) + or len(offsets) != 2 + or any(not isinstance(offset, int) for offset in offsets) + or offsets[0] < 0 + ): + raise TrainerResultInvalid("trainer_safetensors_invalid") + if offsets[1] > payload_size or offsets[1] < offsets[0]: + raise TrainerResultInvalid("trainer_safetensors_invalid") + expected_bytes = math.prod(value["shape"]) * widths[value["dtype"]] + if offsets[1] - offsets[0] != expected_bytes: + raise TrainerResultInvalid("trainer_safetensors_invalid") + ranges.append((offsets[0], offsets[1])) + if sorted(ranges) != ranges or not ranges or ranges[0][0] != 0 or ranges[-1][1] != payload_size: + raise TrainerResultInvalid("trainer_safetensors_invalid") + if any(left[1] != right[0] for left, right in pairwise(ranges)): + raise TrainerResultInvalid("trainer_safetensors_invalid") + + +def _validate_evidence_payload( + data: bytes, + request: TrainingRequest, + step: int, + role: str, + *, + physical_names: tuple[str, ...] = (), +) -> None: + value = json.loads(data) + shared = {"request_id": request.request_id, "run_id": request.run_id, "step": step} + expected: dict[str, object] + if role == "optimizer": + _validate_optimizer_state(value, request, step, physical_names=physical_names) + expected = value + elif role == "trainer_state": + _validate_trainer_state(value, request, step) + expected = value + elif role == "frozen_parameters": + expected = { + "schema_version": "trainer-frozen-parameters.v1", + **shared, + "trainable_targets": list(request.trainable_targets), + "unchanged": True, + } + elif role == "finite_gradient_proof": + if ( + not isinstance(value, dict) + or set(value) + != { + "schema_version", + "request_id", + "run_id", + "step", + "finite", + "loss", + "gradient_norm", + } + or value.get("schema_version") != "trainer-finite-gradient-proof.v1" + or any(value.get(key) != expected_value for key, expected_value in shared.items()) + or value.get("finite") is not True + or not isinstance(value.get("loss"), (int, float)) + or not isinstance(value.get("gradient_norm"), (int, float)) + or not math.isfinite(float(value["loss"])) + or not math.isfinite(float(value["gradient_norm"])) + or float(value["gradient_norm"]) < 0 + ): + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + expected = value + else: + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + if canonical_json(value).encode() != data or value != expected: + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + + +def _validate_trainable_targets_payload( + data: bytes, request: TrainingRequest, step: int +) -> tuple[str, ...]: + value = json.loads(data) + shared = {"request_id": request.request_id, "run_id": request.run_id, "step": step} + if request.backend_id != "diffusers.wan2.2-animegen-lora.mps.v1": + expected = { + "schema_version": "trainer-trainable-targets.v1", + **shared, + "targets": list(request.trainable_targets), + } + if value != expected or canonical_json(value).encode() != data: + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + return () + if not isinstance(value, dict) or set(value) != { + "schema_version", "request_id", "run_id", "step", "targets", "physical_parameters" + }: + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + try: + specs = validate_trainable_parameter_specs( + trainable_parameter_specs_from_records(value["physical_parameters"]), + declared_targets=request.trainable_targets, + ) + except (KeyError, TypeError, ValueError): + raise TrainerResultInvalid("trainer_evidence_payload_invalid") from None + if ( + value.get("schema_version") != "trainer-trainable-targets.v2" + or any(value.get(key) != expected for key, expected in shared.items()) + or value.get("targets") != list(request.trainable_targets) + or canonical_json(value).encode() != data + ): + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + return tuple(item.physical_name for item in specs) + + +def _substantive_numeric_state(value: object) -> tuple[bool, bool]: + if isinstance(value, bool): + return True, False + if isinstance(value, (int, float)): + finite = math.isfinite(float(value)) + return finite, finite + if isinstance(value, str): + return bool(value), False + if isinstance(value, list): + children = tuple(_substantive_numeric_state(item) for item in value) + return bool(children) and all(valid for valid, _ in children), any( + substantive for _, substantive in children + ) + if isinstance(value, dict): + children = tuple(_substantive_numeric_state(item) for item in value.values()) + return ( + bool(children) + and all(isinstance(key, str) and key for key in value) + and all(valid for valid, _ in children), + any(substantive for _, substantive in children), + ) + return False, False + + +def _is_substantive_numeric_state(value: object) -> bool: + valid, substantive = _substantive_numeric_state(value) + return valid and substantive + + +def _is_bound_backend_state(value: object, backend_id: str) -> bool: + return ( + isinstance(value, dict) + and set(value) == {backend_id} + and _is_substantive_numeric_state(value[backend_id]) + ) + + +def _validate_optimizer_state( + value: object, + request: TrainingRequest, + step: int, + *, + physical_names: tuple[str, ...] = (), +) -> None: + config = {item.name: item.value for item in request.optimizer.parameters} + if not isinstance(value, dict) or set(value) != { + "schema_version", + "request_id", + "run_id", + "step", + "algorithm", + "config", + "global_step", + "parameter_groups", + "parameter_state", + "backend_state", + "parent_optimizer_state_digest", + }: + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + groups = value.get("parameter_groups") + states = value.get("parameter_state") + if ( + value.get("schema_version") != "trainer-optimizer-state.v2" + or value.get("request_id") != request.request_id + or value.get("run_id") != request.run_id + or value.get("step") != step + or value.get("global_step") != step + or value.get("algorithm") != request.optimizer.algorithm + or value.get("config") != config + or value.get("parent_optimizer_state_digest") != request.optimizer_state_digest + or not isinstance(groups, list) + or not groups + or not isinstance(states, dict) + or set(states) != set(request.trainable_targets) + or not _is_optimizer_backend_state( + value.get("backend_state"), request, step, config, physical_names=physical_names + ) + ): + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + covered: list[str] = [] + for group in groups: + if ( + not isinstance(group, dict) + or set(group) != {"group_id", "parameters", "hyperparameters"} + or not isinstance(group.get("group_id"), str) + or not isinstance(group.get("parameters"), list) + or group.get("hyperparameters") != config + ): + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + covered.extend(group["parameters"]) + if tuple(covered) != request.trainable_targets: + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + for state in states.values(): + if ( + not isinstance(state, dict) + or set(state) != {"step", "slots"} + or state.get("step") != step + or not _is_substantive_numeric_state(state.get("slots")) + ): + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + + +def _is_optimizer_backend_state( + value: object, + request: TrainingRequest, + step: int, + config: Mapping[str, object], + *, + physical_names: tuple[str, ...] = (), +) -> bool: + try: + if request.backend_id != "diffusers.wan2.2-animegen-lora.mps.v1": + return _is_bound_backend_state(value, request.backend_id) + if not isinstance(value, dict) or set(value) != {request.backend_id}: + return False + state = value[request.backend_id] + if not isinstance(state, dict): + return False + accumulation = config.get("gradient_accumulation_steps", 1) + if type(accumulation) is not int or accumulation <= 0: + return False + residual = step % accumulation + terminal_partial = step == request.total_steps and residual > 0 + position = 0 if residual == 0 or terminal_partial else residual + expected_updates = step // accumulation + int(terminal_partial) + if accumulation == 1: + return state == {"optimizer_updates": expected_updates} + if set(state) != {"optimizer_updates", "pending_gradients"}: + return False + if state["optimizer_updates"] != expected_updates: + return False + validate_pending_gradients( + state["pending_gradients"], + position, + expected_parameters=physical_names, + ) + return True + except (KeyError, TypeError, ValueError): + return False + + +def _validate_trainer_state(value: object, request: TrainingRequest, step: int) -> None: + common_fields = { + "schema_version", + "request_id", + "run_id", + "step", + "backend_id", + "seed", + "total_steps", + "global_step", + "scheduler_state", + "scaler_state", + "backend_state", + } + if not isinstance(value, dict): + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + schema = value.get("schema_version") + rng_field = "rng_states" if schema == "trainer-state.v3" else "rng_state" + if set(value) != common_fields | {rng_field}: + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + if ( + schema not in {"trainer-state.v2", "trainer-state.v3"} + or value.get("request_id") != request.request_id + or value.get("run_id") != request.run_id + or value.get("step") != step + or value.get("global_step") != step + or value.get("backend_id") != request.backend_id + or value.get("seed") != request.seed + or value.get("total_steps") != request.total_steps + or ( + schema == "trainer-state.v2" + and value.get("rng_state") != {"seed": request.seed, "counter": step} + ) + or (schema == "trainer-state.v3" and not _valid_rng_states(value.get("rng_states"))) + or not _is_substantive_numeric_state(value.get("scheduler_state")) + or not _is_substantive_numeric_state(value.get("scaler_state")) + or not _is_bound_backend_state(value.get("backend_state"), request.backend_id) + ): + raise TrainerResultInvalid("trainer_evidence_payload_invalid") + + +def _valid_rng_states(value: object) -> bool: + if not isinstance(value, dict) or not value: + return False + for key, item in value.items(): + if ( + type(key) is not str + or not key + or not isinstance(item, dict) + or set(item) != {"algorithm", "device", "state_hex", "digest"} + or type(item.get("algorithm")) is not str + or not item.get("algorithm") + or item.get("device") != key + ): + return False + state_hex = item.get("state_hex") + digest = item.get("digest") + if ( + type(state_hex) is not str + or not state_hex + or state_hex != state_hex.lower() + or len(state_hex) % 2 + or type(digest) is not str + ): + return False + try: + state = bytes.fromhex(state_hex) + except ValueError: + return False + if not state or "sha256:" + hashlib.sha256(state).hexdigest() != digest: + return False + return True + + +def _validate_telemetry(data: bytes, request: TrainingRequest, checkpoint_step: int) -> None: + value = json.loads(data) + if canonical_json(value).encode() != data or not isinstance(value, list) or not value: + raise TrainerResultInvalid("trainer_telemetry_payload_invalid") + expected_sequence = request.telemetry_sequence_start + expected_step = request.start_step + 1 + for record in value: + if not isinstance(record, dict) or set(record) != {"sequence", "step", "payload"}: + raise TrainerResultInvalid("trainer_telemetry_payload_invalid") + if record["sequence"] != expected_sequence or record["step"] != expected_step: + raise TrainerResultInvalid("trainer_telemetry_payload_invalid") + try: + progress = TrainingProgress.model_validate(record["payload"]) + except ValueError: + raise TrainerResultInvalid("trainer_telemetry_payload_invalid") from None + if progress.step != expected_step or progress.total_steps != request.total_steps: + raise TrainerResultInvalid("trainer_telemetry_payload_invalid") + expected_sequence += 1 + expected_step += 1 + if expected_step - 1 != checkpoint_step: + raise TrainerResultInvalid("trainer_telemetry_payload_invalid") diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/__init__.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/__init__.py new file mode 100644 index 0000000..7d4feac --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/__init__.py @@ -0,0 +1,106 @@ +"""Dependency-free local trainer backend implementations.""" + +from video_lab.trainer_protocol import TrainingCancelled +from video_lab.trainers.fake import FakeTrainerBackend, FakeTrainerFault +from video_lab.trainers.wan_diffusers_runtime import ( + WanDiffusersAdapterPaths, + WanDiffusersTrainingRuntime, +) +from video_lab.trainers.wan_factory import ( + WanPinnedLocalRoot, + WanTrainingAdapterCapability, + WanTrainingAdapterUnsupported, + create_wan_lora_mps_trainer, + local_wan_training_adapter_capability, +) +from video_lab.trainers.wan_mps import ( + WAN_LORA_MPS_BACKEND_ID, + LoRATargetPolicy, + WanBackendStageUnavailable, + WanCalibrationMeasurement, + WanLoRAMpsTrainer, + WanMpsObservation, + WanParameterObservation, + WanPreparedTraining, + WanSourceDescriptor, + WanSourceFile, + WanTrainingRuntime, + WanTrainingSourceLease, + WanTrainingSourceResolver, + WanTrainingSources, +) +from video_lab.trainers.wan_production_models import ( + WanBranchCanaryResultV1, + WanBranchInventoryResultV1, + WanBranchInventoryRole, + WanBranchRecipeV1, + WanCanaryPhase, + WanCapacityResultV1, + WanCapacityResultV2, + WanComponentInventoryV1, + WanNumericalToleranceV1, + WanProductionCanaryReceiptV1, + WanProductionCanaryReceiptV2, + WanProductionCanaryRequestV1, + WanProductionCanaryRequestV2, + WanProductionCanaryUnavailableV2, + WanTensorEnvelopeV2, + WanTransformerBranch, + wan_branch_recipe_digest, + wan_inventory_digest, + wan_inventory_family_digest, + wan_production_receipt_id, + wan_production_receipt_v2_id, + wan_recipe_family_digest, + wan_required_tensor_bytes, +) + +__all__ = ( + "FakeTrainerBackend", + "FakeTrainerFault", + "TrainingCancelled", + "WanDiffusersAdapterPaths", + "WanDiffusersTrainingRuntime", + "WanPinnedLocalRoot", + "WanTrainingAdapterCapability", + "WanTrainingAdapterUnsupported", + "create_wan_lora_mps_trainer", + "local_wan_training_adapter_capability", + "LoRATargetPolicy", + "WAN_LORA_MPS_BACKEND_ID", + "WanBackendStageUnavailable", + "WanCalibrationMeasurement", + "WanBranchCanaryResultV1", + "WanBranchInventoryResultV1", + "WanBranchInventoryRole", + "WanBranchRecipeV1", + "WanCanaryPhase", + "WanCapacityResultV1", + "WanCapacityResultV2", + "WanComponentInventoryV1", + "WanLoRAMpsTrainer", + "WanMpsObservation", + "WanNumericalToleranceV1", + "WanParameterObservation", + "WanPreparedTraining", + "WanProductionCanaryReceiptV1", + "WanProductionCanaryReceiptV2", + "WanProductionCanaryRequestV1", + "WanProductionCanaryRequestV2", + "WanProductionCanaryUnavailableV2", + "WanTensorEnvelopeV2", + "WanSourceDescriptor", + "WanSourceFile", + "WanTrainingRuntime", + "WanTrainingSourceResolver", + "WanTrainingSourceLease", + "WanTrainingSources", + "WanTransformerBranch", + "wan_branch_recipe_digest", + "wan_inventory_digest", + "wan_inventory_family_digest", + "wan_production_receipt_id", + "wan_production_receipt_v2_id", + "wan_recipe_family_digest", + "wan_required_tensor_bytes", +) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/fake.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/fake.py new file mode 100644 index 0000000..2351974 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/fake.py @@ -0,0 +1,434 @@ +"""Deterministic trainer used to pressure-test orchestration without ML dependencies.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import stat +from enum import StrEnum +from pathlib import Path +from typing import Any, cast + +from video_lab.domain import canonical_json +from video_lab.trainer_protocol import ( + CalibrationRequest, + CalibrationResult, + CancellationCheck, + CheckpointResult, + OutputFile, + ProbeTrainingRequest, + TrainerCapabilityReport, + TrainingCancelled, + TrainingProgress, + TrainingProgressCallback, + TrainingRequest, + TrainingResult, +) +from video_lab.training_models import checkpoint_steps + + +class FakeTrainerFault(StrEnum): + NONFINITE_GRADIENT = "nonfinite_gradient" + FROZEN_MUTATION = "frozen_mutation" + CORRUPT_CHECKPOINT = "corrupt_checkpoint" + MISSING_CHECKPOINT = "missing_checkpoint" + EXTRA_CHECKPOINT = "extra_checkpoint" + WRONG_STEP = "wrong_step" + WRONG_FRACTION = "wrong_fraction" + WRONG_PARENT = "wrong_parent" + WRONG_TARGET = "wrong_target" + DEADLINE = "deadline" + CRASH = "crash" + + +def _sha(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def _identity(payload: object) -> str: + return _sha(canonical_json(cast(Any, payload)).encode()) + + +class FakeTrainerBackend: + def __init__( + self, + *, + backend_id: str = "fake", + fault: FakeTrainerFault | None = None, + ) -> None: + self.backend_id = backend_id + self.fault = fault + + @property + def backend_digest(self) -> str: + return _identity({"backend": self.backend_id, "version": 1}) + + def probe(self, request: ProbeTrainingRequest) -> TrainerCapabilityReport: + self._require_backend(request.backend_id) + return TrainerCapabilityReport( + backend_id=self.backend_id, + backend_digest=request.expected_backend_digest, + runtime_digest=request.materials.runtime_digest, + supported=True, + max_steps=1_000_000, + safe_boundaries=(1, 2, 3, 4), + ) + + def calibrate( + self, + request: CalibrationRequest, + progress: TrainingProgressCallback, + cancelled: CancellationCheck, + ) -> CalibrationResult: + self._require_backend(request.backend_id) + for step in range(1, request.steps + 1): + item = TrainingProgress( + step=step, + total_steps=request.steps, + fraction=step / request.steps, + loss=1 / (step + 1), + checkpoint_boundary=step == request.steps, + ) + progress(item) + if cancelled() and step == request.steps: + raise TrainingCancelled(step) + payload = { + "request": request.model_dump(mode="json", exclude={"scratch_root"}), + "backend": self.backend_digest, + } + return CalibrationResult( + request_id=request.request_id, + run_id=request.run_id, + backend_digest=request.materials.backend_digest, + runtime_digest=request.materials.runtime_digest, + calibration_digest=_identity(payload), + finite_gradients=self.fault != FakeTrainerFault.NONFINITE_GRADIENT, + frozen_parameters_unchanged=self.fault != FakeTrainerFault.FROZEN_MUTATION, + changed_targets=request.trainable_targets, + measurements={"final_loss": 1 / (request.steps + 1), "gradient_norm": 1.0}, + ) + + def train( + self, + request: TrainingRequest, + progress: TrainingProgressCallback, + cancelled: CancellationCheck, + ) -> TrainingResult: + self._require_backend(request.backend_id) + root_fd, root_identity = self._open_scratch(request.scratch_root) + try: + if self.fault == FakeTrainerFault.CRASH: + raise RuntimeError("fake_trainer_crash") + checkpoints: list[CheckpointResult] = [] + parent: str | None = request.parent_checkpoint_id + telemetry: list[dict[str, object]] = [] + for step in range(request.start_step + 1, request.total_steps + 1): + loss = 1 / (step + request.seed + 1) + if self.fault == FakeTrainerFault.NONFINITE_GRADIENT and step == 1: + loss = math.nan + try: + item = TrainingProgress( + step=step, + total_steps=request.total_steps, + fraction=step / request.total_steps, + loss=loss, + checkpoint_boundary=step in request.checkpoint_steps, + ) + except ValueError: + item = TrainingProgress( + step=step, + total_steps=request.total_steps, + fraction=step / request.total_steps, + loss=0.0, + checkpoint_boundary=step in request.checkpoint_steps, + ) + progress(item) + telemetry.append(item.model_dump(mode="json")) + if step in request.checkpoint_steps: + ordinal = checkpoint_steps(request.total_steps).index(step) + checkpoint = self._checkpoint( + request, root_fd, ordinal, step, parent, telemetry + ) + checkpoints.append(checkpoint) + parent = checkpoint.checkpoint_id + if cancelled(): + self._reverify_root(request.scratch_root, root_fd, root_identity) + raise TrainingCancelled( + step, + TrainingResult( + request_id=request.request_id, + run_id=request.run_id, + backend_id=self.backend_id, + backend_digest=request.materials.backend_digest, + materials=request.materials, + output_root=request.scratch_root, + checkpoints=tuple(checkpoints), + telemetry_digest=checkpoint.telemetry_digest, + changed_targets=request.trainable_targets, + frozen_parameters_unchanged=True, + finite_gradients=True, + elapsed_seconds=0, + ), + ) + if self.fault == FakeTrainerFault.MISSING_CHECKPOINT: + checkpoints.pop() + elif self.fault == FakeTrainerFault.EXTRA_CHECKPOINT: + checkpoints.append(checkpoints[-1].model_copy(update={"checkpoint_id": "extra"})) + elif self.fault == FakeTrainerFault.WRONG_STEP: + checkpoints[0] = checkpoints[0].model_copy(update={"step": 1}) + elif self.fault == FakeTrainerFault.WRONG_FRACTION: + checkpoints[0] = checkpoints[0].model_copy(update={"fraction": 0.5}) + elif self.fault == FakeTrainerFault.WRONG_PARENT: + checkpoints[1] = checkpoints[1].model_copy(update={"parent_checkpoint_id": "wrong"}) + self._reverify_root(request.scratch_root, root_fd, root_identity) + telemetry_digest = checkpoints[-1].telemetry_digest + return TrainingResult( + request_id=request.request_id, + run_id=request.run_id, + backend_id=self.backend_id, + backend_digest=request.materials.backend_digest, + materials=request.materials, + output_root=request.scratch_root, + checkpoints=tuple(checkpoints), + telemetry_digest=telemetry_digest, + changed_targets=( + ("undeclared.target",) + if self.fault == FakeTrainerFault.WRONG_TARGET + else request.trainable_targets + ), + frozen_parameters_unchanged=self.fault != FakeTrainerFault.FROZEN_MUTATION, + finite_gradients=self.fault != FakeTrainerFault.NONFINITE_GRADIENT, + elapsed_seconds=( + request.deadline_seconds + 1 + if self.fault == FakeTrainerFault.DEADLINE + else float(request.total_steps) + ), + fault=self.fault.value if self.fault else None, + ) + finally: + os.close(root_fd) + + def _checkpoint( + self, + request: TrainingRequest, + root_fd: int, + ordinal: int, + step: int, + parent: str | None, + telemetry: list[dict[str, object]], + ) -> CheckpointResult: + checkpoint_id = f"checkpoint-{ordinal + 1}" + prefix = f"checkpoint-{ordinal + 1}" + roles = ( + "weights", + "optimizer", + "trainer_state", + "telemetry", + "trainable_targets", + "frozen_parameters", + "finite_gradient_proof", + ) + files: list[OutputFile] = [] + for role in roles: + name = f"{prefix}-{role}.safetensors" if role == "weights" else f"{prefix}-{role}.json" + payload = self._file_bytes(request, step, role, telemetry) + if ( + self.fault == FakeTrainerFault.CORRUPT_CHECKPOINT + and ordinal == 0 + and role == "weights" + ): + payload = b"corrupt" + self._write_file(root_fd, name, payload) + files.append( + OutputFile( + relative_path=name, + digest=_sha(payload), + size_bytes=len(payload), + role=role, + ) + ) + by_role = {item.role: item.digest for item in files} + return CheckpointResult( + checkpoint_id=checkpoint_id, + step=step, + fraction=step / request.total_steps, + parent_checkpoint_id=parent, + files=tuple(files), + telemetry_digest=by_role["telemetry"], + frozen_parameters_digest=by_role["frozen_parameters"], + trainable_targets_digest=by_role["trainable_targets"], + finite_gradient_proof_digest=by_role["finite_gradient_proof"], + ) + + def _file_bytes( + self, request: TrainingRequest, step: int, role: str, telemetry: list[dict[str, object]] + ) -> bytes: + if role == "telemetry": + payload = canonical_json( + cast( + Any, + [ + {"sequence": index, "step": item["step"], "payload": item} + for index, item in enumerate( + telemetry, start=request.telemetry_sequence_start + ) + ], + ) + ).encode() + elif role == "optimizer": + config = {item.name: item.value for item in request.optimizer.parameters} + payload = canonical_json( + { + "schema_version": "trainer-optimizer-state.v2", + "request_id": request.request_id, + "run_id": request.run_id, + "step": step, + "algorithm": request.optimizer.algorithm, + "config": config, + "global_step": step, + "parameter_groups": [ + { + "group_id": "default", + "parameters": list(request.trainable_targets), + "hyperparameters": config, + } + ], + "parameter_state": { + target: { + "step": step, + "slots": {"primary": step / 10, "secondary": step / 100}, + } + for target in request.trainable_targets + }, + "backend_state": {self.backend_id: {"restored_updates": step}}, + "parent_optimizer_state_digest": request.optimizer_state_digest, + } + ).encode() + elif role == "trainer_state": + payload = canonical_json( + { + "schema_version": "trainer-state.v2", + "request_id": request.request_id, + "run_id": request.run_id, + "step": step, + "backend_id": request.backend_id, + "seed": request.seed, + "total_steps": request.total_steps, + "global_step": step, + "rng_state": {"seed": request.seed, "counter": step}, + "scheduler_state": {"step": step, "rate_scale": 1.0}, + "scaler_state": {"enabled": False, "scale": 1.0}, + "backend_state": {self.backend_id: {"checkpoint_step": step}}, + } + ).encode() + elif role == "trainable_targets": + payload = canonical_json( + { + "schema_version": "trainer-trainable-targets.v1", + "request_id": request.request_id, + "run_id": request.run_id, + "step": step, + "targets": list(request.trainable_targets), + } + ).encode() + elif role == "frozen_parameters": + payload = canonical_json( + { + "schema_version": "trainer-frozen-parameters.v1", + "request_id": request.request_id, + "run_id": request.run_id, + "step": step, + "trainable_targets": list(request.trainable_targets), + "unchanged": True, + } + ).encode() + elif role == "finite_gradient_proof": + latest = telemetry[-1] + payload = canonical_json( + { + "schema_version": "trainer-finite-gradient-proof.v1", + "request_id": request.request_id, + "run_id": request.run_id, + "step": step, + "finite": True, + "loss": latest["loss"], + "gradient_norm": latest["gradient_norm"], + } + ).encode() + elif role == "weights": + payload = canonical_json( + { + "request_id": request.request_id, + "run_id": request.run_id, + "step": step, + "trainable_targets": list(request.trainable_targets), + } + ).encode() + else: + payload = b"" + if role != "weights": + return payload + header = json.dumps( + {"weight": {"dtype": "U8", "shape": [len(payload)], "data_offsets": [0, len(payload)]}}, + sort_keys=True, + separators=(",", ":"), + ).encode() + padding = (-len(header)) % 8 + header += b" " * padding + return len(header).to_bytes(8, "little") + header + payload + + @staticmethod + def _write_file(root_fd: int, name: str, payload: bytes) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(name, flags, 0o600, dir_fd=root_fd) + try: + with os.fdopen(descriptor, "wb", closefd=False) as stream: + stream.write(payload) + stream.flush() + os.fsync(descriptor) + finally: + os.close(descriptor) + + @staticmethod + def _open_scratch(root: Path) -> tuple[int, tuple[int, int]]: + try: + if root.exists() or root.is_symlink(): + info = root.lstat() + if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode): + raise ValueError("unsafe_scratch_root") + else: + root.mkdir(mode=0o700, parents=True) + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(root, flags) + info = os.fstat(descriptor) + return descriptor, (info.st_dev, info.st_ino) + except OSError: + raise ValueError("unsafe_scratch_root") from None + + @staticmethod + def _reverify_root(root: Path, descriptor: int, identity: tuple[int, int]) -> None: + try: + held = os.fstat(descriptor) + current = root.lstat() + except OSError: + raise ValueError("unsafe_scratch_root") from None + if ( + stat.S_ISLNK(current.st_mode) + or (held.st_dev, held.st_ino) != identity + or ( + current.st_dev, + current.st_ino, + ) + != identity + ): + raise ValueError("unsafe_scratch_root") + + def _require_backend(self, requested: str) -> None: + if requested != self.backend_id: + raise ValueError("backend_id_mismatch") diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_batch.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_batch.py new file mode 100644 index 0000000..cd7ae35 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_batch.py @@ -0,0 +1,314 @@ +"""Shared bounded authority for canonical Wan precomputed batch artifacts.""" + +from __future__ import annotations + +import hashlib +import importlib +import json +import math +import struct +from collections.abc import Mapping +from typing import Any, cast + +from video_lab.domain import canonical_json + +WAN_BATCH_SCHEMA = "wan-precomputed-training-batch.v1" +WAN_BATCH_TENSORS = frozenset( + { + "hidden_states", + "timestep", + "encoder_hidden_states", + "clean_latents", + "noise_latents", + "target", + "transformer_selector", + } +) +_DTYPE_BYTES = { + "BOOL": 1, + "U8": 1, + "I8": 1, + "I16": 2, + "U16": 2, + "F16": 2, + "BF16": 2, + "I32": 4, + "U32": 4, + "F32": 4, + "I64": 8, + "U64": 8, + "F64": 8, +} +_MAX_HEADER_BYTES = 16 * 1024 * 1024 +_MAX_RANK = 32 +_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024 * 1024 + + +def validated_safetensors_header( + payload: bytes, *, expected_names: set[str] | frozenset[str], batch_size: int +) -> dict[str, dict[str, object]]: + """Parse a bounded, gapless safetensors envelope without allocating tensors.""" + try: + if len(payload) < 9 or len(payload) > _MAX_PAYLOAD_BYTES: + raise ValueError + header_size = struct.unpack(" _MAX_HEADER_BYTES or 8 + header_size > len(payload): + raise ValueError + return validated_safetensors_header_prefix( + payload[: 8 + header_size], + total_size=len(payload), + expected_names=expected_names, + batch_size=batch_size, + ) + except (ValueError, struct.error): + raise ValueError("wan_training_batch_invalid") from None + + +def validated_safetensors_header_prefix( + prefix: bytes, + *, + total_size: int, + expected_names: set[str] | frozenset[str], + batch_size: int, +) -> dict[str, dict[str, object]]: + """Validate an exact bounded header prefix against the declared full artifact size.""" + + def unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError + result[key] = value + return result + + try: + if ( + len(prefix) < 9 + or total_size > _MAX_PAYLOAD_BYTES + or type(batch_size) is not int + or batch_size <= 0 + ): + raise ValueError + header_size = struct.unpack(" _MAX_HEADER_BYTES or len(prefix) != 8 + header_size: + raise ValueError + parsed = json.loads( + prefix[8:].decode().rstrip(" "), object_pairs_hook=unique_object + ) + if not isinstance(parsed, dict): + raise ValueError + metadata = parsed.pop("__metadata__", None) + if metadata is not None and ( + not isinstance(metadata, dict) + or any( + type(key) is not str or type(value) is not str for key, value in metadata.items() + ) + ): + raise ValueError + if set(parsed) != set(expected_names) or len(parsed) > 100_000: + raise ValueError + data_size = total_size - 8 - header_size + spans: list[tuple[int, int]] = [] + for item in parsed.values(): + if not isinstance(item, dict) or set(item) != {"dtype", "shape", "data_offsets"}: + raise ValueError + dtype, shape, offsets = item["dtype"], item["shape"], item["data_offsets"] + if ( + type(dtype) is not str + or dtype not in _DTYPE_BYTES + or not isinstance(shape, list) + or not shape + or len(shape) > _MAX_RANK + or any(type(dim) is not int or dim <= 0 or dim > 2**31 for dim in shape) + or shape[0] != batch_size + or not isinstance(offsets, list) + or len(offsets) != 2 + or any(type(offset) is not int or offset < 0 for offset in offsets) + ): + raise ValueError + count = math.prod(shape) + start, end = offsets + if count > data_size or end - start != count * _DTYPE_BYTES[dtype] or end > data_size: + raise ValueError + spans.append((start, end)) + cursor = 0 + for start, end in sorted(spans): + if start != cursor: + raise ValueError + cursor = end + if cursor != data_size: + raise ValueError + return cast(dict[str, dict[str, object]], parsed) + except (UnicodeError, json.JSONDecodeError, KeyError, TypeError, ValueError, struct.error): + raise ValueError("wan_training_batch_invalid") from None + + +def build_wan_batch_manifest( + payload: bytes, + *, + batch_size: int, + timestep_min: int, + timestep_max: int, + timestep_boundary: int, +) -> bytes: + header = validated_safetensors_header( + payload, expected_names=WAN_BATCH_TENSORS, batch_size=batch_size + ) + header_size = struct.unpack(" None: + try: + if set(tensors) != WAN_BATCH_TENSORS or not 0 <= timestep_min < boundary <= timestep_max: + raise ValueError + if any( + getattr(tensor, "ndim", 0) < 1 or int(tensor.shape[0]) != batch_size + for tensor in tensors.values() + ): + raise ValueError + hidden, clean, noise = ( + tensors["hidden_states"], + tensors["clean_latents"], + tensors["noise_latents"], + ) + target, timestep = tensors["target"], tensors["timestep"] + selector, encoder = tensors["transformer_selector"], tensors["encoder_hidden_states"] + sigma = (timestep / float(timestep_max)).to(dtype=hidden.dtype) + while sigma.ndim < hidden.ndim: + sigma = sigma.unsqueeze(-1) + if ( + tuple(hidden.shape) != tuple(clean.shape) + or tuple(hidden.shape) != tuple(noise.shape) + or tuple(hidden.shape) != tuple(target.shape) + or hidden.ndim < 3 + or int(hidden.shape[1]) != in_channels + or encoder.ndim != 3 + or tuple(timestep.shape) != (batch_size,) + or str(timestep.dtype) != "torch.float32" + or tuple(selector.shape) != (batch_size,) + or str(selector.dtype) != "torch.int64" + or not bool((timestep >= timestep_min).all().item()) + or not bool((timestep <= timestep_max).all().item()) + or not bool(((selector == 0) | (selector == 1)).all().item()) + or not bool((selector == (timestep < boundary).to(dtype=selector.dtype)).all().item()) + or not bool((hidden == sigma * noise + (1.0 - sigma) * clean).all().item()) + or not bool((target == noise - clean).all().item()) + or any( + not bool(tensor.isfinite().all().item()) + for name, tensor in tensors.items() + if name != "transformer_selector" + ) + ): + raise ValueError + except (AttributeError, KeyError, TypeError, ValueError): + raise ValueError("wan_training_batch_invalid") from None + + +def validate_wan_batch_bytes( + manifest_bytes: bytes, payload: bytes, *, torch_module: Any | None = None +) -> dict[str, Any]: + """Independently validate exact serialized bytes, then decode and verify arithmetic.""" + try: + parsed_manifest = json.loads(manifest_bytes) + if canonical_json(parsed_manifest).encode() != manifest_bytes: + raise ValueError + manifest = validate_wan_batch_envelope(manifest_bytes, payload) + batch_size = cast(int, manifest["batch_size"]) + torch_module = importlib.import_module("torch") if torch_module is None else torch_module + safetensors = importlib.import_module("safetensors.torch") + tensors = cast(dict[str, Any], safetensors.load(payload)) + schedule = cast(dict[str, int], manifest["noise_schedule"]) + validate_loaded_wan_batch( + tensors, + batch_size=batch_size, + timestep_min=schedule["timestep_min"], + timestep_max=schedule["timestep_max"], + boundary=schedule["timestep_boundary"], + in_channels=int(tensors["clean_latents"].shape[1]), + ) + return tensors + except (ImportError, KeyError, TypeError, ValueError, json.JSONDecodeError): + raise ValueError("wan_training_batch_invalid") from None + + +def validate_wan_batch_envelope(manifest_bytes: bytes, payload: bytes) -> dict[str, object]: + """Validate canonical manifest and tensor byte inventory without ML imports.""" + try: + manifest = json.loads(manifest_bytes) + if not isinstance(manifest, dict) or set(manifest) != { + "schema_version", + "sha256", + "batch_size", + "tensors", + "objective", + "noise_schedule", + }: + raise ValueError + batch_size = manifest["batch_size"] + schedule = manifest["noise_schedule"] + if ( + type(batch_size) is not int + or manifest["schema_version"] != WAN_BATCH_SCHEMA + or not isinstance(schedule, dict) + or set(schedule) + != {"timestep_min", "timestep_max", "timestep_boundary", "selector_rule"} + ): + raise ValueError + rebuilt = build_wan_batch_manifest( + payload, + batch_size=batch_size, + timestep_min=schedule["timestep_min"], + timestep_max=schedule["timestep_max"], + timestep_boundary=schedule["timestep_boundary"], + ) + if json.loads(rebuilt) != manifest: + raise ValueError + return cast(dict[str, object], manifest) + except (KeyError, TypeError, ValueError, json.JSONDecodeError): + raise ValueError("wan_training_batch_invalid") from None diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_branch_loader.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_branch_loader.py new file mode 100644 index 0000000..9f6d88c --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_branch_loader.py @@ -0,0 +1,623 @@ +"""Inventory-first, local-only loading for one Wan transformer branch.""" + +from __future__ import annotations + +import hashlib +import importlib +import os +import threading +import uuid +from collections.abc import Callable +from contextlib import AbstractContextManager +from dataclasses import dataclass, field, replace +from pathlib import Path, PureWindowsPath +from typing import Any, cast + +from video_lab.trainers.wan_factory import ( + DEFAULT_WAN_SNAPSHOT_COPY_LIMITS, + WanPinnedLocalRoot, + WanRetainedRootFile, + WanRetainedRootLease, + WanSnapshotCopyLimits, + remove_wan_snapshot_at, + require_component_on_mps, + retain_pinned_root, + secure_snapshot_pinned_root_at, +) +from video_lab.trainers.wan_mps import WanPreparedTraining +from video_lab.trainers.wan_production_models import ( + WanBranchInventoryResultV1, + WanBranchInventoryRole, + WanBranchRecipeV1, + WanComponentInventoryV1, + WanComponentManifest, + WanComponentName, + WanTransformerBranch, +) + +_BRANCH_LOAD_LOCK = threading.Lock() +@dataclass(frozen=True, slots=True) +class WanBranchLoadRequest: + """Exact local authority and relative component locations for one branch.""" + + model_root: WanPinnedLocalRoot + snapshot_parent: Path + branch: WanTransformerBranch + scheduler: Path + text_encoder: Path + tokenizer: Path + transformer_config: Path + high_transformer: Path + low_transformer: Path + vae: Path + _authority_signature: tuple[tuple[str, tuple[int, ...]], ...] = field( + init=False, repr=False, compare=False + ) + _snapshot_parent_identity: tuple[int, int] = field( + init=False, repr=False, compare=False + ) + + def __post_init__(self) -> None: + if type(self.model_root) is not WanPinnedLocalRoot: + raise ValueError("wan_branch_request_invalid") + if self.branch not in ("high_noise", "low_noise"): + raise ValueError("wan_branch_request_invalid") + parent = Path(self.snapshot_parent) + if not parent.is_absolute() or not parent.is_dir() or parent.is_symlink(): + raise ValueError("wan_branch_request_invalid") + object.__setattr__(self, "snapshot_parent", parent) + parent_descriptor = _open_directory_descriptor( + parent, error="wan_branch_request_invalid" + ) + try: + parent_info = os.fstat(parent_descriptor) + object.__setattr__( + self, + "_snapshot_parent_identity", + (parent_info.st_dev, parent_info.st_ino), + ) + finally: + os.close(parent_descriptor) + for name in ( + "scheduler", + "text_encoder", + "tokenizer", + "transformer_config", + "high_transformer", + "low_transformer", + "vae", + ): + value = _safe_relative_path(getattr(self, name)) + object.__setattr__(self, name, value) + try: + with retain_pinned_root(self.model_root) as lease: + object.__setattr__(self, "_authority_signature", _lease_signature(lease)) + except ValueError: + raise ValueError("wan_factory_snapshot_invalid") from None + + +class WanLoadedBranch: + """Exclusive owner of one transformer and its retained local authorities.""" + + def __init__( + self, + *, + branch: WanTransformerBranch, + inventory: WanComponentInventoryV1, + source_lease: WanRetainedRootLease, + snapshot: WanPinnedLocalRoot, + snapshot_lease: WanRetainedRootLease, + transformer: object, + torch: Any, + source_scope: AbstractContextManager[WanRetainedRootLease], + snapshot_scope: AbstractContextManager[WanRetainedRootLease], + snapshot_parent_descriptor: int, + snapshot_root_descriptor: int, + snapshot_root_identity: tuple[int, int], + ) -> None: + self.branch = branch + self.inventory = inventory + self.source_lease: WanRetainedRootLease | None = source_lease + self.snapshot = snapshot + self.snapshot_lease: WanRetainedRootLease | None = snapshot_lease + self.transformer: object | None = transformer + self._torch = torch + self._source_scope: AbstractContextManager[WanRetainedRootLease] | None = source_scope + self._snapshot_scope: AbstractContextManager[WanRetainedRootLease] | None = snapshot_scope + self._snapshot_parent_descriptor: int | None = snapshot_parent_descriptor + self._snapshot_root_descriptor: int | None = snapshot_root_descriptor + self._snapshot_root_identity = snapshot_root_identity + self._closed = False + + def close(self) -> None: + """Drop all owned state and exhaust every cleanup boundary exactly once.""" + + if self._closed: + return + self._closed = True + failures: list[BaseException] = [] + self.transformer = None + torch, self._torch = self._torch, None + try: + torch.mps.synchronize() + except BaseException: + failures.append(RuntimeError("wan_branch_mps_synchronize_failed")) + try: + torch.mps.empty_cache() + except BaseException: + failures.append(RuntimeError("wan_branch_mps_cache_cleanup_failed")) + parent_descriptor = self._snapshot_parent_descriptor + root_descriptor = self._snapshot_root_descriptor + self._snapshot_parent_descriptor = None + self._snapshot_root_descriptor = None + if parent_descriptor is not None and root_descriptor is not None: + try: + _remove_snapshot_at( + parent_descriptor=parent_descriptor, + root_descriptor=root_descriptor, + root_identity=self._snapshot_root_identity, + ) + except BaseException: + failures.append(RuntimeError("wan_branch_snapshot_cleanup_failed")) + self.snapshot_lease = None + snapshot_scope, self._snapshot_scope = self._snapshot_scope, None + if snapshot_scope is not None: + try: + snapshot_scope.__exit__(None, None, None) + except BaseException: + failures.append(RuntimeError("wan_branch_snapshot_lease_close_failed")) + self.source_lease = None + source_scope, self._source_scope = self._source_scope, None + if source_scope is not None: + try: + source_scope.__exit__(None, None, None) + except BaseException: + failures.append(RuntimeError("wan_branch_source_lease_close_failed")) + for descriptor, label in ( + (root_descriptor, "wan_branch_snapshot_descriptor_close_failed"), + (parent_descriptor, "wan_branch_snapshot_parent_close_failed"), + ): + if descriptor is not None: + try: + os.close(descriptor) + except OSError: + failures.append(RuntimeError(label)) + try: + _BRANCH_LOAD_LOCK.release() + except BaseException: + failures.append(RuntimeError("wan_branch_load_lease_release_failed")) + if failures: + raise BaseExceptionGroup("wan_branch_cleanup_failed", failures) + + +@dataclass(frozen=True, slots=True) +class WanMpsPreparedFactory: + """Prepare the branch selected by an immutable production recipe.""" + + request: WanBranchLoadRequest + importer: Callable[[str], object] = importlib.import_module + limits: WanSnapshotCopyLimits = DEFAULT_WAN_SNAPSHOT_COPY_LIMITS + + def prepare_branch( + self, + sources: object, + request: object, + branch_recipe: WanBranchRecipeV1, + *, + cancelled: Callable[[], bool] = lambda: False, + ) -> WanPreparedTraining: + from video_lab.trainers.wan_branch_runtime import WanMpsBranchOperations + from video_lab.trainers.wan_mps import WanTrainingSourceLease, WanTrainingSources + + selected = replace(self.request, branch=branch_recipe.branch) + object.__setattr__( + selected, "_authority_signature", self.request._authority_signature + ) + object.__setattr__( + selected, + "_snapshot_parent_identity", + self.request._snapshot_parent_identity, + ) + loaded = load_wan_branch(selected, limits=self.limits, importer=self.importer) + # Preserve the Task 2 loader-only boundary for callers that have not + # supplied the versioned training-source authority yet. + if not isinstance(sources, (WanTrainingSourceLease, WanTrainingSources)): + return WanPreparedTraining(handle=loaded) + try: + if type(sources) is WanTrainingSourceLease: + operations = WanMpsBranchOperations.from_source_lease( + loaded_branch=loaded, + recipe=branch_recipe, + source_lease=sources, + importer=self.importer, + cancelled=cancelled, + ) + else: + assert isinstance(sources, WanTrainingSources) + operations = WanMpsBranchOperations.from_sources( + loaded_branch=loaded, + recipe=branch_recipe, + sources=sources, + importer=self.importer, + cancelled=cancelled, + ) + except BaseException: + loaded.close() + raise + return WanPreparedTraining( + handle=operations + ) + + def __call__( + self, + sources: object, + request: object, + branch_recipe: WanBranchRecipeV1, + *, + cancelled: Callable[[], bool] = lambda: False, + ) -> WanPreparedTraining: + return self.prepare_branch( + sources, request, branch_recipe, cancelled=cancelled + ) + + +def inventory_wan_branch( + request: WanBranchLoadRequest, + *, + limits: WanSnapshotCopyLimits = DEFAULT_WAN_SNAPSHOT_COPY_LIMITS, +) -> WanBranchInventoryResultV1: + """Hash a bounded retained root without importing any ML dependency.""" + + with retain_pinned_root(request.model_root, limits=limits) as lease: + _require_request_authority(lease, request) + lease.reverify() + return _inventory_from_retained_files(lease, request) + + +def load_wan_branch( + request: WanBranchLoadRequest, + *, + limits: WanSnapshotCopyLimits = DEFAULT_WAN_SNAPSHOT_COPY_LIMITS, + importer: Callable[[str], object] = importlib.import_module, +) -> WanLoadedBranch: + """Load exactly one local transformer after a successful inventory preflight.""" + + if not _BRANCH_LOAD_LOCK.acquire(blocking=False): + raise ValueError("wan_branch_concurrent_load") + source_scope: AbstractContextManager[WanRetainedRootLease] | None = None + snapshot_scope: AbstractContextManager[WanRetainedRootLease] | None = None + snapshot: WanPinnedLocalRoot | None = None + snapshot_parent_descriptor: int | None = None + snapshot_root_descriptor: int | None = None + snapshot_root_identity: tuple[int, int] | None = None + snapshot_lease: WanRetainedRootLease | None = None + torch: Any = None + transformer: object | None = None + try: + snapshot_parent_descriptor = _open_snapshot_parent(request) + source_scope = retain_pinned_root(request.model_root, limits=limits) + source_lease = source_scope.__enter__() + _require_request_authority(source_lease, request) + inventory_result = _inventory_from_retained_files(source_lease, request) + source_lease.reverify() + if not inventory_result.available: + raise ValueError("wan_branch_inventory_unavailable") + inventory = inventory_result.inventory + assert inventory is not None + destination = request.snapshot_parent / f"wan-branch-{uuid.uuid4().hex}" + snapshot = secure_snapshot_pinned_root_at( + request.model_root, + destination, + destination_parent_descriptor=snapshot_parent_descriptor, + limits=limits, + lease=source_lease, + ) + source_lease.reverify() + snapshot_root_descriptor = os.open( + destination.name, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=snapshot_parent_descriptor, + ) + snapshot_info = os.fstat(snapshot_root_descriptor) + snapshot_root_identity = (snapshot_info.st_dev, snapshot_info.st_ino) + snapshot_scope = retain_pinned_root(snapshot, limits=limits) + snapshot_lease = snapshot_scope.__enter__() + retained_root_info = os.fstat(snapshot_lease.directory_descriptors[0]) + if (retained_root_info.st_dev, retained_root_info.st_ino) != snapshot_root_identity: + raise ValueError("wan_factory_snapshot_invalid") + snapshot_lease.reverify() + + torch = cast(Any, importer("torch")) + diffusers = cast(Any, importer("diffusers")) + safetensors = cast(Any, importer("safetensors.torch")) + selected = ( + request.high_transformer + if request.branch == "high_noise" + else request.low_transformer + ) + try: + checkpoint = safetensors.load_file( + str(_retained_file_path(snapshot_lease, selected)) + ) + if type(checkpoint) is not dict: + raise ValueError + transformer = diffusers.WanTransformer3DModel.from_single_file( + checkpoint, + config=str( + _retained_file_path(snapshot_lease, request.transformer_config) + ), + local_files_only=True, + torch_dtype=torch.bfloat16, + ) + except BaseException: + raise RuntimeError("wan_branch_load_failed") from None + cast(Any, transformer).to(device="mps", dtype=torch.bfloat16) + require_component_on_mps(transformer) + source_lease.reverify() + snapshot_lease.reverify() + return WanLoadedBranch( + branch=request.branch, + inventory=inventory, + source_lease=source_lease, + snapshot=snapshot, + snapshot_lease=snapshot_lease, + transformer=transformer, + torch=torch, + source_scope=source_scope, + snapshot_scope=snapshot_scope, + snapshot_parent_descriptor=snapshot_parent_descriptor, + snapshot_root_descriptor=snapshot_root_descriptor, + snapshot_root_identity=snapshot_root_identity, + ) + except BaseException as primary: + failures = _cleanup_failed_load( + torch=torch, + transformer=transformer, + source_scope=source_scope, + snapshot_scope=snapshot_scope, + snapshot=snapshot, + snapshot_parent_descriptor=snapshot_parent_descriptor, + snapshot_root_descriptor=snapshot_root_descriptor, + snapshot_root_identity=snapshot_root_identity, + ) + if failures: + raise BaseExceptionGroup( + "wan_branch_load_and_cleanup_failed", (primary, *failures) + ) from None + raise + + +def _inventory_from_retained_files( + lease: WanRetainedRootLease, request: WanBranchLoadRequest +) -> WanBranchInventoryResultV1: + selected = ( + request.high_transformer + if request.branch == "high_noise" + else request.low_transformer + ) + selected_role: WanBranchInventoryRole = ( + "high_transformer" if request.branch == "high_noise" else "low_transformer" + ) + root_locations: dict[WanBranchInventoryRole, Path] = { + "scheduler": request.scheduler, + "text_encoder": request.text_encoder, + "tokenizer": request.tokenizer, + "vae": request.vae, + } + files = {item.relative: item for item in lease.files} + role_files = { + role: { + relative + for relative in files + if relative == location or relative.is_relative_to(location) + } + for role, location in root_locations.items() + } + role_files["transformer_config"] = ( + {request.transformer_config} if request.transformer_config in files else set() + ) + role_files[selected_role] = {selected} if selected in files else set() + missing_roles = tuple(sorted(role for role, matches in role_files.items() if not matches)) + if missing_roles: + return WanBranchInventoryResultV1( + available=False, + inventory=None, + missing_roles=missing_roles, + ) + + component_files: dict[WanComponentName, set[Path]] = { + "scheduler": role_files["scheduler"], + "text_encoder": role_files["text_encoder"], + "tokenizer": role_files["tokenizer"], + "transformer": role_files["transformer_config"] | role_files[selected_role], + "vae": role_files["vae"], + } + manifests: list[WanComponentManifest] = [] + for component in ("scheduler", "text_encoder", "tokenizer", "transformer", "vae"): + records = [ + relative.as_posix().encode() + b"\0" + files[relative].digest + for relative in sorted(component_files[component]) + ] + manifests.append( + (component, "sha256:" + hashlib.sha256(b"\n".join(records)).hexdigest()) + ) + return WanBranchInventoryResultV1( + available=True, + inventory=WanComponentInventoryV1( + schema_version="wan-mps-component-inventory.v1", + component_manifests=tuple(manifests), + ), + missing_roles=(), + ) + + +def _safe_relative_path(value: object) -> Path: + try: + candidate = Path(cast(Path, value)) + windows = PureWindowsPath(str(value)) + if ( + candidate.is_absolute() + or windows.anchor + or windows.drive + or not candidate.parts + or "\\" in str(value) + or any(part in {"", ".", ".."} for part in candidate.parts) + or any(PureWindowsPath(part).is_reserved() for part in candidate.parts) + ): + raise ValueError + return candidate + except (OSError, TypeError, ValueError): + raise ValueError("wan_branch_request_invalid") from None + + +def _lease_signature(lease: WanRetainedRootLease) -> tuple[tuple[str, tuple[int, ...]], ...]: + records: list[tuple[str, tuple[int, ...]]] = [ + (".", tuple(lease.directory_identities[0])) + ] + records.extend( + (f"d:{relative.as_posix()}", tuple(identity)) + for relative, identity in zip( + lease.directories, lease.directory_identities[1:], strict=True + ) + ) + records.extend( + (f"f:{item.relative.as_posix()}", tuple(item.identity)) for item in lease.files + ) + return tuple(records) + + +def _require_request_authority( + lease: WanRetainedRootLease, request: WanBranchLoadRequest +) -> None: + if _lease_signature(lease) != request._authority_signature: + raise ValueError("wan_factory_snapshot_invalid") + + +def _retained_file_path(lease: WanRetainedRootLease, relative: Path) -> Path: + retained_file: WanRetainedRootFile | None = next( + (item for item in lease.files if item.relative == relative), None + ) + if retained_file is None: + raise ValueError("wan_factory_snapshot_invalid") + descriptor_path = Path("/dev/fd") / str(retained_file.descriptor) + probe_descriptor: int | None = None + try: + probe_descriptor = os.open( + descriptor_path, + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0), + ) + if not os.path.samestat( + os.fstat(retained_file.descriptor), os.fstat(probe_descriptor) + ): + raise ValueError + except (OSError, TypeError, ValueError): + raise ValueError("wan_factory_snapshot_invalid") from None + finally: + if probe_descriptor is not None: + os.close(probe_descriptor) + return descriptor_path + + +def _open_directory_descriptor(path: Path, *, error: str) -> int: + descriptor: int | None = None + try: + descriptor = os.open( + path, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + ) + if not os.path.samestat(os.fstat(descriptor), os.stat(path, follow_symlinks=False)): + raise ValueError + return descriptor + except (OSError, TypeError, ValueError): + if descriptor is not None: + os.close(descriptor) + raise ValueError(error) from None + + +def _open_snapshot_parent(request: WanBranchLoadRequest) -> int: + descriptor = _open_directory_descriptor( + request.snapshot_parent, + error="wan_branch_snapshot_parent_invalid", + ) + info = os.fstat(descriptor) + if (info.st_dev, info.st_ino) != request._snapshot_parent_identity: + os.close(descriptor) + raise ValueError("wan_branch_snapshot_parent_invalid") + return descriptor + + +def _cleanup_failed_load( + *, + torch: Any, + transformer: object | None, + source_scope: AbstractContextManager[WanRetainedRootLease] | None, + snapshot_scope: AbstractContextManager[WanRetainedRootLease] | None, + snapshot: WanPinnedLocalRoot | None, + snapshot_parent_descriptor: int | None, + snapshot_root_descriptor: int | None, + snapshot_root_identity: tuple[int, int] | None, +) -> tuple[BaseException, ...]: + del transformer, snapshot + failures: list[BaseException] = [] + if torch is not None: + try: + torch.mps.synchronize() + except BaseException: + failures.append(RuntimeError("wan_branch_mps_synchronize_failed")) + try: + torch.mps.empty_cache() + except BaseException: + failures.append(RuntimeError("wan_branch_mps_cache_cleanup_failed")) + if ( + snapshot_parent_descriptor is not None + and snapshot_root_descriptor is not None + and snapshot_root_identity is not None + ): + try: + _remove_snapshot_at( + parent_descriptor=snapshot_parent_descriptor, + root_descriptor=snapshot_root_descriptor, + root_identity=snapshot_root_identity, + ) + except BaseException: + failures.append(RuntimeError("wan_branch_snapshot_cleanup_failed")) + for scope, label in ( + (snapshot_scope, "wan_branch_snapshot_lease_close_failed"), + (source_scope, "wan_branch_source_lease_close_failed"), + ): + if scope is not None: + try: + scope.__exit__(None, None, None) + except BaseException: + failures.append(RuntimeError(label)) + for descriptor, label in ( + (snapshot_root_descriptor, "wan_branch_snapshot_descriptor_close_failed"), + (snapshot_parent_descriptor, "wan_branch_snapshot_parent_close_failed"), + ): + if descriptor is not None: + try: + os.close(descriptor) + except OSError: + failures.append(RuntimeError(label)) + try: + _BRANCH_LOAD_LOCK.release() + except BaseException: + failures.append(RuntimeError("wan_branch_load_lease_release_failed")) + return tuple(failures) + + +_remove_snapshot_at = remove_wan_snapshot_at + +__all__ = ( + "WanBranchLoadRequest", + "WanLoadedBranch", + "WanMpsPreparedFactory", + "inventory_wan_branch", + "load_wan_branch", +) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_branch_runtime.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_branch_runtime.py new file mode 100644 index 0000000..994caa7 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_branch_runtime.py @@ -0,0 +1,916 @@ +"""One-branch, one-update Wan attention-LoRA production runtime.""" + +from __future__ import annotations + +import hashlib +import math +import random +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import asdict +from typing import Any, Literal, cast + +from pydantic import ConfigDict, Field, StrictFloat, StrictInt, field_validator + +from video_lab.domain import FrozenModel, canonical_json +from video_lab.trainer_protocol import LoRAConfiguration +from video_lab.trainers.wan_branch_loader import WanLoadedBranch +from video_lab.trainers.wan_checkpoint import ( + WanBranchCheckpointManifestV2, + WanBranchCheckpointStateV2, + WanBranchRestoreReceiptV2, + WanBranchTensorSpecV2, + WanCheckpointState, + validate_wan_branch_checkpoint_v2, +) +from video_lab.trainers.wan_mps import ( + WanParameterObservation, + WanRuntimeCheckpointState, + WanRuntimeRestoreReceipt, + WanTrainingSourceLease, + WanTrainingSources, +) +from video_lab.trainers.wan_production_models import ( + WanBranchRecipeV1, + WanTransformerBranch, + wan_branch_recipe_digest, + wan_inventory_digest, +) + +_ADAPTER_NAME = "character" +_LORA_MARKERS = (".lora_A.", ".lora_B.", ".lora_embedding_A.", ".lora_embedding_B.") +_ADAMW_SLOT_NAMES = ("exp_avg", "exp_avg_sq", "step") + + +def _is_mps_device(value: object) -> bool: + device_type = getattr(value, "type", None) + if device_type is not None: + return type(device_type) is str and device_type == "mps" + if type(value) is not str: + return False + if value == "mps": + return True + prefix, separator, index = value.partition(":") + return prefix == "mps" and separator == ":" and index.isdigit() + + +def _is_bfloat16_dtype(value: object) -> bool: + return str(value).removeprefix("torch.") == "bfloat16" + + +class WanMpsOutOfMemoryError(RuntimeError): + """Normalized boundary for a recognized local MPS allocation failure.""" + + +def is_recognized_mps_oom(error: BaseException) -> bool: + """Return whether an exception is the normalized or native MPS OOM condition.""" + + if isinstance(error, WanMpsOutOfMemoryError): + return True + message = str(error).lower() + return isinstance(error, RuntimeError) and "mps" in message and "out of memory" in message + + +class WanBranchStepEvidence(FrozenModel): + """Strict evidence from exactly one complete branch optimizer update.""" + + model_config = ConfigDict( + frozen=True, extra="forbid", strict=True, revalidate_instances="always" + ) + + branch: WanTransformerBranch + selector_count: StrictInt = Field(gt=0) + loss: StrictFloat + gradient_norm: StrictFloat = Field(ge=0) + changed_adapter_inventory: tuple[WanParameterObservation, ...] + unchanged_base_inventory: tuple[WanParameterObservation, ...] + optimizer_update_count: Literal[1] + memory_observations: tuple[tuple[str, StrictInt], ...] + + @field_validator("loss", "gradient_norm") + @classmethod + def finite_measurements(cls, value: float) -> float: + if not math.isfinite(value): + raise ValueError("wan_branch_numeric_nonfinite") + return value + + @field_validator("memory_observations") + @classmethod + def canonical_memory( + cls, value: tuple[tuple[str, StrictInt], ...] + ) -> tuple[tuple[str, StrictInt], ...]: + names = tuple(name for name, _ in value) + if not value or names != tuple(sorted(set(names))): + raise ValueError("wan_branch_memory_invalid") + return value + + +class WanMpsBranchOperations: + """Train exactly one already-loaded transformer with one attention LoRA update.""" + + def __init__( + self, + *, + loaded_branch: WanLoadedBranch, + recipe: WanBranchRecipeV1, + batch: Mapping[str, Any], + batch_digest: str | None = None, + importer: Callable[[str], object], + cancelled: Callable[[], bool] = lambda: False, + ) -> None: + if loaded_branch.branch != recipe.branch or loaded_branch.transformer is None: + raise ValueError("wan_branch_recipe_invalid") + self.loaded_branch = loaded_branch + self.recipe = recipe + self.transformer = cast(Any, loaded_branch.transformer) + self.batch = dict(batch) + self.batch_digest = batch_digest + self.importer = importer + self.torch = cast(Any, importer("torch")) + self.cancelled = cancelled + self.optimizer: Any = None + self.optimizer_updates = 0 + self._baseline: tuple[WanParameterObservation, ...] | None = None + self._injected: tuple[WanParameterObservation, ...] | None = None + self._last_evidence: WanBranchStepEvidence | None = None + self._restored: WanRuntimeRestoreReceipt | None = None + self._validate_batch() + + @classmethod + def from_sources( + cls, + *, + loaded_branch: WanLoadedBranch, + recipe: WanBranchRecipeV1, + sources: WanTrainingSources, + importer: Callable[[str], object], + cancelled: Callable[[], bool] = lambda: False, + ) -> WanMpsBranchOperations: + """Decode the already-authorized precomputed batch without model-side components.""" + + from video_lab.trainers.wan_diffusers_runtime import validate_wan_batch_artifact + + by_name = {item.name: item.path for item in sources.files} + try: + manifest = by_name["batch_manifest"] + tensors = by_name["batch_tensors"] + except KeyError: + raise ValueError("wan_training_batch_invalid") from None + payload = validate_wan_batch_artifact(manifest, tensors) + torch = cast(Any, importer("torch")) + decoded = cast(Any, importer("safetensors.torch")).load(payload) + batch = { + name: tensor.to( + device="mps", + dtype=( + torch.int64 + if name == "transformer_selector" + else torch.float32 + if name == "timestep" + else torch.bfloat16 + ), + ) + for name, tensor in decoded.items() + } + return cls( + loaded_branch=loaded_branch, + recipe=recipe, + batch=batch, + batch_digest=sources.source_digest, + importer=importer, + cancelled=cancelled, + ) + + @classmethod + def from_source_lease( + cls, + *, + loaded_branch: WanLoadedBranch, + recipe: WanBranchRecipeV1, + source_lease: WanTrainingSourceLease, + importer: Callable[[str], object], + cancelled: Callable[[], bool] = lambda: False, + ) -> WanMpsBranchOperations: + """Decode only bytes read from the retained source descriptors.""" + + from video_lab.trainers.wan_batch import validate_wan_batch_envelope + + if type(source_lease) is not WanTrainingSourceLease: + raise ValueError("wan_training_batch_invalid") + by_name = {item.name: item for item in source_lease.sources.files} + try: + manifest_source = by_name["batch_manifest"] + tensor_source = by_name["batch_tensors"] + except KeyError: + raise ValueError("wan_training_batch_invalid") from None + try: + manifest_payload = source_lease.read_file_bytes(manifest_source) + tensor_payload = source_lease.read_file_bytes(tensor_source) + validate_wan_batch_envelope(manifest_payload, tensor_payload) + decoded = cast(Any, importer("safetensors.torch")).load(tensor_payload) + source_lease.reverify() + except ValueError as error: + if str(error) == "wan_source_invalid": + raise + raise ValueError("wan_training_batch_invalid") from None + torch = cast(Any, importer("torch")) + batch = { + name: tensor.to( + device="mps", + dtype=( + torch.int64 + if name == "transformer_selector" + else torch.float32 + if name == "timestep" + else torch.bfloat16 + ), + ) + for name, tensor in decoded.items() + } + source_lease.reverify() + return cls( + loaded_branch=loaded_branch, + recipe=recipe, + batch=batch, + batch_digest=tensor_source.digest, + importer=importer, + cancelled=cancelled, + ) + + @staticmethod + def selector_for_branch(branch: object) -> int: + if branch == "high_noise": + return 0 + if branch == "low_noise": + return 1 + raise ValueError("wan_branch_recipe_invalid") + + def eligible_lora_modules(self) -> tuple[str, ...]: + found: list[str] = [] + suffix_counts = {suffix: 0 for suffix in self.recipe.target_suffixes} + for name, module in self.transformer.named_modules(): + matches = tuple( + suffix for suffix in self.recipe.target_suffixes if _matches_suffix(name, suffix) + ) + if len(matches) > 1: + raise ValueError("wan_branch_targets_invalid") + if matches and hasattr(module, "weight"): + suffix_counts[matches[0]] += 1 + found.append(f"transformer.{name}") + if not found or any(count == 0 for count in suffix_counts.values()): + raise ValueError("wan_branch_targets_invalid") + result = tuple(sorted(found)) + if len(result) != len(set(result)): + raise ValueError("wan_branch_targets_invalid") + return result + + def inject_lora( + self, + *, + targets: tuple[str, ...], + configuration: LoRAConfiguration, + seed: int, + device: str, + dtype: str, + ) -> None: + with self._rng_fence(seed): + self._inject_lora( + targets=targets, + configuration=configuration, + seed=seed, + device=device, + dtype=dtype, + ) + + def _inject_lora( + self, + *, + targets: tuple[str, ...], + configuration: LoRAConfiguration, + seed: int, + device: str, + dtype: str, + ) -> None: + if self.optimizer is not None: + raise ValueError("wan_branch_adapter_already_injected") + if ( + configuration.rank != 2 + or configuration.alpha != self.recipe.alpha + or configuration.dropout != 0.0 + or device != "mps" + or dtype != "bfloat16" + or targets != self.eligible_lora_modules() + ): + raise ValueError("wan_branch_recipe_invalid") + for parameter in self.transformer.parameters(): + parameter.requires_grad_(False) + self._baseline = self.parameter_inventory() + local_targets = [target.removeprefix("transformer.") for target in targets] + peft = cast(Any, self.importer("peft")) + with self._rng_fence(seed): + config = peft.LoraConfig( + r=2, + lora_alpha=self.recipe.alpha, + lora_dropout=0.0, + target_modules=local_targets, + ) + self.transformer.add_adapter(config, adapter_name=_ADAPTER_NAME) + trainable = self._named_trainable() + if not trainable or any(_logical_target(name) not in targets for name in trainable): + raise ValueError("wan_branch_trainable_inventory_invalid") + if set(_logical_target(name) for name in trainable) != set(targets): + raise ValueError("wan_branch_trainable_inventory_invalid") + declared = {id(parameter) for parameter in trainable.values()} + all_trainable = { + id(parameter) for parameter in self.transformer.parameters() if parameter.requires_grad + } + if declared != all_trainable: + raise ValueError("wan_branch_trainable_inventory_invalid") + optimizer_config = _adamw_config(self.recipe.optimizer_parameters) + self.optimizer = self.torch.optim.AdamW(tuple(trainable.values()), **optimizer_config) + owned = { + id(parameter) for group in self.optimizer.param_groups for parameter in group["params"] + } + if owned != declared: + raise ValueError("wan_branch_optimizer_invalid") + self._injected = self.parameter_inventory() + + def parameter_inventory(self) -> tuple[WanParameterObservation, ...]: + named = tuple(sorted(self.transformer.named_parameters())) + parameter_names = frozenset(name for name, _parameter in named) + values = [] + for name, parameter in named: + device = getattr(parameter, "device", None) + dtype = getattr(parameter, "dtype", None) + if not _is_mps_device(device) or not _is_bfloat16_dtype(dtype): + raise ValueError("wan_runtime_placement_invalid") + device_type = getattr(cast(Any, device), "type", None) + device_name = cast( + Literal["mps"], + device_type + if type(device_type) is str + else str(device).partition(":")[0], + ) + dtype_name = cast(Literal["bfloat16"], str(dtype).removeprefix("torch.")) + logical_name = f"transformer.{_inventory_parameter_name(name, parameter_names)}" + values.append( + WanParameterObservation( + name=logical_name, + module=_logical_target(logical_name), + requires_grad=bool(parameter.requires_grad), + digest=self._parameter_digest(parameter), + device=device_name, + dtype=dtype_name, + shape=tuple(int(value) for value in parameter.shape), + ) + ) + if not values: + raise ValueError("wan_branch_parameter_inventory_invalid") + return tuple(values) + + def measure_forward_backward(self) -> object: + raise ValueError("wan_branch_canary_requires_one_optimizer_step") + + def train_step(self, *, step: int, total_steps: int) -> dict[str, float]: + seed = _domain_seed(self.recipe.seed, self.recipe.branch) + with self._rng_fence(seed): + try: + return self._train_step(step=step, total_steps=total_steps) + except RuntimeError as error: + if is_recognized_mps_oom(error): + raise WanMpsOutOfMemoryError("wan_branch_mps_out_of_memory") from error + raise + + def _train_step(self, *, step: int, total_steps: int) -> dict[str, float]: + if (step, total_steps) != (1, 1) or self.optimizer is None: + raise ValueError("wan_branch_canary_requires_one_optimizer_step") + if self.optimizer_updates: + raise ValueError("wan_branch_optimizer_update_invalid") + if self.cancelled(): + raise ValueError("wan_branch_cancelled") + self.optimizer.zero_grad(set_to_none=True) + loss, selector_count = self._loss() + if self.cancelled(): + self.optimizer.zero_grad(set_to_none=True) + raise ValueError("wan_branch_cancelled") + loss.backward() + self._require_finite_gradients() + if self.cancelled(): + self.optimizer.zero_grad(set_to_none=True) + raise ValueError("wan_branch_cancelled") + gradient = self._clip_gradient_once() + if self.cancelled(): + self.optimizer.zero_grad(set_to_none=True) + raise ValueError("wan_branch_cancelled") + self.optimizer.step() + self.optimizer_updates = 1 + self._require_finite_parameters_and_optimizer() + after = self.parameter_inventory() + assert self._baseline is not None and self._injected is not None + baseline = {item.name: item.digest for item in self._baseline} + unchanged = tuple( + item + for item in after + if not item.requires_grad and baseline.get(item.name) == item.digest + ) + base_after = tuple(item for item in after if not item.requires_grad) + if len(unchanged) != len(base_after) or len(base_after) != len(self._baseline): + raise ValueError("wan_branch_base_parameter_changed") + injected = {item.name: item.digest for item in self._injected if item.requires_grad} + changed = tuple( + item for item in after if item.requires_grad and injected.get(item.name) != item.digest + ) + if not changed or any(item.name not in injected for item in changed): + raise ValueError("wan_branch_adapter_parameter_unchanged") + evidence = WanBranchStepEvidence( + branch=self.recipe.branch, + selector_count=selector_count, + loss=float(loss.detach().item()), + gradient_norm=gradient, + changed_adapter_inventory=changed, + unchanged_base_inventory=unchanged, + optimizer_update_count=1, + memory_observations=self._memory_observations(), + ) + self._last_evidence = evidence + return {"loss": evidence.loss, "gradient_norm": evidence.gradient_norm} + + def branch_step_evidence(self) -> WanBranchStepEvidence: + if self._last_evidence is None: + raise ValueError("wan_branch_step_evidence_missing") + return self._last_evidence + + def checkpoint_state(self, *, step: int) -> WanRuntimeCheckpointState: + if step != 1 or self.optimizer_updates != 1: + raise ValueError("wan_branch_checkpoint_state_invalid") + safetensors = cast(Any, self.importer("safetensors.torch")) + weights = { + name: parameter.detach().to("cpu").contiguous() + for name, parameter in self._named_trainable().items() + } + return WanRuntimeCheckpointState( + weights=bytes(safetensors.save(weights)), + optimizer={"schema_version": "wan-branch-optimizer-state.v1", "step": 1}, + trainer_state={ + "schema_version": "wan-branch-trainer-state.v1", + "step": 1, + "branch": self.recipe.branch, + }, + ) + + def checkpoint_state_v2( + self, *, parent_checkpoint_id: str | None + ) -> WanBranchCheckpointStateV2: + """Capture adapter, optimizer, and RNG continuation state at step one.""" + + if self.optimizer_updates != 1 or self.batch_digest is None: + raise ValueError("wan_branch_checkpoint_state_invalid") + named = dict(sorted(self._named_trainable().items())) + safetensors = cast(Any, self.importer("safetensors.torch")) + weights = bytes( + safetensors.save( + { + name: parameter.detach().to("cpu").contiguous() + for name, parameter in named.items() + } + ) + ) + specs = self._adapter_specs(named) + optimizer = self._optimizer_state_v2(named) + self._validate_optimizer_contract_v2(optimizer, named) + cpu_rng = self.torch.random.get_rng_state().detach().to("cpu").numpy().tobytes() + mps_rng = self.torch.mps.get_rng_state().detach().to("cpu").numpy().tobytes() + manifest = WanBranchCheckpointManifestV2( + schema_version="wan-branch-checkpoint.v2", + branch=self.recipe.branch, + selector=self.selector_for_branch(self.recipe.branch), + step=1, + total_steps=1, + parent_checkpoint_id=parent_checkpoint_id, + batch_digest=self.batch_digest, + recipe_digest=wan_branch_recipe_digest(self.recipe), + component_inventory_digest=wan_inventory_digest(self.loaded_branch.inventory), + adapter_tensors=specs, + optimizer_parameter_names=tuple(named), + optimizer_slot_names=_ADAMW_SLOT_NAMES, + cpu_rng_digest="sha256:" + hashlib.sha256(cpu_rng).hexdigest(), + mps_rng_digest="sha256:" + hashlib.sha256(mps_rng).hexdigest(), + finite=True, + ) + return WanBranchCheckpointStateV2( + manifest=manifest, + weights=weights, + optimizer=optimizer, + cpu_rng_state=cpu_rng, + mps_rng_state=mps_rng, + ) + + def restore_checkpoint_v2( + self, + state: WanBranchCheckpointStateV2, + expected: WanBranchCheckpointManifestV2, + ) -> WanBranchRestoreReceiptV2: + """Restore a validated checkpoint into this freshly injected branch runtime.""" + + validate_wan_branch_checkpoint_v2(state, expected) + named = dict(sorted(self._named_trainable().items())) + local_specs = self._adapter_specs(named) + local_expected = self._expected_manifest_v2(expected, local_specs) + if expected != local_expected or self.optimizer_updates != 0: + raise ValueError("wan_branch_checkpoint_v2_invalid") + tensors = cast(Any, self.importer("safetensors.torch")).load(state.weights) + self._validate_optimizer_contract_v2(state.optimizer, named) + decoded_optimizer = self._decode_optimizer_state_v2(state.optimizer, named) + cpu_rng = self._rng_tensor(state.cpu_rng_state) + mps_rng = self._rng_tensor(state.mps_rng_state) + for name, tensor in tensors.items(): + named[name].data.copy_(tensor.to(named[name].device)) + self.optimizer.state.clear() + self.optimizer.state.update(decoded_optimizer) + self.torch.random.set_rng_state(cpu_rng) + self.torch.mps.set_rng_state(mps_rng) + self.optimizer_updates = 1 + observed = self.checkpoint_state_v2(parent_checkpoint_id=expected.parent_checkpoint_id) + expected_optimizer = canonical_json(dict(state.optimizer)).encode() + if ( + observed.weights != state.weights + or canonical_json(dict(observed.optimizer)).encode() != expected_optimizer + or observed.cpu_rng_state != state.cpu_rng_state + or observed.mps_rng_state != state.mps_rng_state + ): + raise ValueError("wan_branch_checkpoint_v2_observation_invalid") + manifest_payload = canonical_json(asdict(expected)).encode() + return WanBranchRestoreReceiptV2( + schema_version="wan-branch-restore-receipt.v2", + manifest_digest="sha256:" + hashlib.sha256(manifest_payload).hexdigest(), + weights_digest="sha256:" + hashlib.sha256(state.weights).hexdigest(), + optimizer_digest="sha256:" + hashlib.sha256(expected_optimizer).hexdigest(), + cpu_rng_digest=expected.cpu_rng_digest, + mps_rng_digest=expected.mps_rng_digest, + ) + + def continuity_probe(self) -> dict[str, Any]: + """Run the preregistered branch forward probe without optimizer mutation.""" + + selector = self.batch["transformer_selector"] + mask = selector == self.selector_for_branch(self.recipe.branch) + result = self.transformer( + hidden_states=self.batch["hidden_states"][mask], + timestep=self.batch["timestep"][mask], + encoder_hidden_states=self.batch["encoder_hidden_states"][mask], + return_dict=True, + ) + sample = result.sample.detach().to("cpu").clone() + if not bool(self.torch.isfinite(sample).all().item()): + raise ValueError("wan_branch_output_nonfinite") + return {"probe.output": sample} + + def restore_checkpoint(self, state: WanCheckpointState) -> None: + if state.step != 1 or state.total_steps != 1: + raise ValueError("wan_branch_checkpoint_restore_invalid") + tensors = cast(Any, self.importer("safetensors.torch")).load(state.weights) + named = self._named_trainable() + if set(tensors) != set(named): + raise ValueError("wan_branch_checkpoint_restore_invalid") + for name, tensor in tensors.items(): + parameter = named[name] + if tuple(tensor.shape) != tuple(parameter.shape) or tensor.dtype != parameter.dtype: + raise ValueError("wan_branch_checkpoint_restore_invalid") + parameter.data.copy_(tensor.to(parameter.device)) + self.optimizer_updates = 1 + self._restored = WanRuntimeRestoreReceipt( + weights_digest="sha256:" + hashlib.sha256(state.weights).hexdigest(), + optimizer_payload=canonical_json(dict(state.optimizer)).encode(), + trainer_state_payload=canonical_json(dict(state.trainer_state)).encode(), + ) + + def observe_restored_state(self) -> WanRuntimeRestoreReceipt: + if self._restored is None: + raise ValueError("wan_branch_checkpoint_restore_missing") + return self._restored + + @property + def last_evidence(self) -> WanBranchStepEvidence | None: + return self._last_evidence + + def close(self) -> None: + seed = _domain_seed(self.recipe.seed, self.recipe.branch) + with self._rng_fence(seed): + self.optimizer = None + self.transformer = None + self.batch.clear() + self._baseline = None + self._injected = None + self.loaded_branch.close() + def _adapter_specs(self, named: Mapping[str, Any]) -> tuple[WanBranchTensorSpecV2, ...]: + dtype_names = { + str(self.torch.float16): "F16", + str(self.torch.bfloat16): "BF16", + str(self.torch.float32): "F32", + } + try: + return tuple( + WanBranchTensorSpecV2( + name=name, + shape=tuple(int(size) for size in parameter.shape), + dtype=dtype_names[str(parameter.dtype)], + ) + for name, parameter in named.items() + ) + except KeyError: + raise ValueError("wan_branch_checkpoint_v2_invalid") from None + + def _expected_manifest_v2( + self, + claimed: WanBranchCheckpointManifestV2, + specs: tuple[WanBranchTensorSpecV2, ...], + ) -> WanBranchCheckpointManifestV2: + if self.batch_digest is None: + raise ValueError("wan_branch_checkpoint_v2_invalid") + return WanBranchCheckpointManifestV2( + schema_version="wan-branch-checkpoint.v2", + branch=self.recipe.branch, + selector=self.selector_for_branch(self.recipe.branch), + step=1, + total_steps=1, + parent_checkpoint_id=claimed.parent_checkpoint_id, + batch_digest=self.batch_digest, + recipe_digest=wan_branch_recipe_digest(self.recipe), + component_inventory_digest=wan_inventory_digest(self.loaded_branch.inventory), + adapter_tensors=specs, + optimizer_parameter_names=tuple(item.name for item in specs), + optimizer_slot_names=_ADAMW_SLOT_NAMES, + cpu_rng_digest=claimed.cpu_rng_digest, + mps_rng_digest=claimed.mps_rng_digest, + finite=True, + ) + + def _optimizer_state_v2(self, named: Mapping[str, Any]) -> dict[str, object]: + parameter_state: dict[str, object] = {} + for name, parameter in named.items(): + slots = self.optimizer.state.get(parameter) + if not isinstance(slots, Mapping) or not slots: + raise ValueError("wan_branch_checkpoint_state_invalid") + encoded: dict[str, object] = {} + for slot_name, value in sorted(slots.items()): + if type(slot_name) is not str or not hasattr(value, "detach"): + raise ValueError("wan_branch_checkpoint_state_invalid") + tensor = value.detach().to("cpu") + encoded[slot_name] = { + "dtype": str(tensor.dtype), + "shape": list(tensor.shape), + "values": tensor.reshape(-1).tolist(), + } + parameter_state[name] = encoded + return { + "schema_version": "wan-branch-optimizer-state.v2", + "parameter_state": parameter_state, + } + + def _decode_optimizer_state_v2( + self, payload: Mapping[str, object], named: Mapping[str, Any] + ) -> dict[Any, dict[str, Any]]: + raw_states = cast(Mapping[str, Mapping[str, object]], payload["parameter_state"]) + result: dict[Any, dict[str, Any]] = {} + for name, parameter in named.items(): + decoded: dict[str, Any] = {} + for slot_name, raw in raw_states[name].items(): + slot = cast(Mapping[str, object], raw) + dtype_name = cast(str, slot["dtype"]).removeprefix("torch.") + dtype = getattr(self.torch, dtype_name, None) + shape = tuple(cast(list[int], slot["shape"])) + values = cast(list[float], slot["values"]) + if dtype is None or math.prod(shape) != len(values): + raise ValueError("wan_branch_checkpoint_v2_invalid") + decoded[slot_name] = ( + self.torch.tensor(values, dtype=dtype).reshape(shape).to(parameter.device) + ) + result[parameter] = decoded + return result + + def _validate_optimizer_contract_v2( + self, payload: Mapping[str, object], named: Mapping[str, Any] + ) -> None: + """Derive exact AdamW continuation slots from trusted runtime parameters.""" + + try: + if self.recipe.optimizer_algorithm != "adamw": + raise ValueError + raw_states = cast(Mapping[str, Mapping[str, object]], payload["parameter_state"]) + if set(raw_states) != set(named): + raise ValueError + for name, parameter in named.items(): + slots = raw_states[name] + if tuple(sorted(slots)) != _ADAMW_SLOT_NAMES: + raise ValueError + for slot_name in _ADAMW_SLOT_NAMES: + raw = slots[slot_name] + if not isinstance(raw, Mapping) or set(raw) != {"dtype", "shape", "values"}: + raise ValueError + dtype = raw["dtype"] + shape = raw["shape"] + values = raw["values"] + expected_dtype = ( + "torch.float32" if slot_name == "step" else str(parameter.dtype) + ) + expected_shape = [] if slot_name == "step" else list(parameter.shape) + if ( + dtype != expected_dtype + or shape != expected_shape + or not isinstance(values, list) + or len(values) != max(1, math.prod(expected_shape)) + or any( + type(value) not in (int, float) or not math.isfinite(float(value)) + for value in values + ) + ): + raise ValueError + except (KeyError, TypeError, ValueError): + raise ValueError("wan_branch_checkpoint_v2_invalid") from None + + def _rng_tensor(self, payload: bytes) -> Any: + return self.torch.frombuffer(bytearray(payload), dtype=self.torch.uint8).clone() + + def _loss(self) -> tuple[Any, int]: + self._require_finite_batch() + selector = self.batch["transformer_selector"] + mask = selector == self.selector_for_branch(self.recipe.branch) + count = int(mask.sum().item()) + if count <= 0: + raise ValueError("wan_branch_selector_empty") + result = self.transformer( + hidden_states=self.batch["hidden_states"][mask], + timestep=self.batch["timestep"][mask], + encoder_hidden_states=self.batch["encoder_hidden_states"][mask], + return_dict=True, + ) + prediction = result.sample + if not bool(self.torch.isfinite(prediction).all().item()): + raise ValueError("wan_branch_output_nonfinite") + target = self.batch["target"][mask] + loss = self.torch.nn.functional.mse_loss(prediction.float(), target.float()) + if not bool(self.torch.isfinite(loss).all().item()): + raise ValueError("wan_branch_loss_nonfinite") + return loss, count + + def _validate_batch(self) -> None: + required = { + "hidden_states", + "timestep", + "encoder_hidden_states", + "clean_latents", + "noise_latents", + "target", + "transformer_selector", + } + if set(self.batch) != required: + raise ValueError("wan_training_batch_invalid") + + def _require_finite_batch(self) -> None: + for name, tensor in self.batch.items(): + if name != "transformer_selector" and not bool( + self.torch.isfinite(tensor).all().item() + ): + raise ValueError("wan_branch_input_nonfinite") + + def _named_trainable(self) -> dict[str, Any]: + return { + f"transformer.{name}": parameter + for name, parameter in self.transformer.named_parameters() + if parameter.requires_grad + } + + def _parameter_digest(self, parameter: Any) -> str: + payload = parameter.detach().to("cpu").contiguous().view(self.torch.uint8).numpy().tobytes() + return "sha256:" + hashlib.sha256(payload).hexdigest() + + def _require_finite_gradients(self) -> None: + named = self._named_trainable() + if not named or any( + parameter.grad is None or not bool(self.torch.isfinite(parameter.grad).all().item()) + for parameter in named.values() + ): + raise ValueError("wan_branch_gradient_nonfinite") + + def _clip_gradient_once(self) -> float: + value = self.torch.nn.utils.clip_grad_norm_( + tuple(self._named_trainable().values()), self.recipe.gradient_clip_norm + ) + result = float(value.detach().item() if hasattr(value, "detach") else value) + if not math.isfinite(result): + raise ValueError("wan_branch_gradient_nonfinite") + return result + + def _require_finite_parameters_and_optimizer(self) -> None: + if any( + not bool(self.torch.isfinite(parameter).all().item()) + for parameter in self._named_trainable().values() + ): + raise ValueError("wan_branch_parameter_nonfinite") + for state in self.optimizer.state.values(): + if not isinstance(state, Mapping): + raise ValueError("wan_branch_optimizer_invalid") + for value in state.values(): + if hasattr(value, "dtype") and not bool(self.torch.isfinite(value).all().item()): + raise ValueError("wan_branch_optimizer_nonfinite") + + def _memory_observations(self) -> tuple[tuple[str, int], ...]: + return tuple( + sorted( + ( + ("allocated_bytes", int(self.torch.mps.current_allocated_memory())), + ("driver_bytes", int(self.torch.mps.driver_allocated_memory())), + ) + ) + ) + + @contextmanager + def _rng_fence(self, seed: int) -> Iterator[None]: + python_state = random.getstate() + numpy: Any = None + numpy_state: object | None = None + try: + numpy = self.importer("numpy") + numpy_state = numpy.random.get_state() + except (ImportError, ModuleNotFoundError): + pass + cpu_state = self.torch.random.get_rng_state() + mps_state = self.torch.mps.get_rng_state() + try: + random.seed(seed) + if numpy is not None: + numpy.random.seed(seed % (2**32)) + self.torch.manual_seed(seed) + self.torch.mps.manual_seed(seed) + yield + finally: + random.setstate(python_state) + if numpy is not None and numpy_state is not None: + numpy.random.set_state(numpy_state) + self.torch.random.set_rng_state(cpu_state) + self.torch.mps.set_rng_state(mps_state) + + +def _matches_suffix(name: str, suffix: str) -> bool: + return name == suffix or name.endswith("." + suffix) + + +def _logical_target(parameter_name: str) -> str: + matches = tuple(marker for marker in _LORA_MARKERS if marker in parameter_name) + if len(matches) > 1: + raise ValueError("wan_branch_trainable_inventory_invalid") + if matches: + return parameter_name.split(matches[0], 1)[0] + return parameter_name.rpartition(".")[0] + + +def _inventory_parameter_name(parameter_name: str, parameter_names: frozenset[str]) -> str: + if any(marker in parameter_name for marker in _LORA_MARKERS): + return parameter_name + prefix, marker, suffix = parameter_name.partition(".base_layer.") + if marker and any( + candidate.startswith(prefix + lora_marker) + for candidate in parameter_names + for lora_marker in _LORA_MARKERS + ): + return f"{prefix}.{suffix}" + return parameter_name + + +def _domain_seed(seed: int, branch: WanTransformerBranch) -> int: + payload = f"wan-mps-branch-seed.v1\0{branch}\0{seed}".encode() + return int.from_bytes(hashlib.sha256(payload).digest()[:8], "big") % (2**63 - 1) + + +def _adamw_config(parameters: tuple[tuple[str, int | float | bool], ...]) -> dict[str, object]: + aliases = {"learning_rate": "lr", "beta1": "betas", "beta2": "betas"} + raw = dict(parameters) + allowed = { + "learning_rate", + "beta1", + "beta2", + "eps", + "weight_decay", + "amsgrad", + "maximize", + "foreach", + "capturable", + "differentiable", + "fused", + } + if set(raw) - allowed or "learning_rate" not in raw: + raise ValueError("wan_branch_optimizer_invalid") + result: dict[str, object] = {} + for name, value in raw.items(): + if name not in {"beta1", "beta2"}: + result[aliases.get(name, name)] = value + result["betas"] = (float(raw.get("beta1", 0.9)), float(raw.get("beta2", 0.999))) + return result + + +__all__ = [ + "WanBranchStepEvidence", + "WanMpsBranchOperations", + "WanMpsOutOfMemoryError", + "is_recognized_mps_oom", +] diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_checkpoint.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_checkpoint.py new file mode 100644 index 0000000..c1dacf9 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_checkpoint.py @@ -0,0 +1,1642 @@ +"""Durable, dependency-free checkpoint publication for the Wan trainer.""" + +from __future__ import annotations + +import ctypes +import errno +import hashlib +import json +import math +import os +import stat +import sys +import unicodedata +from collections.abc import Collection, Mapping +from dataclasses import dataclass, replace +from itertools import pairwise +from pathlib import Path +from typing import Any, cast + +from video_lab.domain import canonical_json +from video_lab.trainer_protocol import CheckpointResult, OutputFile, TrainingProgress +from video_lab.wan_pending import ( + TrainableParameterSpec, + trainable_parameter_specs_from_records, + validate_pending_gradients, + validate_trainable_parameter_specs, +) + +_ROLES = ( + "weights", + "optimizer", + "trainer_state", + "telemetry", + "trainable_targets", + "frozen_parameters", + "finite_gradient_proof", +) + + +@dataclass(frozen=True, slots=True) +class WanCheckpointState: + """Complete in-memory state for one independently provable boundary.""" + + checkpoint_id: str + step: int + total_steps: int + parent_checkpoint_id: str | None + weights: bytes + optimizer: Mapping[str, object] + trainer_state: Mapping[str, object] + telemetry: tuple[Mapping[str, object], ...] + trainable_targets: Mapping[str, object] + frozen_parameters: Mapping[str, object] + finite_gradient_proof: Mapping[str, object] + + +@dataclass(frozen=True, slots=True) +class WanBranchTensorSpecV2: + """Path-free physical identity of one adapter tensor.""" + + name: str + shape: tuple[int, ...] + dtype: str + + +@dataclass(frozen=True, slots=True) +class WanBranchCheckpointManifestV2: + """Explicit v2 authority for production branch continuation.""" + + schema_version: str + branch: str + selector: int + step: int + total_steps: int + parent_checkpoint_id: str | None + batch_digest: str + recipe_digest: str + component_inventory_digest: str + adapter_tensors: tuple[WanBranchTensorSpecV2, ...] + optimizer_parameter_names: tuple[str, ...] + optimizer_slot_names: tuple[str, ...] + cpu_rng_digest: str + mps_rng_digest: str + finite: bool + + +@dataclass(frozen=True, slots=True) +class WanBranchCheckpointStateV2: + """Adapter-only bytes and continuation state validated against a v2 manifest.""" + + manifest: WanBranchCheckpointManifestV2 + weights: bytes + optimizer: Mapping[str, object] + cpu_rng_state: bytes + mps_rng_state: bytes + + +@dataclass(frozen=True, slots=True) +class WanBranchRestoreReceiptV2: + """Observed equality proof published only after a fresh-runtime restore.""" + + schema_version: str + manifest_digest: str + weights_digest: str + optimizer_digest: str + cpu_rng_digest: str + mps_rng_digest: str + + +def validate_wan_branch_checkpoint_v2( + state: WanBranchCheckpointStateV2, + expected: WanBranchCheckpointManifestV2, +) -> WanBranchCheckpointStateV2: + """Validate every identity before a caller is allowed to mutate a runtime.""" + + try: + manifest = state.manifest + specs = manifest.adapter_tensors + names = tuple(item.name for item in specs) + tensors = _inspect_branch_safetensors_v2(state.weights) + expected_tensors = { + item.name: (item.dtype, item.shape) + for item in specs + if type(item) is WanBranchTensorSpecV2 + } + parameter_state = state.optimizer["parameter_state"] + if ( + type(state) is not WanBranchCheckpointStateV2 + or type(expected) is not WanBranchCheckpointManifestV2 + or manifest != expected + or manifest.schema_version != "wan-branch-checkpoint.v2" + or manifest.branch not in {"high_noise", "low_noise"} + or manifest.selector != (0 if manifest.branch == "high_noise" else 1) + or manifest.step != 1 + or manifest.total_steps != 1 + or not manifest.finite + or not specs + or names != tuple(sorted(set(names))) + or tensors != expected_tensors + or manifest.optimizer_parameter_names != names + or not manifest.optimizer_slot_names + or set(state.optimizer) != {"schema_version", "parameter_state"} + or state.optimizer.get("schema_version") != "wan-branch-optimizer-state.v2" + or not isinstance(parameter_state, Mapping) + or set(parameter_state) != set(names) + or any( + not isinstance(parameter_state[name], Mapping) + or set(cast(Mapping[str, object], parameter_state[name])) + != set(manifest.optimizer_slot_names) + for name in names + ) + or not _finite_json(state.optimizer) + or not state.cpu_rng_state + or not state.mps_rng_state + or "sha256:" + hashlib.sha256(state.cpu_rng_state).hexdigest() + != manifest.cpu_rng_digest + or "sha256:" + hashlib.sha256(state.mps_rng_state).hexdigest() + != manifest.mps_rng_digest + or any( + not _is_sha256_digest(value) + for value in ( + manifest.batch_digest, + manifest.recipe_digest, + manifest.component_inventory_digest, + manifest.cpu_rng_digest, + manifest.mps_rng_digest, + ) + ) + ): + raise ValueError + except (AttributeError, KeyError, TypeError, ValueError): + raise ValueError("wan_branch_checkpoint_v2_invalid") from None + return state + + +def _inspect_branch_safetensors_v2(data: bytes) -> dict[str, tuple[str, tuple[int, ...]]]: + """Inspect standard safetensors without imposing the legacy canonical-header encoding.""" + + try: + if len(data) < 9: + raise ValueError + header_size = int.from_bytes(data[:8], "little") + if header_size <= 0 or header_size > len(data) - 8: + raise ValueError + header = json.loads(data[8 : 8 + header_size]) + payload_size = len(data) - 8 - header_size + widths = {"F16": 2, "BF16": 2, "F32": 4} + tensors: dict[str, tuple[str, tuple[int, ...]]] = {} + ranges: list[tuple[int, int]] = [] + if not isinstance(header, dict) or not header: + raise ValueError + for name, value in header.items(): + if name == "__metadata__": + raise ValueError + if ( + type(name) is not str + or not name + or not isinstance(value, dict) + or set(value) != {"dtype", "shape", "data_offsets"} + ): + raise ValueError + dtype, shape, offsets = value["dtype"], value["shape"], value["data_offsets"] + if ( + dtype not in widths + or not isinstance(shape, list) + or not shape + or any(type(size) is not int or size <= 0 for size in shape) + or not isinstance(offsets, list) + or len(offsets) != 2 + or offsets[0] < 0 + or offsets[1] - offsets[0] != math.prod(shape) * widths[dtype] + or offsets[1] > payload_size + ): + raise ValueError + tensors[name] = (dtype, tuple(shape)) + ranges.append((offsets[0], offsets[1])) + raw = data[8 + header_size + offsets[0] : 8 + header_size + offsets[1]] + width = widths[dtype] + exponent_mask = ( + 0x7C00 + if dtype == "F16" + else 0x7F80 + if dtype == "BF16" + else 0x7F800000 + ) + for index in range(0, len(raw), width): + bits = int.from_bytes(raw[index : index + width], "little") + if bits & exponent_mask == exponent_mask: + raise ValueError + if ( + not tensors + or sorted(ranges) != ranges + or ranges[0][0] != 0 + or ranges[-1][1] != payload_size + or any(left[1] != right[0] for left, right in pairwise(ranges)) + ): + raise ValueError + return tensors + except (KeyError, TypeError, ValueError, json.JSONDecodeError): + raise ValueError("wan_branch_checkpoint_v2_invalid") from None + + +OptimizerValue = str | int | float | bool + + +@dataclass(frozen=True, slots=True) +class WanCheckpointValidationContext: + """Immutable request facts that checkpoint payloads cannot self-authorize.""" + + checkpoint_id: str + request_id: str + run_id: str + backend_id: str + seed: int + step: int + total_steps: int + parent_checkpoint_id: str | None + parent_optimizer_state_digest: str | None + trainable_targets: tuple[str, ...] + trainable_parameter_specs: tuple[TrainableParameterSpec, ...] + lora_rank: int + lora_alpha: int + lora_dropout: float + optimizer_algorithm: str + optimizer_config: tuple[tuple[str, OptimizerValue], ...] + start_step: int = 0 + telemetry_sequence_start: int = 1 + + +@dataclass(frozen=True, slots=True) +class WanParentCheckpointFileDescriptor: + """Factory-authorized identity for one immutable parent artifact.""" + + role: str + path: Path + digest: str + size_bytes: int + device: int + inode: int + + +@dataclass(frozen=True, slots=True) +class WanParentCheckpointDescriptor: + """Exact seven-file inventory selected by orchestration for resume.""" + + checkpoint_id: str + checkpoint_digest: str + files: tuple[WanParentCheckpointFileDescriptor, ...] + step: int = 0 + total_steps: int = 0 + parent_checkpoint_id: str | None = None + parent_optimizer_state_digest: str | None = None + + +@dataclass(frozen=True, slots=True) +class _RestoredAncestorLease: + name: str + descriptor: int + device: int + inode: int + + +@dataclass(frozen=True, slots=True) +class _RestoredFileLease: + descriptor: WanParentCheckpointFileDescriptor + file_descriptor: int + path: Path + ancestors: tuple[_RestoredAncestorLease, ...] + + +class WanCheckpointCleanupError(RuntimeError): + """A checkpoint cleanup failed, optionally after a primary operation error.""" + + def __init__( + self, + primary_error: BaseException | None, + cleanup_errors: tuple[BaseException, ...], + ) -> None: + if not cleanup_errors: + raise ValueError("wan_checkpoint_cleanup_errors_missing") + self.primary_error = primary_error + self.cleanup_errors = cleanup_errors + super().__init__("wan_checkpoint_cleanup_failed") + + +class WanCheckpointCodec: + """Publish complete checkpoint directories relative to one retained root.""" + + def __init__(self, scratch_root: Path) -> None: + self._root = scratch_root + self._root_fd, self._root_identity = _open_scratch_root(scratch_root) + self._closed = False + self._restored: tuple[_RestoredFileLease, ...] = () + + def __enter__(self) -> WanCheckpointCodec: + if self._closed: + raise ValueError("wan_checkpoint_codec_closed") + return self + + def __exit__( + self, + _error_type: type[BaseException] | None, + _error: BaseException | None, + _traceback: object | None, + ) -> None: + self.close() + + def close(self) -> None: + if not self._closed: + self._closed = True + for lease in self._restored: + os.close(lease.file_descriptor) + for ancestor in lease.ancestors: + os.close(ancestor.descriptor) + self._restored = () + os.close(self._root_fd) + + def restore( + self, + descriptor: WanParentCheckpointDescriptor, + context: WanCheckpointValidationContext, + ) -> WanCheckpointState: + """Stable-open, rehash, and decode one exact parent inventory.""" + if self._closed or self._restored: + raise ValueError("wan_checkpoint_restore_invalid") + if ( + type(descriptor) is not WanParentCheckpointDescriptor + or descriptor.checkpoint_id != context.checkpoint_id + or not _is_sha256_digest(descriptor.checkpoint_digest) + or type(descriptor.files) is not tuple + or len(descriptor.files) != len(_ROLES) + ): + raise ValueError("wan_checkpoint_restore_invalid") + if any( + type(item) is not WanParentCheckpointFileDescriptor + or type(item.role) is not str + or not isinstance(item.path, Path) + or type(item.digest) is not str + or type(item.size_bytes) is not int + or type(item.device) is not int + or type(item.inode) is not int + for item in descriptor.files + ): + raise ValueError("wan_checkpoint_restore_invalid") + by_role = {item.role: item for item in descriptor.files} + if set(by_role) != set(_ROLES) or len(by_role) != len(descriptor.files): + raise ValueError("wan_checkpoint_restore_invalid") + leases: list[_RestoredFileLease] = [] + payloads: dict[str, bytes] = {} + seen_paths: set[Path] = set() + seen_identities: set[tuple[int, int]] = set() + try: + for role in _ROLES: + item = by_role[role] + if ( + type(item) is not WanParentCheckpointFileDescriptor + or item.size_bytes <= 0 + or not _is_sha256_digest(item.digest) + ): + raise ValueError("wan_checkpoint_restore_invalid") + path = item.path + path = Path(*path.parts) + identity = (item.device, item.inode) + if path in seen_paths or identity in seen_identities: + raise ValueError("wan_checkpoint_restore_invalid") + seen_paths.add(path) + seen_identities.add(identity) + lease = _open_exact_parent_file(path, item) + leases.append(lease) + payload = _read_open_regular(lease.file_descriptor, item.size_bytes) + if "sha256:" + hashlib.sha256(payload).hexdigest() != item.digest: + raise ValueError("wan_checkpoint_restore_invalid") + payloads[role] = payload + state = _state_from_payloads(payloads, context) + historical_context = _historical_validation_context(state, context) + _validated_payloads(state, historical_context) + self._restored = tuple(leases) + self.reverify_restored() + return state + except BaseException: + self._restored = () + for lease in leases: + os.close(lease.file_descriptor) + for ancestor in lease.ancestors: + os.close(ancestor.descriptor) + raise ValueError("wan_checkpoint_restore_invalid") from None + def reverify_restored(self) -> None: + """Prove retained parent handles and their authorized paths still agree.""" + for lease in self._restored: + info = os.fstat(lease.file_descriptor) + try: + _reverify_ancestor_chain(lease) + except OSError: + raise ValueError("wan_checkpoint_restore_invalid") from None + if ( + not stat.S_ISREG(info.st_mode) + or (info.st_dev, info.st_ino) + != (lease.descriptor.device, lease.descriptor.inode) + or info.st_size != lease.descriptor.size_bytes + ): + raise ValueError("wan_checkpoint_restore_invalid") + + def discard(self, checkpoint_id: str) -> None: + """Remove a checkpoint that must not escape a failed training transaction.""" + if self._closed: + return + name = _safe_component(checkpoint_id) + files = tuple( + "weights.safetensors" if role == "weights" else f"{role}.json" for role in _ROLES + ) + _remove_checkpoint(self._root_fd, name, files, -1) + + def write( + self, state: WanCheckpointState, context: WanCheckpointValidationContext + ) -> CheckpointResult: + if self._closed: + raise ValueError("wan_checkpoint_codec_closed") + snapshot, payloads = _validated_payloads(state, context) + checkpoint_name = _safe_component(snapshot.checkpoint_id) + staging_name = f".{checkpoint_name}.staging" + _reverify_root(self._root, self._root_fd, self._root_identity) + staging_fd = -1 + staging_created = False + published = False + written: list[str] = [] + files: list[OutputFile] = [] + operation_error: BaseException | None = None + close_errors: list[BaseException] = [] + try: + os.mkdir(staging_name, 0o700, dir_fd=self._root_fd) + staging_created = True + staging_fd = _open_directory(staging_name, self._root_fd) + for role, payload in payloads: + name = "weights.safetensors" if role == "weights" else f"{role}.json" + _write_exclusive(staging_fd, name, payload) + written.append(name) + reopened = _read_regular(staging_fd, name, len(payload)) + if reopened != payload: + raise OSError("checkpoint bytes changed after fsync") + digest = "sha256:" + hashlib.sha256(reopened).hexdigest() + files.append( + OutputFile( + relative_path=f"{checkpoint_name}/{name}", + digest=digest, + size_bytes=len(reopened), + role=role, + ) + ) + _fsync_directory(staging_fd) + _reverify_root(self._root, self._root_fd, self._root_identity) + _require_destination_absent(self._root_fd, checkpoint_name) + _publish_directory_no_replace(self._root_fd, staging_name, checkpoint_name) + published = True + _fsync_directory(self._root_fd) + _reverify_root(self._root, self._root_fd, self._root_identity) + except BaseException as error: + operation_error = error + finally: + if staging_fd >= 0: + try: + _close_directory(staging_fd) + except BaseException as error: + close_errors.append(error) + staging_fd = -1 + if operation_error is not None or close_errors: + cleanup_errors = list(close_errors) + if staging_created: + cleanup_name = checkpoint_name if published else staging_name + try: + _remove_checkpoint(self._root_fd, cleanup_name, tuple(written), -1) + except WanCheckpointCleanupError as cleanup_error: + cleanup_errors.extend(cleanup_error.cleanup_errors) + if cleanup_errors: + raise WanCheckpointCleanupError( + operation_error, tuple(cleanup_errors) + ) from operation_error + raise ValueError("wan_checkpoint_write_failed") from None + by_role = {item.role: item.digest for item in files} + return CheckpointResult( + checkpoint_id=snapshot.checkpoint_id, + step=snapshot.step, + fraction=snapshot.step / snapshot.total_steps, + parent_checkpoint_id=snapshot.parent_checkpoint_id, + files=tuple(files), + telemetry_digest=by_role["telemetry"], + frozen_parameters_digest=by_role["frozen_parameters"], + trainable_targets_digest=by_role["trainable_targets"], + finite_gradient_proof_digest=by_role["finite_gradient_proof"], + ) + + +def _historical_validation_context( + state: WanCheckpointState, context: WanCheckpointValidationContext +) -> WanCheckpointValidationContext: + """Recover request-local fields from the digest-bound checkpoint payload.""" + request_id = state.trainable_targets.get("request_id") + if type(request_id) is not str or not request_id or not state.telemetry: + raise ValueError("wan_checkpoint_restore_invalid") + first = state.telemetry[0] + if not isinstance(first, Mapping): + raise ValueError("wan_checkpoint_restore_invalid") + first_step = first.get("step") + first_sequence = first.get("sequence") + if type(first_step) is not int or type(first_sequence) is not int: + raise ValueError("wan_checkpoint_restore_invalid") + return replace( + context, + request_id=request_id, + start_step=first_step - 1, + telemetry_sequence_start=first_sequence, + ) + + +def _validated_payloads( + state: WanCheckpointState, context: WanCheckpointValidationContext +) -> tuple[WanCheckpointState, tuple[tuple[str, bytes], ...]]: + if type(state) is not WanCheckpointState: + raise ValueError("wan_checkpoint_state_invalid") + checkpoint_id = state.checkpoint_id + step = state.step + total_steps = state.total_steps + parent_checkpoint_id = state.parent_checkpoint_id + weights = state.weights + optimizer = state.optimizer + trainer_state = state.trainer_state + telemetry = state.telemetry + trainable_targets = state.trainable_targets + frozen_parameters = state.frozen_parameters + finite_gradient_proof = state.finite_gradient_proof + owned_context = _owned_validation_context(context) + if ( + type(step) is not int + or type(total_steps) is not int + or step <= 0 + or total_steps < step + or type(weights) is not bytes + or not telemetry + ): + raise ValueError("wan_checkpoint_state_invalid") + _validate_context(owned_context) + _safe_component(checkpoint_id) + if parent_checkpoint_id is not None: + _safe_component(parent_checkpoint_id) + if ( + checkpoint_id != owned_context.checkpoint_id + or step != owned_context.step + or total_steps != owned_context.total_steps + or parent_checkpoint_id != owned_context.parent_checkpoint_id + ): + raise ValueError("wan_checkpoint_state_invalid") + try: + json_payloads = tuple( + _snapshot_json_payload(value) + for value in ( + optimizer, + trainer_state, + list(telemetry), + trainable_targets, + frozen_parameters, + finite_gradient_proof, + ) + ) + decoded = tuple(json.loads(payload) for payload in json_payloads) + snapshot = WanCheckpointState( + checkpoint_id=checkpoint_id, + step=step, + total_steps=total_steps, + parent_checkpoint_id=parent_checkpoint_id, + weights=weights, + optimizer=cast(dict[str, object], decoded[0]), + trainer_state=cast(dict[str, object], decoded[1]), + telemetry=tuple(cast(list[dict[str, object]], decoded[2])), + trainable_targets=cast(dict[str, object], decoded[3]), + frozen_parameters=cast(dict[str, object], decoded[4]), + finite_gradient_proof=cast(dict[str, object], decoded[5]), + ) + except (TypeError, ValueError, OverflowError, RecursionError): + raise ValueError("wan_checkpoint_state_invalid") from None + if any(not _finite_json(value) for value in decoded): + raise ValueError("wan_checkpoint_state_invalid") + _validate_role_schemas(snapshot, owned_context) + tensors = _validate_safetensors(snapshot.weights) + targets = snapshot.trainable_targets.get("targets") + if ( + not isinstance(targets, list) + or not targets + or any(type(target) is not str or not target for target in targets) + or len(set(targets)) != len(targets) + ): + raise ValueError("wan_checkpoint_state_invalid") + _validate_lora_tensors(tensors, cast(list[str], targets), owned_context.lora_rank) + payloads = tuple(zip(_ROLES, (weights, *json_payloads), strict=True)) + return snapshot, payloads + + +def _owned_validation_context( + context: WanCheckpointValidationContext, +) -> WanCheckpointValidationContext: + if type(context) is not WanCheckpointValidationContext: + raise ValueError("wan_checkpoint_state_invalid") + checkpoint_id = context.checkpoint_id + request_id = context.request_id + run_id = context.run_id + backend_id = context.backend_id + seed = context.seed + step = context.step + total_steps = context.total_steps + parent_checkpoint_id = context.parent_checkpoint_id + parent_optimizer_state_digest = context.parent_optimizer_state_digest + trainable_targets = context.trainable_targets + trainable_parameter_specs = context.trainable_parameter_specs + lora_rank = context.lora_rank + lora_alpha = context.lora_alpha + lora_dropout = context.lora_dropout + optimizer_algorithm = context.optimizer_algorithm + optimizer_config = context.optimizer_config + start_step = context.start_step + telemetry_sequence_start = context.telemetry_sequence_start + if ( + type(trainable_targets) is not tuple + or type(trainable_parameter_specs) is not tuple + or type(optimizer_config) is not tuple + ): + raise ValueError("wan_checkpoint_state_invalid") + owned_targets: list[str] = [] + for target in trainable_targets: + if type(target) is not str: + raise ValueError("wan_checkpoint_state_invalid") + owned_targets.append(target) + owned_config: list[tuple[str, OptimizerValue]] = [] + for entry in optimizer_config: + if type(entry) is not tuple or len(entry) != 2: + raise ValueError("wan_checkpoint_state_invalid") + name, value = entry + if type(name) is not str or not _is_optimizer_value(value): + raise ValueError("wan_checkpoint_state_invalid") + owned_config.append((name, value)) + return WanCheckpointValidationContext( + checkpoint_id=checkpoint_id, + request_id=request_id, + run_id=run_id, + backend_id=backend_id, + seed=seed, + step=step, + total_steps=total_steps, + parent_checkpoint_id=parent_checkpoint_id, + parent_optimizer_state_digest=parent_optimizer_state_digest, + trainable_targets=tuple(owned_targets), + trainable_parameter_specs=validate_trainable_parameter_specs( + trainable_parameter_specs, declared_targets=owned_targets + ), + lora_rank=lora_rank, + lora_alpha=lora_alpha, + lora_dropout=lora_dropout, + optimizer_algorithm=optimizer_algorithm, + optimizer_config=tuple(owned_config), + start_step=start_step, + telemetry_sequence_start=telemetry_sequence_start, + ) + + +def _snapshot_json_payload(value: object) -> bytes: + owned = _owned_json_value(value, set()) + if type(owned) not in {dict, list}: + raise ValueError + return canonical_json(cast(dict[str, Any], owned)).encode() + + +def _state_from_payloads( + payloads: Mapping[str, bytes], context: WanCheckpointValidationContext +) -> WanCheckpointState: + try: + decoded = { + role: json.loads(payloads[role]) + for role in _ROLES + if role != "weights" + } + telemetry = decoded["telemetry"] + if not isinstance(telemetry, list): + raise ValueError + return WanCheckpointState( + checkpoint_id=context.checkpoint_id, + step=context.step, + total_steps=context.total_steps, + parent_checkpoint_id=context.parent_checkpoint_id, + weights=payloads["weights"], + optimizer=cast(dict[str, object], decoded["optimizer"]), + trainer_state=cast(dict[str, object], decoded["trainer_state"]), + telemetry=tuple(cast(list[dict[str, object]], telemetry)), + trainable_targets=cast(dict[str, object], decoded["trainable_targets"]), + frozen_parameters=cast(dict[str, object], decoded["frozen_parameters"]), + finite_gradient_proof=cast( + dict[str, object], decoded["finite_gradient_proof"] + ), + ) + except (KeyError, TypeError, ValueError, json.JSONDecodeError): + raise ValueError("wan_checkpoint_restore_invalid") from None + + +def _owned_json_value(value: object, ancestors: set[int]) -> object: + if value is None or type(value) in {str, int, float, bool}: + return value + if type(value) not in {dict, list}: + raise ValueError + identity = id(value) + if identity in ancestors: + raise ValueError + ancestors.add(identity) + try: + if type(value) is dict: + mapping = cast(dict[object, object], value) + owned: dict[str, object] = {} + for key, item in mapping.items(): + if type(key) is not str: + raise ValueError + owned[key] = _owned_json_value(item, ancestors) + return owned + return [ + _owned_json_value(item, ancestors) for item in cast(list[object], value) + ] + finally: + ancestors.remove(identity) + + +def _validate_context(context: WanCheckpointValidationContext) -> None: + if ( + type(context) is not WanCheckpointValidationContext + or any( + type(value) is not str or not value + for value in ( + context.checkpoint_id, + context.request_id, + context.run_id, + context.backend_id, + context.optimizer_algorithm, + ) + ) + or type(context.seed) is not int + or type(context.step) is not int + or context.step <= 0 + or type(context.start_step) is not int + or not 0 <= context.start_step < context.step + or type(context.telemetry_sequence_start) is not int + or context.telemetry_sequence_start <= 0 + or type(context.total_steps) is not int + or context.total_steps < context.step + or type(context.lora_rank) is not int + or context.lora_rank <= 0 + or type(context.lora_alpha) is not int + or context.lora_alpha <= 0 + or type(context.lora_dropout) is not float + or not math.isfinite(context.lora_dropout) + or not 0 <= context.lora_dropout < 1 + or not context.trainable_targets + or any(type(target) is not str or not target for target in context.trainable_targets) + or len(set(context.trainable_targets)) != len(context.trainable_targets) + or validate_trainable_parameter_specs( + context.trainable_parameter_specs, + declared_targets=context.trainable_targets, + ) + != context.trainable_parameter_specs + or not context.optimizer_config + or tuple(name for name, _value in context.optimizer_config) + != tuple(sorted({name for name, _value in context.optimizer_config})) + or any( + type(name) is not str or not name or not _is_optimizer_value(value) + for name, value in context.optimizer_config + ) + or ( + context.parent_optimizer_state_digest is not None + and not _is_sha256_digest(context.parent_optimizer_state_digest) + ) + ): + raise ValueError("wan_checkpoint_state_invalid") + _safe_component(context.checkpoint_id) + if context.parent_checkpoint_id is not None: + _safe_component(context.parent_checkpoint_id) + + +def _validate_role_schemas( + state: WanCheckpointState, context: WanCheckpointValidationContext +) -> None: + try: + trainable = _exact_mapping( + state.trainable_targets, + { + "schema_version", "request_id", "run_id", "step", "targets", + "physical_parameters", + }, + ) + request_id = trainable.get("request_id") + run_id = trainable.get("run_id") + targets = trainable.get("targets") + if ( + trainable.get("schema_version") != "trainer-trainable-targets.v2" + or request_id != context.request_id + or run_id != context.run_id + or type(trainable.get("step")) is not int + or trainable.get("step") != state.step + or not isinstance(targets, list) + or not targets + or any(type(target) is not str or not target for target in targets) + or len(set(targets)) != len(targets) + or targets != list(context.trainable_targets) + or validate_trainable_parameter_specs( + trainable_parameter_specs_from_records(trainable.get("physical_parameters")), + declared_targets=context.trainable_targets, + ) + != context.trainable_parameter_specs + ): + raise ValueError + shared = { + "request_id": context.request_id, + "run_id": context.run_id, + "step": context.step, + } + _validate_trainer_role(state.trainer_state, state, shared, context) + _validate_optimizer_role(state.optimizer, targets, shared, context) + progress = _validate_telemetry_role( + state.telemetry, + state.step, + state.total_steps, + context.start_step, + context.telemetry_sequence_start, + ) + _validate_frozen_role(state.frozen_parameters, targets, shared) + _validate_finite_proof_role(state.finite_gradient_proof, progress, shared) + except (KeyError, TypeError, ValueError, OverflowError): + raise ValueError("wan_checkpoint_state_invalid") from None + + +def _exact_mapping(value: object, keys: set[str]) -> Mapping[str, object]: + if not isinstance(value, Mapping) or set(value) != keys: + raise ValueError + return value + + +def _validate_trainer_role( + value: object, + state: WanCheckpointState, + shared: Mapping[str, object], + context: WanCheckpointValidationContext, +) -> None: + trainer = _exact_mapping( + value, + { + "schema_version", + "request_id", + "run_id", + "step", + "backend_id", + "seed", + "total_steps", + "global_step", + "rng_states", + "scheduler_state", + "scaler_state", + "backend_state", + }, + ) + backend_id = trainer.get("backend_id") + seed = trainer.get("seed") + if ( + trainer.get("schema_version") != "trainer-state.v3" + or not _matches_shared(trainer, shared) + or backend_id != context.backend_id + or seed != context.seed + or type(trainer.get("total_steps")) is not int + or trainer.get("total_steps") != state.total_steps + or type(trainer.get("global_step")) is not int + or trainer.get("global_step") != state.step + or not _matches_rng_states(trainer.get("rng_states")) + or not _is_substantive_numeric_state(trainer.get("scheduler_state")) + or not _is_substantive_numeric_state(trainer.get("scaler_state")) + or trainer.get("backend_state") + != {context.backend_id: {"data_cursor": context.step}} + ): + raise ValueError + + +def _validate_optimizer_role( + value: object, + targets: list[object], + shared: Mapping[str, object], + context: WanCheckpointValidationContext, +) -> None: + optimizer = _exact_mapping( + value, + { + "schema_version", + "request_id", + "run_id", + "step", + "algorithm", + "config", + "global_step", + "parameter_groups", + "parameter_state", + "backend_state", + "parent_optimizer_state_digest", + }, + ) + config = optimizer.get("config") + groups = optimizer.get("parameter_groups") + parameter_state = optimizer.get("parameter_state") + parent_digest = optimizer.get("parent_optimizer_state_digest") + configured = dict(context.optimizer_config) + accumulation = configured.get("gradient_accumulation_steps", 1) + if type(accumulation) is not int or accumulation <= 0: + raise ValueError + residual = context.step % accumulation + terminal_partial = context.step == context.total_steps and residual > 0 + expected_updates = context.step // accumulation + int(terminal_partial) + expected_position = 0 if residual == 0 or terminal_partial else residual + expected_backend: dict[str, object] = {"optimizer_updates": expected_updates} + if accumulation > 1: + expected_backend["pending_gradients"] = _validated_pending_gradients( + cast( + Mapping[str, object], + cast(Mapping[str, object], optimizer.get("backend_state"))[ + context.backend_id + ], + ).get("pending_gradients"), + expected_position, + expected_parameters=tuple( + item.physical_name for item in context.trainable_parameter_specs + ), + expected_specs={ + item.physical_name: (item.shape, item.dtype, item.device) + for item in context.trainable_parameter_specs + }, + ) + if ( + optimizer.get("schema_version") != "trainer-optimizer-state.v2" + or not _matches_shared(optimizer, shared) + or optimizer.get("algorithm") != context.optimizer_algorithm + or not isinstance(config, Mapping) + or not _matches_optimizer_config(config, context.optimizer_config) + or type(optimizer.get("global_step")) is not int + or optimizer.get("global_step") != context.step + or not isinstance(groups, list) + or not groups + or not isinstance(parameter_state, Mapping) + or set(parameter_state) != set(targets) + or optimizer.get("backend_state") != {context.backend_id: expected_backend} + or parent_digest != context.parent_optimizer_state_digest + ): + raise ValueError + covered: list[object] = [] + group_ids: list[str] = [] + for value_group in groups: + group = _exact_mapping(value_group, {"group_id", "parameters", "hyperparameters"}) + group_id = group.get("group_id") + parameters = group.get("parameters") + if ( + type(group_id) is not str + or not group_id + or not isinstance(parameters, list) + or not parameters + or any(type(parameter) is not str or not parameter for parameter in parameters) + or group.get("hyperparameters") != config + ): + raise ValueError + group_ids.append(group_id) + covered.extend(parameters) + if len(set(group_ids)) != len(group_ids) or covered != targets: + raise ValueError + for target in targets: + parameter = _exact_mapping(parameter_state[target], {"step", "slots"}) + if ( + type(parameter.get("step")) is not int + or parameter.get("step") != context.step + or not _is_substantive_numeric_state(parameter.get("slots")) + ): + raise ValueError + + +def _validated_pending_gradients( + value: object, + position: int, + *, + expected_parameters: Collection[str], + expected_specs: Mapping[str, tuple[tuple[int, ...], str, str]] | None = None, +) -> Mapping[str, object]: + return validate_pending_gradients( + value, + position, + expected_parameters=expected_parameters, + expected_specs=expected_specs, + ) + + +def _validate_telemetry_role( + value: tuple[Mapping[str, object], ...], + step: int, + total_steps: int, + start_step: int, + sequence_start: int, +) -> TrainingProgress: + if len(value) != step - start_step: + raise ValueError + progress: TrainingProgress | None = None + for offset, raw_record in enumerate(value): + expected_step = start_step + offset + 1 + expected_sequence = sequence_start + offset + record = _exact_mapping(raw_record, {"sequence", "step", "payload"}) + if ( + record.get("sequence") != expected_sequence + or record.get("step") != expected_step + ): + raise ValueError + progress = TrainingProgress.model_validate(record.get("payload"), strict=True) + if progress.step != expected_step or progress.total_steps != total_steps: + raise ValueError + if progress is None or progress.step != step or not progress.checkpoint_boundary: + raise ValueError + return progress + + +def _validate_frozen_role( + value: object, targets: list[object], shared: Mapping[str, object] +) -> None: + frozen = _exact_mapping( + value, + { + "schema_version", + "request_id", + "run_id", + "step", + "trainable_targets", + "unchanged", + }, + ) + if ( + frozen.get("schema_version") != "trainer-frozen-parameters.v1" + or not _matches_shared(frozen, shared) + or frozen.get("trainable_targets") != targets + or frozen.get("unchanged") is not True + ): + raise ValueError + + +def _validate_finite_proof_role( + value: object, progress: TrainingProgress, shared: Mapping[str, object] +) -> None: + proof = _exact_mapping( + value, + { + "schema_version", + "request_id", + "run_id", + "step", + "finite", + "loss", + "gradient_norm", + }, + ) + if ( + proof.get("schema_version") != "trainer-finite-gradient-proof.v1" + or not _matches_shared(proof, shared) + or proof.get("finite") is not True + or not _is_finite_number(proof.get("loss")) + or not _is_finite_number(proof.get("gradient_norm")) + or float(cast(int | float, proof["gradient_norm"])) < 0 + or proof.get("loss") != progress.loss + or proof.get("gradient_norm") != progress.gradient_norm + ): + raise ValueError + + +def _is_finite_number(value: object) -> bool: + return type(value) in {int, float} and math.isfinite(float(cast(int | float, value))) + + +def _is_optimizer_value(value: object) -> bool: + return type(value) in {str, int, bool} or ( + type(value) is float and math.isfinite(value) + ) + + +def _matches_optimizer_config( + value: Mapping[str, object], expected: tuple[tuple[str, OptimizerValue], ...] +) -> bool: + if tuple(value) != tuple(name for name, _expected in expected): + return False + return all( + type(value[name]) is type(expected_value) and value[name] == expected_value + for name, expected_value in expected + ) + + +def _matches_shared( + value: Mapping[str, object], expected: Mapping[str, object] +) -> bool: + return ( + type(value.get("request_id")) is str + and value.get("request_id") == expected["request_id"] + and type(value.get("run_id")) is str + and value.get("run_id") == expected["run_id"] + and type(value.get("step")) is int + and value.get("step") == expected["step"] + ) + + +def _matches_rng_states(value: object) -> bool: + if not isinstance(value, Mapping) or set(value) != {"cpu", "mps"}: + return False + for device in ("cpu", "mps"): + item = value.get(device) + if not isinstance(item, Mapping) or set(item) != { + "algorithm", + "device", + "state_hex", + "digest", + }: + return False + algorithm = item.get("algorithm") + state_hex = item.get("state_hex") + digest = item.get("digest") + if ( + type(algorithm) is not str + or not algorithm + or item.get("device") != device + or type(state_hex) is not str + or not state_hex + or state_hex != state_hex.lower() + or len(state_hex) % 2 + or type(digest) is not str + ): + return False + try: + state = bytes.fromhex(state_hex) + except ValueError: + return False + if not state or "sha256:" + hashlib.sha256(state).hexdigest() != digest: + return False + return True + + +def _substantive_numeric_state(value: object) -> tuple[bool, bool]: + if isinstance(value, bool): + return True, False + if type(value) in {int, float}: + finite = math.isfinite(float(cast(int | float, value))) + return finite, finite + if isinstance(value, str): + return bool(value), False + if isinstance(value, list): + children = tuple(_substantive_numeric_state(item) for item in value) + return bool(children) and all(valid for valid, _ in children), any( + substantive for _, substantive in children + ) + if isinstance(value, Mapping): + children = tuple(_substantive_numeric_state(item) for item in value.values()) + return ( + bool(children) + and all(type(key) is str and bool(key) for key in value) + and all(valid for valid, _ in children), + any(substantive for _, substantive in children), + ) + return False, False + + +def _is_substantive_numeric_state(value: object) -> bool: + valid, substantive = _substantive_numeric_state(value) + return valid and substantive + + +def _is_bound_backend_state(value: object, backend_id: str) -> bool: + return ( + isinstance(value, Mapping) + and set(value) == {backend_id} + and _is_substantive_numeric_state(value[backend_id]) + ) + + +def _is_sha256_digest(value: object) -> bool: + return ( + type(value) is str + and len(value) == 71 + and value.startswith("sha256:") + and all(character in "0123456789abcdef" for character in value[7:]) + ) + + +def _finite_json(value: object) -> bool: + if value is None or isinstance(value, (bool, str)): + return True + if isinstance(value, (int, float)) and not isinstance(value, bool): + return math.isfinite(float(value)) + if isinstance(value, list | tuple): + return all(_finite_json(item) for item in value) + if isinstance(value, Mapping): + return all(type(key) is str and _finite_json(item) for key, item in value.items()) + return False + + +def _validate_safetensors(data: bytes) -> dict[str, tuple[str, tuple[int, ...]]]: + try: + if len(data) < 9: + raise ValueError + header_size = int.from_bytes(data[:8], "little") + if header_size <= 0 or header_size > len(data) - 8: + raise ValueError + raw_header = data[8 : 8 + header_size] + header = json.loads(raw_header) + if not isinstance(header, dict) or not header: + raise ValueError + canonical = json.dumps(header, sort_keys=True, separators=(",", ":")).encode() + if canonical + b" " * (header_size - len(canonical)) != raw_header: + raise ValueError + payload_size = len(data) - 8 - header_size + widths = {"F16": 2, "BF16": 2, "F32": 4} + ranges: list[tuple[int, int]] = [] + tensors: dict[str, tuple[str, tuple[int, ...]]] = {} + for name, value in header.items(): + if ( + type(name) is not str + or not name + or not isinstance(value, dict) + or set(value) != {"dtype", "shape", "data_offsets"} + or value["dtype"] not in widths + ): + raise ValueError + shape = value["shape"] + offsets = value["data_offsets"] + if ( + not isinstance(shape, list) + or len(shape) != 2 + or any(type(size) is not int or size <= 0 for size in shape) + or not isinstance(offsets, list) + or len(offsets) != 2 + or any(type(offset) is not int for offset in offsets) + or offsets[0] < 0 + or offsets[1] < offsets[0] + or offsets[1] > payload_size + or offsets[1] - offsets[0] != math.prod(shape) * widths[value["dtype"]] + ): + raise ValueError + ranges.append((offsets[0], offsets[1])) + tensors[name] = (value["dtype"], tuple(shape)) + if ( + sorted(ranges) != ranges + or ranges[0][0] != 0 + or ranges[-1][1] != payload_size + or any(left[1] != right[0] for left, right in pairwise(ranges)) + ): + raise ValueError + return tensors + except (ValueError, TypeError, KeyError, json.JSONDecodeError): + raise ValueError("wan_checkpoint_state_invalid") from None + + +def _validate_lora_tensors( + tensors: Mapping[str, tuple[str, tuple[int, ...]]], targets: list[str], rank: int +) -> None: + expected_names = { + f"{target}.lora_{suffix}.weight" for target in targets for suffix in ("A", "B") + } + if set(tensors) != expected_names: + raise ValueError("wan_checkpoint_state_invalid") + for target in targets: + dtype_a, shape_a = tensors[f"{target}.lora_A.weight"] + dtype_b, shape_b = tensors[f"{target}.lora_B.weight"] + if dtype_a != dtype_b or shape_a[0] != rank or shape_b[1] != rank: + raise ValueError("wan_checkpoint_state_invalid") + + +def _safe_component(value: str) -> str: + if ( + type(value) is not str + or not value + or value in {".", ".."} + or "/" in value + or "\x00" in value + or unicodedata.normalize("NFC", value) != value + ): + raise ValueError("wan_checkpoint_state_invalid") + return value + + +def _open_scratch_root(root: Path) -> tuple[int, tuple[int, int]]: + if not root.is_absolute() or any(part in {"", ".", ".."} for part in root.parts[1:]): + raise ValueError("unsafe_scratch_root") + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + current = os.open(root.parts[0], flags) + try: + for index, part in enumerate(root.parts[1:]): + final = index == len(root.parts[1:]) - 1 + try: + child = os.open(part, flags, dir_fd=current) + except FileNotFoundError: + if not final: + raise ValueError("unsafe_scratch_root") from None + os.mkdir(part, 0o700, dir_fd=current) + child = os.open(part, flags, dir_fd=current) + os.close(current) + current = child + info = os.fstat(current) + if not stat.S_ISDIR(info.st_mode): + raise ValueError("unsafe_scratch_root") + return current, (info.st_dev, info.st_ino) + except Exception: + os.close(current) + raise ValueError("unsafe_scratch_root") from None + + +def _open_exact_parent_file( + path: Path, expected: WanParentCheckpointFileDescriptor +) -> _RestoredFileLease: + if ( + not isinstance(path, Path) + or not path.is_absolute() + or any(part in {"", ".", ".."} for part in path.parts[1:]) + ): + raise ValueError("wan_checkpoint_restore_invalid") + directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC + file_flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK + if hasattr(os, "O_NOFOLLOW"): + directory_flags |= os.O_NOFOLLOW + file_flags |= os.O_NOFOLLOW + current = os.open(path.parts[0], directory_flags) + root_info = os.fstat(current) + ancestors = [ + _RestoredAncestorLease( + name=path.parts[0], + descriptor=current, + device=root_info.st_dev, + inode=root_info.st_ino, + ) + ] + try: + for part in path.parts[1:-1]: + child = os.open(part, directory_flags, dir_fd=current) + current = child + info = os.fstat(current) + if not stat.S_ISDIR(info.st_mode): + raise ValueError("wan_checkpoint_restore_invalid") + ancestors.append( + _RestoredAncestorLease( + name=part, + descriptor=current, + device=info.st_dev, + inode=info.st_ino, + ) + ) + descriptor = os.open(path.name, file_flags, dir_fd=current) + except BaseException: + for ancestor in ancestors: + os.close(ancestor.descriptor) + raise + info = os.fstat(descriptor) + if ( + not stat.S_ISREG(info.st_mode) + or (info.st_dev, info.st_ino) != (expected.device, expected.inode) + or info.st_size != expected.size_bytes + ): + os.close(descriptor) + for ancestor in ancestors: + os.close(ancestor.descriptor) + raise ValueError("wan_checkpoint_restore_invalid") + return _RestoredFileLease(expected, descriptor, path, tuple(ancestors)) + + +def _reverify_ancestor_chain(lease: _RestoredFileLease) -> None: + directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC + file_flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK + if hasattr(os, "O_NOFOLLOW"): + directory_flags |= os.O_NOFOLLOW + file_flags |= os.O_NOFOLLOW + current = os.dup(lease.ancestors[0].descriptor) + try: + root = os.fstat(current) + expected_root = lease.ancestors[0] + if ( + not stat.S_ISDIR(root.st_mode) + or (root.st_dev, root.st_ino) != (expected_root.device, expected_root.inode) + ): + raise OSError("checkpoint root replaced") + for expected in lease.ancestors[1:]: + child = os.open(expected.name, directory_flags, dir_fd=current) + os.close(current) + current = child + info = os.fstat(current) + retained = os.fstat(expected.descriptor) + identity = (expected.device, expected.inode) + if ( + not stat.S_ISDIR(info.st_mode) + or not stat.S_ISDIR(retained.st_mode) + or (info.st_dev, info.st_ino) != identity + or (retained.st_dev, retained.st_ino) != identity + ): + raise OSError("checkpoint ancestor replaced") + leaf = os.open(lease.path.name, file_flags, dir_fd=current) + try: + info = os.fstat(leaf) + if ( + not stat.S_ISREG(info.st_mode) + or (info.st_dev, info.st_ino) + != (lease.descriptor.device, lease.descriptor.inode) + or info.st_size != lease.descriptor.size_bytes + ): + raise OSError("checkpoint file replaced") + finally: + os.close(leaf) + finally: + os.close(current) + + +def _read_open_regular(descriptor: int, expected_size: int) -> bytes: + os.lseek(descriptor, 0, os.SEEK_SET) + data = bytearray() + while len(data) <= expected_size: + chunk = os.read(descriptor, min(1024 * 1024, expected_size + 1 - len(data))) + if not chunk: + break + data.extend(chunk) + info = os.fstat(descriptor) + if ( + len(data) != expected_size + or info.st_size != expected_size + or not stat.S_ISREG(info.st_mode) + ): + raise ValueError("wan_checkpoint_restore_invalid") + return bytes(data) + + +def _open_directory(name: str, parent_fd: int) -> int: + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + return os.open(name, flags, dir_fd=parent_fd) + + +def _require_destination_absent(root_fd: int, destination: str) -> None: + try: + os.stat(destination, dir_fd=root_fd, follow_symlinks=False) + except FileNotFoundError: + return + raise FileExistsError(errno.EEXIST, "checkpoint destination exists", destination) + + +def _publish_directory_no_replace(root_fd: int, source: str, destination: str) -> None: + """Atomically rename one directory without ever replacing a destination.""" + library = ctypes.CDLL(None, use_errno=True) + if sys.platform == "darwin": + function_name = "renameatx_np" + no_replace_flag = 0x00000004 # RENAME_EXCL + elif sys.platform.startswith("linux"): + function_name = "renameat2" + no_replace_flag = 0x00000001 # RENAME_NOREPLACE + else: + raise OSError(errno.ENOTSUP, "atomic no-replace rename unsupported") + try: + rename = getattr(library, function_name) + except AttributeError: + raise OSError(errno.ENOTSUP, "atomic no-replace rename unsupported") from None + rename.argtypes = ( + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ) + rename.restype = ctypes.c_int + ctypes.set_errno(0) + result = rename( + root_fd, + os.fsencode(source), + root_fd, + os.fsencode(destination), + no_replace_flag, + ) + if result == 0: + return + error_number = ctypes.get_errno() + if error_number in {errno.EEXIST, errno.ENOTEMPTY}: + raise FileExistsError(errno.EEXIST, "checkpoint destination exists", destination) + unsupported_errors = { + errno.ENOSYS, + errno.ENOTSUP, + getattr(errno, "EOPNOTSUPP", errno.ENOTSUP), + } + if error_number in unsupported_errors: + raise OSError(errno.ENOTSUP, "atomic no-replace rename unsupported") + raise OSError(error_number, os.strerror(error_number)) + + +def _write_exclusive(directory_fd: int, name: str, payload: bytes) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(name, flags, 0o600, dir_fd=directory_fd) + try: + view = memoryview(payload) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise OSError("short checkpoint write") + view = view[written:] + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _read_regular(directory_fd: int, name: str, expected_size: int) -> bytes: + flags = os.O_RDONLY | os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(name, flags, dir_fd=directory_fd) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_size != expected_size: + raise OSError("checkpoint file replaced") + data = bytearray() + while len(data) <= expected_size: + chunk = os.read(descriptor, min(1024 * 1024, expected_size + 1 - len(data))) + if not chunk: + break + data.extend(chunk) + if len(data) != expected_size or os.fstat(descriptor).st_size != expected_size: + raise OSError("checkpoint file changed") + return bytes(data) + finally: + os.close(descriptor) + + +def _reverify_root(root: Path, descriptor: int, identity: tuple[int, int]) -> None: + try: + held = os.fstat(descriptor) + current = root.lstat() + except OSError: + raise ValueError("unsafe_scratch_root") from None + if ( + not stat.S_ISDIR(held.st_mode) + or stat.S_ISLNK(current.st_mode) + or (held.st_dev, held.st_ino) != identity + or (current.st_dev, current.st_ino) != identity + ): + raise ValueError("unsafe_scratch_root") + + +def _remove_checkpoint( + root_fd: int, directory_name: str, names: tuple[str, ...], open_fd: int +) -> None: + cleanup_errors: list[BaseException] = [] + descriptor = open_fd + owned_descriptor = False + if descriptor < 0: + try: + descriptor = _open_directory(directory_name, root_fd) + owned_descriptor = True + except FileNotFoundError: + descriptor = -1 + except OSError as error: + cleanup_errors.append(error) + descriptor = -1 + if descriptor >= 0: + try: + for name in names: + try: + _unlink_checkpoint_entry(descriptor, name) + except FileNotFoundError: + pass + except OSError as error: + cleanup_errors.append(error) + finally: + if owned_descriptor: + try: + os.close(descriptor) + except OSError as error: + cleanup_errors.append(error) + try: + _remove_checkpoint_directory(root_fd, directory_name) + except FileNotFoundError: + pass + except OSError as error: + cleanup_errors.append(error) + try: + _fsync_directory(root_fd) + except OSError as error: + cleanup_errors.append(error) + if cleanup_errors: + raise WanCheckpointCleanupError(None, tuple(cleanup_errors)) + + +def _unlink_checkpoint_entry(directory_fd: int, name: str) -> None: + os.unlink(name, dir_fd=directory_fd) + + +def _remove_checkpoint_directory(root_fd: int, directory_name: str) -> None: + os.rmdir(directory_name, dir_fd=root_fd) + + +def _fsync_directory(descriptor: int) -> None: + os.fsync(descriptor) + + +def _close_directory(descriptor: int) -> None: + os.close(descriptor) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_continuity.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_continuity.py new file mode 100644 index 0000000..b839535 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_continuity.py @@ -0,0 +1,147 @@ +"""Path-free, preregistered continuity evidence for Wan branch state.""" + +from __future__ import annotations + +import hashlib +import math +from collections.abc import Iterable, Mapping +from typing import Any, Literal, cast + +from pydantic import ConfigDict, Field, StrictBool, StrictFloat, field_validator + +from video_lab.domain import FrozenModel, canonical_json +from video_lab.trainers.wan_production_models import ( + WanBranchRecipeV1, + wan_branch_recipe_digest, +) +from video_lab.training_models import Sha256Digest + + +class WanContinuityComparisonV1(FrozenModel): + """Redacted numerical evidence for an uninterrupted/restored comparison.""" + + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + + schema_version: Literal["wan-branch-continuity-comparison.v1"] = ( + "wan-branch-continuity-comparison.v1" + ) + recipe_digest: Sha256Digest + tolerance_digest: Sha256Digest + accepted: StrictBool + max_absolute_error: StrictFloat = Field(ge=0) + max_relative_error: StrictFloat = Field(ge=0) + failing_tensor_names: tuple[str, ...] + + @field_validator("max_absolute_error", "max_relative_error") + @classmethod + def finite_errors(cls, value: float) -> float: + if not math.isfinite(value): + raise ValueError("wan_continuity_nonfinite") + return value + + @field_validator("failing_tensor_names") + @classmethod + def canonical_names(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if value != tuple(sorted(set(value))) or any(not name for name in value): + raise ValueError("wan_continuity_structure_invalid") + return value + + +def _tensor(value: object) -> tuple[tuple[int, ...], str, tuple[float, ...]]: + candidate: Any = value + dtype: str | None = None + shape: tuple[int, ...] | None = None + if hasattr(candidate, "detach"): + candidate = candidate.detach() + dtype = str(candidate.dtype) + shape = tuple(int(size) for size in candidate.shape) + if hasattr(candidate, "to"): + candidate = candidate.to("cpu") + if shape is not None and hasattr(candidate, "reshape") and hasattr(candidate, "tolist"): + candidate = candidate.reshape(-1).tolist() + if not isinstance(candidate, Iterable) or isinstance(candidate, (str, bytes, Mapping)): + raise ValueError("wan_continuity_structure_invalid") + if shape is None: + def inspect(sequence: object) -> tuple[tuple[int, ...], str, list[float]]: + if type(sequence) in (int, float): + number = float(cast(int | float, sequence)) + if not math.isfinite(number): + raise ValueError("wan_continuity_nonfinite") + return (), "float64" if type(sequence) is float else "int64", [number] + if not isinstance(sequence, (tuple, list)) or not sequence: + raise ValueError("wan_continuity_structure_invalid") + children = [inspect(item) for item in sequence] + if any(child[:2] != children[0][:2] for child in children[1:]): + raise ValueError("wan_continuity_structure_invalid") + return (len(children), *children[0][0]), children[0][1], [ + item for child in children for item in child[2] + ] + + shape, dtype, result = inspect(candidate) + else: + result = [] + for item in candidate: + if type(item) not in (int, float): + raise ValueError("wan_continuity_structure_invalid") + number = float(item) + if not math.isfinite(number): + raise ValueError("wan_continuity_nonfinite") + result.append(number) + if not result: + raise ValueError("wan_continuity_structure_invalid") + assert dtype is not None + return shape, dtype, tuple(result) + + +def compare_wan_branch_continuity( + reference: Mapping[str, object], + candidate: Mapping[str, object], + tolerance: WanBranchRecipeV1, +) -> WanContinuityComparisonV1: + """Compare exact state names/shapes with the recipe's immutable tolerance.""" + + if type(tolerance) is not WanBranchRecipeV1 or set(reference) != set(candidate): + raise ValueError("wan_continuity_structure_invalid") + if not reference or any(type(name) is not str or not name for name in reference): + raise ValueError("wan_continuity_structure_invalid") + atol, rtol = tolerance.tolerance.atol, tolerance.tolerance.rtol + maximum_absolute = 0.0 + maximum_relative = 0.0 + failed: list[str] = [] + for name in sorted(reference): + expected_shape, expected_dtype, expected = _tensor(reference[name]) + observed_shape, observed_dtype, observed = _tensor(candidate[name]) + if (expected_shape, expected_dtype) != (observed_shape, observed_dtype): + raise ValueError("wan_continuity_structure_invalid") + tensor_failed = False + for expected_value, observed_value in zip(expected, observed, strict=True): + absolute = abs(observed_value - expected_value) + relative = absolute / max(abs(expected_value), atol) + maximum_absolute = max(maximum_absolute, absolute) + maximum_relative = max(maximum_relative, relative) + if absolute > atol + rtol * abs(expected_value): + tensor_failed = True + if tensor_failed: + failed.append(name) + recipe_digest = wan_branch_recipe_digest(tolerance) + tolerance_digest = "sha256:" + hashlib.sha256( + canonical_json( + { + "domain": "wan-branch-continuity-tolerance.v1", + "recipe_digest": recipe_digest, + "atol": atol, + "rtol": rtol, + } + ).encode() + ).hexdigest() + return WanContinuityComparisonV1( + recipe_digest=recipe_digest, + tolerance_digest=tolerance_digest, + accepted=not failed, + max_absolute_error=maximum_absolute, + max_relative_error=maximum_relative, + failing_tensor_names=tuple(failed), + ) + + +__all__ = ("WanContinuityComparisonV1", "compare_wan_branch_continuity") diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_diffusers_runtime.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_diffusers_runtime.py new file mode 100644 index 0000000..052d2b8 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_diffusers_runtime.py @@ -0,0 +1,1289 @@ +"""Concrete local-only Diffusers/PEFT runtime for precomputed Wan training batches.""" + +from __future__ import annotations + +import base64 +import hashlib +import importlib +import json +import math +import shutil +import struct +import threading +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from time import monotonic +from typing import Any, Protocol, cast + +from video_lab.domain import canonical_json +from video_lab.trainer_protocol import CalibrationRequest, LoRAConfiguration, TrainingRequest +from video_lab.trainers.wan_batch import ( + validate_loaded_wan_batch, + validate_wan_batch_envelope, +) +from video_lab.trainers.wan_checkpoint import WanCheckpointState +from video_lab.trainers.wan_mps import ( + WAN_LORA_MPS_BACKEND_ID, + WanMpsObservation, + WanPreparedTraining, + WanRuntimeCheckpointState, + WanRuntimeRestoreReceipt, + WanTrainingSources, +) +from video_lab.trainers.wan_production_models import WanBranchRecipeV1 +from video_lab.training_models import Sha256Digest +from video_lab.wan_pending import validate_pending_gradients + +WAN_BATCH_SCHEMA = "wan-precomputed-training-batch.v1" +_WAN_BATCH_TENSORS = { + "hidden_states", + "timestep", + "encoder_hidden_states", + "clean_latents", + "noise_latents", + "target", + "transformer_selector", +} +_SAFETENSORS_DTYPE_BYTES = { + "BOOL": 1, + "U8": 1, + "I8": 1, + "I16": 2, + "U16": 2, + "F16": 2, + "BF16": 2, + "I32": 4, + "U32": 4, + "F32": 4, + "I64": 8, + "U64": 8, + "F64": 8, +} +_MAX_SAFETENSORS_HEADER_BYTES = 16 * 1024 * 1024 +_MAX_SAFETENSORS_RANK = 32 +_ADAMW_KEYS = { + "learning_rate", + "gradient_accumulation_steps", + "beta1", + "beta2", + "eps", + "weight_decay", + "amsgrad", + "maximize", + "foreach", + "fused", + "capturable", + "differentiable", +} +_PRODUCTION_RNG_LOCK = threading.Lock() + + +class _WanBranchPreparedFactory(Protocol): + def prepare_branch( + self, sources: object, request: object, branch_recipe: WanBranchRecipeV1 + ) -> WanPreparedTraining: ... + + +@dataclass(frozen=True, slots=True) +class WanDiffusersAdapterPaths: + wan_snapshot: Path + high_transformer: Path + low_transformer: Path + batch_manifest: Path + batch_tensors: Path + + +class WanDiffusersTrainingRuntime: + """Repository-owned forward/loss/optimizer implementation; imports libraries lazily.""" + + def __init__( + self, + *, + root: Path, + paths: WanDiffusersAdapterPaths, + runtime_digest: Sha256Digest, + importer: Callable[[str], object] = importlib.import_module, + cleanup: Callable[[], None] | None = None, + prepared_factory: _WanBranchPreparedFactory | None = None, + branch_recipe: WanBranchRecipeV1 | None = None, + ) -> None: + self._root, self._paths = root, paths + self._runtime_digest, self._importer = runtime_digest, importer + self._cleanup = cleanup + if (prepared_factory is None) != (branch_recipe is None): + raise ValueError("wan_runtime_branch_factory_invalid") + self._prepared_factory = prepared_factory + self._branch_recipe = branch_recipe + self._branch_prepared: set[int] = set() + self._prepared: set[int] = set() + self._state_lock = threading.Lock() + self._operation_held = False + self._root_identity = _capture_root_identity(root) + + def observe_host(self) -> WanMpsObservation: + torch = cast(Any, self._importer("torch")) + built = bool(torch.backends.mps.is_built()) + available = bool(torch.backends.mps.is_available()) + bf16 = False + if built and available: + bf16 = _is_mps_device( + torch.empty(1, device="mps", dtype=torch.bfloat16).device + ) + maximum = int(torch.mps.recommended_max_memory()) if available else 0 + allocated = int(torch.mps.driver_allocated_memory()) if available else 0 + return WanMpsObservation( + runtime_digest=self._runtime_digest, + built=built, + available=available, + bf16_supported=bf16, + component_placements=tuple( + (name, "mps") for name in ("text_encoder", "transformer", "transformer_2", "vae") + ), + disk_free_bytes=shutil.disk_usage(self._root).free, + disk_required_bytes=1, + memory_headroom_bytes=max(0, maximum - allocated), + memory_required_bytes=1, + ) + + def prepare( + self, sources: WanTrainingSources, request: TrainingRequest | CalibrationRequest + ) -> WanPreparedTraining: + self._acquire_production_operation() + try: + if self._prepared_factory is not None: + assert self._branch_recipe is not None + prepared = self._prepared_factory.prepare_branch( + sources, request, self._branch_recipe + ) + with self._state_lock: + self._prepared.add(id(prepared.handle)) + self._branch_prepared.add(id(prepared.handle)) + return prepared + del sources + self._reverify_root() + operations = _WanDiffusersOperations( + root=self._root, + paths=self._paths, + request=request, + importer=self._importer, + reverify=self._reverify_root, + ) + operations.load() + self._reverify_root() + prepared = WanPreparedTraining(handle=operations) + with self._state_lock: + self._prepared.add(id(operations)) + return prepared + except BaseException: + self._release_production_operation() + raise + + def _reverify_root(self) -> None: + if _capture_root_identity(self._root) != self._root_identity: + raise ValueError("wan_runtime_source_invalid") + + def release(self, prepared: WanPreparedTraining | None) -> None: + release_operation = False + branch_prepared = False + if prepared is not None: + with self._state_lock: + identifier = id(prepared.handle) + release_operation = identifier in self._prepared + branch_prepared = identifier in self._branch_prepared + self._prepared.discard(identifier) + self._branch_prepared.discard(identifier) + if branch_prepared: + assert prepared is not None + try: + close = getattr(prepared.handle, "close", None) + if not callable(close): + raise ValueError("wan_runtime_branch_handle_invalid") + close() + finally: + if release_operation: + self._release_production_operation() + return + try: + torch = cast(Any, self._importer("torch")) + torch.mps.synchronize() + torch.mps.empty_cache() + except Exception: + pass + if release_operation: + self._release_production_operation() + + def close(self) -> None: + with self._state_lock: + if self._prepared or self._operation_held: + raise ValueError("wan_runtime_busy") + cleanup = self._cleanup + if cleanup is not None: + cleanup() + self._cleanup = None + + def _acquire_production_operation(self) -> None: + with self._state_lock: + if self._operation_held or not _PRODUCTION_RNG_LOCK.acquire(blocking=False): + raise ValueError("wan_runtime_concurrent_operation") + self._operation_held = True + + def _release_production_operation(self) -> None: + with self._state_lock: + if not self._operation_held: + raise ValueError("wan_runtime_operation_missing") + self._operation_held = False + _PRODUCTION_RNG_LOCK.release() + + +class _WanDiffusersOperations: + def __init__( + self, + *, + root: Path, + paths: WanDiffusersAdapterPaths, + request: TrainingRequest | CalibrationRequest, + importer: Callable[[str], object], + reverify: Callable[[], None], + ) -> None: + self.root, self.paths, self.request, self.importer = root, paths, request, importer + self.torch: Any = None + self.reverify = reverify + self.components: dict[str, Any] = {} + self.batch: dict[str, Any] = {} + self.optimizer: Any = None + self.step = 0 + self.batch_cursor = 0 + self.accumulation_position = 0 + self.optimizer_updates = 0 + self.scheduler_state: dict[str, object] = {"step": 0, "rate_scale": 1.0} + self.scaler_state: dict[str, object] = { + "schema_version": "torch-amp-scaler-state.v1", + "enabled": False, + "scale": 1.0, + } + self._restored: WanRuntimeRestoreReceipt | None = None + + def load(self) -> None: + self.reverify() + self.torch = cast(Any, self.importer("torch")) + self.torch.manual_seed(self.request.seed) + self.torch.mps.manual_seed(self.request.seed) + self.torch.use_deterministic_algorithms(True) + if self.request.lora.dropout != 0: + raise ValueError("wan_runtime_stochastic_recipe_unsupported") + diffusers = cast(Any, self.importer("diffusers")) + transformers = cast(Any, self.importer("transformers")) + local = {"local_files_only": True, "torch_dtype": self.torch.bfloat16} + wan = self._path(self.paths.wan_snapshot) + self.components = { + "text_encoder": transformers.UMT5EncoderModel.from_pretrained( + str(wan), subfolder="text_encoder", **local + ), + "vae": diffusers.AutoencoderKLWan.from_pretrained(str(wan), subfolder="vae", **local), + "transformer": diffusers.WanTransformer3DModel.from_single_file( + self._path(self.paths.high_transformer), + config=str(wan), + subfolder="transformer", + **local, + ), + "transformer_2": diffusers.WanTransformer3DModel.from_single_file( + self._path(self.paths.low_transformer), + config=str(wan), + subfolder="transformer", + **local, + ), + } + for component in self.components.values(): + component.to(device="mps", dtype=self.torch.bfloat16) + _require_full_mps(component) + self.batch = self._load_batch() + self.reverify() + + def eligible_lora_modules(self) -> tuple[str, ...]: + self.reverify() + names: list[str] = [] + for prefix in ("transformer", "transformer_2"): + for name, module in self.components[prefix].named_modules(): + if name.endswith(("to_q", "to_k", "to_v", "to_out.0")) and hasattr( + module, "weight" + ): + names.append(f"{prefix}.{name}") + return tuple(sorted(names)) + + def inject_lora( + self, + *, + targets: tuple[str, ...], + configuration: LoRAConfiguration, + seed: int, + device: str, + dtype: str, + ) -> None: + self.reverify() + if device != "mps" or dtype != "bfloat16": + raise ValueError("wan_runtime_placement_invalid") + self.torch.manual_seed(seed) + peft = cast(Any, self.importer("peft")) + for component in self.components.values(): + for parameter in component.parameters(): + parameter.requires_grad_(False) + for prefix in ("transformer", "transformer_2"): + selected = tuple( + name.removeprefix(prefix + ".") for name in targets if name.startswith(prefix + ".") + ) + if selected: + config = peft.LoraConfig( + r=configuration.rank, + lora_alpha=configuration.alpha, + lora_dropout=configuration.dropout, + target_modules=list(selected), + ) + self.components[prefix].add_adapter(config, adapter_name="character") + trainable = [ + parameter + for component in self.components.values() + for parameter in component.parameters() + if parameter.requires_grad + ] + if not trainable: + raise ValueError("wan_runtime_targets_invalid") + raw_config = ( + {item.name: item.value for item in self.request.optimizer.parameters} + if isinstance(self.request, TrainingRequest) + else {"learning_rate": 0.001} + ) + config = _normalized_adamw_config(raw_config) + self.gradient_accumulation_steps = cast(int, config.pop("gradient_accumulation_steps")) + self.optimizer = self.torch.optim.AdamW(trainable, **config) + self._verify() + self.reverify() + + def parameter_inventory(self) -> tuple[object, ...]: + self.reverify() + values: list[dict[str, object]] = [] + for prefix, component in self.components.items(): + for name, parameter in component.named_parameters(): + payload = ( + parameter.detach() + .to("cpu") + .contiguous() + .view(self.torch.uint8) + .numpy() + .tobytes() + ) + values.append( + { + "name": f"{prefix}.{name}", + "module": _lora_target_module(prefix, name), + "requires_grad": bool(parameter.requires_grad), + "digest": "sha256:" + hashlib.sha256(payload).hexdigest(), + "device": str(parameter.device), + "dtype": str(parameter.dtype).removeprefix("torch."), + "shape": tuple(parameter.shape), + } + ) + return tuple(values) + + def measure_forward_backward(self) -> object: + self.reverify() + started = monotonic() + loss = self._loss() + loss.backward() + gradient = self._gradient_norm() + self.optimizer.zero_grad(set_to_none=True) + result = self._measurement(loss, gradient, monotonic() - started) + self.reverify() + return result + + def train_step(self, *, step: int, total_steps: int) -> object: + self.reverify() + accumulation = getattr(self, "gradient_accumulation_steps", 1) + if self.accumulation_position == 0: + self.optimizer.zero_grad(set_to_none=True) + loss = self._loss() + loss.backward() + self.accumulation_position += 1 + if _optimizer_update_due( + step=step, total_steps=total_steps, accumulation_steps=accumulation + ): + _normalize_accumulated_gradients( + tuple(self._named_trainable().values()), + accumulation_position=self.accumulation_position, + ) + gradient = self._gradient_norm() + self.optimizer.step() + self.optimizer.zero_grad(set_to_none=True) + self.optimizer_updates += 1 + self.accumulation_position = 0 + else: + gradient = self._gradient_norm() + self.step = step + self.batch_cursor = step + self.scheduler_state = {"step": step, "rate_scale": 1.0} + self._verify() + self.reverify() + return {"loss": float(loss.detach().item()), "gradient_norm": gradient} + + def checkpoint_state(self, *, step: int) -> WanRuntimeCheckpointState: + self.reverify() + safetensors = cast(Any, self.importer("safetensors.torch")) + weights = { + name: parameter.detach().to("cpu").contiguous() + for prefix, component in self.components.items() + for name, parameter in component.named_parameters() + if parameter.requires_grad + for name in (f"{prefix}.{name}",) + } + weight_bytes = bytes(safetensors.save(weights)) + targets = self.request.trainable_targets + config = ( + {item.name: item.value for item in self.request.optimizer.parameters} + if isinstance(self.request, TrainingRequest) + else {"learning_rate": 0.001} + ) + accumulation = int(config.get("gradient_accumulation_steps", 1)) + backend_optimizer: dict[str, object] = {"optimizer_updates": self.optimizer_updates} + if accumulation > 1: + backend_optimizer["pending_gradients"] = _pending_gradient_record( + self._named_trainable(), + accumulation_position=self.accumulation_position, + ) + optimizer = { + "schema_version": "trainer-optimizer-state.v2", + "request_id": self.request.request_id, + "run_id": self.request.run_id, + "step": step, + "algorithm": "adamw", + "config": config, + "global_step": step, + "parameter_groups": [ + {"group_id": "default", "parameters": list(targets), "hyperparameters": config} + ], + "parameter_state": self._optimizer_parameter_state(step, targets), + "backend_state": {WAN_LORA_MPS_BACKEND_ID: backend_optimizer}, + "parent_optimizer_state_digest": getattr(self.request, "optimizer_state_digest", None), + } + trainer = self._observed_trainer_state(step) + return WanRuntimeCheckpointState( + weights=weight_bytes, optimizer=optimizer, trainer_state=trainer + ) + + def restore_checkpoint(self, state: WanCheckpointState) -> None: + self.reverify() + safetensors = cast(Any, self.importer("safetensors.torch")) + self.optimizer_updates = cast( + int, + cast( + Mapping[str, object], + cast(Mapping[str, object], state.optimizer["backend_state"])[ + WAN_LORA_MPS_BACKEND_ID + ], + )["optimizer_updates"], + ) + weights = safetensors.load(state.weights) + for prefix, component in self.components.items(): + by_name = dict(component.named_parameters()) + for qualified, tensor in weights.items(): + if qualified.startswith(prefix + "."): + by_name[qualified.removeprefix(prefix + ".")].data.copy_(tensor.to("mps")) + self._restore_optimizer_parameter_state(state.optimizer) + self._restore_trainer_state(state.trainer_state, state.optimizer) + self.step = state.step + self._verify() + self.reverify() + self._restored = WanRuntimeRestoreReceipt( + weights_digest="sha256:" + hashlib.sha256(state.weights).hexdigest(), + optimizer_payload=canonical_json( + self._observed_optimizer_payload(state.optimizer) + ).encode(), + trainer_state_payload=canonical_json(self._observed_trainer_state(state.step)).encode(), + ) + + def observe_restored_state(self) -> object: + if self._restored is None: + raise ValueError("wan_runtime_restore_missing") + self._verify() + self.reverify() + return self._restored + + def _loss(self) -> Any: + predicted = self.torch.empty_like(self.batch["target"]) + selector = self.batch["transformer_selector"] + for value, name in ((0, "transformer"), (1, "transformer_2")): + mask = selector == value + if bool(mask.any().item()): + result = self.components[name]( + hidden_states=self.batch["hidden_states"][mask], + timestep=self.batch["timestep"][mask], + encoder_hidden_states=self.batch["encoder_hidden_states"][mask], + return_dict=True, + ) + predicted[mask] = result.sample + loss = self.torch.nn.functional.mse_loss(predicted, self.batch["target"]) + if not bool(self.torch.isfinite(loss).item()): + raise ValueError("wan_training_step_invalid") + return loss + + def _named_trainable(self) -> dict[str, Any]: + return { + f"{prefix}.{name}": parameter + for prefix, component in self.components.items() + for name, parameter in component.named_parameters() + if parameter.requires_grad + } + + def _optimizer_parameter_state(self, step: int, targets: tuple[str, ...]) -> dict[str, object]: + safetensors = cast(Any, self.importer("safetensors.torch")) + named = self._named_trainable() + result: dict[str, object] = {} + covered: set[str] = set() + for target in targets: + parameters: dict[str, object] = {} + for name, parameter in sorted(named.items()): + if _lora_target_module(*name.split(".", 1)) != target: + continue + state = self.optimizer.state.get(parameter) + if not isinstance(state, Mapping): + raise ValueError("wan_runtime_optimizer_invalid") + if not state and self.optimizer_updates == 0: + parameters[name] = {"initialized": False} + covered.add(name) + continue + if not state: + raise ValueError("wan_runtime_optimizer_invalid") + group = self.optimizer.param_groups[0] + _validate_adamw_slots( + state, + parameter, + amsgrad=bool(group["amsgrad"]), + capturable=bool(group["capturable"]), + fused=bool(group["fused"]), + ) + slot_tensors = { + str(slot): value.detach().to("cpu").contiguous() + for slot, value in state.items() + if hasattr(value, "detach") + } + if set(slot_tensors) != set(state): + raise ValueError("wan_runtime_optimizer_invalid") + payload = bytes(safetensors.save(slot_tensors)) + parameters[name] = { + "serialized_b64": base64.b64encode(payload).decode(), + "serialized_size": len(payload), + "digest": "sha256:" + hashlib.sha256(payload).hexdigest(), + } + covered.add(name) + if not parameters: + raise ValueError("wan_runtime_optimizer_invalid") + result[target] = { + "step": step, + "slots": {"parameter_count": len(parameters), "parameters": parameters}, + } + if covered != set(named): + raise ValueError("wan_runtime_optimizer_invalid") + return result + + def _restore_optimizer_parameter_state(self, optimizer: Mapping[str, object]) -> None: + safetensors = cast(Any, self.importer("safetensors.torch")) + named = self._named_trainable() + parameter_state = cast(Mapping[str, object], optimizer["parameter_state"]) + if tuple(parameter_state) != self.request.trainable_targets: + raise ValueError("wan_runtime_optimizer_invalid") + restored: set[str] = set() + self.optimizer.state.clear() + for target in self.request.trainable_targets: + record = cast(Mapping[str, object], parameter_state[target]) + slots = cast(Mapping[str, object], record["slots"]) + parameters = cast(Mapping[str, object], slots["parameters"]) + if slots.get("parameter_count") != len(parameters): + raise ValueError("wan_runtime_optimizer_invalid") + for name, raw_record in parameters.items(): + if ( + name in restored + or name not in named + or _lora_target_module(*name.split(".", 1)) != target + ): + raise ValueError("wan_runtime_optimizer_invalid") + encoded = cast(Mapping[str, object], raw_record) + if encoded == {"initialized": False}: + if self.optimizer_updates != 0: + raise ValueError("wan_runtime_optimizer_invalid") + restored.add(name) + continue + payload = base64.b64decode(cast(str, encoded["serialized_b64"]), validate=True) + if ( + encoded.get("serialized_size") != len(payload) + or encoded.get("digest") != "sha256:" + hashlib.sha256(payload).hexdigest() + ): + raise ValueError("wan_runtime_optimizer_invalid") + values = safetensors.load(payload) + group = self.optimizer.param_groups[0] + parameter = named[name] + restored_state = { + slot: tensor.to( + device=( + parameter.device + if slot != "step" or group["capturable"] or group["fused"] + else "cpu" + ) + ) + for slot, tensor in values.items() + } + _validate_adamw_slots( + restored_state, + parameter, + amsgrad=bool(group["amsgrad"]), + capturable=bool(group["capturable"]), + fused=bool(group["fused"]), + ) + self.optimizer.state[parameter] = restored_state + restored.add(name) + if restored != set(named): + raise ValueError("wan_runtime_optimizer_invalid") + + def _observed_optimizer_payload(self, original: Mapping[str, object]) -> dict[str, object]: + observed = dict(original) + observed["parameter_state"] = self._optimizer_parameter_state( + self.step, self.request.trainable_targets + ) + backend: dict[str, object] = {"optimizer_updates": self.optimizer_updates} + if getattr(self, "gradient_accumulation_steps", 1) > 1: + backend["pending_gradients"] = _pending_gradient_record( + self._named_trainable(), + accumulation_position=self.accumulation_position, + ) + observed["backend_state"] = {WAN_LORA_MPS_BACKEND_ID: backend} + return observed + + def _observed_trainer_state(self, step: int) -> dict[str, object]: + return _trainer_state( + self.request, + step, + cpu_rng=_capture_global_rng(self.torch, "cpu"), + mps_rng=_capture_global_rng(self.torch, "mps"), + scheduler_state=self.scheduler_state, + scaler_state=self.scaler_state, + batch_cursor=self.batch_cursor, + ) + + def _restore_trainer_state( + self, trainer: Mapping[str, object], optimizer: Mapping[str, object] + ) -> None: + rng_states = cast(Mapping[str, object], trainer["rng_states"]) + _restore_global_rng(self.torch, rng_states["cpu"], "cpu") + _restore_global_rng(self.torch, rng_states["mps"], "mps") + self.scheduler_state = dict(cast(Mapping[str, object], trainer["scheduler_state"])) + self.scaler_state = dict(cast(Mapping[str, object], trainer["scaler_state"])) + backend = cast(Mapping[str, object], trainer["backend_state"]) + owned = cast(Mapping[str, object], backend[WAN_LORA_MPS_BACKEND_ID]) + self.batch_cursor = cast(int, owned["data_cursor"]) + optimizer_backend = cast( + Mapping[str, object], + cast(Mapping[str, object], optimizer["backend_state"])[WAN_LORA_MPS_BACKEND_ID], + ) + self.optimizer_updates = cast(int, optimizer_backend["optimizer_updates"]) + pending = optimizer_backend.get("pending_gradients") + self.accumulation_position = ( + _restore_pending_gradients( + self.torch, + self._named_trainable(), + cast(Mapping[str, object], pending), + ) + if pending is not None + else 0 + ) + + def _gradient_norm(self) -> float: + squares = [ + parameter.grad.detach().float().pow(2).sum() + for component in self.components.values() + for parameter in component.parameters() + if parameter.requires_grad and parameter.grad is not None + ] + if not squares: + raise ValueError("wan_training_step_invalid") + value = float(self.torch.stack(squares).sum().sqrt().item()) + if not math.isfinite(value): + raise ValueError("wan_training_step_invalid") + return value + + def _measurement(self, loss: Any, gradient: float, elapsed: float) -> dict[str, float]: + allocated = float(self.torch.mps.current_allocated_memory()) + driver = float(self.torch.mps.driver_allocated_memory()) + return { + "loss": float(loss.detach().item()), + "gradient_norm": gradient, + "wall_seconds": elapsed, + "rss_start_bytes": 0.0, + "rss_end_bytes": 0.0, + "mps_allocated_start_bytes": allocated, + "mps_allocated_end_bytes": allocated, + "mps_driver_start_bytes": driver, + "mps_driver_end_bytes": driver, + } + + def _load_batch(self) -> dict[str, Any]: + manifest_path, tensors_path = ( + self._path(self.paths.batch_manifest), + self._path(self.paths.batch_tensors), + ) + payload = validate_wan_batch_artifact(manifest_path, tensors_path) + manifest = cast(dict[str, object], json.loads(manifest_path.read_text())) + tensors = cast(Any, self.importer("safetensors.torch")).load(payload) + schedule = cast(dict[str, object], manifest["noise_schedule"]) + transformer_config = self.components["transformer"].config + validate_loaded_wan_batch( + tensors, + batch_size=cast(int, manifest["batch_size"]), + timestep_min=cast(int, schedule["timestep_min"]), + timestep_max=cast(int, schedule["timestep_max"]), + boundary=cast(int, schedule["timestep_boundary"]), + in_channels=int(transformer_config.in_channels), + ) + return { + name: tensor.to( + device="mps", + dtype=( + self.torch.int64 + if name == "transformer_selector" + else self.torch.float32 + if name == "timestep" + else self.torch.bfloat16 + ), + ) + for name, tensor in tensors.items() + } + + def _path(self, relative: Path) -> Path: + path = self.root / relative + if ( + relative.is_absolute() + or not path.is_relative_to(self.root) + or not path.exists() + or path.is_symlink() + ): + raise ValueError("wan_runtime_source_invalid") + return path + + def _verify(self) -> None: + for component in self.components.values(): + _require_full_mps(component) + + +def _require_full_mps(component: object) -> None: + observed = cast(Any, component) + device_map = getattr(observed, "hf_device_map", {"": "mps"}) + tensors = tuple(observed.parameters()) + tuple(observed.buffers()) + if ( + not isinstance(device_map, Mapping) + or not device_map + or not tensors + or any(not _is_mps_device(value) for value in device_map.values()) + or any(not _is_mps_device(getattr(tensor, "device", None)) for tensor in tensors) + or any( + str(getattr(tensor, "dtype", "")) not in {"bfloat16", "torch.bfloat16"} + for tensor in tensors + ) + ): + raise ValueError("wan_runtime_placement_invalid") + + +def _is_mps_device(value: object) -> bool: + device_type = getattr(value, "type", None) + if device_type is not None: + return type(device_type) is str and device_type == "mps" + if type(value) is not str: + return False + if value == "mps": + return True + prefix, separator, index = value.partition(":") + return prefix == "mps" and separator == ":" and index.isdigit() + + +def _lora_target_module(prefix: str, parameter_name: str) -> str: + for marker in (".lora_A.", ".lora_B.", ".lora_embedding_A.", ".lora_embedding_B."): + if marker in parameter_name: + return f"{prefix}.{parameter_name.split(marker, 1)[0]}" + return f"{prefix}.{parameter_name.rpartition('.')[0]}" + + +def validate_wan_batch_artifact(manifest_path: Path, tensors_path: Path) -> bytes: + """Validate the exact dependency-free envelope before safetensors decoding.""" + + try: + if manifest_path.is_symlink() or tensors_path.is_symlink(): + raise ValueError("wan_training_batch_invalid") + payload = tensors_path.read_bytes() + validate_wan_batch_envelope(manifest_path.read_bytes(), payload) + return payload + except (OSError, UnicodeError, json.JSONDecodeError, ValueError): + raise ValueError("wan_training_batch_invalid") from None + + +def _batch_manifest_matches_payload(manifest: Mapping[str, object], payload: bytes) -> bool: + try: + header_size = struct.unpack(" len(payload): + return False + expected.append( + { + "name": name, + "dtype": dtype, + "rank": len(shape), + "shape": shape, + "byte_offsets": offsets, + "digest": "sha256:" + hashlib.sha256(payload[start:end]).hexdigest(), + "finite": True, + } + ) + objective = manifest.get("objective") + schedule = manifest.get("noise_schedule") + return ( + records == expected + and objective + == { + "target_kind": "flow_noise_minus_clean", + "formula": "target=noise_latents-clean_latents", + "clean_latents_digest": next( + item["digest"] for item in expected if item["name"] == "clean_latents" + ), + "noise_latents_digest": next( + item["digest"] for item in expected if item["name"] == "noise_latents" + ), + "target_digest": next( + item["digest"] for item in expected if item["name"] == "target" + ), + } + and isinstance(schedule, dict) + and set(schedule) + == { + "timestep_min", + "timestep_max", + "timestep_boundary", + "selector_rule", + } + and schedule.get("selector_rule") == "high_if_timestep_gte_boundary_else_low" + ) + except (KeyError, TypeError, ValueError, UnicodeError, json.JSONDecodeError, struct.error): + return False + + +def _validated_safetensors_header( + payload: bytes, *, expected_names: set[str], batch_size: int +) -> dict[str, dict[str, object]]: + """Parse a bounded, canonical safetensors envelope before allocating tensors.""" + + def unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError("wan_training_batch_invalid") + result[key] = value + return result + + try: + if len(payload) < 9 or type(batch_size) is not int or batch_size <= 0: + raise ValueError + header_size = struct.unpack(" _MAX_SAFETENSORS_HEADER_BYTES + or 8 + header_size > len(payload) + ): + raise ValueError + parsed = json.loads( + payload[8 : 8 + header_size].decode().rstrip(" "), + object_pairs_hook=unique_object, + ) + if not isinstance(parsed, dict): + raise ValueError + metadata = parsed.pop("__metadata__", None) + if metadata is not None and ( + not isinstance(metadata, dict) + or any( + type(key) is not str or type(value) is not str for key, value in metadata.items() + ) + ): + raise ValueError + if set(parsed) != expected_names or len(parsed) > 100_000: + raise ValueError + data_size = len(payload) - 8 - header_size + spans: list[tuple[int, int]] = [] + for item in parsed.values(): + if not isinstance(item, dict) or set(item) != {"dtype", "shape", "data_offsets"}: + raise ValueError + dtype, shape, offsets = item["dtype"], item["shape"], item["data_offsets"] + if ( + type(dtype) is not str + or dtype not in _SAFETENSORS_DTYPE_BYTES + or not isinstance(shape, list) + or not shape + or len(shape) > _MAX_SAFETENSORS_RANK + or any(type(dim) is not int or dim <= 0 or dim > 2**31 for dim in shape) + or shape[0] != batch_size + or not isinstance(offsets, list) + or len(offsets) != 2 + or any(type(offset) is not int or offset < 0 for offset in offsets) + ): + raise ValueError + count = math.prod(shape) + byte_count = count * _SAFETENSORS_DTYPE_BYTES[dtype] + start, end = offsets + if count > data_size or end - start != byte_count or end > data_size: + raise ValueError + spans.append((start, end)) + cursor = 0 + for start, end in sorted(spans): + if start != cursor: + raise ValueError + cursor = end + if cursor != data_size: + raise ValueError + return cast(dict[str, dict[str, object]], parsed) + except (UnicodeError, json.JSONDecodeError, KeyError, TypeError, ValueError, struct.error): + raise ValueError("wan_training_batch_invalid") from None + + +def _validate_loaded_batch( + tensors: Mapping[str, Any], + *, + batch_size: int, + timestep_min: int, + timestep_max: int, + boundary: int, + in_channels: int, +) -> None: + """Validate exact Wan flow-training semantics after safetensors decoding.""" + try: + if set(tensors) != _WAN_BATCH_TENSORS or not 0 <= timestep_min < boundary <= timestep_max: + raise ValueError + if any( + getattr(tensor, "ndim", 0) < 1 or int(tensor.shape[0]) != batch_size + for tensor in tensors.values() + ): + raise ValueError + hidden = tensors["hidden_states"] + clean = tensors["clean_latents"] + noise = tensors["noise_latents"] + target = tensors["target"] + timestep = tensors["timestep"] + selector = tensors["transformer_selector"] + encoder = tensors["encoder_hidden_states"] + sigma = (timestep / float(timestep_max)).to(dtype=hidden.dtype) + while sigma.ndim < hidden.ndim: + sigma = sigma.unsqueeze(-1) + expected_hidden = sigma * noise + (1.0 - sigma) * clean + if ( + tuple(hidden.shape) != tuple(clean.shape) + or tuple(hidden.shape) != tuple(noise.shape) + or tuple(hidden.shape) != tuple(target.shape) + or hidden.ndim < 3 + or int(hidden.shape[1]) != in_channels + or encoder.ndim != 3 + or tuple(timestep.shape) != (batch_size,) + or str(timestep.dtype) != "torch.float32" + or tuple(selector.shape) != (batch_size,) + or str(selector.dtype) != "torch.int64" + or not bool((timestep >= timestep_min).all().item()) + or not bool((timestep <= timestep_max).all().item()) + or not bool(((selector == 0) | (selector == 1)).all().item()) + or not bool((selector == (timestep < boundary).to(dtype=selector.dtype)).all().item()) + # FlowMatchEulerDiscreteScheduler.add_noise casts sigma to the + # sample dtype, then computes sigma*noise + (1-sigma)*sample. + or not bool((hidden == expected_hidden).all().item()) + or not bool((target == noise - clean).all().item()) + or any( + not bool(cast(Any, tensor).isfinite().all().item()) + for name, tensor in tensors.items() + if name != "transformer_selector" + ) + ): + raise ValueError + except (AttributeError, KeyError, TypeError, ValueError): + raise ValueError("wan_training_batch_invalid") from None + + +def _capture_root_identity(root: Path) -> tuple[tuple[str, int, int, int, int], ...]: + try: + if not root.is_absolute() or root.is_symlink() or not root.is_dir(): + raise ValueError("wan_runtime_source_invalid") + values: list[tuple[str, int, int, int, int]] = [] + root_info = root.stat(follow_symlinks=False) + values.append( + (".", root_info.st_dev, root_info.st_ino, root_info.st_size, root_info.st_mtime_ns) + ) + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise ValueError("wan_runtime_source_invalid") + info = path.stat(follow_symlinks=False) + values.append( + ( + path.relative_to(root).as_posix(), + info.st_dev, + info.st_ino, + info.st_size, + info.st_mtime_ns, + ) + ) + return tuple(values) + except (OSError, ValueError): + raise ValueError("wan_runtime_source_invalid") from None + + +def _trainer_state( + request: TrainingRequest | CalibrationRequest, + step: int, + *, + cpu_rng: Mapping[str, object], + mps_rng: Mapping[str, object], + scheduler_state: Mapping[str, object], + scaler_state: Mapping[str, object], + batch_cursor: int, +) -> dict[str, object]: + total = request.total_steps if isinstance(request, TrainingRequest) else max(4, request.steps) + return { + "schema_version": "trainer-state.v3", + "request_id": request.request_id, + "run_id": request.run_id, + "step": step, + "backend_id": request.backend_id, + "seed": request.seed, + "total_steps": total, + "global_step": step, + "rng_states": {"cpu": dict(cpu_rng), "mps": dict(mps_rng)}, + "scheduler_state": dict(scheduler_state), + "scaler_state": dict(scaler_state), + "backend_state": {WAN_LORA_MPS_BACKEND_ID: {"data_cursor": batch_cursor}}, + } + + +def _torch_rng_state(state: Any, device: str) -> dict[str, object]: + raw = bytes(state.detach().to("cpu").contiguous().tolist()) + return { + "algorithm": "torch.global_rng", + "device": device, + "state_hex": raw.hex(), + "digest": "sha256:" + hashlib.sha256(raw).hexdigest(), + } + + +def _decode_torch_rng(torch: Any, record: object, device: str) -> Any: + try: + payload = cast(Mapping[str, object], record) + raw = bytes.fromhex(cast(str, payload["state_hex"])) + if ( + payload.get("algorithm") != "torch.global_rng" + or payload.get("device") != device + or payload.get("digest") != "sha256:" + hashlib.sha256(raw).hexdigest() + or not raw + ): + raise ValueError + return torch.tensor(tuple(raw), dtype=torch.uint8, device="cpu") + except (KeyError, TypeError, ValueError): + raise ValueError("wan_runtime_restore_invalid") from None + + +def _capture_global_rng(torch: Any, device: str) -> dict[str, object]: + state = torch.get_rng_state() if device == "cpu" else torch.mps.get_rng_state() + return _torch_rng_state(state, device) + + +def _restore_global_rng(torch: Any, record: object, device: str) -> None: + state = _decode_torch_rng(torch, record, device) + if device == "cpu": + torch.set_rng_state(state) + else: + torch.mps.set_rng_state(state) + + +def _normalized_adamw_config(raw: Mapping[str, object]) -> dict[str, object]: + try: + if set(raw) - _ADAMW_KEYS: + raise ValueError + values: dict[str, Any] = { + "lr": raw.get("learning_rate", 0.001), + "gradient_accumulation_steps": raw.get("gradient_accumulation_steps", 1), + "betas": (raw.get("beta1", 0.9), raw.get("beta2", 0.999)), + "eps": raw.get("eps", 1e-8), + "weight_decay": raw.get("weight_decay", 0.01), + "amsgrad": raw.get("amsgrad", False), + "maximize": raw.get("maximize", False), + "foreach": raw.get("foreach", False), + "fused": raw.get("fused", False), + "capturable": raw.get("capturable", False), + "differentiable": raw.get("differentiable", False), + } + lr = values["lr"] + accumulation = values["gradient_accumulation_steps"] + betas = values["betas"] + if ( + type(lr) not in (int, float) + or not math.isfinite(float(lr)) + or float(lr) <= 0 + or type(accumulation) is not int + or accumulation <= 0 + or any(type(beta) not in (int, float) or not 0 <= float(beta) < 1 for beta in betas) + or any( + type(values[key]) is not bool + for key in ( + "amsgrad", + "maximize", + "foreach", + "fused", + "capturable", + "differentiable", + ) + ) + ): + raise ValueError + values["lr"] = float(lr) + values["betas"] = tuple(float(beta) for beta in betas) + values["eps"] = float(values["eps"]) + values["weight_decay"] = float(values["weight_decay"]) + return values + except (TypeError, ValueError, OverflowError): + raise ValueError("wan_runtime_optimizer_config_invalid") from None + + +def _validate_adamw_slots( + state: Mapping[str, Any], + parameter: Any, + *, + amsgrad: bool, + capturable: bool, + fused: bool, +) -> None: + expected = {"step", "exp_avg", "exp_avg_sq"} + if amsgrad: + expected.add("max_exp_avg_sq") + try: + step = state["step"] + expected_step_device = str(parameter.device) if capturable or fused else "cpu" + if ( + set(state) != expected + or tuple(step.shape) != () + or str(step.dtype) != "torch.float32" + or str(step.device) != expected_step_device + or bool(step.requires_grad) + ): + raise ValueError + for name in expected - {"step"}: + slot = state[name] + if ( + tuple(slot.shape) != tuple(parameter.shape) + or slot.dtype != parameter.dtype + or slot.device != parameter.device + or bool(slot.requires_grad) + ): + raise ValueError + except (AttributeError, KeyError, TypeError, ValueError): + raise ValueError("wan_runtime_optimizer_invalid") from None + + +def _optimizer_update_due(*, step: int, total_steps: int, accumulation_steps: int) -> bool: + if not 1 <= step <= total_steps or accumulation_steps <= 0: + raise ValueError("wan_runtime_optimizer_config_invalid") + return step % accumulation_steps == 0 or step == total_steps + + +def _normalize_accumulated_gradients( + parameters: tuple[Any, ...], *, accumulation_position: int +) -> None: + if accumulation_position <= 0: + raise ValueError("wan_runtime_optimizer_invalid") + for parameter in parameters: + gradient = parameter.grad + if gradient is not None: + gradient.div_(accumulation_position) + + +def _pending_gradient_record( + parameters: Mapping[str, Any], *, accumulation_position: int +) -> dict[str, object]: + if accumulation_position < 0: + raise ValueError("wan_runtime_optimizer_invalid") + records: dict[str, object] = {} + for name, parameter in sorted(parameters.items()): + gradient = parameter.grad + if gradient is None: + if accumulation_position: + raise ValueError("wan_runtime_optimizer_invalid") + continue + # Untyped storage avoids NumPy dtype support gaps such as bfloat16. + payload = bytes(gradient.detach().to("cpu").contiguous().untyped_storage()) + records[name] = { + "digest": "sha256:" + hashlib.sha256(payload).hexdigest(), + "serialized_b64": base64.b64encode(payload).decode(), + "shape": list(gradient.shape), + "dtype": str(gradient.dtype).removeprefix("torch."), + "device": str(gradient.device), + } + if bool(records) != bool(accumulation_position): + raise ValueError("wan_runtime_optimizer_invalid") + return {"accumulation_position": accumulation_position, "parameters": records} + + +def _restore_pending_gradients( + torch: Any, parameters: Mapping[str, Any], record: Mapping[str, object] +) -> int: + try: + position = record["accumulation_position"] + if type(position) is not int or position < 0: + raise ValueError + specs = { + name: ( + tuple(parameter.shape), + str(parameter.dtype).removeprefix("torch."), + str(parameter.device), + ) + for name, parameter in parameters.items() + } + validated = validate_pending_gradients( + record, + position, + expected_parameters=parameters, + expected_specs=specs, + ) + raw_parameters = cast(Mapping[str, object], validated["parameters"]) + for name, parameter in parameters.items(): + if not position: + parameter.grad = None + continue + item = cast(Mapping[str, object], raw_parameters[name]) + payload = base64.b64decode(cast(str, item["serialized_b64"]), validate=True) + gradient = torch.frombuffer(bytearray(payload), dtype=parameter.dtype).clone() + parameter.grad = gradient.reshape(parameter.shape).to(parameter.device) + return position + except (KeyError, TypeError, ValueError): + raise ValueError("wan_runtime_optimizer_invalid") from None diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_factory.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_factory.py new file mode 100644 index 0000000..7918adb --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_factory.py @@ -0,0 +1,1069 @@ +"""Explicit, dependency-free construction for the local Wan MPS trainer.""" + +from __future__ import annotations + +import hashlib +import importlib +import os +import stat +import tempfile +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path + +from video_lab.domain import canonical_json +from video_lab.trainer_protocol import TrainerBackend, TrainingRequest +from video_lab.trainers.wan_checkpoint import WanParentCheckpointDescriptor +from video_lab.trainers.wan_diffusers_runtime import ( + WanDiffusersAdapterPaths, + WanDiffusersTrainingRuntime, +) +from video_lab.trainers.wan_mps import ( + WanLoRAMpsTrainer, + WanSourceDescriptor, + WanTrainingRuntime, + WanTrainingSourceLease, + WanTrainingSourceResolver, +) +from video_lab.training_models import Sha256Digest + +WAN_ANIMEGEN_ADAPTER_ID = "diffusers.wan2.2-animegen-precomputed-batch-lora.v1" + + +def _is_mps_device(device: object) -> bool: + observed_type = getattr(device, "type", None) + if observed_type is not None: + return type(observed_type) is str and observed_type == "mps" + text = str(device) + return text == "mps" or text.startswith("mps:") + + +def is_bfloat16_dtype(value: object) -> bool: + return str(value).removeprefix("torch.") == "bfloat16" + + +@dataclass(frozen=True, slots=True) +class WanPinnedLocalRoot: + """A root whose revision is bound to a canonical complete file manifest.""" + + path: Path + revision: str + manifest_digest: Sha256Digest | None = None + + +@dataclass(frozen=True, slots=True) +class WanSnapshotCopyLimits: + max_files: int = 100_000 + max_bytes: int = 64 * 1024 * 1024 * 1024 + max_depth: int = 32 + + +DEFAULT_WAN_SNAPSHOT_COPY_LIMITS = WanSnapshotCopyLimits() + + +@dataclass(frozen=True, slots=True) +class WanRetainedRootFile: + relative: Path + descriptor: int + identity: tuple[int, int, int, int, int, int, int] + digest: bytes + + +@dataclass(frozen=True, slots=True) +class WanRetainedRootLease: + source: WanPinnedLocalRoot + descriptors: tuple[int, ...] + descriptor_identities: tuple[tuple[int, int], ...] + directory_descriptors: tuple[int, ...] + directory_identities: tuple[tuple[int, int, int, int, int, int, int], ...] + directories: tuple[Path, ...] + files: tuple[WanRetainedRootFile, ...] + manifest_digest: Sha256Digest + + def reverify(self) -> None: + if ( + tuple((os.fstat(fd).st_dev, os.fstat(fd).st_ino) for fd in self.descriptors) + != self.descriptor_identities + ): + raise ValueError("wan_factory_snapshot_invalid") + if ( + tuple(_manifest_identity(os.fstat(fd)) for fd in self.directory_descriptors) + != self.directory_identities + ): + raise ValueError("wan_factory_snapshot_invalid") + for item in self.files: + info = os.fstat(item.descriptor) + if ( + _manifest_identity(info) != item.identity + or hashlib.sha256(os.pread(item.descriptor, info.st_size, 0)).digest() + != item.digest + ): + raise ValueError("wan_factory_snapshot_invalid") + if root_manifest_digest(self.source.path) != self.manifest_digest: + raise ValueError("wan_factory_snapshot_invalid") + + +@contextmanager +def retain_pinned_root( + source: WanPinnedLocalRoot, + *, + limits: WanSnapshotCopyLimits = DEFAULT_WAN_SNAPSHOT_COPY_LIMITS, +) -> Iterator[WanRetainedRootLease]: + """Retain every ancestor, directory, and file before snapshot copying starts.""" + + descriptors: list[int] = [] + files: list[WanRetainedRootFile] = [] + directories: list[Path] = [] + directory_descriptors: list[int] = [] + records: list[bytes] = [] + byte_count = 0 + primary: BaseException | None = None + lease: WanRetainedRootLease | None = None + try: + if type(source) is not WanPinnedLocalRoot or source.manifest_digest is None: + raise ValueError + else: + descriptors.extend(_open_directory_chain(source.path)) + root_fd = descriptors[-1] + directory_descriptors.append(root_fd) + + def retain_directory(directory_fd: int, relative: Path, depth: int) -> None: + nonlocal byte_count + if depth > limits.max_depth: + raise ValueError + for name in sorted(os.listdir(directory_fd)): + if name in {"", ".", ".."} or "/" in name: + raise ValueError + child_relative = relative / name + info = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + if stat.S_ISDIR(info.st_mode): + child = os.open( + name, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=directory_fd, + ) + descriptors.append(child) + if _manifest_identity(os.fstat(child)) != _manifest_identity(info): + raise ValueError + directory_descriptors.append(child) + directories.append(child_relative) + retain_directory(child, child_relative, depth + 1) + continue + if not stat.S_ISREG(info.st_mode) or info.st_nlink < 1: + raise ValueError + descriptor = _open_regular_file_at(directory_fd, name) + descriptors.append(descriptor) + identity = _manifest_identity(os.fstat(descriptor)) + if identity != _manifest_identity(info): + raise ValueError + byte_count += info.st_size + if len(files) >= limits.max_files or byte_count > limits.max_bytes: + raise ValueError + digest = hashlib.sha256(os.pread(descriptor, info.st_size, 0)).digest() + if _manifest_identity(os.fstat(descriptor)) != identity: + raise ValueError + files.append( + WanRetainedRootFile(child_relative, descriptor, identity, digest) + ) + records.append( + child_relative.as_posix().encode() + + b"\0" + + str(info.st_size).encode() + + b"\0" + + digest + ) + + retain_directory(root_fd, Path(), 0) + manifest = "sha256:" + hashlib.sha256(b"\n".join(records)).hexdigest() + if manifest != source.manifest_digest: + raise ValueError + lease = WanRetainedRootLease( + source, + tuple(descriptors), + tuple((os.fstat(fd).st_dev, os.fstat(fd).st_ino) for fd in descriptors), + tuple(directory_descriptors), + tuple(_manifest_identity(os.fstat(fd)) for fd in directory_descriptors), + tuple(directories), + tuple(files), + manifest, + ) + lease.reverify() + except (OSError, TypeError, ValueError): + primary = ValueError("wan_factory_snapshot_invalid") + if primary is None: + assert lease is not None + try: + yield lease + except BaseException as error: + primary = error + close_failures: list[Exception] = [] + for descriptor in reversed(descriptors): + try: + os.close(descriptor) + except OSError: + close_failures.append(RuntimeError("wan_retained_root_close_failed")) + cleanup: BaseException | None = None + if close_failures: + cleanup = ExceptionGroup("wan_retained_root_close_failed", close_failures) + if primary is not None and cleanup is not None: + raise BaseExceptionGroup("wan_retained_root_primary_and_close_failed", (primary, cleanup)) + if cleanup is not None: + raise cleanup + if primary is not None: + raise primary from None + + +@dataclass(frozen=True, slots=True) +class WanTrainingAdapterCapability: + """Repository-owned declaration of the exact adapter contract available locally.""" + + adapter_id: str + supported: bool + reason: str + required_batch_format: str + + +class WanTrainingAdapterUnsupported(RuntimeError): + def __init__(self, reason: str) -> None: + self.reason = reason + super().__init__(f"wan_training_adapter_unsupported:{reason}") + + +def local_wan_training_adapter_capability() -> WanTrainingAdapterCapability: + """Report the honest local boundary without importing ML libraries.""" + + return WanTrainingAdapterCapability( + adapter_id=WAN_ANIMEGEN_ADAPTER_ID, + supported=True, + reason="supported", + required_batch_format="wan-precomputed-training-batch.v1", + ) + + +def require_component_on_mps( + component: object, + *, + parameterless_kind: str | None = None, +) -> None: + """Reject hidden CPU/offload state; inspect every executable tensor.""" + + device_map = getattr(component, "hf_device_map", None) + if device_map is not None and ( + not isinstance(device_map, Mapping) + or not device_map + or any(not _is_mps_device(device) for device in device_map.values()) + ): + raise ValueError("wan_runtime_placement_invalid") + declared_device = getattr(component, "device", None) + if declared_device is not None and not _is_mps_device(declared_device): + raise ValueError("wan_runtime_placement_invalid") + parameters = getattr(component, "parameters", None) + buffers = getattr(component, "buffers", None) + if not callable(parameters) or not callable(buffers): + raise ValueError("wan_runtime_placement_invalid") + parameter_tensors = tuple(parameters()) + buffer_tensors = tuple(buffers()) + parameter_devices = tuple(getattr(item, "device", None) for item in parameter_tensors) + buffer_devices = tuple(getattr(item, "device", None) for item in buffer_tensors) + dtypes = tuple(getattr(item, "dtype", None) for item in parameter_tensors + buffer_tensors) + if not parameter_tensors and not buffer_tensors: + if ( + parameterless_kind not in {"tokenizer", "scheduler"} + or not _is_mps_device(getattr(component, "device", None)) + or not is_bfloat16_dtype(getattr(component, "dtype", None)) + ): + raise ValueError("wan_runtime_placement_invalid") + return + if any(not _is_mps_device(device) for device in parameter_devices + buffer_devices) or any( + not is_bfloat16_dtype(dtype) for dtype in dtypes + ): + raise ValueError("wan_runtime_placement_invalid") + + +def create_wan_lora_mps_trainer( + *, + backend_digest: Sha256Digest, + material_digest: Sha256Digest, + orchestration_source_digest: Sha256Digest | None = None, + descriptor_source_digest: Sha256Digest | None = None, + source: WanPinnedLocalRoot, + cache: WanPinnedLocalRoot, + artifact_root: Path, + source_descriptors: tuple[WanSourceDescriptor, ...], + runtime: WanTrainingRuntime | None = None, + adapter_paths: WanDiffusersAdapterPaths | None = None, + runtime_digest: Sha256Digest | None = None, + library_importer: Callable[[str], object] = importlib.import_module, + checkpoint_resolver: Callable[[TrainingRequest], WanParentCheckpointDescriptor] | None = None, +) -> TrainerBackend: + """Compose a tested runtime, or fail rather than claim absent production support.""" + + source_root = _require_pinned_root(source) + _require_pinned_root(cache) + _require_local_directory(artifact_root, "wan_factory_artifact_root_invalid") + if ( + not source_descriptors + or orchestration_source_digest is None + or descriptor_source_digest is None + ): + raise ValueError("wan_factory_source_invalid") + try: + for descriptor in source_descriptors: + candidate = Path(descriptor.path()) + if not candidate.is_absolute() or not candidate.is_relative_to(source_root): + raise ValueError("wan_factory_source_invalid") + except (OSError, TypeError, ValueError): + raise ValueError("wan_factory_source_invalid") from None + snapshot_root: Path | None = None + ownership_transferred = False + try: + try: + with WanTrainingSourceResolver(source_descriptors).acquire() as lease: + if lease.sources.source_digest != descriptor_source_digest: + raise ValueError("wan_factory_source_invalid") + if runtime is None: + if adapter_paths is None or runtime_digest is None: + raise WanTrainingAdapterUnsupported( + "adapter_paths_and_runtime_digest_required" + ) + snapshot_root = _snapshot_adapter_inputs( + source_root, adapter_paths, artifact_root, lease=lease + ) + except ValueError: + raise ValueError("wan_factory_source_invalid") from None + if runtime is None: + assert adapter_paths is not None and runtime_digest is not None + assert snapshot_root is not None + runtime = WanDiffusersTrainingRuntime( + root=snapshot_root, + paths=adapter_paths, + runtime_digest=runtime_digest, + importer=library_importer, + cleanup=lambda: _remove_snapshot(snapshot_root), + ) + elif adapter_paths is not None or runtime_digest is not None: + raise ValueError("wan_factory_runtime_invalid") + trainer = WanLoRAMpsTrainer( + backend_digest=backend_digest, + material_digest=material_digest, + orchestration_source_digest=orchestration_source_digest, + descriptor_source_digest=descriptor_source_digest, + source_resolver=WanTrainingSourceResolver(source_descriptors), + runtime=runtime, + checkpoint_resolver=checkpoint_resolver, + ) + ownership_transferred = True + return trainer + finally: + if snapshot_root is not None and not ownership_transferred: + _remove_snapshot(snapshot_root) + + +def root_manifest_digest( + root: Path, + *, + max_files: int | None = None, + max_bytes: int | None = None, + max_depth: int | None = None, +) -> Sha256Digest: + """Canonical content identity for every regular file below an exact local root.""" + base = _require_local_directory(root, "wan_factory_root_invalid") + descriptors: list[int] = [] + try: + descriptors = _open_directory_chain(base) + ancestor_identities = tuple( + (os.fstat(item).st_dev, os.fstat(item).st_ino) for item in descriptors[:-1] + ) + root_identity = _manifest_identity(os.fstat(descriptors[-1])) + root_descriptor = descriptors[-1] + records: list[bytes] = [] + observed_bytes = 0 + for path in sorted(base.rglob("*")): + relative = path.relative_to(base) + info = path.stat(follow_symlinks=False) + if max_depth is not None and len(relative.parts) > max_depth: + raise ValueError("wan_factory_root_invalid") + if stat.S_ISREG(info.st_mode): + observed_bytes += info.st_size + if (max_files is not None and len(records) >= max_files) or ( + max_bytes is not None and observed_bytes > max_bytes + ): + raise ValueError("wan_factory_root_invalid") + record = _manifest_file_record(root_descriptor, relative) + if record is not None: + records.append(record) + if ( + tuple((os.fstat(item).st_dev, os.fstat(item).st_ino) for item in descriptors[:-1]) + != ancestor_identities + or _manifest_identity(os.fstat(descriptors[-1])) != root_identity + ): + raise ValueError("wan_factory_root_invalid") + return "sha256:" + hashlib.sha256(b"\n".join(records)).hexdigest() + except (OSError, ValueError): + raise ValueError("wan_factory_root_invalid") from None + finally: + for descriptor in reversed(descriptors): + os.close(descriptor) + + +def secure_snapshot_pinned_root( + source: WanPinnedLocalRoot, + destination: Path, + *, + limits: WanSnapshotCopyLimits = DEFAULT_WAN_SNAPSHOT_COPY_LIMITS, + lease: WanRetainedRootLease | None = None, + after_retain: Callable[[], None] | None = None, +) -> WanPinnedLocalRoot: + """Copy one pinned root exclusively from pre-retained file descriptors.""" + if ( + type(source) is not WanPinnedLocalRoot + or source.manifest_digest is None + or limits.max_files <= 0 + or limits.max_bytes <= 0 + or limits.max_depth <= 0 + ): + raise ValueError("wan_factory_snapshot_invalid") + destination = Path(destination) + if destination.exists() or not destination.parent.is_dir(): + raise ValueError("wan_factory_snapshot_invalid") + if lease is None: + with retain_pinned_root(source, limits=limits) as retained: + return secure_snapshot_pinned_root( + source, + destination, + limits=limits, + lease=retained, + after_retain=after_retain, + ) + retained = lease + try: + if retained.source != source or retained.manifest_digest != source.manifest_digest: + raise ValueError + retained.reverify() + if after_retain is not None: + after_retain() + retained.reverify() + destination.mkdir(mode=0o700) + for relative in sorted(retained.directories, key=lambda item: len(item.parts)): + (destination / relative).mkdir(mode=0o700) + for retained_file in retained.files: + retained.reverify() + target_file = os.open( + destination / retained_file.relative, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0), + 0o400, + ) + try: + _copy_retained_file_to_descriptor(retained_file, target_file) + finally: + os.close(target_file) + retained.reverify() + copied_digest = root_manifest_digest( + destination, + max_files=limits.max_files, + max_bytes=limits.max_bytes, + max_depth=limits.max_depth, + ) + if copied_digest != source.manifest_digest: + raise ValueError + for snapshot_item in sorted(destination.rglob("*"), reverse=True): + os.chmod( + snapshot_item, + 0o500 if snapshot_item.is_dir() else 0o400, + follow_symlinks=False, + ) + os.chmod(destination, 0o500, follow_symlinks=False) + return WanPinnedLocalRoot(destination, source.revision, copied_digest) + except BaseException as error: + primary: BaseException = ( + ValueError("wan_factory_snapshot_invalid") + if isinstance(error, (OSError, TypeError, ValueError)) + else error + ) + try: + _remove_snapshot(destination) + except BaseException as cleanup: + raise BaseExceptionGroup( + "wan_snapshot_construction_and_cleanup_failed", (primary, cleanup) + ) from None + raise primary from None + + +def secure_snapshot_pinned_root_at( + source: WanPinnedLocalRoot, + destination: Path, + *, + destination_parent_descriptor: int, + limits: WanSnapshotCopyLimits = DEFAULT_WAN_SNAPSHOT_COPY_LIMITS, + lease: WanRetainedRootLease | None = None, +) -> WanPinnedLocalRoot: + """Copy a retained root beneath one already-open destination parent.""" + + if ( + type(source) is not WanPinnedLocalRoot + or source.manifest_digest is None + or limits.max_files <= 0 + or limits.max_bytes <= 0 + or limits.max_depth <= 0 + ): + raise ValueError("wan_factory_snapshot_invalid") + destination = Path(destination) + name = destination.name + if not name or name in {".", ".."} or Path(name).parts != (name,): + raise ValueError("wan_factory_snapshot_invalid") + try: + parent_info = os.fstat(destination_parent_descriptor) + if not stat.S_ISDIR(parent_info.st_mode): + raise ValueError + os.stat(name, dir_fd=destination_parent_descriptor, follow_symlinks=False) + except FileNotFoundError: + pass + except (OSError, TypeError, ValueError): + raise ValueError("wan_factory_snapshot_invalid") from None + else: + raise ValueError("wan_factory_snapshot_invalid") + if lease is None: + with retain_pinned_root(source, limits=limits) as retained: + return secure_snapshot_pinned_root_at( + source, + destination, + destination_parent_descriptor=destination_parent_descriptor, + limits=limits, + lease=retained, + ) + + retained = lease + root_descriptor: int | None = None + directory_descriptors: dict[Path, int] = {} + root_identity: tuple[int, int] | None = None + primary: BaseException | None = None + try: + if retained.source != source or retained.manifest_digest != source.manifest_digest: + raise ValueError + retained.reverify() + os.mkdir(name, mode=0o700, dir_fd=destination_parent_descriptor) + created_info = os.stat( + name, + dir_fd=destination_parent_descriptor, + follow_symlinks=False, + ) + if not stat.S_ISDIR(created_info.st_mode): + raise ValueError + root_identity = (created_info.st_dev, created_info.st_ino) + root_descriptor = os.open( + name, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=destination_parent_descriptor, + ) + root_info = os.fstat(root_descriptor) + if (root_info.st_dev, root_info.st_ino) != root_identity: + raise ValueError + directory_descriptors[Path()] = root_descriptor + for relative in sorted(retained.directories, key=lambda item: len(item.parts)): + parent_descriptor = directory_descriptors[relative.parent] + os.mkdir(relative.name, mode=0o700, dir_fd=parent_descriptor) + directory_descriptors[relative] = os.open( + relative.name, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=parent_descriptor, + ) + for retained_file in retained.files: + retained.reverify() + target_file = os.open( + retained_file.relative.name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0), + 0o400, + dir_fd=directory_descriptors[retained_file.relative.parent], + ) + try: + _copy_retained_file_to_descriptor(retained_file, target_file) + os.fchmod(target_file, 0o400) + finally: + os.close(target_file) + retained.reverify() + for relative, descriptor in sorted( + directory_descriptors.items(), key=lambda item: len(item[0].parts), reverse=True + ): + del relative + os.fchmod(descriptor, 0o500) + os.fsync(descriptor) + retained.reverify() + return WanPinnedLocalRoot(destination, source.revision, source.manifest_digest) + except BaseException as error: + primary = ( + ValueError("wan_factory_snapshot_invalid") + if isinstance(error, (OSError, TypeError, ValueError)) + else error + ) + finally: + for relative, descriptor in sorted( + directory_descriptors.items(), key=lambda item: len(item[0].parts), reverse=True + ): + if relative != Path(): + try: + os.close(descriptor) + except OSError: + if primary is None: + primary = RuntimeError("wan_snapshot_descriptor_close_failed") + if primary is None and root_descriptor is not None: + os.close(root_descriptor) + assert primary is not None + cleanup: BaseException | None = None + cleanup_descriptor = root_descriptor + if cleanup_descriptor is None and root_identity is not None: + try: + cleanup_descriptor = _open_snapshot_root_by_identity( + destination_parent_descriptor, root_identity + ) + except BaseException as error: + cleanup = error + if cleanup_descriptor is not None and root_identity is not None: + try: + remove_wan_snapshot_at( + parent_descriptor=destination_parent_descriptor, + root_descriptor=cleanup_descriptor, + root_identity=root_identity, + ) + except BaseException as error: + cleanup = error + try: + os.close(cleanup_descriptor) + except OSError: + close_failure = RuntimeError("wan_snapshot_descriptor_close_failed") + cleanup = ( + close_failure + if cleanup is None + else BaseExceptionGroup( + "wan_snapshot_cleanup_and_close_failed", (cleanup, close_failure) + ) + ) + if cleanup is not None: + raise BaseExceptionGroup( + "wan_snapshot_construction_and_cleanup_failed", (primary, cleanup) + ) from None + raise primary from None + + +def _open_snapshot_root_by_identity( + parent_descriptor: int, root_identity: tuple[int, int] +) -> int: + try: + matching_names: list[str] = [] + for name in os.listdir(parent_descriptor): + info = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if (info.st_dev, info.st_ino) == root_identity: + matching_names.append(name) + if len(matching_names) != 1: + raise ValueError + descriptor = os.open( + matching_names[0], + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=parent_descriptor, + ) + info = os.fstat(descriptor) + if (info.st_dev, info.st_ino) != root_identity: + os.close(descriptor) + raise ValueError + return descriptor + except (OSError, TypeError, ValueError): + raise ValueError("wan_snapshot_cleanup_authority_invalid") from None + + +def remove_wan_snapshot_at( + *, + parent_descriptor: int, + root_descriptor: int, + root_identity: tuple[int, int], +) -> None: + """Clear only the retained snapshot inode and unlink that inode from its parent.""" + + failures: list[Exception] = [] + try: + root_info = os.fstat(root_descriptor) + if (root_info.st_dev, root_info.st_ino) != root_identity: + raise ValueError + os.fchmod(root_descriptor, 0o700) + except (OSError, TypeError, ValueError): + raise ValueError("wan_snapshot_cleanup_authority_invalid") from None + _clear_directory_descriptor(root_descriptor, failures) + matching_names: list[str] = [] + try: + for name in os.listdir(parent_descriptor): + info = os.stat( + name, + dir_fd=parent_descriptor, + follow_symlinks=False, + ) + if (info.st_dev, info.st_ino) == root_identity: + matching_names.append(name) + except OSError: + failures.append(RuntimeError("wan_snapshot_cleanup_inventory_failed")) + if len(matching_names) == 1: + try: + os.rmdir(matching_names[0], dir_fd=parent_descriptor) + except OSError: + failures.append(RuntimeError("wan_snapshot_cleanup_root_failed")) + else: + failures.append(RuntimeError("wan_snapshot_cleanup_root_missing")) + try: + os.fsync(parent_descriptor) + except OSError: + failures.append(RuntimeError("wan_snapshot_cleanup_fsync_failed")) + if failures: + raise ExceptionGroup("wan_snapshot_cleanup_failed", failures) + + +def _copy_retained_file_to_descriptor( + retained_file: WanRetainedRootFile, target_file: int +) -> None: + offset = 0 + while offset < retained_file.identity[4]: + chunk = os.pread( + retained_file.descriptor, + min(1 << 20, retained_file.identity[4] - offset), + offset, + ) + if not chunk or os.write(target_file, chunk) != len(chunk): + raise ValueError + offset += len(chunk) + os.fsync(target_file) + + +def _clear_directory_descriptor(descriptor: int, failures: list[Exception]) -> None: + try: + names = tuple(os.listdir(descriptor)) + except OSError: + failures.append(RuntimeError("wan_snapshot_cleanup_inventory_failed")) + return + for name in names: + try: + info = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if stat.S_ISDIR(info.st_mode): + child = os.open( + name, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=descriptor, + ) + try: + os.fchmod(child, 0o700) + _clear_directory_descriptor(child, failures) + finally: + os.close(child) + os.rmdir(name, dir_fd=descriptor) + else: + os.unlink(name, dir_fd=descriptor) + except OSError: + failures.append(RuntimeError("wan_snapshot_cleanup_remove_failed")) + + +def _open_directory_chain(path: Path) -> list[int]: + flags = ( + os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0) + ) + descriptors = [os.open(path.anchor, flags)] + try: + for part in path.parts[1:]: + descriptors.append(os.open(part, flags, dir_fd=descriptors[-1])) + return descriptors + except Exception: + for descriptor in reversed(descriptors): + os.close(descriptor) + raise + + +def _manifest_identity(info: os.stat_result) -> tuple[int, int, int, int, int, int, int]: + return ( + info.st_dev, + info.st_ino, + info.st_mode, + info.st_nlink, + info.st_size, + info.st_mtime_ns, + info.st_ctime_ns, + ) + + +def _manifest_file_record(root_descriptor: int, relative: Path) -> bytes | None: + directory_flags = ( + os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0) + ) + opened: list[int] = [] + current = root_descriptor + try: + ancestor_identities: list[tuple[int, int, int, int, int, int, int]] = [] + for part in relative.parts[:-1]: + current = os.open(part, directory_flags, dir_fd=current) + opened.append(current) + ancestor_identities.append(_manifest_identity(os.fstat(current))) + leaf_info = os.stat(relative.name, dir_fd=current, follow_symlinks=False) + if stat.S_ISDIR(leaf_info.st_mode): + return None + descriptor = _open_regular_file_at(current, relative.name) + opened.append(descriptor) + before = os.fstat(descriptor) + digest = hashlib.sha256() + size = 0 + while size < before.st_size: + chunk = os.read(descriptor, min(1 << 20, before.st_size - size)) + if not chunk: + raise ValueError + digest.update(chunk) + size += len(chunk) + if size != before.st_size or _manifest_identity(os.fstat(descriptor)) != _manifest_identity( + before + ): + raise ValueError + for ancestor, identity in zip(opened[:-1], ancestor_identities, strict=True): + if _manifest_identity(os.fstat(ancestor)) != identity: + raise ValueError + relative_bytes = relative.as_posix().encode() + return relative_bytes + b"\0" + str(size).encode() + b"\0" + digest.digest() + finally: + for descriptor in reversed(opened): + os.close(descriptor) + + +def _open_regular_file_at(directory_fd: int, name: str) -> int: + """Open a leaf nonblocking/no-follow and reject it before reading unless regular.""" + descriptor = os.open( + name, + os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0), + dir_fd=directory_fd, + ) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_nlink < 1: + raise ValueError + return descriptor + except Exception: + os.close(descriptor) + raise + + +def _require_pinned_root(root: WanPinnedLocalRoot) -> Path: + if ( + type(root) is not WanPinnedLocalRoot + or len(root.revision) != 40 + or any(character not in "0123456789abcdef" for character in root.revision) + or root.manifest_digest is None + or root.manifest_digest != root_manifest_digest(root.path) + ): + raise ValueError("wan_factory_root_invalid") + return _require_local_directory(root.path, "wan_factory_root_invalid") + + +def _require_local_directory(path: Path, error: str) -> Path: + try: + candidate = Path(path) + if not candidate.is_absolute() or not candidate.is_dir() or candidate.is_symlink(): + raise ValueError(error) + if os.stat(candidate, follow_symlinks=False).st_nlink < 1: + raise ValueError(error) + return candidate + except (OSError, TypeError, ValueError): + raise ValueError(error) from None + + +def _snapshot_adapter_inputs( + source_root: Path, + paths: WanDiffusersAdapterPaths, + artifact_root: Path, + lease: WanTrainingSourceLease, +) -> Path: + """Stable-open exact adapter inputs into a private immutable loader tree.""" + snapshot = Path(tempfile.mkdtemp(prefix="wan-adapter-", dir=artifact_root)) + try: + _fsync_directory(artifact_root) + relative_files: set[Path] = set() + for relative in ( + paths.high_transformer, + paths.low_transformer, + paths.batch_manifest, + paths.batch_tensors, + ): + relative_files.add(_safe_relative(relative)) + approved = { + source_file.path.relative_to(source_root): source_file + for source_file in lease.sources.files + } + relative_files.update( + relative + for relative in approved + if relative.is_relative_to(_safe_relative(paths.wan_snapshot)) + ) + if relative_files != set(approved): + raise ValueError("wan_factory_source_invalid") + manifest_records: list[dict[str, object]] = [] + for relative in sorted(relative_files): + destination = snapshot / relative + destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + descriptor = os.open( + destination, + os.O_RDWR | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0), + 0o400, + ) + try: + digest, size = lease.copy_file_to(approved[relative], descriptor) + manifest_records.append( + {"relative_path": relative.as_posix(), "digest": digest, "size": size} + ) + finally: + os.close(descriptor) + manifest = snapshot / ".wan-snapshot-manifest.json" + manifest.write_text( + canonical_json( + { + "schema_version": "wan-private-snapshot.v1", + "source_digest": lease.sources.source_digest, + "files": manifest_records, + } + ) + ) + with manifest.open("rb") as manifest_file: + os.fsync(manifest_file.fileno()) + for directory in sorted( + (item for item in snapshot.rglob("*") if item.is_dir()), reverse=True + ): + os.chmod(directory, 0o500, follow_symlinks=False) + os.chmod(snapshot, 0o500, follow_symlinks=False) + directories = [snapshot, *(item for item in snapshot.rglob("*") if item.is_dir())] + for directory in sorted(directories, key=lambda item: len(item.parts), reverse=True): + _fsync_directory(directory) + _fsync_directory(artifact_root) + return snapshot + except Exception: + _remove_snapshot(snapshot) + raise ValueError("wan_factory_source_invalid") from None + + +def _safe_relative(path: Path) -> Path: + candidate = Path(path) + if ( + candidate.is_absolute() + or not candidate.parts + or any(part in {"", ".", ".."} for part in candidate.parts) + ): + raise ValueError("wan_factory_source_invalid") + return candidate + + +def _stable_read_relative(root: Path, relative: Path) -> bytes: + no_follow = getattr(os, "O_NOFOLLOW", 0) + directory_flags = os.O_RDONLY | os.O_DIRECTORY | no_follow | getattr(os, "O_CLOEXEC", 0) + current = os.open(root, directory_flags) + opened: list[int] = [current] + try: + for part in relative.parts[:-1]: + current = os.open(part, directory_flags, dir_fd=current) + opened.append(current) + descriptor = os.open( + relative.name, + os.O_RDONLY | no_follow | getattr(os, "O_CLOEXEC", 0), + dir_fd=current, + ) + opened.append(descriptor) + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode) or before.st_nlink < 1: + raise ValueError("wan_factory_source_invalid") + chunks: list[bytes] = [] + remaining = before.st_size + while remaining: + chunk = os.read(descriptor, min(1 << 20, remaining)) + if not chunk: + raise ValueError("wan_factory_source_invalid") + chunks.append(chunk) + remaining -= len(chunk) + after = os.fstat(descriptor) + if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ): + raise ValueError("wan_factory_source_invalid") + return b"".join(chunks) + finally: + for descriptor in reversed(opened): + os.close(descriptor) + + +def _remove_snapshot(root: Path) -> None: + if not root.exists(): + return + failures: list[Exception] = [] + try: + nested = tuple(root.rglob("*")) + except OSError: + nested = () + failures.append(RuntimeError("wan_snapshot_cleanup_inventory_failed")) + for path in (root, *nested): + try: + os.chmod(path, 0o700 if path.is_dir() else 0o600, follow_symlinks=False) + except OSError: + failures.append(RuntimeError("wan_snapshot_cleanup_mode_failed")) + for path in sorted(nested, reverse=True): + try: + path.rmdir() if path.is_dir() else path.unlink() + except OSError: + failures.append(RuntimeError("wan_snapshot_cleanup_remove_failed")) + try: + root.rmdir() + except OSError: + failures.append(RuntimeError("wan_snapshot_cleanup_root_failed")) + try: + _fsync_directory(root.parent) + except OSError: + failures.append(RuntimeError("wan_snapshot_cleanup_fsync_failed")) + if failures: + raise ExceptionGroup("wan_snapshot_cleanup_failed", failures) + + +def remove_wan_snapshot(root: Path) -> None: + """Remove a private Wan snapshot with normalized path-free failures.""" + + _remove_snapshot(root) + + +@contextmanager +def wan_snapshot_cleanup_scope(root: Path) -> Iterator[None]: + """Always clean a private snapshot while preserving BaseException semantics.""" + + primary: BaseException | None = None + try: + yield + except BaseException as error: + primary = error + try: + _remove_snapshot(root) + except BaseException as cleanup: + if primary is not None: + raise BaseExceptionGroup( + "wan_snapshot_primary_and_cleanup_failed", (primary, cleanup) + ) from None + raise + if primary is not None: + raise primary + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_mps.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_mps.py new file mode 100644 index 0000000..eccb7cf --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_mps.py @@ -0,0 +1,1442 @@ +"""Strictly local source and capability boundary for Wan LoRA training.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import stat +import threading +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from time import monotonic +from typing import Literal, Protocol, cast, runtime_checkable + +from pydantic import ConfigDict, Field, StrictBool, StrictFloat, StrictInt, field_validator + +from video_lab.domain import FrozenModel, canonical_json +from video_lab.trainer_protocol import ( + CalibrationRequest, + CalibrationResult, + CancellationCheck, + CheckpointResult, + LoRAConfiguration, + ProbeTrainingRequest, + TrainerCapabilityReport, + TrainingCancelled, + TrainingMaterialHandles, + TrainingProgress, + TrainingProgressCallback, + TrainingRequest, + TrainingResult, + validate_cancelled_training_result, + validate_training_result, +) +from video_lab.trainers.wan_checkpoint import ( + WanCheckpointCleanupError, + WanCheckpointCodec, + WanCheckpointState, + WanCheckpointValidationContext, + WanParentCheckpointDescriptor, +) +from video_lab.training_models import Sha256Digest +from video_lab.wan_pending import ( + TrainableParameterSpec, + trainable_parameter_spec_records, + validate_trainable_parameter_specs, +) + +WAN_LORA_MPS_BACKEND_ID = "diffusers.wan2.2-animegen-lora.mps.v1" +_SAFE_BOUNDARIES = (1, 2, 3, 4) +_MAX_STEPS = 1_000_000 +_EXECUTABLE_COMPONENTS = ("text_encoder", "transformer", "transformer_2", "vae") + + +@dataclass(frozen=True, slots=True) +class WanSourceDescriptor: + """Factory-authorized identity for one required local source file.""" + + name: str + path: Callable[[], Path] + expected_digest: Sha256Digest + expected_size: int + expected_device: int + expected_inode: int + + +@dataclass(frozen=True, slots=True) +class WanSourceFile: + name: str + path: Path + digest: Sha256Digest + size_bytes: int + device: int + inode: int + + +@dataclass(frozen=True, slots=True) +class WanTrainingSources: + files: tuple[WanSourceFile, ...] + base_model_digest: Sha256Digest + source_digest: Sha256Digest + + +class WanMpsObservation(FrozenModel): + model_config = ConfigDict( + frozen=True, extra="forbid", strict=True, revalidate_instances="always" + ) + + runtime_digest: Sha256Digest + built: StrictBool + available: StrictBool + bf16_supported: StrictBool + component_placements: tuple[tuple[str, str], ...] + disk_free_bytes: StrictInt = Field(ge=0) + disk_required_bytes: StrictInt = Field(ge=0) + memory_headroom_bytes: StrictInt = Field(ge=0) + memory_required_bytes: StrictInt = Field(ge=0) + + @field_validator("component_placements", mode="before") + @classmethod + def canonical_placements(cls, value: object) -> object: + if type(value) is not tuple: + raise ValueError("wan_component_placements_invalid") + placements: list[tuple[str, str]] = [] + for item in value: + if ( + type(item) is not tuple + or len(item) != 2 + or type(item[0]) is not str + or type(item[1]) is not str + or not item[0] + or not item[1] + ): + raise ValueError("wan_component_placements_invalid") + placements.append((item[0], item[1])) + canonical = tuple(placements) + if canonical != tuple(sorted(canonical)) or len({name for name, _ in canonical}) != len( + canonical + ): + raise ValueError("wan_component_placements_invalid") + return canonical + + +class WanParameterObservation(FrozenModel): + """One independently observable prepared parameter identity.""" + + model_config = ConfigDict( + frozen=True, extra="forbid", strict=True, revalidate_instances="always" + ) + + name: str = Field(min_length=1) + module: str = Field(min_length=1) + requires_grad: StrictBool + digest: Sha256Digest + device: Literal["mps"] + dtype: Literal["bfloat16"] + shape: tuple[StrictInt, ...] + + @field_validator("shape", mode="before") + @classmethod + def exact_positive_shape(cls, value: object) -> object: + if ( + type(value) is not tuple + or not value + or any(type(item) is not int or item <= 0 for item in value) + ): + raise ValueError("wan_parameter_shape_invalid") + return value + + +class WanCalibrationMeasurement(FrozenModel): + """Measured forward/backward and endpoint resource telemetry.""" + + model_config = ConfigDict( + frozen=True, extra="forbid", strict=True, revalidate_instances="always" + ) + + loss: StrictFloat + gradient_norm: StrictFloat = Field(ge=0) + wall_seconds: StrictFloat = Field(ge=0) + rss_start_bytes: StrictFloat = Field(ge=0) + rss_end_bytes: StrictFloat = Field(ge=0) + mps_allocated_start_bytes: StrictFloat = Field(ge=0) + mps_allocated_end_bytes: StrictFloat = Field(ge=0) + mps_driver_start_bytes: StrictFloat = Field(ge=0) + mps_driver_end_bytes: StrictFloat = Field(ge=0) + + @field_validator("*") + @classmethod + def finite_values(cls, value: float) -> float: + if not math.isfinite(value): + raise ValueError("wan_calibration_measurement_invalid") + return value + + +class WanTrainingStepMeasurement(FrozenModel): + """One finite optimizer step observed through the injected runtime boundary.""" + + model_config = ConfigDict( + frozen=True, extra="forbid", strict=True, revalidate_instances="always" + ) + + loss: StrictFloat + gradient_norm: StrictFloat = Field(ge=0) + + @field_validator("*") + @classmethod + def finite_values(cls, value: float) -> float: + if not math.isfinite(value): + raise ValueError("wan_training_step_invalid") + return value + + +@dataclass(frozen=True, slots=True) +class WanRuntimeCheckpointState: + """Runtime-owned trainable bytes and substantive continuation state.""" + + weights: bytes + optimizer: Mapping[str, object] + trainer_state: Mapping[str, object] + + +@dataclass(frozen=True, slots=True) +class WanRuntimeRestoreReceipt: + """Canonical snapshot proving the runtime applied the exact parent state.""" + + weights_digest: Sha256Digest + optimizer_payload: bytes + trainer_state_payload: bytes + + +@runtime_checkable +class WanPreparedCommonOperations(Protocol): + def eligible_lora_modules(self) -> tuple[str, ...]: ... + + def inject_lora( + self, + *, + targets: tuple[str, ...], + configuration: LoRAConfiguration, + seed: int, + device: str, + dtype: str, + ) -> None: ... + + def parameter_inventory(self) -> tuple[object, ...]: ... + + + +@runtime_checkable +class WanPreparedOperations(WanPreparedCommonOperations, Protocol): + def measure_forward_backward(self) -> object: ... + + +@runtime_checkable +class WanPreparedTrainingOperations(WanPreparedCommonOperations, Protocol): + def train_step(self, *, step: int, total_steps: int) -> object: ... + + def checkpoint_state(self, *, step: int) -> object: ... + + def restore_checkpoint(self, state: WanCheckpointState) -> None: ... + + def observe_restored_state(self) -> object: ... + + +class _WanPreparedBoundaryInvalid(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class WanPreparedTraining: + """Runtime-owned prepared state with a narrow, ML-free calibration boundary.""" + + handle: object + + def eligible_lora_modules(self) -> tuple[str, ...]: + operations = self._operations() + value = operations.eligible_lora_modules() + if type(value) is not tuple or any(type(item) is not str or not item for item in value): + raise _WanPreparedBoundaryInvalid("targets") + return value + + def inject_lora( + self, + *, + targets: tuple[str, ...], + configuration: LoRAConfiguration, + seed: int, + ) -> None: + self._operations().inject_lora( + targets=targets, + configuration=configuration, + seed=seed, + device="mps", + dtype="bfloat16", + ) + + def parameter_inventory(self) -> tuple[WanParameterObservation, ...]: + raw = self._operations().parameter_inventory() + if type(raw) is not tuple: + raise _WanPreparedBoundaryInvalid("inventory") + try: + return tuple(WanParameterObservation.model_validate(item, strict=True) for item in raw) + except Exception: + raise _WanPreparedBoundaryInvalid("inventory") from None + + + def measure_forward_backward(self) -> WanCalibrationMeasurement: + raw = self._calibration_operations().measure_forward_backward() + try: + return WanCalibrationMeasurement.model_validate(raw, strict=True) + except Exception: + raise _WanPreparedBoundaryInvalid("measurement") from None + + def train_step(self, *, step: int, total_steps: int) -> WanTrainingStepMeasurement: + raw = self._training_operations().train_step(step=step, total_steps=total_steps) + try: + return WanTrainingStepMeasurement.model_validate(raw, strict=True) + except Exception: + raise _WanPreparedBoundaryInvalid("training_step") from None + + def checkpoint_state(self, *, step: int) -> WanRuntimeCheckpointState: + raw = self._training_operations().checkpoint_state(step=step) + if ( + type(raw) is not WanRuntimeCheckpointState + or type(raw.weights) is not bytes + or not isinstance(raw.optimizer, Mapping) + or not isinstance(raw.trainer_state, Mapping) + ): + raise _WanPreparedBoundaryInvalid("checkpoint_state") + return raw + + def restore_checkpoint(self, state: WanCheckpointState) -> WanRuntimeRestoreReceipt: + if type(state) is not WanCheckpointState: + raise _WanPreparedBoundaryInvalid("restore_state") + operations = self._training_operations() + apply_restore = cast( + Callable[[WanCheckpointState], object], operations.restore_checkpoint + ) + result = apply_restore(state) + if result is not None: + raise _WanPreparedBoundaryInvalid("restore_receipt") + raw = operations.observe_restored_state() + if type(raw) is not WanRuntimeRestoreReceipt: + raise _WanPreparedBoundaryInvalid("restore_receipt") + expected_optimizer = canonical_json(dict(state.optimizer)).encode() + expected_trainer = canonical_json(dict(state.trainer_state)).encode() + expected_weights = "sha256:" + hashlib.sha256(state.weights).hexdigest() + if ( + raw.weights_digest != expected_weights + or type(raw.optimizer_payload) is not bytes + or raw.optimizer_payload != expected_optimizer + or type(raw.trainer_state_payload) is not bytes + or raw.trainer_state_payload != expected_trainer + ): + raise _WanPreparedBoundaryInvalid("restore_receipt") + return raw + + def _operations(self) -> WanPreparedCommonOperations: + if not isinstance(self.handle, WanPreparedCommonOperations): + raise _WanPreparedBoundaryInvalid("operations") + return self.handle + + def _training_operations(self) -> WanPreparedTrainingOperations: + if not isinstance(self.handle, WanPreparedTrainingOperations): + raise _WanPreparedBoundaryInvalid("training_operations") + return self.handle + + def _calibration_operations(self) -> WanPreparedOperations: + if not isinstance(self.handle, WanPreparedOperations): + raise _WanPreparedBoundaryInvalid("calibration_operations") + return self.handle + + +@dataclass(frozen=True, slots=True) +class LoRATargetPolicy: + """Fail-closed exact-module policy for isolated Wan LoRA injection.""" + + declared_targets: tuple[str, ...] + + def __post_init__(self) -> None: + targets = self.declared_targets + if ( + type(targets) is not tuple + or not targets + or any(type(item) is not str or not item for item in targets) + or targets != tuple(sorted(set(targets))) + ): + raise ValueError("wan_lora_targets_invalid") + + def require_exact_eligible(self, eligible: tuple[str, ...]) -> None: + if eligible != self.declared_targets: + raise ValueError("wan_lora_targets_invalid") + + def require_isolated_inventory( + self, + before: tuple[WanParameterObservation, ...], + injected: tuple[WanParameterObservation, ...], + ) -> None: + before_by_name = _parameter_identities(before) + injected_by_name = _parameter_identities(injected) + if any(item.requires_grad for item in before): + raise ValueError("wan_parameter_inventory_invalid") + trainable = tuple(item for item in injected if item.requires_grad) + frozen = {item.name: item.digest for item in injected if not item.requires_grad} + if ( + not trainable + or {item.module for item in trainable} != set(self.declared_targets) + or any(item.module not in self.declared_targets for item in trainable) + or frozen != before_by_name + or len(injected_by_name) != len(injected) + ): + raise ValueError("wan_parameter_inventory_invalid") + + @staticmethod + def require_frozen_unchanged( + baseline: tuple[WanParameterObservation, ...], + after: tuple[WanParameterObservation, ...], + ) -> None: + expected = {item.name: item.digest for item in baseline if not item.requires_grad} + actual = {item.name: item.digest for item in after if not item.requires_grad} + if expected != actual: + raise ValueError("wan_frozen_parameters_changed") + + +def _trainable_specs_from_inventory( + inventory: tuple[WanParameterObservation, ...], + declared_targets: tuple[str, ...], +) -> tuple[TrainableParameterSpec, ...]: + specs: list[TrainableParameterSpec] = [] + for item in inventory: + if not item.requires_grad: + continue + role: Literal["A", "B"] | None = None + logical_name: str | None = None + for marker, candidate in ((".lora_A.", "A"), (".lora_B.", "B")): + if marker in item.name: + role = cast(Literal["A", "B"], candidate) + logical_name = item.name.split(marker, 1)[0] + break + if role is None or logical_name is None or item.module != logical_name: + raise ValueError("wan_parameter_inventory_invalid") + specs.append( + TrainableParameterSpec( + physical_name=item.name, + logical_target=item.module, + role=role, + shape=item.shape, + dtype=item.dtype, + device=item.device, + ) + ) + try: + return validate_trainable_parameter_specs( + specs, declared_targets=declared_targets + ) + except ValueError: + raise ValueError("wan_parameter_inventory_invalid") from None + + +def _parameter_identities( + inventory: tuple[WanParameterObservation, ...], +) -> dict[str, Sha256Digest]: + identities = {item.name: item.digest for item in inventory} + if not inventory or len(identities) != len(inventory): + raise ValueError("wan_parameter_inventory_invalid") + return identities + + +@runtime_checkable +class WanTrainingRuntime(Protocol): + def observe_host(self) -> WanMpsObservation: ... + + def prepare( + self, + sources: WanTrainingSources, + request: TrainingRequest | CalibrationRequest, + ) -> WanPreparedTraining: ... + + def release(self, prepared: WanPreparedTraining | None) -> None: ... + + +@dataclass(frozen=True, slots=True) +class _LeasedSource: + file: WanSourceFile + descriptor: int + + +class WanTrainingSourceLease: + """Backend-owned retained descriptors for one verified source snapshot.""" + + def __init__(self, sources: WanTrainingSources, leased: tuple[_LeasedSource, ...]) -> None: + self._sources = sources + self._leased = leased + self._closed = False + + @property + def sources(self) -> WanTrainingSources: + return self._sources + + def __enter__(self) -> WanTrainingSourceLease: + if self._closed: + raise ValueError("wan_source_lease_closed") + return self + + def __exit__( + self, + _error_type: type[BaseException] | None, + _error: BaseException | None, + _traceback: object | None, + ) -> None: + self.close() + + def reverify(self) -> None: + if self._closed: + raise ValueError("wan_source_lease_closed") + try: + for leased in self._leased: + _verify_open_file(leased.descriptor, leased.file) + current = _open_no_follow(leased.file.path) + try: + _verify_open_file(current, leased.file) + finally: + os.close(current) + except (OSError, ValueError): + raise ValueError("wan_source_invalid") from None + + def _verify_authority(self) -> None: + for leased in self._leased: + retained = os.fstat(leased.descriptor) + if not _matches_source(retained, leased.file): + raise ValueError("wan_source_invalid") + current = _open_no_follow(leased.file.path) + try: + if _file_identity(os.fstat(current)) != _file_identity(retained): + raise ValueError("wan_source_invalid") + finally: + os.close(current) + + def copy_file_to(self, source: WanSourceFile, destination: int) -> tuple[str, int]: + """Stream one retained source into an exclusive destination descriptor.""" + if self._closed: + raise ValueError("wan_source_lease_closed") + leased = next((item for item in self._leased if item.file == source), None) + if leased is None: + raise ValueError("wan_source_invalid") + before = os.fstat(leased.descriptor) + if not _matches_source(before, leased.file): + raise ValueError("wan_source_invalid") + os.lseek(leased.descriptor, 0, os.SEEK_SET) + digest = hashlib.sha256() + size = 0 + while size < source.size_bytes: + chunk = os.read(leased.descriptor, min(1024 * 1024, source.size_bytes - size)) + if not chunk: + raise ValueError("wan_source_invalid") + view = memoryview(chunk) + while view: + written = os.write(destination, view) + if written <= 0: + raise OSError("wan snapshot short write") + view = view[written:] + digest.update(chunk) + size += len(chunk) + if size != source.size_bytes or "sha256:" + digest.hexdigest() != source.digest: + raise ValueError("wan_source_invalid") + os.fsync(destination) + os.lseek(destination, 0, os.SEEK_SET) + copied = hashlib.sha256() + copied_size = 0 + while copied_size < size: + chunk = os.read(destination, min(1024 * 1024, size - copied_size)) + if not chunk: + raise ValueError("wan_source_invalid") + copied.update(chunk) + copied_size += len(chunk) + if copied_size != size or copied.digest() != digest.digest(): + raise ValueError("wan_source_invalid") + after = os.fstat(leased.descriptor) + if ( + not _matches_source(after, leased.file) + or _file_identity(before) != _file_identity(after) + ): + raise ValueError("wan_source_invalid") + return source.digest, size + + def read_file_bytes(self, source: WanSourceFile) -> bytes: + """Read exact bytes from the retained descriptor, never from its path.""" + + if self._closed: + raise ValueError("wan_source_lease_closed") + leased = next((item for item in self._leased if item.file == source), None) + if leased is None: + raise ValueError("wan_source_invalid") + try: + before = os.fstat(leased.descriptor) + if not _matches_source(before, source): + raise ValueError + payload = os.pread(leased.descriptor, source.size_bytes, 0) + after = os.fstat(leased.descriptor) + if ( + len(payload) != source.size_bytes + or "sha256:" + hashlib.sha256(payload).hexdigest() != source.digest + or _file_identity(before) != _file_identity(after) + or not _matches_source(after, source) + ): + raise ValueError + return payload + except (OSError, ValueError): + raise ValueError("wan_source_invalid") from None + + def close(self) -> None: + if self._closed: + return + self._closed = True + for item in self._leased: + try: + os.close(item.descriptor) + except OSError: + pass + + +class WanTrainingSourceResolver: + """Acquire retained, exact-byte leases for factory-authorized local artifacts.""" + + def __init__(self, descriptors: tuple[WanSourceDescriptor, ...]) -> None: + self._descriptors = descriptors + + def acquire(self) -> WanTrainingSourceLease: + names = tuple(item.name for item in self._descriptors) + if ( + not names + or not all(names) + or len(names) != len(set(names)) + or "base_model" not in names + ): + raise ValueError("wan_source_invalid") + leased: list[_LeasedSource] = [] + try: + for descriptor in sorted(self._descriptors, key=lambda item: item.name): + leased.append(self._acquire_one(descriptor)) + files = tuple(item.file for item in leased) + sources = _training_sources(files) + lease = WanTrainingSourceLease(sources, tuple(leased)) + lease._verify_authority() + return lease + except Exception: + for leased_source in leased: + try: + os.close(leased_source.descriptor) + except OSError: + pass + raise ValueError("wan_source_invalid") from None + + @staticmethod + def _acquire_one(descriptor: WanSourceDescriptor) -> _LeasedSource: + try: + path = Path(descriptor.path()) + expected_identity = ( + descriptor.expected_device, + descriptor.expected_inode, + descriptor.expected_size, + ) + if descriptor.expected_size < 0: + raise ValueError("wan_source_invalid") + file_descriptor = _open_no_follow(path) + try: + info = os.fstat(file_descriptor) + if ( + not stat.S_ISREG(info.st_mode) + or (info.st_dev, info.st_ino, info.st_size) != expected_identity + or _digest_open_file(file_descriptor, descriptor.expected_size) + != descriptor.expected_digest + ): + raise ValueError("wan_source_invalid") + after = os.fstat(file_descriptor) + if _file_identity(info) != _file_identity(after): + raise ValueError("wan_source_invalid") + except Exception: + os.close(file_descriptor) + raise + except (OSError, ValueError, TypeError): + raise ValueError("wan_source_invalid") from None + return _LeasedSource( + file=WanSourceFile( + name=descriptor.name, + path=path, + digest=descriptor.expected_digest, + size_bytes=descriptor.expected_size, + device=info.st_dev, + inode=info.st_ino, + ), + descriptor=file_descriptor, + ) + + +def _training_sources(files: tuple[WanSourceFile, ...]) -> WanTrainingSources: + payload = { + "files": [ + { + "name": item.name, + "sha256": item.digest, + "size_bytes": item.size_bytes, + } + for item in files + ] + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + source_digest = "sha256:" + hashlib.sha256(encoded).hexdigest() + base_model_digest = next(item.digest for item in files if item.name == "base_model") + return WanTrainingSources( + files=files, base_model_digest=base_model_digest, source_digest=source_digest + ) + + +def _open_no_follow(path: Path) -> int: + no_follow = getattr(os, "O_NOFOLLOW", None) + directory = getattr(os, "O_DIRECTORY", None) + if no_follow is None or directory is None or not path.is_absolute(): + raise ValueError("wan_source_invalid") + parts = path.parts + if len(parts) < 2 or any(part in {"", ".", ".."} for part in parts[1:]): + raise ValueError("wan_source_invalid") + directory_flags = os.O_RDONLY | directory | no_follow | getattr(os, "O_CLOEXEC", 0) + current = os.open(parts[0], directory_flags) + try: + for part in parts[1:-1]: + child = os.open(part, directory_flags, dir_fd=current) + os.close(current) + current = child + file_flags = ( + os.O_RDONLY | no_follow | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NONBLOCK", 0) + ) + return os.open(parts[-1], file_flags, dir_fd=current) + finally: + os.close(current) + + +def _digest_open_file(file_descriptor: int, expected_size: int) -> str: + os.lseek(file_descriptor, 0, os.SEEK_SET) + digest = hashlib.sha256() + size = 0 + while size <= expected_size: + chunk = os.read(file_descriptor, min(1024 * 1024, expected_size + 1 - size)) + if not chunk: + break + digest.update(chunk) + size += len(chunk) + if size != expected_size: + raise ValueError("wan_source_invalid") + return "sha256:" + digest.hexdigest() + + +def _file_identity(info: os.stat_result) -> tuple[int, int, int, int]: + return info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns + + +def _matches_source(info: os.stat_result, source: WanSourceFile) -> bool: + return stat.S_ISREG(info.st_mode) and (info.st_dev, info.st_ino, info.st_size) == ( + source.device, + source.inode, + source.size_bytes, + ) + + +def _verify_open_file(file_descriptor: int, source: WanSourceFile) -> None: + info = os.fstat(file_descriptor) + if ( + not _matches_source(info, source) + or _digest_open_file(file_descriptor, source.size_bytes) != source.digest + ): + raise ValueError("wan_source_invalid") + + +class WanBackendStageUnavailable(RuntimeError): + def __init__(self, stage: Literal["calibrate", "train"]) -> None: + self.stage = stage + super().__init__("wan_backend_stage_unavailable") + + +class WanCalibrationRuntimeError(RuntimeError): + def __init__(self) -> None: + super().__init__("wan_calibration_runtime_failed") + + +class WanTrainingRuntimeError(RuntimeError): + def __init__(self) -> None: + super().__init__("wan_training_runtime_failed") + + +class WanTrainingDeadlineExceeded(RuntimeError): + def __init__(self) -> None: + super().__init__("wan_training_deadline_exceeded") + + +class WanLoRAMpsTrainer: + backend_id = WAN_LORA_MPS_BACKEND_ID + + def __init__( + self, + *, + backend_digest: Sha256Digest, + material_digest: Sha256Digest, + orchestration_source_digest: Sha256Digest | None = None, + descriptor_source_digest: Sha256Digest | None = None, + source_resolver: WanTrainingSourceResolver, + runtime: WanTrainingRuntime, + checkpoint_resolver: Callable[[TrainingRequest], WanParentCheckpointDescriptor] + | None = None, + clock: Callable[[], float] = monotonic, + ) -> None: + self._backend_digest = backend_digest + self._material_digest = material_digest + self._orchestration_source_digest = orchestration_source_digest + self._descriptor_source_digest = descriptor_source_digest + self._source_resolver = source_resolver + self._runtime = runtime + self._checkpoint_resolver = checkpoint_resolver + self._clock = clock + self._closed = False + self._active_operations = 0 + self._condition = threading.Condition() + + def __enter__(self) -> WanLoRAMpsTrainer: + self._require_open() + return self + + def __exit__(self, *_error: object) -> None: + self.close() + + def close(self) -> None: + with self._condition: + if self._closed: + return + if self._active_operations: + raise ValueError("wan_trainer_busy") + close = getattr(self._runtime, "close", None) + if callable(close): + close() + self._closed = True + + def _require_open(self) -> None: + with self._condition: + if self._closed: + raise ValueError("wan_trainer_closed") + + @contextmanager + def _operation(self) -> Iterator[None]: + with self._condition: + if self._closed: + raise ValueError("wan_trainer_closed") + self._active_operations += 1 + try: + yield + finally: + with self._condition: + self._active_operations -= 1 + self._condition.notify_all() + + def probe(self, request: ProbeTrainingRequest) -> TrainerCapabilityReport: + with self._operation(): + return self._probe(request) + + def _probe(self, request: ProbeTrainingRequest) -> TrainerCapabilityReport: + lease: WanTrainingSourceLease | None = None + try: + lease = self._source_resolver.acquire() + observed = self._runtime.observe_host() + observation = WanMpsObservation.model_validate(observed, strict=True) + lease.reverify() + return self._report( + observation.runtime_digest, + self._supports(request, lease.sources, observation), + ) + except Exception: + return self._report(request.materials.runtime_digest, False) + finally: + if lease is not None: + lease.close() + + def calibrate( + self, + request: CalibrationRequest, + progress: TrainingProgressCallback, + cancelled: CancellationCheck, + ) -> CalibrationResult: + with self._operation(): + return self._calibrate(request, progress, cancelled) + + def _calibrate( + self, + request: CalibrationRequest, + progress: TrainingProgressCallback, + cancelled: CancellationCheck, + ) -> CalibrationResult: + lease: WanTrainingSourceLease | None = None + prepared: WanPreparedTraining | None = None + primary_error: BaseException | None = None + try: + lease = self._source_resolver.acquire() + observed = self._runtime.observe_host() + observation = WanMpsObservation.model_validate(observed, strict=True) + if not self._supports_calibration(request, lease.sources, observation): + raise ValueError("wan_calibration_unsupported") + lease.reverify() + prepared = self._runtime.prepare(lease.sources, request) + if type(prepared) is not WanPreparedTraining: + raise _WanPreparedBoundaryInvalid("prepared") + lease.reverify() + if cancelled(): + raise TrainingCancelled(0) + + policy = LoRATargetPolicy(request.trainable_targets) + policy.require_exact_eligible(prepared.eligible_lora_modules()) + before = prepared.parameter_inventory() + prepared.inject_lora( + targets=policy.declared_targets, + configuration=request.lora, + seed=request.seed, + ) + injected = prepared.parameter_inventory() + policy.require_isolated_inventory(before, injected) + if cancelled(): + raise TrainingCancelled(0) + measurement = prepared.measure_forward_backward() + after = prepared.parameter_inventory() + policy.require_isolated_inventory(before, after) + policy.require_frozen_unchanged(injected, after) + if request.steps > 0: + progress( + TrainingProgress( + step=request.steps, + total_steps=request.steps, + fraction=1.0, + loss=measurement.loss, + gradient_norm=measurement.gradient_norm, + checkpoint_boundary=True, + ) + ) + lease.reverify() + calibration_digest = _calibration_digest( + request=request, + backend_digest=self._backend_digest, + runtime_digest=observation.runtime_digest, + before=before, + injected=injected, + after=after, + measurement=measurement, + ) + return CalibrationResult( + request_id=request.request_id, + run_id=request.run_id, + backend_digest=self._backend_digest, + runtime_digest=observation.runtime_digest, + calibration_digest=calibration_digest, + finite_gradients=True, + frozen_parameters_unchanged=True, + changed_targets=policy.declared_targets, + measurements=measurement.model_dump(mode="python"), + ) + except TrainingCancelled as error: + primary_error = error + raise + except _WanPreparedBoundaryInvalid as error: + primary_error = error + if str(error) == "measurement": + raise ValueError("wan_calibration_measurement_invalid") from None + raise ValueError("wan_parameter_inventory_invalid") from None + except ValueError as error: + primary_error = error + raise + except BaseException as error: + primary_error = error + raise WanCalibrationRuntimeError() from None + finally: + cleanup_failed = False + try: + self._runtime.release(prepared) + except BaseException: + cleanup_failed = True + if lease is not None: + try: + lease.close() + except BaseException: + cleanup_failed = True + if cleanup_failed and primary_error is None: + raise WanCalibrationRuntimeError() from None + + def train( + self, + request: TrainingRequest, + progress: TrainingProgressCallback, + cancelled: CancellationCheck, + ) -> TrainingResult: + with self._operation(): + return self._train(request, progress, cancelled) + + def _train( + self, + request: TrainingRequest, + progress: TrainingProgressCallback, + cancelled: CancellationCheck, + ) -> TrainingResult: + resume = request.start_step > 0 + if (not resume and request.telemetry_sequence_start != 1) or ( + resume and self._checkpoint_resolver is None + ): + raise ValueError("wan_training_resume_invalid") + + started = self._clock() + lease: WanTrainingSourceLease | None = None + prepared: WanPreparedTraining | None = None + codec: WanCheckpointCodec | None = None + checkpoints: list[CheckpointResult] = [] + restored: WanCheckpointState | None = None + descriptor: WanParentCheckpointDescriptor | None = None + primary_error: BaseException | None = None + try: + lease = self._source_resolver.acquire() + self._training_observation(request, lease.sources) + lease.reverify() + codec = WanCheckpointCodec(request.scratch_root) + if resume: + assert self._checkpoint_resolver is not None + descriptor = self._checkpoint_resolver(request) + if ( + descriptor.checkpoint_id != request.parent_checkpoint_id + or descriptor.checkpoint_digest != request.parent_checkpoint_digest + or descriptor.step != request.start_step + or descriptor.total_steps != request.total_steps + ): + raise ValueError("wan_training_resume_invalid") + optimizer_files = tuple( + item for item in descriptor.files if item.role == "optimizer" + ) + if ( + len(optimizer_files) != 1 + or optimizer_files[0].digest != request.optimizer_state_digest + ): + raise ValueError("wan_training_resume_invalid") + prepared = self._runtime.prepare(lease.sources, request) + if type(prepared) is not WanPreparedTraining: + raise _WanPreparedBoundaryInvalid("prepared") + lease.reverify() + + policy = LoRATargetPolicy(request.trainable_targets) + eligible_modules = prepared.eligible_lora_modules() + lease.reverify() + policy.require_exact_eligible(eligible_modules) + before = prepared.parameter_inventory() + lease.reverify() + prepared.inject_lora( + targets=policy.declared_targets, + configuration=request.lora, + seed=request.seed, + ) + lease.reverify() + injected = prepared.parameter_inventory() + lease.reverify() + policy.require_isolated_inventory(before, injected) + trainable_specs = _trainable_specs_from_inventory( + injected, policy.declared_targets + ) + telemetry: list[dict[str, object]] = [] + parent = request.parent_checkpoint_id + + if resume: + assert descriptor is not None + restored = codec.restore( + descriptor, + WanCheckpointValidationContext( + checkpoint_id=descriptor.checkpoint_id, + request_id=request.request_id, + run_id=request.run_id, + backend_id=request.backend_id, + seed=request.seed, + step=request.start_step, + total_steps=request.total_steps, + parent_checkpoint_id=descriptor.parent_checkpoint_id, + parent_optimizer_state_digest=descriptor.parent_optimizer_state_digest, + trainable_targets=request.trainable_targets, + trainable_parameter_specs=trainable_specs, + lora_rank=request.lora.rank, + lora_alpha=request.lora.alpha, + lora_dropout=request.lora.dropout, + optimizer_algorithm=request.optimizer.algorithm, + optimizer_config=tuple( + (item.name, item.value) for item in request.optimizer.parameters + ), + ), + ) + codec.reverify_restored() + prepared.restore_checkpoint(restored) + codec.reverify_restored() + if request.telemetry_sequence_start != request.start_step + 1: + raise ValueError("wan_training_resume_invalid") + + for step in range(request.start_step + 1, request.total_steps + 1): + self._require_deadline(started, request.deadline_seconds) + if resume and step == request.start_step + 1: + codec.reverify_restored() + self._training_observation(request, lease.sources) + lease.reverify() + measurement = prepared.train_step(step=step, total_steps=request.total_steps) + lease.reverify() + after = prepared.parameter_inventory() + lease.reverify() + policy.require_isolated_inventory(before, after) + policy.require_frozen_unchanged(injected, after) + item = TrainingProgress( + step=step, + total_steps=request.total_steps, + fraction=step / request.total_steps, + loss=measurement.loss, + gradient_norm=measurement.gradient_norm, + checkpoint_boundary=step in request.checkpoint_steps, + ) + progress(item) + lease.reverify() + telemetry.append( + { + "sequence": request.telemetry_sequence_start + len(telemetry), + "step": step, + "payload": item.model_dump(mode="json"), + } + ) + self._require_deadline(started, request.deadline_seconds) + if step not in request.checkpoint_steps: + continue + + runtime_state = prepared.checkpoint_state(step=step) + lease.reverify() + checkpoint_id = _checkpoint_id(request, step, parent) + checkpoint = codec.write( + WanCheckpointState( + checkpoint_id=checkpoint_id, + step=step, + total_steps=request.total_steps, + parent_checkpoint_id=parent, + weights=runtime_state.weights, + optimizer=runtime_state.optimizer, + trainer_state=runtime_state.trainer_state, + telemetry=tuple(telemetry), + trainable_targets={ + "schema_version": "trainer-trainable-targets.v2", + "request_id": request.request_id, + "run_id": request.run_id, + "step": step, + "targets": list(request.trainable_targets), + "physical_parameters": trainable_parameter_spec_records( + trainable_specs + ), + }, + frozen_parameters={ + "schema_version": "trainer-frozen-parameters.v1", + "request_id": request.request_id, + "run_id": request.run_id, + "step": step, + "trainable_targets": list(request.trainable_targets), + "unchanged": True, + }, + finite_gradient_proof={ + "schema_version": "trainer-finite-gradient-proof.v1", + "request_id": request.request_id, + "run_id": request.run_id, + "step": step, + "finite": True, + "loss": measurement.loss, + "gradient_norm": measurement.gradient_norm, + }, + ), + WanCheckpointValidationContext( + checkpoint_id=checkpoint_id, + request_id=request.request_id, + run_id=request.run_id, + backend_id=request.backend_id, + seed=request.seed, + step=step, + total_steps=request.total_steps, + parent_checkpoint_id=parent, + parent_optimizer_state_digest=request.optimizer_state_digest, + trainable_targets=request.trainable_targets, + trainable_parameter_specs=trainable_specs, + lora_rank=request.lora.rank, + lora_alpha=request.lora.alpha, + lora_dropout=request.lora.dropout, + optimizer_algorithm=request.optimizer.algorithm, + optimizer_config=tuple( + (item.name, item.value) for item in request.optimizer.parameters + ), + start_step=request.start_step, + telemetry_sequence_start=request.telemetry_sequence_start, + ), + ) + checkpoints.append(checkpoint) + lease.reverify() + parent = checkpoint.checkpoint_id + result = self._training_result( + request=request, + checkpoints=tuple(checkpoints), + elapsed=self._elapsed(started), + ) + self._require_deadline(started, request.deadline_seconds) + validate_cancelled_training_result(request, result, step) + lease.reverify() + is_cancelled = cancelled() + lease.reverify() + if is_cancelled: + raise TrainingCancelled(step, result) + + result = self._training_result( + request=request, + checkpoints=tuple(checkpoints), + elapsed=self._elapsed(started), + ) + validate_training_result(request, result) + lease.reverify() + return result + except TrainingCancelled as error: + primary_error = error + raise + except WanCheckpointCleanupError as error: + primary_error = error + raise + except _WanPreparedBoundaryInvalid as error: + primary_error = error + if str(error) in {"restore_state", "restore_receipt"}: + raise ValueError("wan_training_restore_invalid") from None + raise ValueError("wan_parameter_inventory_invalid") from None + except (ValueError, WanTrainingDeadlineExceeded) as error: + primary_error = error + raise + except BaseException: + bounded_error = WanTrainingRuntimeError() + primary_error = bounded_error + raise bounded_error from None + finally: + cleanup_errors: list[BaseException] = [] + final_source_error: BaseException | None = None + try: + self._runtime.release(prepared) + except BaseException as error: + cleanup_errors.append(error) + if lease is not None: + try: + lease.reverify() + except BaseException as error: + final_source_error = error + discard_required = ( + primary_error is not None + and not isinstance(primary_error, TrainingCancelled) + ) or final_source_error is not None or bool(cleanup_errors) + if codec is not None: + if discard_required: + for checkpoint in reversed(checkpoints): + try: + codec.discard(checkpoint.checkpoint_id) + except WanCheckpointCleanupError as error: + cleanup_errors.extend(error.cleanup_errors) + except BaseException as error: + cleanup_errors.append(error) + try: + codec.close() + except BaseException as error: + cleanup_errors.append(error) + if lease is not None: + try: + lease.close() + except BaseException as error: + cleanup_errors.append(error) + if ( + isinstance(primary_error, TrainingCancelled) + and not discard_required + and cleanup_errors + ): + recovery_codec: WanCheckpointCodec | None = None + try: + recovery_codec = WanCheckpointCodec(request.scratch_root) + for checkpoint in reversed(checkpoints): + try: + recovery_codec.discard(checkpoint.checkpoint_id) + except WanCheckpointCleanupError as error: + cleanup_errors.extend(error.cleanup_errors) + except BaseException as error: + cleanup_errors.append(error) + except BaseException as error: + cleanup_errors.append(error) + finally: + if recovery_codec is not None: + try: + recovery_codec.close() + except BaseException as error: + cleanup_errors.append(error) + if isinstance(primary_error, WanCheckpointCleanupError): + cleanup_errors[0:0] = primary_error.cleanup_errors + primary_error = primary_error.primary_error + if cleanup_errors: + raise WanCheckpointCleanupError( + final_source_error or primary_error, tuple(cleanup_errors) + ) from primary_error + if final_source_error is not None: + raise final_source_error from primary_error + + def _training_observation( + self, request: TrainingRequest, sources: WanTrainingSources + ) -> WanMpsObservation: + observed = self._runtime.observe_host() + observation = WanMpsObservation.model_validate(observed, strict=True) + if not self._supports_common( + backend_id=request.backend_id, + materials=request.materials, + sources=sources, + observation=observation, + ): + raise ValueError("wan_training_placement_invalid") + return observation + + def _elapsed(self, started: float) -> float: + elapsed = self._clock() - started + if not math.isfinite(elapsed) or elapsed < 0: + raise WanTrainingDeadlineExceeded() + return elapsed + + def _require_deadline(self, started: float, deadline_seconds: int) -> None: + if self._elapsed(started) > deadline_seconds: + raise WanTrainingDeadlineExceeded() + + def _training_result( + self, + *, + request: TrainingRequest, + checkpoints: tuple[CheckpointResult, ...], + elapsed: float, + ) -> TrainingResult: + if not checkpoints: + raise ValueError("wan_training_checkpoint_missing") + latest = checkpoints[-1] + return TrainingResult( + request_id=request.request_id, + run_id=request.run_id, + backend_id=self.backend_id, + backend_digest=self._backend_digest, + materials=request.materials, + output_root=request.scratch_root, + checkpoints=checkpoints, + telemetry_digest=latest.telemetry_digest, + changed_targets=request.trainable_targets, + frozen_parameters_unchanged=True, + finite_gradients=True, + elapsed_seconds=elapsed, + ) + + def _report(self, runtime_digest: Sha256Digest, supported: bool) -> TrainerCapabilityReport: + return TrainerCapabilityReport( + backend_id=self.backend_id, + backend_digest=self._backend_digest, + runtime_digest=runtime_digest, + supported=supported, + max_steps=_MAX_STEPS, + safe_boundaries=_SAFE_BOUNDARIES, + ) + + def _supports( + self, + request: ProbeTrainingRequest, + sources: WanTrainingSources, + observation: WanMpsObservation, + ) -> bool: + return request.expected_backend_digest == self._backend_digest and self._supports_common( + backend_id=request.backend_id, + materials=request.materials, + sources=sources, + observation=observation, + ) + + def _supports_calibration( + self, + request: CalibrationRequest, + sources: WanTrainingSources, + observation: WanMpsObservation, + ) -> bool: + return self._supports_common( + backend_id=request.backend_id, + materials=request.materials, + sources=sources, + observation=observation, + ) + + def _supports_common( + self, + *, + backend_id: str, + materials: TrainingMaterialHandles, + sources: WanTrainingSources, + observation: WanMpsObservation, + ) -> bool: + placements = observation.component_placements + source_bindings_match = ( + materials.source_digest == sources.source_digest + if self._orchestration_source_digest is None + and self._descriptor_source_digest is None + else materials.source_digest == self._orchestration_source_digest + and sources.source_digest == self._descriptor_source_digest + ) + return ( + backend_id == self.backend_id + and materials.backend_digest == self._backend_digest + and materials.material_digest == self._material_digest + and materials.base_model_digest == sources.base_model_digest + and source_bindings_match + and materials.runtime_digest == observation.runtime_digest + and observation.built + and observation.available + and observation.bf16_supported + and tuple(name for name, _device in placements) == _EXECUTABLE_COMPONENTS + and all(device == "mps" for _name, device in placements) + and observation.disk_required_bytes > 0 + and observation.disk_free_bytes >= observation.disk_required_bytes + and observation.memory_required_bytes > 0 + and observation.memory_headroom_bytes >= observation.memory_required_bytes + ) + + +def _calibration_digest( + *, + request: CalibrationRequest, + backend_digest: Sha256Digest, + runtime_digest: Sha256Digest, + before: tuple[WanParameterObservation, ...], + injected: tuple[WanParameterObservation, ...], + after: tuple[WanParameterObservation, ...], + measurement: WanCalibrationMeasurement, +) -> Sha256Digest: + payload = { + "request": request.model_dump(mode="json", exclude={"scratch_root"}), + "backend_digest": backend_digest, + "runtime_digest": runtime_digest, + "before": [item.model_dump(mode="json") for item in before], + "injected": [item.model_dump(mode="json") for item in injected], + "after": [item.model_dump(mode="json") for item in after], + "measurement": measurement.model_dump(mode="json"), + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def _checkpoint_id(request: TrainingRequest, step: int, parent_checkpoint_id: str | None) -> str: + payload = { + "schema_version": "wan-checkpoint-identity.v1", + "request_id": request.request_id, + "run_id": request.run_id, + "backend_id": request.backend_id, + "backend_digest": request.materials.backend_digest, + "material_digest": request.materials.material_digest, + "step": step, + "parent_checkpoint_id": parent_checkpoint_id, + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return "wan-" + hashlib.sha256(encoded).hexdigest() diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_production_models.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_production_models.py new file mode 100644 index 0000000..b7b92da --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/trainers/wan_production_models.py @@ -0,0 +1,735 @@ +"""Immutable local-only evidence contracts for Wan MPS production canaries.""" + +from __future__ import annotations + +import hashlib +import math +from collections.abc import Mapping +from copy import deepcopy +from pathlib import PurePath, PureWindowsPath +from typing import Annotated, Any, Literal, Self + +from pydantic import ( + ConfigDict, + Field, + StrictBool, + StrictFloat, + StrictInt, + field_validator, + model_validator, +) + +from video_lab.domain import FrozenModel, canonical_json +from video_lab.training_models import Sha256Digest + +WanTransformerBranch = Literal["high_noise", "low_noise"] +WanCanaryPhase = Literal[ + "inventory", + "load_trained", + "step", + "checkpoint", + "close_trained", + "memory_release_trained", + "load_fresh_restore", + "restore", + "continuity", + "close_restored", + "memory_release_final", +] +WanComponentName = Literal["scheduler", "text_encoder", "tokenizer", "transformer", "vae"] +WanComponentManifest = tuple[WanComponentName, Sha256Digest] +WanBranchInventoryRole = Literal[ + "high_transformer", + "low_transformer", + "scheduler", + "text_encoder", + "tokenizer", + "transformer_config", + "vae", +] +WanTensorRole = Literal["activation", "optimizer", "parameter"] +WanTensorDtype = Literal["bfloat16", "float32", "int64"] +_TARGET_SUFFIXES: tuple[Literal["to_k", "to_out.0", "to_q", "to_v"], ...] = ( + "to_k", + "to_out.0", + "to_q", + "to_v", +) +_COMPONENT_NAMES: tuple[WanComponentName, ...] = ( + "scheduler", + "text_encoder", + "tokenizer", + "transformer", + "vae", +) + + +class _StrictWanV2Model(FrozenModel): + """V2 evidence base that validates construction and every model copy.""" + + model_config = ConfigDict( + frozen=True, + extra="forbid", + strict=True, + revalidate_instances="always", + hide_input_in_errors=True, + ) + + def model_copy( + self, *, update: Mapping[str, Any] | None = None, deep: bool = False + ) -> Self: + values = {name: getattr(self, name) for name in type(self).model_fields} + if deep: + values = deepcopy(values) + if update is not None: + values.update(update) + return type(self).model_validate(values, strict=True) + + +def _exact_tuple(value: object, *, label: str) -> tuple[object, ...]: + if type(value) is not tuple: + raise ValueError(f"{label} must be an exact tuple") + return value + + +def _revalidate_exact_model(value: object, expected: type[FrozenModel]) -> FrozenModel: + if type(value) is expected: + payload = value.model_dump(mode="python") + elif type(value) is dict: + payload = value + else: + raise ValueError("nested model must have the exact declared type") + return expected.model_validate(payload, strict=True) + + +def _domain_digest(domain: str, payload: dict[str, object]) -> str: + serialized = canonical_json({"domain": domain, "payload": payload}) + return f"sha256:{hashlib.sha256(serialized.encode()).hexdigest()}" + + +def _canonical_pairs( + value: tuple[tuple[str, object], ...], *, label: str +) -> tuple[tuple[str, object], ...]: + names = tuple(item[0] for item in value) + if not value or names != tuple(sorted(set(names))): + raise ValueError(f"{label} must be sorted and unique") + return value + + +def _builtin_int(value: object) -> int: + if type(value) is not int: + raise ValueError("value must be an exact built-in int") + return value + + +def _builtin_float(value: object) -> float: + if type(value) is not float: + raise ValueError("value must be an exact built-in float") + return value + + +def _relative_path(value: str) -> str: + if type(value) is not str: + raise ValueError("path must be a string") + path = PurePath(value) + windows_path = PureWindowsPath(value) + if ( + not value + or "\\" in value + or path.is_absolute() + or windows_path.anchor != "" + or windows_path.drive != "" + or ".." in path.parts + or ".." in windows_path.parts + or value == "." + or value.startswith("./") + or value.endswith("/.") + or "/./" in value + or any(PureWindowsPath(component).is_reserved() for component in path.parts) + ): + raise ValueError("path must be relative and contained") + return value + + +class WanNumericalToleranceV1(FrozenModel): + schema_version: Literal["wan-numerical-tolerance.v1"] = "wan-numerical-tolerance.v1" + atol: Annotated[StrictFloat, Field(gt=0, allow_inf_nan=False)] + rtol: Annotated[StrictFloat, Field(gt=0, allow_inf_nan=False)] + + @field_validator("atol", "rtol", mode="before") + @classmethod + def validate_builtin_float(cls, value: object) -> float: + return _builtin_float(value) + + +class WanBranchRecipeV1(FrozenModel): + schema_version: Literal["wan-mps-branch-recipe.v1"] + branch: WanTransformerBranch + rank: Literal[2] + alpha: Annotated[StrictInt, Field(gt=0)] + dropout: Literal[0.0] # type: ignore[valid-type] + target_suffixes: tuple[Literal["to_k", "to_out.0", "to_q", "to_v"], ...] + optimizer_algorithm: Literal["adamw"] + optimizer_parameters: tuple[tuple[str, StrictInt | StrictFloat | StrictBool], ...] + gradient_clip_norm: Annotated[StrictFloat, Field(gt=0, allow_inf_nan=False)] + seed: StrictInt + tolerance: WanNumericalToleranceV1 + + @field_validator("rank", "alpha", "seed", mode="before") + @classmethod + def validate_builtin_int(cls, value: object) -> int: + return _builtin_int(value) + + @field_validator("dropout", "gradient_clip_norm", mode="before") + @classmethod + def validate_builtin_float(cls, value: object) -> float: + return _builtin_float(value) + + @field_validator("target_suffixes") + @classmethod + def validate_target_suffixes( + cls, value: tuple[Literal["to_k", "to_out.0", "to_q", "to_v"], ...] + ) -> tuple[Literal["to_k", "to_out.0", "to_q", "to_v"], ...]: + if value != _TARGET_SUFFIXES: + raise ValueError("target suffixes must be the exact canonical attention policy") + return value + + @field_validator("optimizer_parameters") + @classmethod + def validate_optimizer_parameters( + cls, value: tuple[tuple[str, StrictInt | StrictFloat | StrictBool], ...] + ) -> tuple[tuple[str, StrictInt | StrictFloat | StrictBool], ...]: + _canonical_pairs(value, label="optimizer parameters") + if any(not name for name, _ in value): + raise ValueError("optimizer parameter names must be non-empty") + return value + + @field_validator("optimizer_parameters", mode="before") + @classmethod + def validate_optimizer_builtin_values(cls, value: object) -> object: + if type(value) is not tuple: + raise ValueError("optimizer parameters must be an exact tuple") + for entry in value: + if type(entry) is not tuple or len(entry) != 2: + raise ValueError("optimizer parameter entries must be exact pairs") + parameter_name, parameter_value = entry + if type(parameter_name) is not str: + raise ValueError("optimizer parameter names must use exact built-in strings") + if type(parameter_value) not in (int, float, bool): + raise ValueError("optimizer values must use exact built-in scalar types") + return value + + +class WanComponentInventoryV1(FrozenModel): + schema_version: Literal["wan-mps-component-inventory.v1"] + component_manifests: tuple[WanComponentManifest, ...] + + @field_validator("component_manifests") + @classmethod + def validate_component_manifests( + cls, + value: tuple[WanComponentManifest, ...], + ) -> tuple[WanComponentManifest, ...]: + names = tuple(item[0] for item in value) + if names != _COMPONENT_NAMES: + raise ValueError("component manifests must contain every canonical component once") + return value + + +class WanBranchInventoryResultV1(FrozenModel): + """Complete inventory or path-free roles required for local acquisition.""" + + model_config = ConfigDict( + frozen=True, + extra="forbid", + strict=True, + revalidate_instances="always", + hide_input_in_errors=True, + ) + + available: StrictBool + inventory: WanComponentInventoryV1 | None + missing_roles: tuple[WanBranchInventoryRole, ...] + + @field_validator("missing_roles", mode="before") + @classmethod + def validate_exact_role_tuple(cls, value: object) -> object: + if type(value) is not tuple or any(type(role) is not str for role in value): + raise ValueError("missing roles must be an exact tuple of logical roles") + return value + + @field_validator("missing_roles") + @classmethod + def validate_canonical_roles( + cls, value: tuple[WanBranchInventoryRole, ...] + ) -> tuple[WanBranchInventoryRole, ...]: + if value != tuple(sorted(set(value))): + raise ValueError("missing roles must be sorted, unique, and nonempty when unavailable") + return value + + @model_validator(mode="after") + def validate_availability(self) -> WanBranchInventoryResultV1: + if self.available: + if self.inventory is None or self.missing_roles: + raise ValueError("available inventory result must contain one complete inventory") + elif self.inventory is not None or not self.missing_roles: + raise ValueError("unavailable inventory result must contain missing logical roles") + return self + + +class WanProductionCanaryRequestV1(FrozenModel): + schema_version: Literal["wan-mps-production-canary-request.v1"] + recipe: WanBranchRecipeV1 + inventory: WanComponentInventoryV1 + device: Literal["mps"] + dtype: Literal["bfloat16"] + batch_digest: Sha256Digest + relative_work_path: str = Field(min_length=1) + + @field_validator("relative_work_path") + @classmethod + def validate_relative_work_path(cls, value: str) -> str: + return _relative_path(value) + + +class WanBranchCanaryResultV1(FrozenModel): + schema_version: Literal["wan-mps-branch-canary-result.v1"] + branch: WanTransformerBranch + recipe_digest: Sha256Digest + output_digest: Sha256Digest + max_absolute_error: Annotated[StrictFloat, Field(ge=0, allow_inf_nan=False)] + max_relative_error: Annotated[StrictFloat, Field(ge=0, allow_inf_nan=False)] + + @field_validator("max_absolute_error", "max_relative_error", mode="before") + @classmethod + def validate_builtin_float(cls, value: object) -> float: + return _builtin_float(value) + + +class WanProductionCanaryReceiptV1(FrozenModel): + schema_version: Literal["wan-mps-production-canary-receipt.v1"] + branch_results: tuple[WanBranchCanaryResultV1, WanBranchCanaryResultV1] + compilation_receipt_digest: Sha256Digest + batch_digest: Sha256Digest + recipe_family_digest: Sha256Digest + dependency_versions: tuple[tuple[str, str], ...] + hardware_identity: tuple[tuple[str, str], ...] + cleanup_state: Literal["cleaned"] + + @model_validator(mode="after") + def validate_receipt_bindings(self) -> WanProductionCanaryReceiptV1: + if tuple(result.branch for result in self.branch_results) != ("high_noise", "low_noise"): + raise ValueError("branch results must bind high_noise then low_noise exactly once") + _canonical_pairs(self.dependency_versions, label="dependency versions") + _canonical_pairs(self.hardware_identity, label="hardware identity") + if any(not value for _, value in self.dependency_versions + self.hardware_identity): + raise ValueError("receipt identity values must be non-empty") + return self + + +class WanCapacityResultV1(FrozenModel): + schema_version: Literal["wan-mps-capacity-result.v1"] + device: Literal["mps"] + dtype: Literal["bfloat16"] + total_memory_bytes: Annotated[StrictInt, Field(ge=0)] + available_memory_bytes: Annotated[StrictInt, Field(ge=0)] + required_memory_bytes: Annotated[StrictInt, Field(ge=0)] + fits: StrictBool + + @field_validator( + "total_memory_bytes", "available_memory_bytes", "required_memory_bytes", mode="before" + ) + @classmethod + def validate_builtin_int(cls, value: object) -> int: + return _builtin_int(value) + + @field_validator("fits", mode="before") + @classmethod + def validate_builtin_bool(cls, value: object) -> bool: + if type(value) is not bool: + raise ValueError("fits must be an exact built-in bool") + return value + + @model_validator(mode="after") + def validate_capacity(self) -> WanCapacityResultV1: + if self.available_memory_bytes > self.total_memory_bytes: + raise ValueError("available memory cannot exceed total memory") + if self.fits != (self.required_memory_bytes <= self.available_memory_bytes): + raise ValueError("capacity result must match required and available memory") + return self + + +class WanProductionCanaryRequestV2(_StrictWanV2Model): + """Complete immutable authority for the two sequential branch runs.""" + + schema_version: Literal["wan-mps-production-canary-request.v2"] + recipes: tuple[WanBranchRecipeV1, WanBranchRecipeV1] + inventories: tuple[WanComponentInventoryV1, WanComponentInventoryV1] + compilation_receipt_digest: Sha256Digest + batch_digest: Sha256Digest + primary_run_id: str = Field(min_length=8) + independent_run_id: str = Field(min_length=8) + requested_tensor_envelope: tuple[WanTensorEnvelopeV2, ...] + memory_release_bound_bytes: Annotated[StrictInt, Field(ge=0)] + required_artifact_bytes: Annotated[StrictInt, Field(gt=0)] + dependency_versions: tuple[tuple[str, str], ...] + hardware_identity: tuple[tuple[str, str], ...] + + @field_validator( + "recipes", + "inventories", + "requested_tensor_envelope", + "dependency_versions", + "hardware_identity", + mode="before", + ) + @classmethod + def validate_exact_containers(cls, value: object) -> object: + return _exact_tuple(value, label="v2 request container") + + @field_validator("recipes", mode="before") + @classmethod + def revalidate_recipes(cls, value: object) -> object: + items = _exact_tuple(value, label="recipes") + return tuple(_revalidate_exact_model(item, WanBranchRecipeV1) for item in items) + + @field_validator("inventories", mode="before") + @classmethod + def revalidate_inventories(cls, value: object) -> object: + items = _exact_tuple(value, label="inventories") + return tuple(_revalidate_exact_model(item, WanComponentInventoryV1) for item in items) + + @field_validator("requested_tensor_envelope", mode="before") + @classmethod + def revalidate_envelope(cls, value: object) -> object: + items = _exact_tuple(value, label="requested tensor envelope") + return tuple(_revalidate_exact_model(item, WanTensorEnvelopeV2) for item in items) + + @field_validator( + "memory_release_bound_bytes", "required_artifact_bytes", mode="before" + ) + @classmethod + def validate_exact_ints(cls, value: object) -> int: + return _builtin_int(value) + + @model_validator(mode="after") + def validate_authority(self) -> WanProductionCanaryRequestV2: + if tuple(recipe.branch for recipe in self.recipes) != ("high_noise", "low_noise"): + raise ValueError("wan_canary_branch_order_invalid") + if self.primary_run_id == self.independent_run_id: + raise ValueError("wan_canary_independent_run_invalid") + names = tuple(item.name for item in self.requested_tensor_envelope) + roles = {item.role for item in self.requested_tensor_envelope} + if ( + not names + or names != tuple(sorted(set(names))) + or roles != {"activation", "optimizer", "parameter"} + ): + raise ValueError("wan_canary_tensor_envelope_invalid") + _canonical_pairs(self.dependency_versions, label="dependency versions") + _canonical_pairs(self.hardware_identity, label="hardware identity") + if any(not value for _, value in self.dependency_versions + self.hardware_identity): + raise ValueError("wan_canary_identity_invalid") + hardware = dict(self.hardware_identity) + if ( + set(hardware) != {"device", "machine", "system"} + or hardware["device"] != "mps" + or hardware["system"] != "Darwin" + ): + raise ValueError("wan_canary_hardware_identity_invalid") + return self + + +class WanCapacityResultV2(_StrictWanV2Model): + schema_version: Literal["wan-mps-capacity-result.v2"] + branch: WanTransformerBranch + phase: WanCanaryPhase + device: Literal["mps"] + dtype: Literal["bfloat16"] + requested_tensor_envelope: tuple[WanTensorEnvelopeV2, ...] + driver_allocated_before_bytes: Annotated[StrictInt, Field(ge=0)] + driver_allocated_after_failure_bytes: Annotated[StrictInt, Field(ge=0)] + driver_allocated_after_cleanup_bytes: Annotated[StrictInt, Field(ge=0)] + recommended_maximum_bytes: Annotated[StrictInt, Field(ge=0)] + required_memory_bytes: Annotated[StrictInt, Field(gt=0)] + recipe_digest: Sha256Digest + inventory_digest: Sha256Digest + fits: Literal[False] + classification: Literal["recognized_mps_allocation_failure"] + + @field_validator("requested_tensor_envelope", mode="before") + @classmethod + def revalidate_envelope(cls, value: object) -> object: + items = _exact_tuple(value, label="requested tensor envelope") + return tuple(_revalidate_exact_model(item, WanTensorEnvelopeV2) for item in items) + + @field_validator( + "driver_allocated_before_bytes", + "driver_allocated_after_failure_bytes", + "driver_allocated_after_cleanup_bytes", + "recommended_maximum_bytes", + "required_memory_bytes", + mode="before", + ) + @classmethod + def validate_exact_ints(cls, value: object) -> int: + return _builtin_int(value) + + @field_validator("fits", mode="before") + @classmethod + def validate_exact_false(cls, value: object) -> bool: + if type(value) is not bool or value: + raise ValueError("fits must be exact false") + return value + + @model_validator(mode="after") + def validate_capacity(self) -> WanCapacityResultV2: + names = tuple(item.name for item in self.requested_tensor_envelope) + roles = {item.role for item in self.requested_tensor_envelope} + if ( + not names + or names != tuple(sorted(set(names))) + or roles != {"activation", "optimizer", "parameter"} + ): + raise ValueError("wan_capacity_envelope_invalid") + if self.required_memory_bytes != wan_required_tensor_bytes( + self.requested_tensor_envelope + ): + raise ValueError("wan_capacity_required_bytes_invalid") + return self + + +class WanTensorEnvelopeV2(_StrictWanV2Model): + schema_version: Literal["wan-mps-tensor-envelope.v2"] = "wan-mps-tensor-envelope.v2" + name: str = Field(min_length=1) + role: WanTensorRole + shape: tuple[StrictInt, ...] + dtype: WanTensorDtype + copies: Annotated[StrictInt, Field(gt=0)] + + @field_validator("copies", mode="before") + @classmethod + def validate_exact_copies(cls, value: object) -> int: + return _builtin_int(value) + + @field_validator("shape", mode="before") + @classmethod + def validate_shape(cls, value: object) -> object: + if ( + type(value) is not tuple + or not value + or any(type(item) is not int or item <= 0 for item in value) + ): + raise ValueError("wan_tensor_envelope_shape_invalid") + return value + + +class WanProductionCanaryUnavailableV2(_StrictWanV2Model): + schema_version: Literal["wan-mps-production-canary-unavailable.v2"] + branch: WanTransformerBranch + missing_roles: tuple[WanBranchInventoryRole, ...] + + @field_validator("missing_roles", mode="before") + @classmethod + def validate_exact_roles(cls, value: object) -> object: + roles = _exact_tuple(value, label="missing roles") + if any(type(role) is not str for role in roles): + raise ValueError("wan_canary_missing_roles_invalid") + return roles + + @field_validator("missing_roles") + @classmethod + def validate_roles( + cls, value: tuple[WanBranchInventoryRole, ...] + ) -> tuple[WanBranchInventoryRole, ...]: + if not value or value != tuple(sorted(set(value))): + raise ValueError("wan_canary_missing_roles_invalid") + return value + + +class WanBranchNumericalProbeV2(_StrictWanV2Model): + schema_version: Literal["wan-mps-branch-numerical-probe.v2"] = ( + "wan-mps-branch-numerical-probe.v2" + ) + branch: WanTransformerBranch + recipe_digest: Sha256Digest + tensor_name: Literal["probe.output"] + shape: tuple[StrictInt, ...] + dtype: Literal["bfloat16", "float32"] + values: tuple[StrictFloat, ...] + + @field_validator("shape", mode="before") + @classmethod + def validate_shape(cls, value: object) -> object: + items = _exact_tuple(value, label="probe shape") + if not items or any(type(item) is not int or item <= 0 for item in items): + raise ValueError("wan_canary_probe_invalid") + return items + + @field_validator("values", mode="before") + @classmethod + def validate_values(cls, value: object) -> object: + items = _exact_tuple(value, label="probe values") + if not items or any(type(item) is not float for item in items): + raise ValueError("wan_canary_probe_invalid") + return items + + @model_validator(mode="after") + def validate_probe(self) -> WanBranchNumericalProbeV2: + if math.prod(self.shape) != len(self.values) or any( + not math.isfinite(value) for value in self.values + ): + raise ValueError("wan_canary_probe_invalid") + return self + + +class WanProductionCanaryReceiptV2(_StrictWanV2Model): + schema_version: Literal["wan-mps-production-canary-receipt.v2"] + branch_results: tuple[WanBranchCanaryResultV1, WanBranchCanaryResultV1] + branch_probes: tuple[WanBranchNumericalProbeV2, WanBranchNumericalProbeV2] + compilation_receipt_digest: Sha256Digest + batch_digest: Sha256Digest + inventory_family_digest: Sha256Digest + recipe_family_digest: Sha256Digest + primary_run_id: str = Field(min_length=8) + independent_run_id: str = Field(min_length=8) + dependency_versions: tuple[tuple[str, str], ...] + hardware_identity: tuple[tuple[str, str], ...] + memory_release_bytes: tuple[tuple[str, StrictInt], ...] + cleanup_state: Literal["cleaned"] + + @field_validator( + "branch_results", + "branch_probes", + "dependency_versions", + "hardware_identity", + "memory_release_bytes", + mode="before", + ) + @classmethod + def validate_exact_containers(cls, value: object) -> object: + return _exact_tuple(value, label="v2 receipt container") + + @field_validator("branch_results", mode="before") + @classmethod + def revalidate_results(cls, value: object) -> object: + items = _exact_tuple(value, label="branch results") + return tuple(_revalidate_exact_model(item, WanBranchCanaryResultV1) for item in items) + + @field_validator("branch_probes", mode="before") + @classmethod + def revalidate_probes(cls, value: object) -> object: + items = _exact_tuple(value, label="branch probes") + return tuple(_revalidate_exact_model(item, WanBranchNumericalProbeV2) for item in items) + + @model_validator(mode="after") + def validate_receipt(self) -> WanProductionCanaryReceiptV2: + if tuple(result.branch for result in self.branch_results) != ("high_noise", "low_noise"): + raise ValueError("wan_canary_branch_order_invalid") + if tuple(probe.branch for probe in self.branch_probes) != ("high_noise", "low_noise"): + raise ValueError("wan_canary_branch_order_invalid") + if any( + probe.recipe_digest != result.recipe_digest + for probe, result in zip(self.branch_probes, self.branch_results, strict=True) + ): + raise ValueError("wan_canary_probe_invalid") + if self.primary_run_id == self.independent_run_id: + raise ValueError("wan_canary_independent_run_invalid") + if tuple(name for name, _ in self.memory_release_bytes) != ( + "high_noise.trained", + "high_noise.restored", + "low_noise.trained", + "low_noise.restored", + ): + raise ValueError("wan_canary_memory_release_invalid") + if any(value < 0 for _, value in self.memory_release_bytes): + raise ValueError("wan_canary_memory_release_invalid") + _canonical_pairs(self.dependency_versions, label="dependency versions") + _canonical_pairs(self.hardware_identity, label="hardware identity") + hardware = dict(self.hardware_identity) + if ( + set(hardware) != {"device", "machine", "system"} + or hardware["device"] != "mps" + or hardware["system"] != "Darwin" + ): + raise ValueError("wan_canary_hardware_identity_invalid") + return self + + +def wan_branch_recipe_digest(recipe: WanBranchRecipeV1) -> Sha256Digest: + return _domain_digest("wan-mps-branch-recipe-digest.v1", recipe.model_dump(mode="json")) + + +def wan_inventory_digest(inventory: WanComponentInventoryV1) -> Sha256Digest: + return _domain_digest( + "wan-mps-component-inventory-digest.v1", inventory.model_dump(mode="json") + ) + + +def wan_inventory_family_digest( + inventories: tuple[WanComponentInventoryV1, WanComponentInventoryV1], +) -> Sha256Digest: + return _domain_digest( + "wan-mps-inventory-family-digest.v2", + {"inventories": [item.model_dump(mode="json") for item in inventories]}, + ) + + +def wan_recipe_family_digest( + recipes: tuple[WanBranchRecipeV1, WanBranchRecipeV1], +) -> Sha256Digest: + return _domain_digest( + "wan-mps-recipe-family-digest.v2", + {"recipes": [item.model_dump(mode="json") for item in recipes]}, + ) + + +def wan_required_tensor_bytes(envelope: tuple[WanTensorEnvelopeV2, ...]) -> int: + widths = {"bfloat16": 2, "float32": 4, "int64": 8} + total = 0 + for item in envelope: + elements = 1 + for extent in item.shape: + elements *= extent + total += elements * widths[item.dtype] * item.copies + return total + + +def wan_production_receipt_id(receipt: WanProductionCanaryReceiptV1) -> Sha256Digest: + return _domain_digest( + "wan-mps-production-canary-receipt-id.v1", receipt.model_dump(mode="json") + ) + + +def wan_production_receipt_v2_id(receipt: WanProductionCanaryReceiptV2) -> Sha256Digest: + return _domain_digest( + "wan-mps-production-canary-receipt-id.v2", receipt.model_dump(mode="json") + ) + + +__all__ = ( + "WanBranchCanaryResultV1", + "WanBranchNumericalProbeV2", + "WanBranchInventoryResultV1", + "WanBranchInventoryRole", + "WanBranchRecipeV1", + "WanCanaryPhase", + "WanCapacityResultV1", + "WanCapacityResultV2", + "WanComponentInventoryV1", + "WanNumericalToleranceV1", + "WanProductionCanaryReceiptV1", + "WanProductionCanaryReceiptV2", + "WanProductionCanaryUnavailableV2", + "WanProductionCanaryRequestV1", + "WanProductionCanaryRequestV2", + "WanTransformerBranch", + "WanTensorEnvelopeV2", + "wan_branch_recipe_digest", + "wan_inventory_digest", + "wan_inventory_family_digest", + "wan_production_receipt_id", + "wan_production_receipt_v2_id", + "wan_recipe_family_digest", + "wan_required_tensor_bytes", +) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_data.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_data.py new file mode 100644 index 0000000..c35963a --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_data.py @@ -0,0 +1,181 @@ +"""Content-addressed training data with caller-selected readiness policy.""" + +from __future__ import annotations + +import hashlib +from collections import defaultdict +from collections.abc import Iterable +from fractions import Fraction +from typing import Annotated, Literal, Protocol + +from pydantic import Field, model_validator + +from video_lab.domain import FrozenModel, canonical_json + +Digest = Annotated[str, Field(pattern=r"^sha256:[0-9a-f]{64}$")] + + +class DatasetMedia(FrozenModel): + """Media identity and metadata; source paths are not persisted.""" + + schema_version: Literal["training-media.v1"] = "training-media.v1" + sha256: Digest + media_type: Literal["image/jpeg", "image/png", "image/webp", "video/mp4"] + byte_size: int = Field(gt=0) + width: int = Field(gt=0) + height: int = Field(gt=0) + duration_seconds: float | None = Field(default=None, gt=0, allow_inf_nan=False) + frame_count: int | None = Field(default=None, gt=0) + fps: float | None = Field(default=None, gt=0, allow_inf_nan=False) + + @model_validator(mode="after") + def validate_kind_metadata(self) -> DatasetMedia: + values = (self.duration_seconds, self.frame_count, self.fps) + if self.media_type == "video/mp4": + if any(value is None for value in values): + raise ValueError("MP4 metadata requires duration, frame_count, and fps") + elif any(value is not None for value in values): + raise ValueError("still metadata must not contain video fields") + return self + + +class DatasetSample(FrozenModel): + sample_id: str = Field(min_length=1) + source_lineage: str = Field(min_length=1) + caption: str = Field(min_length=1) + split: Literal["train", "validation", "sealed_holdout"] + media: DatasetMedia + weight: float = Field(default=1.0, gt=0, allow_inf_nan=False) + synthetic: bool = False + + @property + def sample_digest(self) -> str: + return self.media.sha256 + + +class DatasetPolicy(FrozenModel): + """Optional composition limits; no character or campaign requirements.""" + + min_samples: int = Field(default=1, ge=1) + min_clip_seconds: float | None = Field(default=None, gt=0, allow_inf_nan=False) + max_clip_seconds: float | None = Field(default=None, gt=0, allow_inf_nan=False) + max_synthetic_share: float | None = Field( + default=None, ge=0, le=1, allow_inf_nan=False + ) + + @model_validator(mode="after") + def validate_duration_range(self) -> DatasetPolicy: + if ( + self.min_clip_seconds is not None + and self.max_clip_seconds is not None + and self.min_clip_seconds > self.max_clip_seconds + ): + raise ValueError("min_clip_seconds must not exceed max_clip_seconds") + return self + + +class TrainingDataset(FrozenModel): + schema_version: Literal["training-dataset.v1"] = "training-dataset.v1" + samples: tuple[DatasetSample, ...] + policy: DatasetPolicy = Field(default_factory=DatasetPolicy) + + +class DatasetNotReady(ValueError): + def __init__(self, reasons: Iterable[str]) -> None: + self.reason_codes = tuple(sorted(set(reasons))) + super().__init__( + "training dataset is not ready: " + ", ".join(self.reason_codes) + ) + + +class _SampleIdentity(Protocol): + sample_id: str + sample_digest: str + source_lineage: str + split: str + + +def dataset_integrity_reasons(samples: Iterable[_SampleIdentity]) -> set[str]: + """Shared integrity checks independent of any training policy or media backend.""" + items = tuple(samples) + reasons: set[str] = set() + if len({sample.sample_id for sample in items}) != len(items): + reasons.add("duplicate_sample_id") + if len({sample.sample_digest for sample in items}) != len(items): + reasons.add("duplicate_content") + lineages: dict[str, set[str]] = defaultdict(set) + for sample in items: + lineages[sample.source_lineage].add(str(sample.split)) + if any(len(splits) > 1 for splits in lineages.values()): + reasons.add("lineage_split_leakage") + return reasons + + +def _readiness_reasons(dataset: TrainingDataset) -> set[str]: + reasons = dataset_integrity_reasons(dataset.samples) + if len(dataset.samples) < dataset.policy.min_samples: + reasons.add("sample_count") + if not any(sample.split == "train" for sample in dataset.samples): + reasons.add("training_split_empty") + for sample in dataset.samples: + duration = sample.media.duration_seconds + if duration is None: + continue + if ( + dataset.policy.min_clip_seconds is not None + and duration < dataset.policy.min_clip_seconds + ) or ( + dataset.policy.max_clip_seconds is not None + and duration > dataset.policy.max_clip_seconds + ): + reasons.add("clip_duration") + cap = dataset.policy.max_synthetic_share + if cap is not None and dataset.samples: + total = sum( + (Fraction(str(sample.weight)) for sample in dataset.samples), Fraction(0) + ) + synthetic = sum( + ( + Fraction(str(sample.weight)) + for sample in dataset.samples + if sample.synthetic + ), + Fraction(0), + ) + if synthetic > Fraction(str(cap)) * total: + reasons.add("synthetic_share") + return reasons + + +def dataset_digest(dataset: TrainingDataset) -> str: + payload = dataset.model_dump(mode="json", exclude={"dataset_digest"}) + payload["samples"] = sorted( + payload["samples"], key=lambda sample: sample["sample_id"] + ) + return ( + "sha256:" + hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() + ) + + +class FrozenDataset(TrainingDataset): + dataset_digest: Digest + + @model_validator(mode="after") + def validate_snapshot(self) -> FrozenDataset: + reasons = _readiness_reasons(self) + if reasons: + raise DatasetNotReady(reasons) + if dataset_digest(self) != self.dataset_digest: + raise ValueError("dataset_digest does not match dataset content") + return self + + +def freeze_dataset(dataset: TrainingDataset) -> FrozenDataset: + """Validate and freeze the media, captions, splits, weights, and readiness policy.""" + reasons = _readiness_reasons(dataset) + if reasons: + raise DatasetNotReady(reasons) + return FrozenDataset( + **dataset.model_dump(mode="python"), + dataset_digest=dataset_digest(dataset), + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_data_cli.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_data_cli.py new file mode 100644 index 0000000..ef3a6ed --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_data_cli.py @@ -0,0 +1,66 @@ +"""Read-only CLI for generic training dataset metadata and snapshots.""" + +from __future__ import annotations + +import argparse +import sys +from collections import Counter +from collections.abc import Sequence +from pathlib import Path + +from video_lab.dataset_cli import DatasetCliError, DatasetExit, load_request +from video_lab.domain import canonical_json +from video_lab.training_data import ( + DatasetNotReady, + FrozenDataset, + TrainingDataset, + freeze_dataset, +) + + +class _Parser(argparse.ArgumentParser): + def error(self, message: str) -> None: + raise DatasetCliError("usage_error", message, DatasetExit.USAGE) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = _Parser(description="Validate and freeze generic training dataset metadata.") + # Shared lab options are accepted; these read-only commands do not open a store. + parser.add_argument("--database") + parser.add_argument("--artifacts") + parser.add_argument("operation", choices=("training-data",)) + parser.add_argument("action", choices=("validate", "freeze", "inspect")) + parser.add_argument("--request-root", type=Path, required=True) + parser.add_argument("--request-file", required=True) + parser.add_argument("--json", action="store_true") + try: + args = parser.parse_args(argv) + model = FrozenDataset if args.action == "inspect" else TrainingDataset + dataset = load_request(args.request_root, args.request_file, model) + frozen = dataset if isinstance(dataset, FrozenDataset) else freeze_dataset(dataset) + if args.action == "freeze": + data = frozen.model_dump(mode="json") + data["samples"] = sorted(data["samples"], key=lambda sample: sample["sample_id"]) + else: + data = { + "dataset_digest": frozen.dataset_digest, + "sample_count": len(frozen.samples), + "split_counts": dict(sorted(Counter(sample.split for sample in frozen.samples).items())), + } + envelope = {"schema_version": "training-data-cli.v1", "ok": True, "data": data} + code = 0 + except DatasetNotReady as error: + envelope = {"schema_version": "training-data-cli.v1", "ok": False, + "error": {"code": "dataset_not_ready", "message": str(error), + "reason_codes": list(error.reason_codes)}} + code = int(DatasetExit.REJECTED) + except DatasetCliError as error: + envelope = {"schema_version": "training-data-cli.v1", "ok": False, + "error": {"code": error.code, "message": error.safe_message}} + code = int(error.exit_code) + print(canonical_json(envelope)) + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_dataset.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_dataset.py new file mode 100644 index 0000000..e2e9f55 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_dataset.py @@ -0,0 +1,206 @@ +"""Pure dataset readiness and freezing rules for character LoRA training.""" + +from __future__ import annotations + +from collections.abc import Iterable +from decimal import Decimal + +from video_lab.training_data import dataset_integrity_reasons +from video_lab.training_models import ( + DatasetCoverage, + FrozenTrainingDataset, + TrainingDatasetDraft, + TrainingRecipe, + TrainingRecipeValidationEvidence, + TrainingSample, + TrainingSourceTier, + training_dataset_id, + training_recipe_id, +) + + +class TrainingDatasetNotReady(ValueError): + """A deliberately path-free readiness failure with stable machine reason codes.""" + + def __init__(self, reason_codes: Iterable[str]) -> None: + self.reason_codes = tuple(sorted(set(reason_codes))) + super().__init__("training dataset is not ready: " + ", ".join(self.reason_codes)) + + +class TrainingRecipeDatasetValidationError(ValueError): + """A stable trainer-neutral recipe/dataset contract failure.""" + + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +def evaluate_training_dataset(draft: TrainingDatasetDraft) -> DatasetCoverage: + """Calculate accepted-data counts and the complete curated coverage matrix.""" + samples = tuple(sample for sample in draft.samples if sample.accepted) + gold_stills = _count(samples, TrainingSourceTier.GOLD_STILL) + gold_clips = _count(samples, TrainingSourceTier.GOLD_CLIP) + synthetic_clips = _count(samples, TrainingSourceTier.SYNTHETIC_CLIP) + clips = gold_clips + synthetic_clips + synthetic_weight = float( + _synthetic_effective_share( + gold_clip_count=gold_clips, + synthetic_clip_count=synthetic_clips, + gold_clip_weight=draft.source_tier_weights[TrainingSourceTier.GOLD_CLIP], + synthetic_clip_weight=draft.source_tier_weights[TrainingSourceTier.SYNTHETIC_CLIP], + ) + ) + annotations = tuple(sample.annotation for sample in samples) + clip_annotations = tuple(sample.annotation for sample in _clips(samples)) + return DatasetCoverage( + accepted_gold_stills=gold_stills, + accepted_gold_clips=gold_clips, + accepted_synthetic_clips=synthetic_clips, + accepted_clips=clips, + synthetic_effective_weight=synthetic_weight, + frontal=any(annotation.head_angle == "front" for annotation in annotations), + three_quarter=any(annotation.head_angle == "three_quarter" for annotation in annotations), + profile=any(annotation.head_angle == "profile" for annotation in annotations), + portrait=any(annotation.framing == "portrait" for annotation in annotations), + full_body=any(annotation.framing == "full_body" for annotation in annotations), + slow_motion=any(annotation.motion_speed == "slow" for annotation in clip_annotations), + fast_motion=any(annotation.motion_speed == "fast" for annotation in clip_annotations), + static_camera=any(annotation.camera_motion == "static" for annotation in clip_annotations), + moving_camera=any(annotation.camera_motion == "moving" for annotation in clip_annotations), + expression_change=any(annotation.expression_change for annotation in clip_annotations), + partial_occlusion=any(annotation.partial_occlusion for annotation in annotations), + ordinary_lighting=any(annotation.lighting == "ordinary" for annotation in annotations), + directional_lighting=any( + annotation.lighting == "directional" for annotation in annotations + ), + visible_outfit_details=any(annotation.visible_outfit_details for annotation in annotations), + ) + + +def freeze_training_dataset(draft: TrainingDatasetDraft) -> FrozenTrainingDataset: + """Freeze a ready draft under a canonical content digest, otherwise raise stable codes.""" + coverage = evaluate_training_dataset(draft) + reasons = _readiness_reasons(draft, coverage) + if reasons: + raise TrainingDatasetNotReady(reasons) + return FrozenTrainingDataset( + **draft.model_dump(mode="python"), + dataset_digest=training_dataset_id(draft), + coverage=coverage, + ) + + +def validate_training_recipe_dataset( + recipe: TrainingRecipe, dataset: FrozenTrainingDataset +) -> TrainingRecipeValidationEvidence: + """Bind a recipe to frozen data and enforce its composition-weighted clip cap.""" + current_draft = TrainingDatasetDraft.model_validate( + dataset.model_dump(mode="python", exclude={"coverage", "dataset_digest"}) + ) + if training_dataset_id(current_draft) != dataset.dataset_digest: + raise TrainingRecipeDatasetValidationError("dataset_digest_mismatch") + if recipe.dataset_digest != dataset.dataset_digest: + raise TrainingRecipeDatasetValidationError("dataset_digest_mismatch") + if recipe.character_pack_digest != dataset.character_pack_digest: + raise TrainingRecipeDatasetValidationError("character_pack_digest_mismatch") + + accepted = tuple(sample for sample in dataset.samples if sample.accepted) + gold_clip_count = _count(accepted, TrainingSourceTier.GOLD_CLIP) + synthetic_clip_count = _count(accepted, TrainingSourceTier.SYNTHETIC_CLIP) + gold_clip_weight = recipe.source_tier_weights[TrainingSourceTier.GOLD_CLIP] + synthetic_clip_weight = recipe.source_tier_weights[TrainingSourceTier.SYNTHETIC_CLIP] + effective_share = _synthetic_effective_share( + gold_clip_count=gold_clip_count, + synthetic_clip_count=synthetic_clip_count, + gold_clip_weight=gold_clip_weight, + synthetic_clip_weight=synthetic_clip_weight, + ) + if effective_share > Decimal("0.5"): + raise TrainingRecipeDatasetValidationError("synthetic_clip_weight") + + return TrainingRecipeValidationEvidence( + recipe_digest=training_recipe_id(recipe), + dataset_digest=dataset.dataset_digest, + character_pack_digest=dataset.character_pack_digest, + accepted_gold_clip_count=gold_clip_count, + accepted_synthetic_clip_count=synthetic_clip_count, + gold_clip_weight=gold_clip_weight, + synthetic_clip_weight=synthetic_clip_weight, + synthetic_effective_share=float(effective_share), + ) + + +def _readiness_reasons(draft: TrainingDatasetDraft, coverage: DatasetCoverage) -> set[str]: + accepted = tuple(sample for sample in draft.samples if sample.accepted) + reasons: set[str] = set() + if not 12 <= coverage.accepted_gold_stills <= 20: + reasons.add("gold_still_count") + if not 12 <= coverage.accepted_clips <= 24: + reasons.add("clip_count") + if coverage.accepted_gold_clips < 8: + reasons.add("gold_clip_count") + if _synthetic_effective_share( + gold_clip_count=_count(accepted, TrainingSourceTier.GOLD_CLIP), + synthetic_clip_count=_count(accepted, TrainingSourceTier.SYNTHETIC_CLIP), + gold_clip_weight=draft.source_tier_weights[TrainingSourceTier.GOLD_CLIP], + synthetic_clip_weight=draft.source_tier_weights[TrainingSourceTier.SYNTHETIC_CLIP], + ) > Decimal("0.5"): + reasons.add("synthetic_clip_weight") + reasons.update(dataset_integrity_reasons(accepted)) + if any( + sample.annotation.canonical_outfit_id != draft.canonical_outfit_id for sample in accepted + ): + reasons.add("canonical_outfit") + if any(draft.reserved_token not in sample.caption for sample in accepted): + reasons.add("reserved_token") + if any( + sample.media.duration_seconds is None or not 2.0 <= sample.media.duration_seconds <= 5.0 + for sample in _clips(accepted) + ): + reasons.add("clip_duration") + if not all( + getattr(coverage, field) + for field in ( + "frontal", + "three_quarter", + "profile", + "portrait", + "full_body", + "slow_motion", + "fast_motion", + "static_camera", + "moving_camera", + "expression_change", + "partial_occlusion", + "ordinary_lighting", + "directional_lighting", + "visible_outfit_details", + ) + ): + reasons.add("coverage_incomplete") + return reasons + + +def _count(samples: tuple[TrainingSample, ...], tier: TrainingSourceTier) -> int: + return sum(sample.source_tier == tier for sample in samples) + + +def _clips(samples: tuple[TrainingSample, ...]) -> tuple[TrainingSample, ...]: + return tuple( + sample for sample in samples if sample.source_tier != TrainingSourceTier.GOLD_STILL + ) + + +def _synthetic_effective_share( + *, + gold_clip_count: int, + synthetic_clip_count: int, + gold_clip_weight: float, + synthetic_clip_weight: float, +) -> Decimal: + weighted_gold = Decimal(gold_clip_count) * Decimal(str(gold_clip_weight)) + weighted_synthetic = Decimal(synthetic_clip_count) * Decimal(str(synthetic_clip_weight)) + total_clip_weight = weighted_gold + weighted_synthetic + if total_clip_weight == 0: + return Decimal(0) + return weighted_synthetic / total_clip_weight diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_media.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_media.py new file mode 100644 index 0000000..03aa99d --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_media.py @@ -0,0 +1,586 @@ +"""Safe, bounded inspection of local training-media bytes.""" + +from __future__ import annotations + +import hashlib +import json +import os +import signal +import stat +import subprocess +import sys +import tempfile +import unicodedata +import warnings +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal + +from PIL import Image, ImageOps, UnidentifiedImageError + +from video_lab.training_models import TrainingMedia + +_CHUNK_SIZE: Final = 1024 * 1024 +_MP4_HEADER_BYTES: Final = 4096 +MAX_STILL_BYTES: Final = 100 * 1024 * 1024 +MAX_VIDEO_BYTES: Final = 2 * 1024 * 1024 * 1024 +MAX_STILL_EDGE: Final = 8192 +MAX_STILL_PIXELS: Final = 40_000_000 +MAX_VIDEO_EDGE: Final = 3840 +MAX_VIDEO_PIXELS: Final = 8_294_400 +_MIN_VIDEO_EDGE: Final = 16 +_MIN_VIDEO_FPS: Final = 1.0 +_MAX_VIDEO_FPS: Final = 120.0 +_MIN_VIDEO_DURATION: Final = 2.0 +_MAX_VIDEO_DURATION: Final = 5.0 +_MIN_VIDEO_FRAMES: Final = 2 +_MAX_VIDEO_FRAMES: Final = 600 +_PROBE_TIMEOUT_SECONDS: Final = 30 +_PROBE_CODE: Final = """ +import json +import sys + +import imageio_ffmpeg + +reader = imageio_ffmpeg.read_frames(sys.argv[1]) +try: + metadata = next(reader) +finally: + reader.close() +fps = float(metadata["fps"]) +duration = float(metadata["duration"]) +width, height = (int(value) for value in metadata["size"]) +if ( + not 16 <= width <= 3840 + or not 16 <= height <= 3840 + or width * height > 8294400 + or not 1.0 <= fps <= 120.0 + or not 2.0 <= duration <= 5.0 +): + print(json.dumps({"status": "out_of_bounds"})) + raise SystemExit(0) +frame_count, _ = imageio_ffmpeg.count_frames_and_secs(sys.argv[1]) +frame_count = int(frame_count) +if not 2 <= frame_count <= 600: + print(json.dumps({"status": "out_of_bounds"})) + raise SystemExit(0) +print( + json.dumps( + { + "status": "ok", + "fps": fps, + "duration": duration, + "size": [width, height], + "frame_count": frame_count, + } + ) +) +""" +_STILL_FORMATS: Final[dict[str, Literal["image/jpeg", "image/png", "image/webp"]]] = { + "JPEG": "image/jpeg", + "PNG": "image/png", + "WEBP": "image/webp", +} +_MP4_BRANDS: Final = {b"isom", b"iso2", b"avc1", b"mp41", b"mp42", b"M4V ", b"hvc1"} + + +class TrainingMediaInspectionError(ValueError): + """Path-free media safety failure with a stable reason code.""" + + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +@dataclass(frozen=True, slots=True) +class TrainingMediaSnapshot: + """Inspected metadata paired with its private, context-bound exact bytes.""" + + media: TrainingMedia + path: Path + + +@dataclass(frozen=True, slots=True) +class TrainingMediaCollectionSnapshot: + """One exact, normalized collection observed through a single root descriptor.""" + + entries: tuple[tuple[str, TrainingMediaSnapshot], ...] + + +def inspect_training_media(path: Path) -> TrainingMedia: + """Safely inspect one regular local file without retaining its source location.""" + with open_training_media_snapshot(path) as snapshot: + return snapshot.media + + +def open_training_media_snapshot(path: Path) -> AbstractContextManager[TrainingMediaSnapshot]: + """Open verified exact bytes whose path remains valid only inside the context.""" + return _open_training_media_snapshot(path) + + +def open_training_media_collection_snapshot( + root: Path, +) -> AbstractContextManager[TrainingMediaCollectionSnapshot]: + """Snapshot a media tree without ever resolving discovered entries by pathname.""" + return _open_training_media_collection_snapshot(root) + + +@contextmanager +def _open_training_media_collection_snapshot( + root: Path, +) -> Iterator[TrainingMediaCollectionSnapshot]: + flags = ( + os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + ) + root_fd = -1 + try: + root_fd = os.open(root, flags) + root_identity = _identity(os.fstat(root_fd)) + with tempfile.TemporaryDirectory(prefix="video-lab-collection-") as directory: + snapshots: list[tuple[str, TrainingMediaSnapshot]] = [] + _snapshot_collection_directory(root_fd, "", Path(directory), snapshots) + if not snapshots: + raise TrainingMediaInspectionError("empty_media_folder") + current = os.stat(root, follow_symlinks=False) + if _identity(current) != root_identity or not stat.S_ISDIR(current.st_mode): + raise TrainingMediaInspectionError("source_changed") + yield TrainingMediaCollectionSnapshot(entries=tuple(snapshots)) + except TrainingMediaInspectionError: + raise + except OSError: + raise TrainingMediaInspectionError("unsafe_path") from None + finally: + if root_fd >= 0: + os.close(root_fd) + + +def _snapshot_collection_directory( + directory_fd: int, + prefix: str, + temporary_root: Path, + snapshots: list[tuple[str, TrainingMediaSnapshot]], +) -> None: + before = _identity(os.fstat(directory_fd)) + names = sorted(os.listdir(directory_fd), key=lambda value: _normalized_entry_key(prefix, value)) + if len({_normalized_entry_key(prefix, name)[1] for name in names}) != len(names): + raise TrainingMediaInspectionError("normalization_collision") + for name in names: + if name in {"", ".", ".."} or os.sep in name: + raise TrainingMediaInspectionError("unsafe_path") + entry = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + relative = f"{prefix}/{name}" if prefix else name + normalized = unicodedata.normalize("NFC", relative) + if stat.S_ISDIR(entry.st_mode): + child = os.open( + name, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + dir_fd=directory_fd, + ) + try: + _snapshot_collection_directory(child, normalized, temporary_root, snapshots) + if _identity(os.fstat(child)) != _identity(entry): + raise TrainingMediaInspectionError("source_changed") + finally: + os.close(child) + elif stat.S_ISREG(entry.st_mode): + suffix = Path(name).suffix.lower() + if suffix not in {".jpg", ".jpeg", ".png", ".webp", ".mp4"}: + raise TrainingMediaInspectionError("unsupported_media") + fd = os.open( + name, + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + dir_fd=directory_fd, + ) + try: + snapshot = temporary_root / f"{len(snapshots):08d}{suffix}" + digest, size = _snapshot_open_descriptor(fd, snapshot, suffix == ".mp4") + if _identity(os.fstat(fd)) != _identity(entry): + raise TrainingMediaInspectionError("source_changed") + current = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + if _identity(current) != _identity(entry): + raise TrainingMediaInspectionError("source_changed") + media = ( + _inspect_mp4(snapshot, digest, size, timeout_seconds=_PROBE_TIMEOUT_SECONDS) + if suffix == ".mp4" + else _inspect_still(snapshot, digest, size) + ) + os.chmod(snapshot, 0o400) + snapshots.append((normalized, TrainingMediaSnapshot(media=media, path=snapshot))) + finally: + os.close(fd) + else: + raise TrainingMediaInspectionError("unsafe_path") + if ( + _identity(os.fstat(directory_fd)) != before + or sorted(os.listdir(directory_fd), key=lambda value: _normalized_entry_key(prefix, value)) + != names + ): + raise TrainingMediaInspectionError("source_changed") + + +def _normalized_entry_key(prefix: str, name: str) -> tuple[str, str]: + value = unicodedata.normalize("NFC", f"{prefix}/{name}" if prefix else name) + return value.casefold(), value + + +def _snapshot_open_descriptor(fd: int, destination: Path, require_mp4: bool) -> tuple[str, int]: + identity = os.fstat(fd) + maximum = MAX_VIDEO_BYTES if require_mp4 else MAX_STILL_BYTES + if not stat.S_ISREG(identity.st_mode) or identity.st_size > maximum: + raise TrainingMediaInspectionError("media_out_of_bounds") + digest = hashlib.sha256() + total = 0 + with destination.open("xb") as output: + while total <= maximum: + chunk = os.read(fd, min(_CHUNK_SIZE, maximum + 1 - total)) + if not chunk: + break + output.write(chunk) + digest.update(chunk) + total += len(chunk) + if total > maximum: + raise TrainingMediaInspectionError("media_out_of_bounds") + if require_mp4: + with destination.open("rb") as stream: + if not _is_mp4_container_header(stream.read(_MP4_HEADER_BYTES)): + raise TrainingMediaInspectionError("unsupported_container") + return f"sha256:{digest.hexdigest()}", total + + +def _inspect_training_media( + path: Path, + *, + before_component: Callable[[str], None] | None = None, + after_chunk: Callable[[int], None] | None = None, + before_entry_stat: Callable[[], None] | None = None, + probe_timeout_seconds: float | None = None, +) -> TrainingMedia: + with _open_training_media_snapshot( + path, + before_component=before_component, + after_chunk=after_chunk, + before_entry_stat=before_entry_stat, + probe_timeout_seconds=probe_timeout_seconds, + ) as snapshot: + return snapshot.media + + +@contextmanager +def _open_training_media_snapshot( + path: Path, + *, + before_component: Callable[[str], None] | None = None, + after_chunk: Callable[[int], None] | None = None, + before_entry_stat: Callable[[], None] | None = None, + probe_timeout_seconds: float | None = None, + after_snapshot: Callable[[Path], None] | None = None, +) -> Iterator[TrainingMediaSnapshot]: + source = Path(path) + suffix = source.suffix.lower() + if suffix not in {".jpg", ".jpeg", ".png", ".webp", ".mp4"}: + raise TrainingMediaInspectionError("unsupported_media") + video = suffix == ".mp4" + max_bytes = MAX_VIDEO_BYTES if video else MAX_STILL_BYTES + with tempfile.TemporaryDirectory(prefix="video-lab-media-") as directory: + os.chmod(directory, 0o700) + snapshot = Path(directory) / ("clip.mp4" if video else "still.bin") + if after_snapshot is not None: + after_snapshot(snapshot) + digest, byte_size = _snapshot_source( + source, + snapshot, + max_bytes=max_bytes, + require_mp4=video, + before_component=before_component, + after_chunk=after_chunk, + before_entry_stat=before_entry_stat, + ) + if video: + timeout = ( + _PROBE_TIMEOUT_SECONDS if probe_timeout_seconds is None else probe_timeout_seconds + ) + media = _inspect_mp4(snapshot, digest, byte_size, timeout_seconds=timeout) + else: + media = _inspect_still(snapshot, digest, byte_size) + try: + os.chmod(snapshot, 0o400) + except OSError: + raise TrainingMediaInspectionError("unsafe_path") from None + yield TrainingMediaSnapshot(media=media, path=snapshot) + + +def _inspect_still(snapshot: Path, digest: str, byte_size: int) -> TrainingMedia: + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(snapshot) as image: + media_type = _STILL_FORMATS.get(image.format or "") + if media_type is None: + raise TrainingMediaInspectionError("unsupported_media") + _require_still_bounds(*image.size) + normalized = ImageOps.exif_transpose(image) + try: + normalized.load() + width, height = normalized.size + finally: + if normalized is not image: + normalized.close() + except TrainingMediaInspectionError: + raise + except (Image.DecompressionBombWarning, Image.DecompressionBombError): + raise TrainingMediaInspectionError("media_out_of_bounds") from None + except (OSError, UnidentifiedImageError, ValueError): + raise TrainingMediaInspectionError("corrupt_media") from None + _require_still_bounds(width, height) + return TrainingMedia( + sha256=digest, + media_type=media_type, + byte_size=byte_size, + width=width, + height=height, + ) + + +def _require_still_bounds(width: int, height: int) -> None: + if ( + not 1 <= width <= MAX_STILL_EDGE + or not 1 <= height <= MAX_STILL_EDGE + or width * height > MAX_STILL_PIXELS + ): + raise TrainingMediaInspectionError("media_out_of_bounds") + + +def _inspect_mp4( + snapshot: Path, digest: str, byte_size: int, *, timeout_seconds: float +) -> TrainingMedia: + metadata = _run_mp4_probe(snapshot, timeout_seconds=timeout_seconds) + try: + if metadata.get("status") == "out_of_bounds": + raise TrainingMediaInspectionError("media_out_of_bounds") + fps = _as_float(metadata["fps"]) + duration = _as_float(metadata["duration"]) + size = metadata["size"] + if not isinstance(size, list | tuple) or len(size) != 2: + raise ValueError + width, height = _as_int(size[0]), _as_int(size[1]) + frame_count = _as_int(metadata["frame_count"]) + except TrainingMediaInspectionError: + raise + except (KeyError, TypeError, ValueError): + raise TrainingMediaInspectionError("corrupt_media") from None + if not _video_metadata_in_bounds(width, height, fps, duration, frame_count): + raise TrainingMediaInspectionError("media_out_of_bounds") + return TrainingMedia( + sha256=digest, + media_type="video/mp4", + byte_size=byte_size, + width=width, + height=height, + duration_seconds=duration, + frame_count=frame_count, + fps=fps, + ) + + +def _run_mp4_probe(snapshot: Path, *, timeout_seconds: float) -> dict[str, object]: + try: + process = subprocess.Popen( + [sys.executable, "-c", _PROBE_CODE, os.fspath(snapshot)], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + start_new_session=True, + ) + except OSError: + raise TrainingMediaInspectionError("corrupt_media") from None + try: + stdout, _ = process.communicate(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + raise TrainingMediaInspectionError("probe_timeout") from None + if process.returncode != 0: + raise TrainingMediaInspectionError("corrupt_media") + try: + metadata = json.loads(stdout) + except (TypeError, ValueError, json.JSONDecodeError): + raise TrainingMediaInspectionError("corrupt_media") from None + if not isinstance(metadata, dict): + raise TrainingMediaInspectionError("corrupt_media") + return metadata + + +def _video_metadata_in_bounds( + width: int, height: int, fps: float, duration: float, frame_count: int +) -> bool: + return ( + _MIN_VIDEO_EDGE <= width <= MAX_VIDEO_EDGE + and _MIN_VIDEO_EDGE <= height <= MAX_VIDEO_EDGE + and width * height <= MAX_VIDEO_PIXELS + and _MIN_VIDEO_FPS <= fps <= _MAX_VIDEO_FPS + and _MIN_VIDEO_DURATION <= duration <= _MAX_VIDEO_DURATION + and _MIN_VIDEO_FRAMES <= frame_count <= _MAX_VIDEO_FRAMES + ) + + +def _as_float(value: object) -> float: + if not isinstance(value, str | int | float): + raise TypeError + return float(value) + + +def _as_int(value: object) -> int: + if not isinstance(value, str | int): + raise TypeError + return int(value) + + +def _snapshot_source( + path: Path, + snapshot: Path, + *, + max_bytes: int, + require_mp4: bool, + before_component: Callable[[str], None] | None, + after_chunk: Callable[[int], None] | None, + before_entry_stat: Callable[[], None] | None, +) -> tuple[str, int]: + directory_fd, name = _open_parent_fd(path, before_component=before_component) + file_fd = -1 + snapshot_fd = -1 + try: + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0) + try: + file_fd = os.open(name, flags, dir_fd=directory_fd) + except OSError: + raise TrainingMediaInspectionError( + "symlink" if path.is_symlink() else "unsafe_path" + ) from None + before_stat = os.fstat(file_fd) + if not stat.S_ISREG(before_stat.st_mode): + raise TrainingMediaInspectionError("unsafe_path") + if before_stat.st_size > max_bytes: + raise TrainingMediaInspectionError("media_out_of_bounds") + snapshot_fd = os.open( + snapshot, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0), + 0o600, + ) + digest = hashlib.sha256() + total = 0 + if require_mp4: + header = os.read(file_fd, min(_MP4_HEADER_BYTES, max_bytes)) + total = _write_snapshot_chunk(snapshot_fd, digest, header, total) + if after_chunk is not None: + after_chunk(total) + if not _is_mp4_container_header(header): + raise TrainingMediaInspectionError("unsupported_container") + while total < max_bytes: + chunk = os.read(file_fd, min(_CHUNK_SIZE, max_bytes - total)) + if not chunk: + break + total = _write_snapshot_chunk(snapshot_fd, digest, chunk, total) + if after_chunk is not None: + after_chunk(total) + after_stat = os.fstat(file_fd) + if after_stat.st_size > max_bytes: + raise TrainingMediaInspectionError("media_out_of_bounds") + if before_entry_stat is not None: + before_entry_stat() + entry_stat = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + if _identity(before_stat) != _identity(after_stat) or _identity(entry_stat) != _identity( + after_stat + ): + raise TrainingMediaInspectionError("source_changed") + os.fsync(snapshot_fd) + return f"sha256:{digest.hexdigest()}", total + except OSError: + raise TrainingMediaInspectionError("unsafe_path") from None + finally: + if snapshot_fd >= 0: + os.close(snapshot_fd) + if file_fd >= 0: + os.close(file_fd) + os.close(directory_fd) + + +def _write_snapshot_chunk(snapshot_fd: int, digest: hashlib._Hash, chunk: bytes, total: int) -> int: + view = memoryview(chunk) + while view: + written = os.write(snapshot_fd, view) + if written <= 0: + raise OSError("snapshot write failed") + view = view[written:] + digest.update(chunk) + return total + len(chunk) + + +def _open_parent_fd( + path: Path, *, before_component: Callable[[str], None] | None +) -> tuple[int, str]: + absolute = os.path.abspath(path) + parts = absolute.split(os.sep) + if not absolute.startswith(os.sep) or any(part in {"", ".", ".."} for part in parts[1:]): + raise TrainingMediaInspectionError("unsafe_path") + root = os.open(os.sep, os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_CLOEXEC", 0)) + current = root + try: + for part in parts[1:-1]: + if before_component is not None: + before_component(part) + next_fd = os.open( + part, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + dir_fd=current, + ) + if current != root: + os.close(current) + current = next_fd + except OSError: + if current != root: + os.close(current) + os.close(root) + raise TrainingMediaInspectionError("unsafe_path") from None + if current != root: + os.close(root) + return current, parts[-1] + + +def _identity(value: os.stat_result) -> tuple[int, int, int, int]: + return value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns + + +def _is_mp4_container_header(data: bytes) -> bool: + offset = 0 + while offset + 8 <= len(data): + box_size = int.from_bytes(data[offset : offset + 4], "big") + box_type = data[offset + 4 : offset + 8] + header_size = 8 + if box_size == 1: + if offset + 16 > len(data): + return False + box_size = int.from_bytes(data[offset + 8 : offset + 16], "big") + header_size = 16 + if box_size < header_size or offset + box_size > len(data): + return False + if box_type == b"ftyp": + brands = { + data[brand_offset : brand_offset + 4] + for brand_offset in range(offset + header_size, offset + box_size, 4) + } + return bool(brands & _MP4_BRANDS) + if box_size == 0: + return False + offset += box_size + return False diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_models.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_models.py new file mode 100644 index 0000000..5052783 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_models.py @@ -0,0 +1,1041 @@ +"""Trainer-neutral, immutable contracts for character-training inputs.""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections.abc import Mapping, Sequence, Set +from datetime import datetime +from enum import StrEnum +from types import MappingProxyType +from typing import Annotated, Any, Literal, cast + +from pydantic import ( + Field, + ValidationInfo, + field_serializer, + field_validator, + model_validator, +) + +from video_lab.domain import FrozenModel, canonical_json +from video_lab.trainer_protocol import LoRAConfiguration +from video_lab.training_data import DatasetMedia + +TRAINING_DATASET_SCHEMA_VERSION = "character-training-dataset.v1" +TRAINING_RECIPE_SCHEMA_VERSION = "character-training-recipe.v1" +TRAINING_MEDIA_SCHEMA_VERSION = "character-training-media.v1" +TRAINING_RECIPE_VALIDATION_SCHEMA_VERSION = "character-training-recipe-validation.v1" +TRAINING_RUN_SCHEMA_VERSION = "character-training-run.v1" +TRAINING_IMPORT_SCHEMA_VERSION = "character-training-import.v1" +TRAINING_CONFIRMATION_SCHEMA_VERSION = "character-training-confirmation.v1" +TRAINING_ORCHESTRATION_OPERATION_SCHEMA_VERSION = "character-training-orchestration-operation.v1" + +Sha256Digest = Annotated[str, Field(pattern=r"^sha256:[0-9a-f]{64}$")] + + +class TrainingOperationKind(StrEnum): + PROBE = "probe" + CALIBRATION = "calibration" + START = "start" + RESUME = "resume" + CANCEL = "cancel" + + +class TrainingOperationStatus(StrEnum): + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + + +class TrainingOperationIdentity(FrozenModel): + request_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + recipe_digest: Sha256Digest + dataset_digest: Sha256Digest + backend_digest: Sha256Digest + runtime_digest: Sha256Digest + material_digest: Sha256Digest + source_digest: Sha256Digest + + +class ProbeOperationRequest(TrainingOperationIdentity): + pass + + +class CalibrationOperationRequest(TrainingOperationIdentity): + probe_operation_id: str = Field(min_length=1) + capability_digest: Sha256Digest + steps: int = Field(ge=0) + trainable_targets: tuple[str, ...] = Field(min_length=1) + lora: LoRAConfiguration + seed: int + + +class StartOperationRequest(TrainingOperationIdentity): + preregistration_id: str = Field(min_length=1) + preregistration_request_digest: Sha256Digest + checkpoint_steps: tuple[int, ...] = Field(min_length=1) + checkpoint_fractions: tuple[Literal[0.25, 0.5, 0.75, 1.0], ...] = Field(min_length=1) # type: ignore[valid-type] + material_artifact_ids: Mapping[str, str] + + +class ResumeInventoryArtifact(FrozenModel): + role: Literal[ + "weights", + "optimizer", + "trainer_state", + "telemetry", + "trainable_targets", + "frozen_parameters", + "finite_gradient_proof", + ] + artifact_id: str = Field(min_length=1) + digest: Sha256Digest + size_bytes: int = Field(gt=0) + + +class ResumeOperationRequest(TrainingOperationIdentity): + parent_checkpoint_id: str = Field(min_length=1) + parent_checkpoint_digest: Sha256Digest + parent_inventory: tuple[ResumeInventoryArtifact, ...] = Field(min_length=7, max_length=7) + + @model_validator(mode="after") + def validate_parent_inventory(self) -> ResumeOperationRequest: + roles = tuple(item.role for item in self.parent_inventory) + if len(set(roles)) != 7: + raise ValueError("resume parent inventory must bind every exact role") + return self + + +class CancelOperationRequest(TrainingOperationIdentity): + cancellation_request_id: str = Field(min_length=1) + request_digest: Sha256Digest + requested_step: int = Field(ge=0) + + +class OperationEvidenceEntry(FrozenModel): + evidence_id: str = Field(min_length=1) + digest: Sha256Digest + + +class OperationEventEntry(OperationEvidenceEntry): + sequence: int = Field(gt=0) + + +class OperationTelemetryEntry(OperationEvidenceEntry): + sequence: int = Field(gt=0) + step: int = Field(ge=0) + + +class OperationCheckpointEntry(OperationEvidenceEntry): + step: int = Field(ge=0) + + +class OperationInventoryEntry(OperationEvidenceEntry): + checkpoint_id: str = Field(min_length=1) + role: str = Field(min_length=1) + + +class OperationArtifactEntry(OperationEvidenceEntry): + size_bytes: int = Field(gt=0) + + +class EvidenceOperationReceipt(FrozenModel): + run_id: str = Field(min_length=1) + lifecycle_version: int = Field(ge=0) + lifecycle_state: str = Field(min_length=1) + evidence_id: str = Field(min_length=1) + evidence_digest: Sha256Digest + + +class CollectionOperationReceipt(FrozenModel): + run_id: str = Field(min_length=1) + lifecycle_version: int = Field(ge=0) + lifecycle_state: str = Field(min_length=1) + events: tuple[OperationEventEntry, ...] + telemetry: tuple[OperationTelemetryEntry, ...] + checkpoints: tuple[OperationCheckpointEntry, ...] + artifacts: tuple[OperationArtifactEntry, ...] + inventory: tuple[OperationInventoryEntry, ...] + evaluation: OperationEvidenceEntry | None = None + + @model_validator(mode="after") + def validate_unique_receipt_ids(self) -> CollectionOperationReceipt: + for entries in ( + self.events, + self.telemetry, + self.checkpoints, + self.artifacts, + self.inventory, + ): + identities = tuple(entry.evidence_id for entry in entries) + if len(set(identities)) != len(identities): + raise ValueError("receipt evidence identities must be unique per kind") + return self + + +class StartOperationReceipt(CollectionOperationReceipt): + pass + + +class ResumeOperationReceipt(CollectionOperationReceipt): + resume: OperationEvidenceEntry + + +class CancelOperationReceipt(FrozenModel): + run_id: str = Field(min_length=1) + lifecycle_version: int = Field(ge=0) + lifecycle_state: Literal["cancelled"] + cancellation: OperationEvidenceEntry + terminal_event: OperationEventEntry + boundary_checkpoint: OperationCheckpointEntry + cancellation_request_id: str = Field(min_length=1) + request_digest: Sha256Digest + requested_step: int = Field(ge=0) + boundary_step: int = Field(ge=0) + + @model_validator(mode="after") + def validate_boundary_checkpoint(self) -> CancelOperationReceipt: + if self.boundary_checkpoint.step != self.boundary_step: + raise ValueError("boundary checkpoint step must equal cancellation boundary") + return self + + +OperationRequest = Annotated[ + ProbeOperationRequest + | CalibrationOperationRequest + | StartOperationRequest + | ResumeOperationRequest + | CancelOperationRequest, + Field(union_mode="left_to_right"), +] + + +def training_operation_payload_digest(payload: Mapping[str, object]) -> str: + serialized = canonical_json(cast(Any, _json_value(dict(payload)))) + return f"sha256:{hashlib.sha256(serialized.encode()).hexdigest()}" + + +class TrainingOrchestrationOperation(FrozenModel): + """Immutable request identity plus the latest persisted execution receipt.""" + + schema_version: Literal["character-training-orchestration-operation.v1"] = ( + "character-training-orchestration-operation.v1" + ) + operation_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + recipe_id: str = Field(min_length=1) + operation_kind: TrainingOperationKind + idempotency_key: str = Field(min_length=1) + request_digest: Sha256Digest + request_payload: Mapping[str, object] + lifecycle_version: int = Field(ge=0) + lifecycle_state: Literal[ + "draft", + "probed", + "calibrated", + "preregistered", + "training", + "evaluating", + "qualified", + "rejected", + "failed", + "cancelled", + ] + status: TrainingOperationStatus + attempt: int = Field(ge=0) + claimant_id: str | None = Field(default=None, min_length=1) + claim_token: str | None = Field(default=None, min_length=1) + lease_expires_at: datetime | None = None + receipt_digest: Sha256Digest | None = None + receipt_payload: Mapping[str, object] | None = None + result_digest: Sha256Digest | None = None + result_payload: Mapping[str, object] | None = None + failure_digest: Sha256Digest | None = None + created_at: datetime | None = None + updated_at: datetime | None = None + + @field_validator("request_payload", "receipt_payload", "result_payload") + @classmethod + def freeze_operation_payload( + cls, value: Mapping[str, object] | None + ) -> Mapping[str, object] | None: + if value is None: + return None + return cast(Mapping[str, object], _freeze_mapping(cast(Mapping[object, object], value))) + + @field_serializer("request_payload", "receipt_payload", "result_payload") + def serialize_operation_payload( + self, value: Mapping[str, object] | None + ) -> dict[str, object] | None: + return ( + None if value is None else {str(key): _json_value(item) for key, item in value.items()} + ) + + @model_validator(mode="after") + def validate_operation_receipt(self) -> TrainingOrchestrationOperation: + creation_states = { + TrainingOperationKind.PROBE: "draft", + TrainingOperationKind.CALIBRATION: "probed", + TrainingOperationKind.START: "preregistered", + TrainingOperationKind.CANCEL: "training", + } + valid_creation_states = ( + {"training", "cancelled"} + if self.operation_kind is TrainingOperationKind.RESUME + else {creation_states[self.operation_kind]} + ) + if self.lifecycle_state not in valid_creation_states: + raise ValueError("operation creation lifecycle does not match operation kind") + if self.request_digest != training_operation_payload_digest(self.request_payload): + raise ValueError("request digest does not match canonical request payload") + running = self.status == TrainingOperationStatus.RUNNING + if running != ( + self.claimant_id is not None + and self.claim_token is not None + and self.lease_expires_at is not None + ): + raise ValueError("running operation requires claimant, claim token, and lease") + completed = self.status == TrainingOperationStatus.COMPLETED + if completed != (self.receipt_digest is not None and self.receipt_payload is not None): + raise ValueError("completed operation requires an exact receipt") + if completed and self.receipt_digest != training_operation_payload_digest( + cast(Mapping[str, object], self.receipt_payload) + ): + raise ValueError("receipt digest does not match canonical receipt payload") + if completed != (self.result_digest is not None and self.result_payload is not None): + raise ValueError("completed operation requires an exact durable result") + if completed and self.result_digest != training_operation_payload_digest( + cast(Mapping[str, object], self.result_payload) + ): + raise ValueError("result digest does not match canonical result payload") + if (self.status == TrainingOperationStatus.FAILED) != (self.failure_digest is not None): + raise ValueError("failed operation requires a failure digest") + return self + + +class TrainingSourceTier(StrEnum): + GOLD_STILL = "gold_still" + GOLD_CLIP = "gold_clip" + SYNTHETIC_CLIP = "synthetic_clip" + + +class DatasetSplit(StrEnum): + TRAIN = "train" + VALIDATION = "validation" + SEALED_HOLDOUT = "sealed_holdout" + + +class TrainingMedia(DatasetMedia): + """Versioned media metadata for persisted character-training records.""" + + schema_version: Literal["character-training-media.v1"] = "character-training-media.v1" + + +class TrainingAnnotation(FrozenModel): + """Curated coverage labels, intentionally independent from trainer internals.""" + + canonical_outfit_id: str = Field(min_length=1) + head_angle: Literal["front", "three_quarter", "profile"] + framing: Literal["portrait", "full_body"] + motion_speed: Literal["slow", "fast"] + camera_motion: Literal["static", "moving"] + expression_change: bool + partial_occlusion: bool + lighting: Literal["ordinary", "directional"] + visible_outfit_details: bool + + +class TrainingCurationSuggestions(FrozenModel): + """Untrusted machine-authored values; absence never implies confirmation.""" + + source_tier: TrainingSourceTier | None = None + source_lineage: str | None = Field(default=None, min_length=1) + caption: str | None = Field(default=None, min_length=1) + split: DatasetSplit | None = None + annotation: TrainingAnnotation | None = None + + +class ImportedTrainingSample(FrozenModel): + schema_version: Literal["character-training-import.v1"] = "character-training-import.v1" + import_id: str = Field(min_length=1) + reviewer_id: str = Field(min_length=1) + character_id: str = Field(min_length=1) + lineage_id: str = Field(min_length=1) + artifact_id: str = Field(min_length=1) + artifact_digest: Sha256Digest + artifact_byte_size: int = Field(gt=0) + media: TrainingMedia + source_lineage: str = Field(min_length=1) + suggestions: TrainingCurationSuggestions | None = None + imported_at: datetime + + @model_validator(mode="after") + def validate_artifact_identity(self) -> ImportedTrainingSample: + if self.artifact_digest != self.media.sha256: + raise ValueError("artifact_digest must match media sha256") + if self.artifact_byte_size != self.media.byte_size: + raise ValueError("artifact_byte_size must match media byte_size") + if self.import_id != imported_training_sample_id(self): + raise ValueError("import_id does not match imported sample content") + return self + + +class CurationDecision(StrEnum): + ACCEPT = "accept" + REJECT = "reject" + + +class TrainingCurationConfirmation(FrozenModel): + schema_version: Literal["character-training-confirmation.v1"] = ( + "character-training-confirmation.v1" + ) + confirmation_id: str = Field(min_length=1) + import_id: str = Field(min_length=1) + reviewer_id: str = Field(min_length=1) + decision: CurationDecision + confirmed_at: datetime + version: int = Field(gt=0) + sample_id: str | None = Field(default=None, min_length=1) + source_tier: TrainingSourceTier | None = None + source_lineage: str | None = Field(default=None, min_length=1) + caption: str | None = Field(default=None, min_length=1) + split: DatasetSplit | None = None + annotation: TrainingAnnotation | None = None + + @model_validator(mode="after") + def validate_decision_fields(self) -> TrainingCurationConfirmation: + curated = ( + self.sample_id, + self.source_tier, + self.source_lineage, + self.caption, + self.split, + self.annotation, + ) + if self.decision == CurationDecision.ACCEPT and any(value is None for value in curated): + raise ValueError("accepted confirmation requires every curated sample field") + if self.decision == CurationDecision.REJECT and any(value is not None for value in curated): + raise ValueError("rejected confirmation must not contain curated sample fields") + if self.confirmation_id != training_confirmation_id(self): + raise ValueError("confirmation_id does not match confirmation content") + return self + + def accepted_sample(self, imported: ImportedTrainingSample) -> TrainingSample: + if ( + self.decision != CurationDecision.ACCEPT + or self.import_id != imported.import_id + or self.reviewer_id != imported.reviewer_id + ): + raise ValueError("confirmation does not accept this imported sample") + return TrainingSample( + sample_id=cast(str, self.sample_id), + source_tier=cast(TrainingSourceTier, self.source_tier), + sample_digest=imported.artifact_digest, + source_lineage=cast(str, self.source_lineage), + caption=cast(str, self.caption), + split=cast(DatasetSplit, self.split), + media=imported.media, + annotation=cast(TrainingAnnotation, self.annotation), + ) + + +class CuratedEvidenceReferenceV1(FrozenModel): + """Accepted, content-addressed curated validation evidence for one sequence.""" + + schema_version: Literal["curated-training-evidence-reference.v1"] = ( + "curated-training-evidence-reference.v1" + ) + curated_dataset_digest: Sha256Digest + validation_report_digest: Sha256Digest + sequence_id: str = Field(pattern=r"^[a-z][a-z0-9_]*$") + validation_state: Literal["accepted"] + + +class TrainingSample(FrozenModel): + """A content-addressed accepted item. Raw source paths are never retained.""" + + sample_id: str = Field(min_length=1) + source_tier: TrainingSourceTier + sample_digest: Sha256Digest + source_lineage: str = Field(min_length=1) + caption: str = Field(min_length=1) + split: DatasetSplit + media: TrainingMedia + annotation: TrainingAnnotation + accepted: bool = True + curated_evidence: CuratedEvidenceReferenceV1 | None = None + + @model_validator(mode="after") + def validate_digest_matches_media(self) -> TrainingSample: + if self.sample_digest != self.media.sha256: + raise ValueError("sample_digest must match media sha256") + if ( + self.source_tier == TrainingSourceTier.GOLD_STILL + and self.media.media_type == "video/mp4" + ): + raise ValueError("gold_still samples must be still images") + if ( + self.source_tier != TrainingSourceTier.GOLD_STILL + and self.media.media_type != "video/mp4" + ): + raise ValueError("clip samples must be MP4 videos") + return self + + +class DatasetCoverage(FrozenModel): + accepted_gold_stills: int = Field(ge=0) + accepted_gold_clips: int = Field(ge=0) + accepted_synthetic_clips: int = Field(ge=0) + accepted_clips: int = Field(ge=0) + synthetic_effective_weight: float = Field(ge=0) + frontal: bool + three_quarter: bool + profile: bool + portrait: bool + full_body: bool + slow_motion: bool + fast_motion: bool + static_camera: bool + moving_camera: bool + expression_change: bool + partial_occlusion: bool + ordinary_lighting: bool + directional_lighting: bool + visible_outfit_details: bool + + +class TrainingDatasetDraft(FrozenModel): + schema_version: Literal["character-training-dataset.v1"] = "character-training-dataset.v1" + character_id: str = Field(min_length=1) + character_pack_digest: Sha256Digest + canonical_outfit_id: str = Field(min_length=1) + reserved_token: str = Field(pattern=r"^<[^<>\s]+>$") + source_tier_weights: Mapping[TrainingSourceTier, float] = Field( + default_factory=lambda: MappingProxyType( + { + TrainingSourceTier.GOLD_STILL: 1.0, + TrainingSourceTier.GOLD_CLIP: 1.0, + TrainingSourceTier.SYNTHETIC_CLIP: 1.0, + } + ) + ) + samples: tuple[TrainingSample, ...] + imported_sample_ids: tuple[str, ...] = () + curation_confirmation_ids: tuple[str, ...] = () + + @field_validator("imported_sample_ids", "curation_confirmation_ids") + @classmethod + def normalize_ledger_ids(cls, value: tuple[str, ...], info: ValidationInfo) -> tuple[str, ...]: + field_name = info.field_name + prefix = ( + "training_import_" if field_name == "imported_sample_ids" else "training_confirmation_" + ) + pattern = re.compile(rf"^{prefix}[0-9a-f]{{64}}$") + if any(pattern.fullmatch(item) is None for item in value): + raise ValueError("dataset ledger contains malformed content identity") + if len(set(value)) != len(value): + raise ValueError("dataset ledger contains duplicate content identity") + return tuple(sorted(value)) + + @field_validator("source_tier_weights") + @classmethod + def freeze_source_tier_weights( + cls, value: Mapping[TrainingSourceTier, float] + ) -> Mapping[TrainingSourceTier, float]: + if set(value) != set(TrainingSourceTier): + raise ValueError("source_tier_weights must include every source tier") + if any(weight <= 0 or not math.isfinite(weight) for weight in value.values()): + raise ValueError("source tier weights must be finite and positive") + return MappingProxyType(dict(value)) + + @field_serializer("source_tier_weights") + def serialize_source_tier_weights( + self, value: Mapping[TrainingSourceTier, float] + ) -> dict[str, float]: + return {tier.value: weight for tier, weight in value.items()} + + +class FrozenTrainingDataset(TrainingDatasetDraft): + dataset_digest: Sha256Digest + coverage: DatasetCoverage + + +class CheckpointPlan(FrozenModel): + fractions: tuple[Literal[0.25, 0.5, 0.75, 1.0], ...] = ( # type: ignore[valid-type] + 0.25, + 0.5, + 0.75, + 1.0, + ) + + @field_validator("fractions") + @classmethod + def require_complete_checkpoint_plan(cls, value: tuple[float, ...]) -> tuple[float, ...]: + if value != (0.25, 0.5, 0.75, 1.0): + raise ValueError("checkpoint fractions must be 0.25, 0.5, 0.75, and 1.0") + return value + + +class TrainingRecipe(FrozenModel): + schema_version: Literal["character-training-recipe.v1"] + dataset_digest: Sha256Digest + character_pack_digest: Sha256Digest + stage_2_manifest_digest: Sha256Digest + stage_2_evidence_digest: Sha256Digest + stage_3_evidence_digest: Sha256Digest + base_model_digest: Sha256Digest + lightning_adapter_digests: Mapping[str, Sha256Digest] + source_code_digest: Sha256Digest + runtime_identity: Mapping[str, str] + backend_id: str + backend_config: Mapping[str, object] + trainable_targets: tuple[str, ...] + lora_rank: int = Field(gt=0) + lora_alpha: int = Field(gt=0) + lora_dropout: float = Field(ge=0, lt=1) + optimizer: str + learning_rate: float = Field(gt=0) + total_steps: int = Field(gt=0) + gradient_accumulation_steps: int = Field(gt=0) + source_tier_weights: Mapping[TrainingSourceTier, float] + checkpoint_plan: CheckpointPlan + calibration_steps: int = Field(ge=0) + wall_clock_limit_seconds: Literal[28800, 57600] + seed: int + + @field_validator( + "lightning_adapter_digests", + "runtime_identity", + "backend_config", + "source_tier_weights", + ) + @classmethod + def freeze_mappings(cls, value: Mapping[object, object]) -> Mapping[object, object]: + return _freeze_mapping(value) + + @field_serializer( + "lightning_adapter_digests", + "runtime_identity", + "backend_config", + "source_tier_weights", + ) + def serialize_mappings(self, value: Mapping[object, object]) -> dict[str, object]: + return {str(key): _json_value(item) for key, item in value.items()} + + @model_validator(mode="after") + def validate_tier_weights(self) -> TrainingRecipe: + if set(self.source_tier_weights) != set(TrainingSourceTier): + raise ValueError("source_tier_weights must include every source tier") + if any( + weight <= 0 or not math.isfinite(weight) for weight in self.source_tier_weights.values() + ): + raise ValueError("source tier weights must be finite and positive") + if not self.trainable_targets or any(not target for target in self.trainable_targets): + raise ValueError("trainable_targets must not be empty") + checkpoint_steps(self.total_steps) + return self + + +class TrainingRecipeValidationEvidence(FrozenModel): + """Composition-derived evidence binding an operative recipe to frozen data.""" + + schema_version: Literal["character-training-recipe-validation.v1"] = ( + "character-training-recipe-validation.v1" + ) + recipe_digest: Sha256Digest + dataset_digest: Sha256Digest + character_pack_digest: Sha256Digest + accepted_gold_clip_count: int = Field(ge=0) + accepted_synthetic_clip_count: int = Field(ge=0) + gold_clip_weight: float = Field(gt=0) + synthetic_clip_weight: float = Field(gt=0) + synthetic_effective_share: float = Field(ge=0, le=0.5) + + +class TrainingRun(FrozenModel): + schema_version: Literal["character-training-run.v1"] = "character-training-run.v1" + run_id: str = Field(min_length=1) + recipe_id: str = Field(min_length=1) + dataset_digest: Sha256Digest + total_steps: int = Field(gt=0) + + +class TrainingRunEvent(FrozenModel): + sequence: int = Field(gt=0) + previous_state: Literal[ + "draft", + "probed", + "calibrated", + "preregistered", + "training", + "evaluating", + "qualified", + "rejected", + "failed", + "cancelled", + ] + new_state: Literal[ + "probed", + "calibrated", + "preregistered", + "training", + "evaluating", + "qualified", + "rejected", + "failed", + "cancelled", + ] + payload: Mapping[str, object] + + @field_validator("payload") + @classmethod + def freeze_payload(cls, value: Mapping[str, object]) -> Mapping[str, object]: + return cast(Mapping[str, object], _freeze_mapping(cast(Mapping[object, object], value))) + + @field_serializer("payload") + def serialize_payload(self, value: Mapping[str, object]) -> dict[str, object]: + return {str(key): _json_value(item) for key, item in value.items()} + + @model_validator(mode="after") + def validate_edge(self) -> TrainingRunEvent: + legal = { + ("draft", "probed"), + ("probed", "calibrated"), + ("calibrated", "preregistered"), + ("preregistered", "training"), + ("cancelled", "training"), + ("training", "evaluating"), + ("evaluating", "qualified"), + ("evaluating", "rejected"), + } + failure_edge = self.new_state == "failed" and self.previous_state in { + "draft", + "probed", + "calibrated", + "preregistered", + "training", + "evaluating", + } + cancellation_edge = self.new_state == "cancelled" and self.previous_state == "training" + if (self.previous_state, self.new_state) not in legal and not ( + failure_edge or cancellation_edge + ): + raise ValueError("invalid scientific training lifecycle edge") + return self + + +class TrainingProbeEvidence(FrozenModel): + evidence_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + recipe_digest: Sha256Digest + dataset_digest: Sha256Digest + backend_digest: Sha256Digest + runtime_digest: Sha256Digest + material_digest: Sha256Digest + source_digest: Sha256Digest + capability_digest: Sha256Digest + succeeded: bool + + +class TrainingCalibrationEvidence(FrozenModel): + evidence_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + probe_evidence_id: str = Field(min_length=1) + recipe_digest: Sha256Digest + dataset_digest: Sha256Digest + backend_digest: Sha256Digest + runtime_digest: Sha256Digest + material_digest: Sha256Digest + source_digest: Sha256Digest + calibration_digest: Sha256Digest + succeeded: bool + + +class TrainingCancellationEvidence(FrozenModel): + cancellation_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + cancellation_request_id: str = Field(min_length=1) + request_digest: Sha256Digest + requested_step: int = Field(ge=0) + boundary_step: int = Field(ge=0) + checkpoint_digest: Sha256Digest + + @model_validator(mode="after") + def boundary_follows_request(self) -> TrainingCancellationEvidence: + if self.boundary_step < self.requested_step: + raise ValueError("cancellation boundary precedes request") + return self + + +class TrainingCheckpointArtifact(FrozenModel): + """One immutable file in a checkpoint's complete artifact inventory.""" + + relative_path: str = Field(min_length=1) + digest: Sha256Digest + size_bytes: int = Field(gt=0) + role: Literal[ + "weights", + "optimizer", + "trainer_state", + "telemetry", + "trainable_targets", + "frozen_parameters", + "finite_gradient_proof", + ] + + @field_validator("relative_path") + @classmethod + def validate_relative_path(cls, value: str) -> str: + if value.startswith("/") or ".." in value.split("/"): + raise ValueError("checkpoint inventory path must be relative and contained") + return value + + +class TrainingCheckpointEvidence(FrozenModel): + schema_version: Literal["character-training-checkpoint-evidence.v1"] = ( + "character-training-checkpoint-evidence.v1" + ) + run_id: str = Field(min_length=1) + recipe_digest: Sha256Digest + dataset_digest: Sha256Digest + backend_digest: Sha256Digest + runtime_digest: Sha256Digest + material_digest: Sha256Digest + source_digest: Sha256Digest + step: int = Field(gt=0) + fraction: Literal[0.25, 0.5, 0.75, 1.0] # type: ignore[valid-type] + parent_checkpoint_digest: Sha256Digest | None + safetensors_digest: Sha256Digest + safetensors_size_bytes: int = Field(gt=0) + optimizer_state_digest: Sha256Digest + trainer_state_digest: Sha256Digest + telemetry_digest: Sha256Digest + telemetry_sequence_start: int = Field(gt=0) + telemetry_sequence_end: int = Field(gt=0) + trainable_targets_digest: Sha256Digest + frozen_parameters_digest: Sha256Digest + finite_gradient_proof_digest: Sha256Digest + cancellation_boundary: bool + inventory: tuple[TrainingCheckpointArtifact, ...] = Field(min_length=1) + + @model_validator(mode="after") + def validate_checkpoint_evidence(self) -> TrainingCheckpointEvidence: + if self.telemetry_sequence_end < self.telemetry_sequence_start: + raise ValueError("checkpoint telemetry range is reversed") + paths = tuple(item.relative_path for item in self.inventory) + if len(paths) != len(set(paths)): + raise ValueError("checkpoint inventory paths must be unique") + expected_digests = { + "weights": self.safetensors_digest, + "optimizer": self.optimizer_state_digest, + "trainer_state": self.trainer_state_digest, + "telemetry": self.telemetry_digest, + "trainable_targets": self.trainable_targets_digest, + "frozen_parameters": self.frozen_parameters_digest, + "finite_gradient_proof": self.finite_gradient_proof_digest, + } + by_role = {item.role: item for item in self.inventory} + if len(self.inventory) != len(expected_digests): + raise ValueError("checkpoint inventory is incomplete") + if set(by_role) != set(expected_digests) or len(by_role) != len(self.inventory): + raise ValueError("checkpoint inventory roles are not exact") + if any(item.digest != expected_digests[item.role] for item in self.inventory): + raise ValueError("checkpoint inventory role binding mismatch") + # The weights file has a separately declared byte size. Other role files have no + # independent size claim in v1; their canonical semantics are a strictly positive, + # inventory-owned byte count (enforced by TrainingCheckpointArtifact). + if by_role["weights"].size_bytes != self.safetensors_size_bytes: + raise ValueError("checkpoint inventory role binding size mismatch") + return self + + +class TrainingResumeEvidence(FrozenModel): + resume_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + parent_checkpoint_id: str = Field(min_length=1) + parent_checkpoint_digest: Sha256Digest + parent_artifact_digest: Sha256Digest + recipe_digest: Sha256Digest + dataset_digest: Sha256Digest + backend_digest: Sha256Digest + runtime_digest: Sha256Digest + material_digest: Sha256Digest + source_digest: Sha256Digest + optimizer_state_digest: Sha256Digest + + +class TrainingEvaluationEvidence(FrozenModel): + evidence_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + disposition: Literal["qualified", "rejected"] + evidence_digest: Sha256Digest + + +class TrainingTelemetry(FrozenModel): + sequence: int = Field(gt=0) + step: int = Field(ge=0) + payload: Mapping[str, object] + + @field_validator("payload") + @classmethod + def freeze_payload(cls, value: Mapping[str, object]) -> Mapping[str, object]: + return cast(Mapping[str, object], _freeze_mapping(cast(Mapping[object, object], value))) + + @field_serializer("payload") + def serialize_payload(self, value: Mapping[str, object]) -> dict[str, object]: + return {str(key): _json_value(item) for key, item in value.items()} + + +class TrainingCheckpoint(FrozenModel): + checkpoint_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + step: int = Field(gt=0) + fraction: Literal[0.25, 0.5, 0.75, 1.0] # type: ignore[valid-type] + parent_checkpoint_id: str | None + artifact_digest: Sha256Digest + manifest_digest: Sha256Digest + manifest: Mapping[str, object] + + @field_validator("manifest") + @classmethod + def freeze_manifest(cls, value: Mapping[str, object]) -> Mapping[str, object]: + required = { + "schema_version", + "run_id", + "reviewer_id", + "recipe_id", + "dataset_digest", + "character_id", + "step", + "fraction", + "parent_checkpoint_id", + "artifact_digest", + "safetensors_size_bytes", + "backend_digest", + "runtime_digest", + "material_digest", + "source_digest", + "optimizer_state_digest", + "trainer_state_digest", + "telemetry_digest", + "telemetry_sequence_start", + "telemetry_sequence_end", + "trainable_targets_digest", + "frozen_parameters_digest", + "finite_gradient_proof_digest", + "cancellation_boundary", + "inventory", + } + if set(value) != required: + raise ValueError("checkpoint manifest is incomplete") + return cast(Mapping[str, object], _freeze_mapping(cast(Mapping[object, object], value))) + + @model_validator(mode="after") + def validate_manifest_identity(self) -> TrainingCheckpoint: + if ( + self.manifest["run_id"] != self.run_id + or self.manifest["step"] != self.step + or self.manifest["fraction"] != self.fraction + or self.manifest["parent_checkpoint_id"] != self.parent_checkpoint_id + or self.manifest["artifact_digest"] != self.artifact_digest + or _content_digest(dict(self.manifest)) != self.manifest_digest + ): + raise ValueError("checkpoint manifest identity or digest mismatch") + return self + + @field_serializer("manifest") + def serialize_manifest(self, value: Mapping[str, object]) -> dict[str, object]: + return {str(key): _json_value(item) for key, item in value.items()} + + +class TrainingFailure(FrozenModel): + failure_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + code: str = Field(min_length=1, max_length=64) + message: str = Field(min_length=1, max_length=1000) + + +def checkpoint_steps(total_steps: int) -> tuple[int, int, int, int]: + """Return collision-free 25/50/75/100% integer milestones.""" + if total_steps < 4: + raise ValueError("total_steps must represent four checkpoint milestones") + steps = tuple((total_steps * numerator + 3) // 4 for numerator in (1, 2, 3, 4)) + if len(set(steps)) != 4 or steps[-1] != total_steps: + raise ValueError("checkpoint milestones collide") + return cast(tuple[int, int, int, int], steps) + + +def training_recipe_id(recipe: TrainingRecipe) -> str: + """Return the versioned canonical SHA-256 identity for a training recipe.""" + payload = recipe.model_dump(mode="json") + payload["trainable_targets"] = sorted(set(recipe.trainable_targets)) + return _content_digest(payload) + + +def training_dataset_id(draft: TrainingDatasetDraft) -> str: + """Order-independent identity that binds every accepted-sample contract field.""" + payload = draft.model_dump(mode="json") + for sample in payload["samples"]: + if sample["curated_evidence"] is None: + # Preserve the v1 identity of legacy samples while binding opt-in evidence. + del sample["curated_evidence"] + payload["samples"] = sorted(payload["samples"], key=_sample_order) + payload["imported_sample_ids"] = sorted(set(payload["imported_sample_ids"])) + payload["curation_confirmation_ids"] = sorted(set(payload["curation_confirmation_ids"])) + return _content_digest(payload) + + +def imported_training_sample_id(sample: ImportedTrainingSample) -> str: + payload = sample.model_dump(mode="json", exclude={"import_id"}) + return "training_import_" + _content_digest(payload).removeprefix("sha256:") + + +def training_confirmation_id(confirmation: TrainingCurationConfirmation) -> str: + payload = confirmation.model_dump(mode="json", exclude={"confirmation_id"}) + return "training_confirmation_" + _content_digest(payload).removeprefix("sha256:") + + +def _sample_order(sample: object) -> str: + return json.dumps(sample, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def _content_digest(payload: object) -> str: + encoded = json.dumps( + _json_value(payload), sort_keys=True, separators=(",", ":"), ensure_ascii=False + ) + return f"sha256:{hashlib.sha256(encoded.encode('utf-8')).hexdigest()}" + + +def _freeze_mapping(value: Mapping[object, object]) -> Mapping[object, object]: + return MappingProxyType({key: _freeze_json(item) for key, item in value.items()}) + + +def _freeze_json(value: object) -> object: + if isinstance(value, float) and not math.isfinite(value): + raise ValueError("unsupported non-finite JSON number") + if value is None or isinstance(value, str | int | float | bool | StrEnum): + return value + if isinstance(value, Mapping): + if not all(isinstance(key, str | StrEnum) for key in value): + raise TypeError("mapping keys must be strings") + return _freeze_mapping(value) + if isinstance(value, Set): + return tuple(sorted((_freeze_json(item) for item in value), key=_sample_order)) + if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): + return tuple(_freeze_json(item) for item in value) + raise TypeError(f"unsupported JSON value: {type(value).__name__}") + + +def _json_value(value: object) -> object: + if isinstance(value, StrEnum): + return value.value + if isinstance(value, Mapping): + return {str(key): _json_value(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_json_value(item) for item in value] + return value diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_repository.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_repository.py new file mode 100644 index 0000000..2000c53 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_repository.py @@ -0,0 +1,1822 @@ +"""Transactional persistence for trainer-neutral character-training evidence.""" + +from __future__ import annotations + +import hashlib +import json +import re +import sqlite3 +import unicodedata +import uuid +from collections.abc import Callable, Mapping +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Literal, cast + +from video_lab.database import Database +from video_lab.domain import FrozenModel, canonical_json, utc_now +from video_lab.training_dataset import validate_training_recipe_dataset +from video_lab.training_models import ( + FrozenTrainingDataset, + ImportedTrainingSample, + TrainingCalibrationEvidence, + TrainingCancellationEvidence, + TrainingCheckpoint, + TrainingCheckpointEvidence, + TrainingCurationConfirmation, + TrainingDatasetDraft, + TrainingEvaluationEvidence, + TrainingFailure, + TrainingOperationKind, + TrainingOperationStatus, + TrainingOrchestrationOperation, + TrainingProbeEvidence, + TrainingRecipe, + TrainingResumeEvidence, + TrainingRun, + TrainingRunEvent, + TrainingSample, + TrainingTelemetry, + training_dataset_id, + training_operation_payload_digest, + training_recipe_id, +) + + +class TrainingNotFound(LookupError): + pass + + +class TrainingConflict(RuntimeError): + pass + + +class DatasetRecord(FrozenModel): + storage_id: str + dataset_id: str + reviewer_id: str + character_id: str + state: Literal["draft", "sealed"] + dataset_digest: str | None + draft: TrainingDatasetDraft + frozen_dataset: FrozenTrainingDataset | None = None + + +class RecipeMaterialBinding(FrozenModel): + """Persistence-only handles for independently resolving recipe evidence.""" + + stage_3_campaign_id: str + stage_3_result_id: str + base_model_artifact_id: str + lightning_adapter_artifact_ids: Mapping[str, str] + source_snapshot_artifact_id: str + runtime_digest: str + + +def recipe_material_digest(binding: RecipeMaterialBinding) -> str: + payload = canonical_json(binding.model_dump(mode="json")) + return f"sha256:{hashlib.sha256(payload.encode()).hexdigest()}" + + +class RecipeRecord(FrozenModel): + storage_id: str + recipe_id: str + reviewer_id: str + state: Literal["draft", "preregistered"] + recipe: TrainingRecipe + binding: RecipeMaterialBinding + + +class RunRecord(FrozenModel): + run: TrainingRun + reviewer_id: str + state: Literal[ + "draft", + "probed", + "calibrated", + "preregistered", + "training", + "evaluating", + "qualified", + "rejected", + "failed", + "cancelled", + ] + created_at: str + terminal_at: str | None + + +class EventRecord(FrozenModel): + run_id: str + reviewer_id: str + event: TrainingRunEvent + created_at: str + + +class TelemetryRecord(FrozenModel): + run_id: str + reviewer_id: str + sample: TrainingTelemetry + created_at: str + + +class CheckpointRecord(FrozenModel): + reviewer_id: str + checkpoint: TrainingCheckpoint + created_at: str + + +class FailureRecord(FrozenModel): + reviewer_id: str + failure: TrainingFailure + created_at: str + + +class TrainingOperationClaim(FrozenModel): + operation: TrainingOrchestrationOperation + claimed: bool + + +class DatasetLineageRecord(FrozenModel): + lineage_id: str + reviewer_id: str + character_id: str + current_dataset_id: str + head_version: int + state: Literal["draft", "sealed"] + created_at: str + + +class DatasetHeadEventRecord(FrozenModel): + lineage_id: str + version: int + previous_dataset_id: str | None + dataset_id: str + state: Literal["draft", "sealed"] + created_at: str + + +class TrainingRepository: + def __init__(self, database: Database, *, reviewer_id: str) -> None: + reviewer = unicodedata.normalize("NFKC", reviewer_id).strip() + if not reviewer or reviewer != reviewer_id: + raise ValueError("reviewer_id must be normalized non-empty text") + self.database = database + self.reviewer_id = reviewer + + def create_dataset_lineage(self, lineage_id: str, dataset_id: str) -> DatasetLineageRecord: + current = self.get_dataset(dataset_id) + with self.database.connect() as connection: + declared = connection.execute( + "SELECT lineage_id FROM training_datasets WHERE storage_id=?", + (current.storage_id,), + ).fetchone() + if declared is None or str(declared["lineage_id"] or "") != lineage_id: + raise TrainingConflict("dataset snapshot did not declare this lineage at creation") + return self.get_dataset_lineage(lineage_id) + + def create_dataset_for_lineage( + self, lineage_id: str, character_id: str, draft: TrainingDatasetDraft + ) -> DatasetLineageRecord: + self.create_dataset(character_id, draft, lineage_id=lineage_id) + return self.get_dataset_lineage(lineage_id) + + def get_dataset_lineage(self, lineage_id: str) -> DatasetLineageRecord: + with self.database.connect() as connection: + row = connection.execute( + "SELECT l.*,d.id dataset_id FROM training_dataset_lineages l " + "JOIN training_datasets d " + "ON d.storage_id=l.current_dataset_storage_id WHERE l.id=? AND l.reviewer_id=?", + (lineage_id, self.reviewer_id), + ).fetchone() + if row is None: + raise TrainingNotFound("training dataset lineage not found") + return DatasetLineageRecord( + lineage_id=str(row["id"]), + reviewer_id=str(row["reviewer_id"]), + character_id=str(row["character_id"]), + current_dataset_id=str(row["dataset_id"]), + head_version=int(row["head_version"]), + state=cast(Literal["draft", "sealed"], str(row["state"])), + created_at=str(row["created_at"]), + ) + + def list_dataset_lineages(self) -> tuple[DatasetLineageRecord, ...]: + with self.database.connect() as connection: + ids = tuple( + str(r[0]) + for r in connection.execute( + "SELECT id FROM training_dataset_lineages WHERE reviewer_id=? " + "ORDER BY created_at,id", + (self.reviewer_id,), + ) + ) + return tuple(self.get_dataset_lineage(value) for value in ids) + + def list_dataset_head_history(self, lineage_id: str) -> tuple[DatasetHeadEventRecord, ...]: + self.get_dataset_lineage(lineage_id) + with self.database.connect() as connection: + rows = connection.execute( + "SELECT e.*,p.id previous_id,d.id dataset_id FROM training_dataset_head_events e " + "LEFT JOIN training_datasets p ON p.storage_id=e.previous_dataset_storage_id " + "JOIN training_datasets d ON d.storage_id=e.dataset_storage_id " + "WHERE e.lineage_id=? AND e.reviewer_id=? ORDER BY e.version", + (lineage_id, self.reviewer_id), + ).fetchall() + return tuple( + DatasetHeadEventRecord( + lineage_id=lineage_id, + version=int(r["version"]), + previous_dataset_id=cast(str | None, r["previous_id"]), + dataset_id=str(r["dataset_id"]), + state=cast(Literal["draft", "sealed"], str(r["state"])), + created_at=str(r["created_at"]), + ) + for r in rows + ) + + def persist_imported_sample_with_connection( + self, connection: sqlite3.Connection, imported: ImportedTrainingSample + ) -> None: + self._require_owned_connection(connection) + if imported.reviewer_id != self.reviewer_id: + raise TrainingConflict("import reviewer mismatch") + state = connection.execute( + "SELECT state FROM training_dataset_lineages WHERE id=? AND reviewer_id=?", + (imported.lineage_id, self.reviewer_id), + ).fetchone() + if state is not None and str(state["state"]) == "sealed": + raise TrainingConflict("sealed training lineage rejects imports") + try: + connection.execute( + "INSERT INTO training_imported_samples VALUES (?,?,?,?,?,?,?,?)", + ( + imported.import_id, + imported.reviewer_id, + imported.character_id, + imported.lineage_id, + imported.artifact_id, + imported.artifact_digest, + canonical_json(imported.model_dump(mode="json")), + imported.model_dump(mode="json")["imported_at"], + ), + ) + except sqlite3.DatabaseError: + raise TrainingConflict("imported sample identity conflict") from None + + def persist_confirmation_with_connection( + self, connection: sqlite3.Connection, confirmation: TrainingCurationConfirmation + ) -> None: + self._require_owned_connection(connection) + if confirmation.reviewer_id != self.reviewer_id: + raise TrainingConflict("confirmation reviewer mismatch") + state = connection.execute( + "SELECT lineage.state FROM training_imported_samples imported " + "JOIN training_dataset_lineages lineage ON lineage.id=imported.lineage_id " + "WHERE imported.id=? AND lineage.reviewer_id=?", + (confirmation.import_id, self.reviewer_id), + ).fetchone() + if state is not None and str(state["state"]) == "sealed": + raise TrainingConflict("sealed training lineage rejects confirmations") + try: + connection.execute( + "INSERT INTO training_curation_confirmations VALUES (?,?,?,?,?,?,?)", + ( + confirmation.confirmation_id, + confirmation.import_id, + confirmation.reviewer_id, + confirmation.version, + confirmation.decision.value, + canonical_json(confirmation.model_dump(mode="json")), + confirmation.model_dump(mode="json")["confirmed_at"], + ), + ) + except sqlite3.DatabaseError: + raise TrainingConflict("training confirmation conflict") from None + + def list_imported_samples(self, lineage_id: str) -> tuple[ImportedTrainingSample, ...]: + self.get_dataset_lineage(lineage_id) + with self.database.connect() as connection: + rows = connection.execute( + "SELECT payload_json FROM training_imported_samples " + "WHERE lineage_id=? AND reviewer_id=? ORDER BY created_at,id", + (lineage_id, self.reviewer_id), + ).fetchall() + return tuple(ImportedTrainingSample.model_validate_json(str(r[0])) for r in rows) + + def list_confirmations(self, import_id: str) -> tuple[TrainingCurationConfirmation, ...]: + with self.database.connect() as connection: + rows = connection.execute( + "SELECT c.payload_json FROM training_curation_confirmations c " + "JOIN training_imported_samples i ON i.id=c.import_id " + "WHERE c.import_id=? AND c.reviewer_id=? AND i.reviewer_id=? " + "ORDER BY c.version", + (import_id, self.reviewer_id, self.reviewer_id), + ).fetchall() + return tuple(TrainingCurationConfirmation.model_validate_json(str(r[0])) for r in rows) + + def publish_dataset_head_with_connection( + self, + connection: sqlite3.Connection, + *, + lineage_id: str, + expected_head_version: int, + draft: TrainingDatasetDraft | FrozenTrainingDataset, + seal: bool = False, + ) -> str: + """Publish one immutable snapshot and CAS its lineage head without committing.""" + self._require_owned_connection(connection) + base = self._draft_from(draft) + state = "sealed" if seal else "draft" + if seal and not isinstance(draft, FrozenTrainingDataset): + raise TrainingConflict("sealing requires a frozen dataset") + head = connection.execute( + "SELECT * FROM training_dataset_lineages WHERE id=? AND reviewer_id=?", + (lineage_id, self.reviewer_id), + ).fetchone() + if ( + head is None + or int(head["head_version"]) != expected_head_version + or str(head["state"]) != "draft" + or str(head["character_id"]) != base.character_id + ): + raise TrainingConflict("stale or invalid dataset head") + previous_storage_id = str(head["current_dataset_storage_id"]) + storage_id = previous_storage_id if seal else f"training_dataset_storage_{uuid.uuid4().hex}" + dataset_id = training_dataset_id(base) + pack = connection.execute( + "SELECT storage_id FROM character_packs WHERE character_id=? " + "AND reviewer_id=? AND state='preregistered' " + "AND canonical_json_digest(manifest_json)=?", + (base.character_id, self.reviewer_id, base.character_pack_digest), + ).fetchone() + if pack is None: + raise TrainingConflict("dataset character pack provenance mismatch") + manifest = draft.model_dump(mode="json") if seal else base.model_dump(mode="json") + digest = draft.dataset_digest if isinstance(draft, FrozenTrainingDataset) and seal else None + now = utc_now().isoformat() + try: + if seal: + current = connection.execute( + "SELECT id,manifest_json FROM training_datasets WHERE storage_id=?", + (previous_storage_id,), + ).fetchone() + if current is None or str(current["id"]) != dataset_id: + raise TrainingConflict("frozen dataset is not the current head") + connection.execute( + "UPDATE training_datasets SET state='sealed',dataset_digest=?," + "manifest_json=?,sealed_at=? WHERE storage_id=?", + (digest, canonical_json(manifest), now, storage_id), + ) + else: + connection.execute( + "INSERT INTO training_datasets(storage_id,id,reviewer_id,character_id," + "character_pack_storage_id,character_pack_digest,state,canonical_outfit_id," + "reserved_token,source_tier_weights_json,manifest_json,dataset_digest," + "created_at,sealed_at,lineage_id) " + "VALUES (?,?,?,?,?,?,'draft',?,?,?,?,NULL,?,NULL,?)", + ( + storage_id, + dataset_id, + self.reviewer_id, + base.character_id, + str(pack["storage_id"]), + base.character_pack_digest, + base.canonical_outfit_id, + base.reserved_token, + canonical_json( + { + tier.value: weight + for tier, weight in base.source_tier_weights.items() + } + ), + canonical_json(base.model_dump(mode="json")), + now, + lineage_id, + ), + ) + version = expected_head_version + 1 + cursor = connection.execute( + "UPDATE training_dataset_lineages SET current_dataset_storage_id=?," + "head_version=?,state=? WHERE id=? AND reviewer_id=? " + "AND head_version=? AND state='draft'", + ( + storage_id, + version, + state, + lineage_id, + self.reviewer_id, + expected_head_version, + ), + ) + if cursor.rowcount != 1: + raise TrainingConflict("stale dataset head") + except TrainingConflict: + raise + except sqlite3.DatabaseError: + raise TrainingConflict("dataset head publication conflict") from None + return str(digest or dataset_id) + + def _require_owned_connection(self, connection: sqlite3.Connection) -> None: + try: + rows = connection.execute("PRAGMA database_list").fetchall() + except sqlite3.Error: + raise TrainingConflict("connection is closed") from None + main = next((str(row[2]) for row in rows if str(row[1]) == "main"), "") + if not main or Path(main).resolve() != self.database.path.resolve(): + raise TrainingConflict("connection belongs to a foreign database") + + def create_dataset( + self, + character_id: str, + draft: TrainingDatasetDraft | FrozenTrainingDataset, + *, + lineage_id: str | None = None, + ) -> DatasetRecord: + base = self._draft_from(draft) + if base.character_id != character_id: + raise TrainingConflict("dataset character identity mismatch") + dataset_id = training_dataset_id(base) + storage_id = f"training_dataset_storage_{uuid.uuid4().hex}" + created_at = utc_now().isoformat() + tier_weights = {tier.value: weight for tier, weight in base.source_tier_weights.items()} + try: + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + owner = connection.execute( + "SELECT reviewer_id FROM characters WHERE id=?", (character_id,) + ).fetchone() + if owner is None or str(owner["reviewer_id"]) != self.reviewer_id: + raise TrainingConflict("dataset owner or character mismatch") + pack = connection.execute( + "SELECT storage_id FROM character_packs WHERE character_id=? " + "AND reviewer_id=? AND state='preregistered' " + "AND canonical_json_digest(manifest_json)=?", + (character_id, self.reviewer_id, base.character_pack_digest), + ).fetchone() + if pack is None: + raise TrainingConflict("dataset character pack provenance mismatch") + connection.execute( + "INSERT INTO training_datasets(storage_id,id,reviewer_id,character_id," + "character_pack_storage_id,character_pack_digest,state,canonical_outfit_id," + "reserved_token,source_tier_weights_json,manifest_json,dataset_digest," + "created_at,sealed_at,lineage_id) " + "VALUES (?,?,?,?,?,?,'draft',?,?,?,?,NULL,?,NULL,?)", + ( + storage_id, + dataset_id, + self.reviewer_id, + character_id, + str(pack["storage_id"]), + base.character_pack_digest, + base.canonical_outfit_id, + base.reserved_token, + canonical_json(tier_weights), + canonical_json(base.model_dump(mode="json")), + created_at, + lineage_id, + ), + ) + if lineage_id is not None: + connection.execute( + "INSERT INTO training_dataset_lineages VALUES (?,?,?,?,1,'draft',?)", + (lineage_id, self.reviewer_id, character_id, storage_id, created_at), + ) + connection.commit() + except TrainingConflict: + raise + except TrainingNotFound: + raise TrainingConflict("recipe dataset identity mismatch") from None + except sqlite3.DatabaseError: + raise TrainingConflict("dataset identity conflict") from None + return self.get_dataset(dataset_id) + + def replace_draft_sample(self, dataset_id: str, sample: TrainingSample) -> DatasetRecord: + current = self.get_dataset(dataset_id) + if current.state != "draft": + raise TrainingConflict("sealed dataset is immutable") + samples = tuple( + item for item in current.draft.samples if item.sample_id != sample.sample_id + ) + changed = current.draft.model_copy(update={"samples": samples + (sample,)}) + return self.create_dataset(changed.character_id, changed) + + def seal_dataset(self, dataset_id: str, frozen: FrozenTrainingDataset) -> DatasetRecord: + current = self.get_dataset(dataset_id) + expected = self._draft_from(frozen) + if current.state == "sealed": + if current.frozen_dataset != frozen: + raise TrainingConflict("sealed dataset replay mismatch") + return current + if current.draft != expected or frozen.dataset_digest != training_dataset_id(expected): + raise TrainingConflict("frozen dataset identity mismatch") + now = utc_now().isoformat() + try: + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + connection.execute( + "UPDATE training_datasets SET state='sealed',dataset_digest=?," + "manifest_json=?,sealed_at=? WHERE storage_id=?", + ( + frozen.dataset_digest, + canonical_json(frozen.model_dump(mode="json")), + now, + current.storage_id, + ), + ) + connection.commit() + except sqlite3.DatabaseError: + raise TrainingConflict("dataset sealing conflict") from None + return self.get_dataset(frozen.dataset_digest) + + def get_dataset(self, dataset_id: str) -> DatasetRecord: + with self.database.connect() as connection: + row = connection.execute( + "SELECT dataset.* FROM training_datasets dataset " + "LEFT JOIN training_dataset_aliases alias " + "ON alias.dataset_storage_id=dataset.storage_id " + "WHERE (dataset.id=? OR dataset.dataset_digest=? OR alias.alias_id=?) " + "AND dataset.reviewer_id=? AND dataset.state IN ('draft','sealed') " + "AND alias.alias_id=dataset.id " + "AND (SELECT count(*) FROM training_dataset_aliases exact_alias " + "WHERE exact_alias.dataset_storage_id=dataset.storage_id)=1 " + "AND ((dataset.state='draft' AND training_draft_identity_is_valid(" + "dataset.storage_id,dataset.id,dataset.manifest_json)=1) OR " + "(dataset.state='sealed' AND training_dataset_identity_is_valid(" + "dataset.storage_id,dataset.dataset_digest,dataset.manifest_json)=1)) LIMIT 1", + (dataset_id, dataset_id, dataset_id, self.reviewer_id), + ).fetchone() + if row is None: + raise TrainingNotFound("training dataset not found") + manifest = json.loads(str(row["manifest_json"])) + if str(row["state"]) == "sealed": + frozen = FrozenTrainingDataset.model_validate(manifest) + draft = self._draft_from(frozen) + else: + frozen = None + draft = TrainingDatasetDraft.model_validate(manifest) + with self.database.connect() as connection: + imported_ids = tuple( + str(item[0]) + for item in connection.execute( + "SELECT import_id FROM training_dataset_snapshot_imports " + "WHERE dataset_storage_id=? ORDER BY import_id", + (row["storage_id"],), + ) + ) + confirmation_ids = tuple( + str(item[0]) + for item in connection.execute( + "SELECT confirmation_id FROM training_dataset_snapshot_confirmations " + "WHERE dataset_storage_id=? ORDER BY confirmation_id", + (row["storage_id"],), + ) + ) + if ( + imported_ids != draft.imported_sample_ids + or confirmation_ids != draft.curation_confirmation_ids + ): + raise TrainingNotFound("training dataset ledger bindings do not match manifest") + return DatasetRecord( + storage_id=str(row["storage_id"]), + dataset_id=str(row["id"]), + reviewer_id=str(row["reviewer_id"]), + character_id=str(row["character_id"]), + state=cast(Literal["draft", "sealed"], str(row["state"])), + dataset_digest=cast(str | None, row["dataset_digest"]), + draft=draft, + frozen_dataset=frozen, + ) + + def create_recipe( + self, + recipe: TrainingRecipe, + binding: RecipeMaterialBinding, + *, + connection: sqlite3.Connection | None = None, + ) -> RecipeRecord: + if connection is not None: + self._require_owned_connection(connection) + return self._create_recipe_with_connection(connection, recipe, binding) + with self.database.connect() as owned: + owned.execute("BEGIN IMMEDIATE") + try: + result = self._create_recipe_with_connection(owned, recipe, binding) + owned.commit() + return result + except Exception: + owned.rollback() + raise + + def _create_recipe_with_connection( + self, + connection: sqlite3.Connection, + recipe: TrainingRecipe, + binding: RecipeMaterialBinding, + ) -> RecipeRecord: + recipe_id = training_recipe_id(recipe) + try: + dataset = self.get_dataset(recipe.dataset_digest) + if dataset.frozen_dataset is None: + raise TrainingConflict("recipe requires an existing sealed dataset") + validate_training_recipe_dataset(recipe, dataset.frozen_dataset) + storage_id = f"training_recipe_storage_{uuid.uuid4().hex}" + provenance = connection.execute( + "SELECT benchmark.id FROM training_datasets dataset " + "JOIN character_benchmark_campaigns benchmark " + "ON benchmark.character_pack_storage_id=dataset.character_pack_storage_id " + "AND benchmark.reviewer_id=? JOIN character_benchmark_results result " + "ON result.campaign_id=benchmark.id WHERE dataset.storage_id=? " + "AND character_benchmark_provenance_is_valid(benchmark.id)=1 " + "AND result.content_digest=? " + "AND json_extract(benchmark.definition_json,'$.character_pack_digest')=? " + "AND json_extract(benchmark.definition_json,'$.stage_2_manifest_digest')=? " + "AND json_extract(benchmark.definition_json,'$.stage_2_evidence_digest')=? " + "AND (SELECT new_state FROM character_benchmark_events " + "WHERE campaign_id=benchmark.id ORDER BY sequence DESC LIMIT 1)='completed'", + ( + self.reviewer_id, + dataset.storage_id, + recipe.stage_3_evidence_digest, + recipe.character_pack_digest, + recipe.stage_2_manifest_digest, + recipe.stage_2_evidence_digest, + ), + ).fetchone() + if provenance is None or str(provenance["id"]) != binding.stage_3_campaign_id: + raise TrainingConflict("recipe prerequisite provenance mismatch") + result = connection.execute( + "SELECT content_digest FROM character_benchmark_results WHERE campaign_id=?", + (binding.stage_3_campaign_id,), + ).fetchone() + if result is None or str(result["content_digest"]) != binding.stage_3_result_id: + raise TrainingConflict("recipe prerequisite provenance mismatch") + self._require_recipe_artifact( + connection, binding.base_model_artifact_id, recipe.base_model_digest + ) + self._require_recipe_artifact( + connection, binding.source_snapshot_artifact_id, recipe.source_code_digest + ) + if set(binding.lightning_adapter_artifact_ids) != set(recipe.lightning_adapter_digests): + raise TrainingConflict("recipe material provenance mismatch") + for name, artifact_id in binding.lightning_adapter_artifact_ids.items(): + self._require_recipe_artifact( + connection, artifact_id, recipe.lightning_adapter_digests[name] + ) + connection.execute( + "INSERT INTO training_recipes VALUES (?,?,?,?,?,?,?,?,?,?,?)", + ( + storage_id, + recipe_id, + self.reviewer_id, + dataset.storage_id, + recipe.dataset_digest, + dataset.character_id, + recipe.character_pack_digest, + "draft", + canonical_json(recipe.model_dump(mode="json")), + utc_now().isoformat(), + None, + ), + ) + connection.execute( + "INSERT INTO training_recipe_materials VALUES (?,?,?,?,?,?,?,?)", + ( + storage_id, + self.reviewer_id, + binding.base_model_artifact_id, + canonical_json(dict(binding.lightning_adapter_artifact_ids)), + binding.source_snapshot_artifact_id, + binding.stage_3_campaign_id, + binding.stage_3_result_id, + binding.runtime_digest, + ), + ) + connection.executemany( + "INSERT INTO training_recipe_lightning_materials VALUES (?,?,?,?)", + ( + (storage_id, self.reviewer_id, name, artifact_id) + for name, artifact_id in sorted(binding.lightning_adapter_artifact_ids.items()) + ), + ) + except TrainingConflict: + raise + except TrainingNotFound: + raise TrainingConflict("recipe dataset identity mismatch") from None + except sqlite3.DatabaseError: + raise TrainingConflict("recipe dataset identity conflict") from None + except (TypeError, ValueError): + raise TrainingConflict("recipe dataset identity mismatch") from None + return self._get_recipe_with_connection(connection, recipe_id) + + @staticmethod + def _require_recipe_artifact( + connection: sqlite3.Connection, artifact_id: str, digest: str + ) -> None: + artifact = connection.execute( + "SELECT sha256 FROM artifacts WHERE id=?", (artifact_id,) + ).fetchone() + if artifact is None or "sha256:" + str(artifact["sha256"]) != digest: + raise TrainingConflict("recipe material provenance mismatch") + + def preregister_recipe( + self, recipe_id: str, *, connection: sqlite3.Connection | None = None + ) -> RecipeRecord: + if connection is not None: + self._require_owned_connection(connection) + return self._preregister_recipe_with_connection(connection, recipe_id) + with self.database.connect() as owned: + owned.execute("BEGIN IMMEDIATE") + try: + result = self._preregister_recipe_with_connection(owned, recipe_id) + owned.commit() + return result + except Exception: + owned.rollback() + raise + + def _preregister_recipe_with_connection( + self, connection: sqlite3.Connection, recipe_id: str + ) -> RecipeRecord: + current = self._get_recipe_with_connection(connection, recipe_id) + if current.state == "preregistered": + return current + try: + connection.execute( + "UPDATE training_recipes SET state='preregistered',preregistered_at=? " + "WHERE storage_id=?", + (utc_now().isoformat(), current.storage_id), + ) + except sqlite3.DatabaseError: + raise TrainingConflict("recipe preregistration conflict") from None + return self._get_recipe_with_connection(connection, recipe_id) + + def get_recipe(self, recipe_id: str) -> RecipeRecord: + with self.database.connect() as connection: + return self._get_recipe_with_connection(connection, recipe_id) + + def _get_recipe_with_connection( + self, connection: sqlite3.Connection, recipe_id: str + ) -> RecipeRecord: + row = connection.execute( + "SELECT recipe.*,materials.base_model_artifact_id," + "materials.lightning_adapter_artifact_ids_json," + "materials.source_snapshot_artifact_id,materials.stage_3_campaign_id," + "materials.stage_3_result_id,materials.runtime_digest " + "FROM training_recipes recipe JOIN training_recipe_materials materials " + "ON materials.recipe_storage_id=recipe.storage_id " + "WHERE (recipe.id=? OR recipe.storage_id=?) AND recipe.reviewer_id=?", + (recipe_id, recipe_id, self.reviewer_id), + ).fetchone() + if row is None: + raise TrainingNotFound("training recipe not found") + recipe = TrainingRecipe.model_validate_json(str(row["recipe_json"])) + binding = RecipeMaterialBinding( + base_model_artifact_id=str(row["base_model_artifact_id"]), + lightning_adapter_artifact_ids=json.loads( + str(row["lightning_adapter_artifact_ids_json"]) + ), + source_snapshot_artifact_id=str(row["source_snapshot_artifact_id"]), + stage_3_campaign_id=str(row["stage_3_campaign_id"]), + stage_3_result_id=str(row["stage_3_result_id"]), + runtime_digest=str(row["runtime_digest"]), + ) + child_bindings = { + str(item["material_name"]): str(item["artifact_id"]) + for item in connection.execute( + "SELECT material_name,artifact_id FROM training_recipe_lightning_materials " + "WHERE recipe_storage_id=? ORDER BY material_name", + (row["storage_id"],), + ) + } + if child_bindings != dict(binding.lightning_adapter_artifact_ids): + raise TrainingConflict("recipe material binding mismatch") + return RecipeRecord( + storage_id=str(row["storage_id"]), + recipe_id=str(row["id"]), + reviewer_id=str(row["reviewer_id"]), + state=cast(Literal["draft", "preregistered"], str(row["state"])), + recipe=recipe, + binding=binding, + ) + + def list_recipes(self) -> tuple[RecipeRecord, ...]: + with self.database.connect() as connection: + identifiers = tuple( + str(row["id"]) + for row in connection.execute( + "SELECT id FROM training_recipes WHERE reviewer_id=? ORDER BY created_at,id", + (self.reviewer_id,), + ) + ) + return tuple(self.get_recipe(identifier) for identifier in identifiers) + + def list_datasets(self) -> tuple[DatasetRecord, ...]: + with self.database.connect() as connection: + identifiers = tuple( + str(row["id"]) + for row in connection.execute( + "SELECT dataset.id FROM training_datasets dataset " + "WHERE dataset.reviewer_id=? AND dataset.state IN ('draft','sealed') " + "AND EXISTS(SELECT 1 FROM training_dataset_aliases alias " + "WHERE alias.dataset_storage_id=dataset.storage_id " + "AND alias.alias_id=dataset.id) " + "AND (SELECT count(*) FROM training_dataset_aliases alias " + "WHERE alias.dataset_storage_id=dataset.storage_id)=1 " + "ORDER BY dataset.created_at,dataset.id", + (self.reviewer_id,), + ) + ) + return tuple(self.get_dataset(identifier) for identifier in identifiers) + + def get_run(self, run_id: str) -> RunRecord: + with self.database.connect() as connection: + return self.get_run_with_connection(connection, run_id) + + def get_run_with_connection(self, connection: sqlite3.Connection, run_id: str) -> RunRecord: + row = connection.execute( + "SELECT run.*,recipe.id AS recipe_id,recipe.dataset_digest " + "FROM training_runs run JOIN training_recipes recipe " + "ON recipe.storage_id=run.recipe_storage_id " + "WHERE run.id=? AND run.reviewer_id=?", + (run_id, self.reviewer_id), + ).fetchone() + if row is None: + raise TrainingNotFound("training run not found") + return RunRecord( + run=TrainingRun( + run_id=str(row["id"]), + recipe_id=str(row["recipe_id"]), + dataset_digest=str(row["dataset_digest"]), + total_steps=int(row["total_steps"]), + ), + reviewer_id=str(row["reviewer_id"]), + state=cast( + Literal[ + "draft", + "probed", + "calibrated", + "preregistered", + "training", + "evaluating", + "qualified", + "rejected", + "failed", + "cancelled", + ], + str(row["lifecycle_state"]), + ), + created_at=str(row["created_at"]), + terminal_at=cast(str | None, row["terminal_at"]), + ) + + def list_runs(self) -> tuple[RunRecord, ...]: + with self.database.connect() as connection: + ids = tuple( + str(row["id"]) + for row in connection.execute( + "SELECT id FROM training_runs WHERE reviewer_id=? ORDER BY created_at,id", + (self.reviewer_id,), + ) + ) + return tuple(self.get_run(run_id) for run_id in ids) + + def reopen_operation_with_connection( + self, + connection: sqlite3.Connection, + operation_kind: TrainingOperationKind, + idempotency_key: str, + ) -> TrainingOrchestrationOperation: + self._require_owned_connection(connection) + row = connection.execute( + "SELECT operation.*,recipe.id recipe_id " + "FROM training_orchestration_operations operation " + "JOIN training_recipes recipe ON recipe.storage_id=operation.recipe_storage_id " + "WHERE operation.reviewer_id=? AND operation.operation_kind=? " + "AND operation.idempotency_key=?", + (self.reviewer_id, operation_kind.value, idempotency_key), + ).fetchone() + if row is None: + raise TrainingNotFound("training orchestration operation not found") + return self._operation_from_row(row) + + def get_operation_with_connection( + self, connection: sqlite3.Connection, operation_id: str + ) -> TrainingOrchestrationOperation: + self._require_owned_connection(connection) + row = connection.execute( + "SELECT operation.*,recipe.id recipe_id " + "FROM training_orchestration_operations operation " + "JOIN training_recipes recipe ON recipe.storage_id=operation.recipe_storage_id " + "WHERE operation.id=? AND operation.reviewer_id=?", + (operation_id, self.reviewer_id), + ).fetchone() + if row is None: + raise TrainingNotFound("training orchestration operation not found") + return self._operation_from_row(row) + + def begin_operation_with_connection( + self, + connection: sqlite3.Connection, + *, + operation_id: str, + run_id: str, + operation_kind: TrainingOperationKind, + idempotency_key: str, + request_payload: Mapping[str, object], + ) -> TrainingOrchestrationOperation: + """Create or exactly reopen an immutable operation request.""" + self._require_owned_connection(connection) + if request_payload.get("run_id") != run_id: + raise TrainingConflict("training operation request run mismatch") + request_json = canonical_json(dict(request_payload)) + request_digest = training_operation_payload_digest(request_payload) + existing = connection.execute( + "SELECT request_digest FROM training_orchestration_operations WHERE reviewer_id=? " + "AND operation_kind=? AND idempotency_key=?", + (self.reviewer_id, operation_kind.value, idempotency_key), + ).fetchone() + if existing is not None: + if str(existing["request_digest"]) != request_digest: + raise TrainingConflict("training operation request digest conflict") + return self.reopen_operation_with_connection( + connection, operation_kind, idempotency_key + ) + run = connection.execute( + "SELECT recipe_storage_id,lifecycle_version,lifecycle_state FROM training_runs " + "WHERE id=? AND reviewer_id=?", + (run_id, self.reviewer_id), + ).fetchone() + if run is None: + raise TrainingConflict("training operation run mismatch") + now = utc_now().isoformat() + try: + connection.execute( + "INSERT INTO training_orchestration_operations " + "(id,reviewer_id,run_id,recipe_storage_id,operation_kind,idempotency_key," + "request_digest,request_payload_json,lifecycle_version,lifecycle_state,status," + "attempt,claimant_id,lease_expires_at,receipt_digest,receipt_payload_json," + "failure_digest,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,'pending',0," + "NULL,NULL,NULL,NULL,NULL,?,?)", + ( + operation_id, + self.reviewer_id, + run_id, + run["recipe_storage_id"], + operation_kind.value, + idempotency_key, + request_digest, + request_json, + run["lifecycle_version"], + run["lifecycle_state"], + now, + now, + ), + ) + except sqlite3.IntegrityError: + # A concurrent exact creator is reopened; an ID/digest collision fails closed. + existing = connection.execute( + "SELECT request_digest FROM training_orchestration_operations WHERE reviewer_id=? " + "AND operation_kind=? AND idempotency_key=?", + (self.reviewer_id, operation_kind.value, idempotency_key), + ).fetchone() + if existing is None or str(existing["request_digest"]) != request_digest: + raise TrainingConflict("training operation identity conflict") from None + return self.reopen_operation_with_connection(connection, operation_kind, idempotency_key) + + def claim_operation_with_connection( + self, + connection: sqlite3.Connection, + *, + operation_kind: TrainingOperationKind, + idempotency_key: str, + claimant_id: str, + lease_seconds: int, + ) -> TrainingOperationClaim: + self._require_owned_connection(connection) + if not claimant_id or lease_seconds <= 0: + raise ValueError("claimant and positive lease are required") + lease = (utc_now() + timedelta(seconds=lease_seconds)).isoformat() + claim_token = uuid.uuid4().hex + cursor = connection.execute( + "UPDATE training_orchestration_operations SET status='running',attempt=attempt+1," + "claimant_id=?,claim_token=?,lease_expires_at=?,updated_at=? WHERE reviewer_id=? " + "AND operation_kind=? AND idempotency_key=? AND (status='pending' OR " + "(status='running' AND julianday(lease_expires_at)<=julianday('now')))", + ( + claimant_id, + claim_token, + lease, + utc_now().isoformat(), + self.reviewer_id, + operation_kind.value, + idempotency_key, + ), + ) + operation = self.reopen_operation_with_connection( + connection, operation_kind, idempotency_key + ) + return TrainingOperationClaim(operation=operation, claimed=cursor.rowcount == 1) + + def complete_operation_with_connection( + self, + connection: sqlite3.Connection, + *, + operation_kind: TrainingOperationKind, + idempotency_key: str, + claimant_id: str, + claim_token: str, + attempt: int, + receipt_payload: Mapping[str, object], + result_payload: Mapping[str, object] | None = None, + ) -> TrainingOrchestrationOperation: + self._require_owned_connection(connection) + self._validate_operation_receipt_evidence(connection, operation_kind, receipt_payload) + receipt_json = canonical_json(dict(receipt_payload)) + receipt_digest = training_operation_payload_digest(receipt_payload) + durable_result = receipt_payload if result_payload is None else result_payload + result_json = canonical_json(dict(durable_result)) + result_digest = training_operation_payload_digest(durable_result) + cursor = connection.execute( + "UPDATE training_orchestration_operations SET status='completed',claimant_id=NULL," + "claim_token=NULL,lease_expires_at=NULL,receipt_digest=?,receipt_payload_json=?," + "result_digest=?,result_payload_json=?,updated_at=? " + "WHERE reviewer_id=? AND operation_kind=? AND idempotency_key=? " + "AND status='running' AND claimant_id=? AND claim_token=? AND attempt=?", + ( + receipt_digest, + receipt_json, + result_digest, + result_json, + utc_now().isoformat(), + self.reviewer_id, + operation_kind.value, + idempotency_key, + claimant_id, + claim_token, + attempt, + ), + ) + if cursor.rowcount != 1: + raise TrainingConflict("training operation completion claim mismatch") + return self.reopen_operation_with_connection(connection, operation_kind, idempotency_key) + + def link_operation_evidence_with_connection( + self, + connection: sqlite3.Connection, + *, + operation_id: str, + attempt: int, + claim_token: str, + evidence_kind: str, + evidence_id: str, + evidence_digest: str, + ) -> None: + """Fence already-published evidence to the exact live operation attempt.""" + self._require_owned_connection(connection) + try: + if evidence_kind == "calibration": + operation = connection.execute( + "SELECT operation_kind,request_digest,request_payload_json " + "FROM training_orchestration_operations WHERE id=?", + (operation_id,), + ).fetchone() + if operation is None or str(operation["operation_kind"]) != "calibration": + raise sqlite3.IntegrityError("calibration operation mismatch") + request = json.loads(str(operation["request_payload_json"])) + connection.execute( + "INSERT INTO training_calibration_operation_bindings " + "(evidence_id,operation_id,request_digest,attempt,claim_token,steps," + "trainable_targets_json,seed,created_at) VALUES (?,?,?,?,?,?,?,?,?)", + ( + evidence_id, + operation_id, + operation["request_digest"], + attempt, + claim_token, + request["steps"], + canonical_json(request["trainable_targets"]), + request["seed"], + utc_now().isoformat(), + ), + ) + connection.execute( + "INSERT INTO training_operation_evidence_links VALUES (?,?,?,?,?,?,?)", + ( + operation_id, + attempt, + claim_token, + evidence_kind, + evidence_id, + evidence_digest, + utc_now().isoformat(), + ), + ) + except sqlite3.DatabaseError: + raise TrainingConflict("training operation evidence fence conflict") from None + + def fail_operation_with_connection( + self, + connection: sqlite3.Connection, + *, + operation_kind: TrainingOperationKind, + idempotency_key: str, + claimant_id: str, + claim_token: str, + attempt: int, + failure_digest: str, + ) -> TrainingOrchestrationOperation: + self._require_owned_connection(connection) + cursor = connection.execute( + "UPDATE training_orchestration_operations SET status='failed',claimant_id=NULL," + "claim_token=NULL,lease_expires_at=NULL,failure_digest=?,updated_at=? " + "WHERE reviewer_id=? " + "AND operation_kind=? AND idempotency_key=? AND status='running' AND claimant_id=? " + "AND claim_token=? AND attempt=?", + ( + failure_digest, + utc_now().isoformat(), + self.reviewer_id, + operation_kind.value, + idempotency_key, + claimant_id, + claim_token, + attempt, + ), + ) + if cursor.rowcount != 1: + raise TrainingConflict("training operation failure claim mismatch") + return self.reopen_operation_with_connection(connection, operation_kind, idempotency_key) + + def _operation_from_row(self, row: sqlite3.Row) -> TrainingOrchestrationOperation: + return TrainingOrchestrationOperation( + operation_id=str(row["id"]), + run_id=str(row["run_id"]), + recipe_id=str(row["recipe_id"]), + operation_kind=TrainingOperationKind(str(row["operation_kind"])), + idempotency_key=str(row["idempotency_key"]), + request_digest=str(row["request_digest"]), + request_payload=json.loads(str(row["request_payload_json"])), + lifecycle_version=int(row["lifecycle_version"]), + lifecycle_state=cast(Any, str(row["lifecycle_state"])), + status=TrainingOperationStatus(str(row["status"])), + attempt=int(row["attempt"]), + claimant_id=cast(str | None, row["claimant_id"]), + claim_token=cast(str | None, row["claim_token"]), + lease_expires_at=( + datetime.fromisoformat(str(row["lease_expires_at"])) + if row["lease_expires_at"] is not None + else None + ), + receipt_digest=cast(str | None, row["receipt_digest"]), + receipt_payload=( + json.loads(str(row["receipt_payload_json"])) + if row["receipt_payload_json"] is not None + else None + ), + result_digest=cast(str | None, row["result_digest"]), + result_payload=( + json.loads(str(row["result_payload_json"])) + if row["result_payload_json"] is not None + else None + ), + failure_digest=cast(str | None, row["failure_digest"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + updated_at=datetime.fromisoformat(str(row["updated_at"])), + ) + + def _validate_operation_receipt_evidence( + self, + connection: sqlite3.Connection, + kind: TrainingOperationKind, + payload: Mapping[str, object], + ) -> None: + run_id = payload.get("run_id") + version = payload.get("lifecycle_version") + if not isinstance(run_id, str) or not isinstance(version, int): + raise TrainingConflict("training operation receipt is incomplete") + run = connection.execute( + "SELECT lifecycle_version FROM training_runs WHERE id=? AND reviewer_id=?", + (run_id, self.reviewer_id), + ).fetchone() + if run is None or int(run[0]) != version: + raise TrainingConflict("training operation receipt lifecycle mismatch") + resume = payload.get("resume") + cancellation = payload.get("cancellation") + table_and_identity = { + TrainingOperationKind.PROBE: ("training_probe_evidence", payload.get("evidence_id")), + TrainingOperationKind.CALIBRATION: ( + "training_calibration_evidence", + payload.get("evidence_id"), + ), + TrainingOperationKind.RESUME: ( + "training_resume_evidence", + resume.get("evidence_id") if isinstance(resume, Mapping) else None, + ), + TrainingOperationKind.CANCEL: ( + "training_cancellation_evidence", + cancellation.get("evidence_id") if isinstance(cancellation, Mapping) else None, + ), + } + if kind in table_and_identity: + table, identity = table_and_identity[kind] + if ( + not isinstance(identity, str) + or connection.execute( + f"SELECT 1 FROM {table} " # noqa: S608 + "WHERE id=? AND reviewer_id=? AND run_id=?", + (identity, self.reviewer_id, run_id), + ).fetchone() + is None + ): + raise TrainingConflict("training operation receipt evidence is incomplete") + + def list_events(self, run_id: str) -> tuple[EventRecord, ...]: + self.get_run(run_id) + with self.database.connect() as connection: + rows = connection.execute( + "SELECT * FROM training_run_events WHERE run_id=? AND reviewer_id=? " + "ORDER BY sequence", + (run_id, self.reviewer_id), + ).fetchall() + return tuple( + EventRecord( + run_id=run_id, + reviewer_id=str(row["reviewer_id"]), + event=TrainingRunEvent( + sequence=int(row["sequence"]), + previous_state=cast(Any, str(row["previous_state"])), + new_state=cast(Any, str(row["new_state"])), + payload=json.loads(str(row["payload_json"])), + ), + created_at=str(row["created_at"]), + ) + for row in rows + ) + + def list_telemetry(self, run_id: str) -> tuple[TelemetryRecord, ...]: + self.get_run(run_id) + with self.database.connect() as connection: + rows = connection.execute( + "SELECT * FROM training_telemetry WHERE run_id=? AND reviewer_id=? " + "ORDER BY sequence", + (run_id, self.reviewer_id), + ).fetchall() + return tuple( + TelemetryRecord( + run_id=run_id, + reviewer_id=str(row["reviewer_id"]), + sample=TrainingTelemetry( + sequence=int(row["sequence"]), + step=int(row["step"]), + payload=json.loads(str(row["payload_json"])), + ), + created_at=str(row["created_at"]), + ) + for row in rows + ) + + def list_checkpoints(self, run_id: str) -> tuple[CheckpointRecord, ...]: + self.get_run(run_id) + with self.database.connect() as connection: + rows = connection.execute( + "SELECT * FROM training_checkpoints WHERE run_id=? AND reviewer_id=? " + "AND publication_state='finalized' ORDER BY step", + (run_id, self.reviewer_id), + ).fetchall() + return tuple( + CheckpointRecord( + reviewer_id=str(row["reviewer_id"]), + checkpoint=TrainingCheckpoint( + checkpoint_id=str(row["id"]), + run_id=run_id, + step=int(row["step"]), + fraction=float(row["fraction"]), + parent_checkpoint_id=cast(str | None, row["parent_checkpoint_id"]), + artifact_digest=str(row["artifact_digest"]), + manifest_digest=str(row["manifest_digest"]), + manifest=json.loads(str(row["manifest_json"])), + ), + created_at=str(row["created_at"]), + ) + for row in rows + ) + + def list_failures(self, run_id: str) -> tuple[FailureRecord, ...]: + self.get_run(run_id) + with self.database.connect() as connection: + rows = connection.execute( + "SELECT * FROM training_failures WHERE run_id=? AND reviewer_id=? " + "ORDER BY created_at,id", + (run_id, self.reviewer_id), + ).fetchall() + return tuple( + FailureRecord( + reviewer_id=str(row["reviewer_id"]), + failure=TrainingFailure( + failure_id=str(row["id"]), + run_id=run_id, + code=str(row["code"]), + message=str(row["message"]), + ), + created_at=str(row["created_at"]), + ) + for row in rows + ) + + def create_run(self, run: TrainingRun) -> RunRecord: + recipe = self.get_recipe(run.recipe_id) + if recipe.state != "preregistered" or recipe.recipe.dataset_digest != run.dataset_digest: + raise TrainingConflict("training run recipe or dataset identity mismatch") + dataset = self.get_dataset(run.dataset_digest) + try: + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + self.create_run_with_connection( + connection, run, recipe.storage_id, dataset.storage_id + ) + connection.commit() + except sqlite3.DatabaseError: + raise TrainingConflict("training run identity conflict") from None + return self.get_run(run.run_id) + + def create_run_with_connection( + self, + connection: sqlite3.Connection, + run: TrainingRun, + recipe_storage_id: str, + dataset_storage_id: str, + ) -> None: + """Insert a draft run without owning the caller's transaction.""" + connection.execute( + "INSERT INTO training_runs(id,reviewer_id,recipe_storage_id,dataset_storage_id," + "state,total_steps,created_at,terminal_at) VALUES (?,?,?,?,?,?,?,NULL)", + ( + run.run_id, + self.reviewer_id, + recipe_storage_id, + dataset_storage_id, + "created", + run.total_steps, + utc_now().isoformat(), + ), + ) + + def append_event( + self, run_id: str, event: TrainingRunEvent, *, expected_version: int | None = None + ) -> None: + self._atomic_evidence( + lambda connection: self.append_event_with_connection( + connection, run_id, event, expected_version=expected_version + ), + "training event conflict", + ) + + def append_event_with_connection( + self, + connection: sqlite3.Connection, + run_id: str, + event: TrainingRunEvent, + *, + expected_version: int | None = None, + ) -> None: + if expected_version is not None: + current = connection.execute( + "SELECT lifecycle_version FROM training_runs WHERE id=? AND reviewer_id=?", + (run_id, self.reviewer_id), + ).fetchone() + if current is None or int(current[0]) != expected_version: + raise TrainingConflict("training run lifecycle compare-and-swap conflict") + connection.execute( + "INSERT INTO training_run_events VALUES (?,?,?,?,?,?,?)", + ( + run_id, + self.reviewer_id, + event.sequence, + event.previous_state, + event.new_state, + canonical_json(dict(event.payload)), + utc_now().isoformat(), + ), + ) + + def record_probe_with_connection( + self, connection: sqlite3.Connection, evidence: TrainingProbeEvidence + ) -> None: + connection.execute( + "INSERT INTO training_probe_evidence VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + ( + evidence.evidence_id, + evidence.run_id, + self.reviewer_id, + evidence.recipe_digest, + evidence.dataset_digest, + evidence.backend_digest, + evidence.runtime_digest, + evidence.material_digest, + evidence.source_digest, + evidence.capability_digest, + int(evidence.succeeded), + utc_now().isoformat(), + ), + ) + + def record_probe(self, evidence: TrainingProbeEvidence) -> None: + self._atomic_evidence( + lambda connection: self.record_probe_with_connection(connection, evidence), + "training probe evidence conflict", + ) + + def record_calibration_with_connection( + self, connection: sqlite3.Connection, evidence: TrainingCalibrationEvidence + ) -> None: + connection.execute( + "INSERT INTO training_calibration_evidence VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + evidence.evidence_id, + evidence.run_id, + self.reviewer_id, + evidence.probe_evidence_id, + evidence.calibration_digest, + evidence.recipe_digest, + evidence.dataset_digest, + evidence.backend_digest, + evidence.runtime_digest, + evidence.material_digest, + evidence.source_digest, + int(evidence.succeeded), + utc_now().isoformat(), + ), + ) + + def record_calibration(self, evidence: TrainingCalibrationEvidence) -> None: + self._atomic_evidence( + lambda connection: self.record_calibration_with_connection(connection, evidence), + "training calibration evidence conflict", + ) + + def record_cancellation_with_connection( + self, connection: sqlite3.Connection, evidence: TrainingCancellationEvidence + ) -> None: + connection.execute( + "INSERT INTO training_cancellation_evidence " + "(id,run_id,reviewer_id,requested_step,boundary_step,checkpoint_digest,created_at," + "cancellation_request_id,request_digest) VALUES (?,?,?,?,?,?,?,?,?)", + ( + evidence.cancellation_id, + evidence.run_id, + self.reviewer_id, + evidence.requested_step, + evidence.boundary_step, + evidence.checkpoint_digest, + utc_now().isoformat(), + evidence.cancellation_request_id, + evidence.request_digest, + ), + ) + + def record_cancellation(self, evidence: TrainingCancellationEvidence) -> None: + self._atomic_evidence( + lambda connection: self.record_cancellation_with_connection(connection, evidence), + "training cancellation evidence conflict", + ) + + def atomic(self, callback: Callable[[sqlite3.Connection], object]) -> object: + """Run a complete evidence publication under one caller-visible transaction.""" + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + result = callback(connection) + connection.commit() + return result + except BaseException: + connection.rollback() + raise + + def _atomic_evidence( + self, callback: Callable[[sqlite3.Connection], object], message: str + ) -> None: + try: + self.atomic(callback) + except sqlite3.DatabaseError: + raise TrainingConflict(message) from None + + def append_telemetry(self, run_id: str, sample: TrainingTelemetry) -> None: + self._atomic_evidence( + lambda connection: self.append_telemetry_with_connection(connection, run_id, sample), + "training telemetry conflict", + ) + + def append_telemetry_with_connection( + self, connection: sqlite3.Connection, run_id: str, sample: TrainingTelemetry + ) -> None: + connection.execute( + "INSERT INTO training_telemetry VALUES (?,?,?,?,?,?)", + ( + run_id, + self.reviewer_id, + sample.sequence, + sample.step, + canonical_json(dict(sample.payload)), + utc_now().isoformat(), + ), + ) + + def telemetry_digest_with_connection( + self, connection: sqlite3.Connection, run_id: str, start: int, end: int + ) -> str: + rows = connection.execute( + "SELECT sequence,step,payload_json FROM training_telemetry " + "WHERE run_id=? AND reviewer_id=? AND sequence BETWEEN ? AND ? ORDER BY sequence", + (run_id, self.reviewer_id, start, end), + ).fetchall() + if len(rows) != end - start + 1 or start < 1 or end < start: + raise TrainingConflict("training telemetry range is incomplete") + payload = [ + { + "sequence": int(row["sequence"]), + "step": int(row["step"]), + "payload": json.loads(str(row["payload_json"])), + } + for row in rows + ] + serialized = canonical_json(cast(Any, payload)) + return f"sha256:{hashlib.sha256(serialized.encode()).hexdigest()}" + + def add_checkpoint(self, checkpoint: TrainingCheckpoint) -> None: + raise TrainingConflict("scientific checkpoint evidence is required") + + def add_checkpoint_with_connection( + self, + connection: sqlite3.Connection, + checkpoint: TrainingCheckpoint, + evidence: TrainingCheckpointEvidence, + ) -> None: + row = connection.execute( + "SELECT recipe_storage_id,dataset_storage_id FROM training_runs " + "WHERE id=? AND reviewer_id=?", + (checkpoint.run_id, self.reviewer_id), + ).fetchone() + if row is None or evidence.run_id != checkpoint.run_id or evidence.step != checkpoint.step: + raise TrainingConflict("training checkpoint run mismatch") + evidence_json = canonical_json(evidence.model_dump(mode="json")) + evidence_digest = f"sha256:{hashlib.sha256(evidence_json.encode()).hexdigest()}" + connection.execute( + "INSERT INTO training_checkpoints(id,run_id,reviewer_id,recipe_storage_id," + "dataset_storage_id,step,fraction,parent_checkpoint_id,artifact_digest," + "manifest_digest,manifest_json,created_at," + "evidence_json,evidence_digest,publication_state) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,'finalized')", + ( + checkpoint.checkpoint_id, + checkpoint.run_id, + self.reviewer_id, + row["recipe_storage_id"], + row["dataset_storage_id"], + checkpoint.step, + checkpoint.fraction, + checkpoint.parent_checkpoint_id, + checkpoint.artifact_digest, + checkpoint.manifest_digest, + canonical_json(checkpoint.model_dump(mode="json")["manifest"]), + utc_now().isoformat(), + evidence_json, + evidence_digest, + ), + ) + + def request_cancellation_with_connection( + self, + connection: sqlite3.Connection, + *, + request_id: str, + run_id: str, + requested_step: int, + request_digest: str, + ) -> None: + connection.execute( + "INSERT INTO training_cancellation_requests VALUES (?,?,?,?,?,?)", + ( + request_id, + run_id, + self.reviewer_id, + requested_step, + request_digest, + utc_now().isoformat(), + ), + ) + + def bind_checkpoint_inventory_artifacts_with_connection( + self, + connection: sqlite3.Connection, + *, + checkpoint_id: str, + artifact_ids_by_role: Mapping[str, str], + ) -> None: + """Bind each checkpoint role to one exact committed artifact identity.""" + self._require_owned_connection(connection) + rows = connection.execute( + "SELECT inventory.ordinal,inventory.role,inventory.artifact_digest," + "inventory.size_bytes FROM training_checkpoint_inventory inventory " + "JOIN training_checkpoints checkpoint ON checkpoint.id=inventory.checkpoint_id " + "WHERE inventory.checkpoint_id=? AND checkpoint.reviewer_id=? " + "ORDER BY inventory.ordinal", + (checkpoint_id, self.reviewer_id), + ).fetchall() + roles = {str(row["role"]) for row in rows} + if len(rows) != 7 or set(artifact_ids_by_role) != roles: + raise TrainingConflict("training checkpoint artifact role inventory mismatch") + try: + for row in rows: + role = str(row["role"]) + connection.execute( + "INSERT INTO training_checkpoint_inventory_artifacts VALUES (?,?,?,?,?,?)", + ( + checkpoint_id, + row["ordinal"], + artifact_ids_by_role[role], + role, + row["artifact_digest"], + row["size_bytes"], + ), + ) + except sqlite3.DatabaseError: + raise TrainingConflict("training checkpoint artifact binding conflict") from None + + def bind_idempotency_with_connection( + self, + connection: sqlite3.Connection, + *, + run_id: str, + idempotency_key: str, + request_digest: str, + probe_evidence_id: str, + calibration_evidence_id: str, + ) -> str: + row = connection.execute( + "SELECT binding.run_id,binding.request_digest FROM training_run_idempotency binding " + "JOIN training_runs run ON run.id=binding.run_id " + "WHERE binding.reviewer_id=? AND binding.recipe_storage_id=run.recipe_storage_id " + "AND binding.idempotency_key=? AND run.recipe_storage_id=" + "(SELECT recipe_storage_id FROM training_runs WHERE id=? AND reviewer_id=?)", + (self.reviewer_id, idempotency_key, run_id, self.reviewer_id), + ).fetchone() + if row is not None: + if str(row["request_digest"]) != request_digest: + raise TrainingConflict("training idempotency request digest conflict") + return str(row["run_id"]) + recipe = connection.execute( + "SELECT recipe_storage_id FROM training_runs WHERE id=? AND reviewer_id=?", + (run_id, self.reviewer_id), + ).fetchone() + if recipe is None: + raise TrainingConflict("training idempotency run mismatch") + connection.execute( + "INSERT INTO training_run_idempotency VALUES (?,?,?,?,?,?,?,?)", + ( + self.reviewer_id, + recipe["recipe_storage_id"], + idempotency_key, + request_digest, + run_id, + probe_evidence_id, + calibration_evidence_id, + utc_now().isoformat(), + ), + ) + return run_id + + def record_resume_with_connection( + self, connection: sqlite3.Connection, evidence: TrainingResumeEvidence + ) -> None: + connection.execute( + "INSERT INTO training_resume_evidence VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + evidence.resume_id, + evidence.run_id, + self.reviewer_id, + evidence.parent_checkpoint_id, + evidence.parent_checkpoint_digest, + evidence.parent_artifact_digest, + evidence.recipe_digest, + evidence.dataset_digest, + evidence.backend_digest, + evidence.runtime_digest, + evidence.material_digest, + evidence.source_digest, + evidence.optimizer_state_digest, + utc_now().isoformat(), + ), + ) + + def record_resume(self, evidence: TrainingResumeEvidence) -> None: + self._atomic_evidence( + lambda connection: self.record_resume_with_connection(connection, evidence), + "training resume evidence conflict", + ) + + def record_evaluation_with_connection( + self, connection: sqlite3.Connection, evidence: TrainingEvaluationEvidence + ) -> None: + connection.execute( + "INSERT INTO training_evaluation_evidence VALUES (?,?,?,?,?,?)", + ( + evidence.evidence_id, + evidence.run_id, + self.reviewer_id, + evidence.disposition, + evidence.evidence_digest, + utc_now().isoformat(), + ), + ) + + def record_evaluation(self, evidence: TrainingEvaluationEvidence) -> None: + self._atomic_evidence( + lambda connection: self.record_evaluation_with_connection(connection, evidence), + "training evaluation evidence conflict", + ) + + def record_failure_with_connection( + self, connection: sqlite3.Connection, failure: TrainingFailure + ) -> None: + if _contains_local_path(failure.message): + raise TrainingConflict("training failure message contains local path material") + connection.execute( + "INSERT INTO training_failures VALUES (?,?,?,?,?,?)", + ( + failure.failure_id, + failure.run_id, + self.reviewer_id, + failure.code, + failure.message, + utc_now().isoformat(), + ), + ) + + def record_failure(self, failure: TrainingFailure) -> None: + self._atomic_evidence( + lambda connection: self.record_failure_with_connection(connection, failure), + "training failure run or owner conflict", + ) + + def _append(self, sql: str, parameters: tuple[object, ...], message: str) -> None: + try: + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + connection.execute(sql, parameters) + connection.commit() + except sqlite3.DatabaseError: + raise TrainingConflict(message) from None + + @staticmethod + def _draft_from(value: FrozenTrainingDataset | TrainingDatasetDraft) -> TrainingDatasetDraft: + return TrainingDatasetDraft.model_validate( + value.model_dump(mode="python", exclude={"coverage", "dataset_digest"}) + ) + + @staticmethod + def _insert_samples( + connection: sqlite3.Connection, + storage_id: str, + samples: tuple[TrainingSample, ...], + *, + ordinal_start: int = 0, + ) -> None: + for ordinal, sample in enumerate(samples, ordinal_start): + connection.execute( + "INSERT INTO training_samples VALUES (?,?,?,?,?,?,?,?,?,?)", + ( + storage_id, + sample.sample_id, + sample.source_tier.value, + sample.sample_digest, + sample.source_lineage, + sample.caption, + sample.split.value, + canonical_json(sample.media.model_dump(mode="json")), + int(sample.accepted), + ordinal, + ), + ) + connection.execute( + "INSERT INTO training_sample_annotations VALUES (?,?,?)", + ( + storage_id, + sample.sample_id, + canonical_json(sample.annotation.model_dump(mode="json")), + ), + ) + + +_WINDOWS_DRIVE_PATH = re.compile(r"(?i)(?:^|\s)[a-z]:[\\/]") + + +def _contains_local_path(message: str) -> bool: + lowered = message.lower() + return ( + "file://" in lowered + or message.startswith("/") + or " /" in message + or "\\\\" in message + or bool(_WINDOWS_DRIVE_PATH.search(message)) + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_service.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_service.py new file mode 100644 index 0000000..fc9820d --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/training_service.py @@ -0,0 +1,2759 @@ +"""Fail-closed application boundary for character-training evidence.""" + +from __future__ import annotations + +import functools +import hashlib +import json +import os +import platform +import sqlite3 +import stat +import sys +import tempfile +import unicodedata +import uuid +from collections.abc import Callable, Mapping +from contextlib import ExitStack +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Literal, ParamSpec, Protocol, TypeVar, cast + +from pydantic import Field + +from video_lab.artifacts import ( + Artifact, + ArtifactError, + ArtifactStore, + StagedArtifact, + VerifiedArtifactHandle, +) +from video_lab.domain import FrozenModel, canonical_json +from video_lab.trainer_protocol import ( + CalibrationRequest, + CalibrationResult, + LoRAConfiguration, + OptimizerConfiguration, + OptimizerParameter, + ProbeTrainingRequest, + TrainerBackend, + TrainerCapabilityReport, + TrainerRegistry, + TrainerResultInvalid, + TrainingCancelled, + TrainingMaterialHandles, + TrainingProgress, + TrainingPublicationReceipt, + TrainingRequest, + TrainingResult, + validate_cancelled_training_result, + validate_training_result, +) +from video_lab.training_dataset import ( + evaluate_training_dataset, + freeze_training_dataset, + validate_training_recipe_dataset, +) +from video_lab.training_media import ( + TrainingMediaInspectionError, + open_training_media_collection_snapshot, +) +from video_lab.training_models import ( + CalibrationOperationRequest, + CancelOperationRequest, + CheckpointPlan, + CurationDecision, + DatasetCoverage, + DatasetSplit, + FrozenTrainingDataset, + ImportedTrainingSample, + ProbeOperationRequest, + ResumeInventoryArtifact, + ResumeOperationRequest, + StartOperationRequest, + TrainingAnnotation, + TrainingCalibrationEvidence, + TrainingCancellationEvidence, + TrainingCheckpoint, + TrainingCheckpointArtifact, + TrainingCheckpointEvidence, + TrainingCurationConfirmation, + TrainingCurationSuggestions, + TrainingDatasetDraft, + TrainingEvaluationEvidence, + TrainingFailure, + TrainingMedia, + TrainingOperationKind, + TrainingOperationStatus, + TrainingOrchestrationOperation, + TrainingProbeEvidence, + TrainingRecipe, + TrainingResumeEvidence, + TrainingRun, + TrainingRunEvent, + TrainingSourceTier, + TrainingTelemetry, + checkpoint_steps, + imported_training_sample_id, + training_confirmation_id, + training_operation_payload_digest, + training_recipe_id, +) +from video_lab.training_repository import ( + DatasetLineageRecord, + RecipeMaterialBinding, + RecipeRecord, + TrainingConflict, + TrainingNotFound, + TrainingRepository, +) + +_SUPPORTED_MEDIA = {".jpg", ".jpeg", ".png", ".webp", ".mp4"} +_SOURCE_LIMIT = 512 * 1024 * 1024 + + +class TrainingServiceError(RuntimeError): + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +_P = ParamSpec("_P") +_R = TypeVar("_R") + + +def _public_guard(function: Callable[_P, _R]) -> Callable[_P, _R]: # noqa: UP047 + """Ensure the application boundary never exposes local implementation details.""" + + @functools.wraps(function) + def guarded(*args: _P.args, **kwargs: _P.kwargs) -> _R: + try: + return function(*args, **kwargs) + except TrainingServiceError as error: + raise TrainingServiceError(error.code) from None + except Exception: + raise TrainingServiceError("training_service_failed") from None + + return guarded + + +class RuntimeIdentityProvider(Protocol): + def identity(self) -> Mapping[str, str]: ... + + +class LocalRuntimeIdentity: + def identity(self) -> Mapping[str, str]: + return { + "machine": platform.machine(), + "platform": platform.system(), + "python": platform.python_version(), + "implementation": sys.implementation.name, + } + + +class DatasetCreateInput(FrozenModel): + character_pack_id: str = Field(min_length=1) + canonical_outfit_id: str = Field(min_length=1) + reserved_token: str = Field(pattern=r"^<[^<>\s]+>$") + + +class CurationInput(FrozenModel): + decision: CurationDecision + sample_id: str | None = Field(default=None, min_length=1) + source_tier: TrainingSourceTier | None = None + source_lineage: str | None = Field(default=None, min_length=1) + caption: str | None = Field(default=None, min_length=1) + split: DatasetSplit | None = None + annotation: TrainingAnnotation | None = None + + +@dataclass(frozen=True, slots=True) +class _PreparedTrainingExecution: + record: RecipeRecord + backend: TrainerBackend + request: TrainingRequest + resume_evidence: TrainingResumeEvidence | None + telemetry_offset: int + + +class RecipeInput(FrozenModel): + stage_3_campaign_id: str = Field(min_length=1) + stage_3_result_id: str = Field(min_length=1) + base_model_artifact_id: str = Field(min_length=1) + lightning_adapter_artifact_ids: Mapping[str, str] + backend_id: str = Field(min_length=1) + backend_config: Mapping[str, object] + trainable_targets: tuple[str, ...] = Field(min_length=1) + lora_rank: int = Field(gt=0) + lora_alpha: int = Field(gt=0) + lora_dropout: float = Field(ge=0, lt=1) + optimizer: str = Field(min_length=1) + learning_rate: float = Field(gt=0) + total_steps: int = Field(ge=4) + gradient_accumulation_steps: int = Field(gt=0) + source_tier_weights: Mapping[TrainingSourceTier, float] + checkpoint_plan: CheckpointPlan = CheckpointPlan() + calibration_steps: int = Field(ge=0) + wall_clock_limit_seconds: Literal[28800, 57600] + seed: int + + +class TrainingDatasetView(FrozenModel): + lineage_id: str + dataset_id: str + head_version: int + state: Literal["draft", "sealed"] + character_id: str + imported: tuple[ImportedTrainingSample, ...] + coverage: DatasetCoverage + + +class TrainingRecipeView(FrozenModel): + recipe_id: str + state: Literal["preview", "draft", "preregistered"] + recipe: TrainingRecipe + + +class TrainingRecipePreview(FrozenModel): + state: Literal["preview"] = "preview" + preview_token: str + source_snapshot_digest: str + source_snapshot_byte_size: int + source_snapshot_artifact_id: None = None + base_model_digest: str + lightning_adapter_digests: Mapping[str, str] + + +def discover_training_media(source: Path) -> tuple[Path, ...]: + try: + root = source.lstat() + if stat.S_ISLNK(root.st_mode) or not stat.S_ISDIR(root.st_mode): + raise TrainingServiceError("unsafe_path") + result: list[Path] = [] + for current, directories, files in os.walk(source, followlinks=False): + for name in directories: + if (Path(current) / name).is_symlink(): + raise TrainingServiceError("unsafe_path") + for name in files: + candidate = Path(current) / name + mode = candidate.lstat().st_mode + if stat.S_ISLNK(mode) or not stat.S_ISREG(mode): + raise TrainingServiceError("unsafe_path") + if candidate.suffix.lower() not in _SUPPORTED_MEDIA: + raise TrainingServiceError("unsupported_entry") + result.append(candidate) + if not result: + raise TrainingServiceError("empty_media_folder") + return tuple(sorted(result, key=lambda item: _sort_key(item.relative_to(source)))) + except TrainingServiceError: + raise + except OSError: + raise TrainingServiceError("unsafe_path") from None + + +class TrainingService: + def __init__( + self, + repository: TrainingRepository, + artifacts: ArtifactStore, + character_packs: object, + *, + reviewer_id: str, + source_root: Path, + runtime: RuntimeIdentityProvider | None = None, + trainer_registry: TrainerRegistry | None = None, + ) -> None: + if repository.reviewer_id != reviewer_id: + raise ValueError("reviewer identity mismatch") + self.repository = repository + self.artifacts = artifacts + self.character_packs = character_packs + self.reviewer_id = reviewer_id + self.source_root = source_root + self.runtime = runtime or LocalRuntimeIdentity() + self.trainer_registry = trainer_registry or TrainerRegistry(reviewer_id=reviewer_id) + + @_public_guard + def probe_backend( + self, + recipe_id: str, + *, + run_id: str, + request_id: str, + scratch_root: Path, + ) -> TrainerCapabilityReport: + record = self.repository.get_recipe(recipe_id) + if record.state != "preregistered": + raise TrainingServiceError("recipe_not_preregistered") + recipe = record.recipe + materials = self._training_materials(record) + expected_backend_digest = _json_digest( + recipe.model_dump(mode="json")["backend_config"] + ) + request_payload = ProbeOperationRequest( + request_id=request_id, + run_id=run_id, + recipe_digest=materials.recipe_digest, + dataset_digest=materials.dataset_digest, + backend_digest=materials.backend_digest, + runtime_digest=materials.runtime_digest, + material_digest=materials.material_digest, + source_digest=materials.source_digest, + ).model_dump(mode="json") + dataset = self.repository.get_dataset(recipe.dataset_digest) + + def begin( + connection: sqlite3.Connection, + ) -> tuple[TrainingOrchestrationOperation, TrainingOrchestrationOperation | None]: + try: + self.repository.get_run_with_connection(connection, run_id) + except TrainingNotFound: + self.repository.create_run_with_connection( + connection, + TrainingRun( + run_id=run_id, + recipe_id=record.recipe_id, + dataset_digest=recipe.dataset_digest, + total_steps=recipe.total_steps, + ), + record.storage_id, + dataset.storage_id, + ) + operation = self.repository.begin_operation_with_connection( + connection, + operation_id=f"operation_probe_{_json_digest(request_payload)[7:]}", + run_id=run_id, + operation_kind=TrainingOperationKind.PROBE, + idempotency_key=request_id, + request_payload=request_payload, + ) + if operation.status is TrainingOperationStatus.COMPLETED: + return operation, None + claim = self.repository.claim_operation_with_connection( + connection, + operation_kind=TrainingOperationKind.PROBE, + idempotency_key=request_id, + claimant_id=f"training-service:{self.reviewer_id}", + lease_seconds=3600, + ) + if not claim.claimed: + raise TrainingServiceError("operation_in_progress") + return operation, claim.operation + + operation, claim = cast( + tuple[TrainingOrchestrationOperation, TrainingOrchestrationOperation | None], + self.repository.atomic(begin), + ) + if claim is None: + payload = operation.result_payload + if not isinstance(payload, Mapping) or not isinstance(payload.get("report"), Mapping): + raise TrainingServiceError("operation_receipt_invalid") + return TrainerCapabilityReport.model_validate(payload["report"]) + backend = self.trainer_registry.resolve(self.reviewer_id, recipe.backend_id) + report = backend.probe( + ProbeTrainingRequest( + request_id=request_id, + backend_id=recipe.backend_id, + expected_backend_digest=expected_backend_digest, + materials=materials, + scratch_root=scratch_root, + ) + ) + if ( + report.backend_id != recipe.backend_id + or report.backend_digest != expected_backend_digest + or report.runtime_digest != materials.runtime_digest + or not report.supported + or report.max_steps < recipe.total_steps + ): + raise TrainingServiceError("backend_probe_invalid") + capability_digest = _json_digest(report.model_dump(mode="json")) + evidence = TrainingProbeEvidence( + evidence_id=f"probe_{capability_digest[7:]}", + run_id=run_id, + recipe_digest=training_recipe_id(recipe), + dataset_digest=recipe.dataset_digest, + backend_digest=expected_backend_digest, + runtime_digest=materials.runtime_digest, + material_digest=materials.material_digest, + source_digest=materials.source_digest, + capability_digest=capability_digest, + succeeded=True, + ) + + def publish(connection: sqlite3.Connection) -> None: + self.repository.record_probe_with_connection(connection, evidence) + self.repository.append_event_with_connection( + connection, + run_id, + TrainingRunEvent( + sequence=1, + previous_state="draft", + new_state="probed", + payload={"probe_evidence_id": evidence.evidence_id}, + ), + expected_version=0, + ) + evidence_digest = str( + connection.execute( + "SELECT training_operation_evidence_digest(?,?,?)", + (operation.operation_id, "probe", evidence.evidence_id), + ).fetchone()[0] + ) + self.repository.link_operation_evidence_with_connection( + connection, + operation_id=operation.operation_id, + attempt=claim.attempt, + claim_token=cast(str, claim.claim_token), + evidence_kind="probe", + evidence_id=evidence.evidence_id, + evidence_digest=evidence_digest, + ) + self.repository.complete_operation_with_connection( + connection, + operation_kind=TrainingOperationKind.PROBE, + idempotency_key=request_id, + claimant_id=cast(str, claim.claimant_id), + claim_token=cast(str, claim.claim_token), + attempt=claim.attempt, + receipt_payload={ + "run_id": run_id, + "lifecycle_version": 1, + "lifecycle_state": "probed", + "evidence_id": evidence.evidence_id, + "evidence_digest": evidence_digest, + }, + result_payload={ + "schema_version": "trainer-capability-report.v1", + "report": report.model_dump(mode="json"), + }, + ) + + self.repository.atomic(publish) + return report + + @_public_guard + def start_calibration( + self, run_id: str, *, request_id: str, scratch_root: Path + ) -> CalibrationResult: + with self.repository.database.connect() as connection: + try: + replay = self.repository.reopen_operation_with_connection( + connection, TrainingOperationKind.CALIBRATION, request_id + ) + except TrainingNotFound: + replay = None + if replay is not None: + if replay.run_id != run_id: + raise TrainingServiceError("operation_request_conflict") + run = self.repository.get_run(run_id) + if replay is None and run.state != "probed": + raise TrainingServiceError("calibration_state_invalid") + record = self.repository.get_recipe(run.run.recipe_id) + recipe = record.recipe + materials = self._training_materials(record) + with self.repository.database.connect() as connection: + probe = connection.execute( + "SELECT evidence.id,evidence.capability_digest,operation.id operation_id " + "FROM training_probe_evidence evidence " + "JOIN training_orchestration_operations operation " + "ON operation.run_id=evidence.run_id " + "AND operation.operation_kind='probe' AND operation.status='completed' " + "WHERE evidence.run_id=? AND evidence.reviewer_id=?", + (run_id, self.reviewer_id), + ).fetchone() + if probe is None: + raise TrainingServiceError("probe_evidence_missing") + request_payload = CalibrationOperationRequest( + request_id=request_id, + run_id=run_id, + recipe_digest=materials.recipe_digest, + dataset_digest=materials.dataset_digest, + backend_digest=materials.backend_digest, + runtime_digest=materials.runtime_digest, + material_digest=materials.material_digest, + source_digest=materials.source_digest, + probe_operation_id=str(probe["operation_id"]), + capability_digest=str(probe["capability_digest"]), + steps=recipe.calibration_steps, + trainable_targets=recipe.trainable_targets, + lora=LoRAConfiguration( + rank=recipe.lora_rank, + alpha=recipe.lora_alpha, + dropout=recipe.lora_dropout, + ), + seed=recipe.seed, + ).model_dump(mode="json") + if replay is not None: + if not _calibration_replay_is_equivalent(replay, request_payload): + raise TrainingServiceError("operation_request_conflict") + if replay.status is not TrainingOperationStatus.COMPLETED: + raise TrainingServiceError("operation_in_progress") + payload = replay.result_payload + if not isinstance(payload, Mapping) or not isinstance(payload.get("result"), Mapping): + raise TrainingServiceError("operation_receipt_invalid") + return CalibrationResult.model_validate(payload["result"]) + + def begin(connection: sqlite3.Connection) -> TrainingOrchestrationOperation: + operation = self.repository.begin_operation_with_connection( + connection, + operation_id=f"operation_calibration_{_json_digest(request_payload)[7:]}", + run_id=run_id, + operation_kind=TrainingOperationKind.CALIBRATION, + idempotency_key=request_id, + request_payload=request_payload, + ) + if operation.status is TrainingOperationStatus.COMPLETED: + return operation + claim = self.repository.claim_operation_with_connection( + connection, + operation_kind=TrainingOperationKind.CALIBRATION, + idempotency_key=request_id, + claimant_id=f"training-service:{self.reviewer_id}", + lease_seconds=3600, + ) + if not claim.claimed: + raise TrainingServiceError("operation_in_progress") + return claim.operation + + operation = cast(TrainingOrchestrationOperation, self.repository.atomic(begin)) + if operation.status is TrainingOperationStatus.COMPLETED: + payload = operation.result_payload + if not isinstance(payload, Mapping) or not isinstance(payload.get("result"), Mapping): + raise TrainingServiceError("operation_receipt_invalid") + return CalibrationResult.model_validate(payload["result"]) + backend = self.trainer_registry.resolve(self.reviewer_id, recipe.backend_id) + result = backend.calibrate( + CalibrationRequest( + request_id=request_id, + run_id=run_id, + backend_id=recipe.backend_id, + materials=materials, + scratch_root=scratch_root, + steps=recipe.calibration_steps, + trainable_targets=recipe.trainable_targets, + lora=LoRAConfiguration( + rank=recipe.lora_rank, + alpha=recipe.lora_alpha, + dropout=recipe.lora_dropout, + ), + seed=recipe.seed, + deadline_seconds=recipe.wall_clock_limit_seconds, + ), + lambda progress: None, + lambda: False, + ) + if ( + result.request_id != request_id + or result.run_id != run_id + or result.backend_digest != materials.backend_digest + or result.runtime_digest != materials.runtime_digest + or result.changed_targets != recipe.trainable_targets + or not result.finite_gradients + or not result.frozen_parameters_unchanged + ): + raise TrainingServiceError("calibration_invalid") + return result + + @_public_guard + def complete_calibration(self, run_id: str, result: CalibrationResult) -> None: + run = self.repository.get_run(run_id) + if run.state != "probed" or result.run_id != run_id: + raise TrainingServiceError("calibration_state_invalid") + record = self.repository.get_recipe(run.run.recipe_id) + recipe = record.recipe + materials = self._training_materials(record) + with self.repository.database.connect() as connection: + probe = connection.execute( + "SELECT id FROM training_probe_evidence WHERE run_id=? AND reviewer_id=? " + "AND succeeded=1 ORDER BY created_at DESC LIMIT 1", + (run_id, self.reviewer_id), + ).fetchone() + if probe is None: + raise TrainingServiceError("probe_evidence_missing") + with self.repository.database.connect() as connection: + operation = self.repository.reopen_operation_with_connection( + connection, TrainingOperationKind.CALIBRATION, result.request_id + ) + if ( + operation.status is not TrainingOperationStatus.RUNNING + or operation.request_payload.get("run_id") != run_id + or operation.request_payload.get("recipe_digest") != materials.recipe_digest + or operation.request_payload.get("dataset_digest") != materials.dataset_digest + or operation.request_payload.get("backend_digest") != result.backend_digest + or operation.request_payload.get("runtime_digest") != result.runtime_digest + or tuple(cast(tuple[str, ...], operation.request_payload.get("trainable_targets", ()))) + != result.changed_targets + or not result.finite_gradients + or not result.frozen_parameters_unchanged + ): + raise TrainingServiceError("calibration_invalid") + evidence = TrainingCalibrationEvidence( + evidence_id=f"calibration_{result.calibration_digest[7:]}", + run_id=run_id, + probe_evidence_id=str(probe[0]), + recipe_digest=training_recipe_id(recipe), + dataset_digest=recipe.dataset_digest, + backend_digest=_json_digest(recipe.model_dump(mode="json")["backend_config"]), + runtime_digest=materials.runtime_digest, + material_digest=materials.material_digest, + source_digest=materials.source_digest, + calibration_digest=result.calibration_digest, + succeeded=True, + ) + + def publish(connection: sqlite3.Connection) -> None: + self.repository.record_calibration_with_connection(connection, evidence) + self.repository.append_event_with_connection( + connection, + run_id, + TrainingRunEvent( + sequence=2, + previous_state="probed", + new_state="calibrated", + payload={"calibration_evidence_id": evidence.evidence_id}, + ), + expected_version=1, + ) + evidence_digest = str( + connection.execute( + "SELECT training_operation_evidence_digest(?,?,?)", + (operation.operation_id, "calibration", evidence.evidence_id), + ).fetchone()[0] + ) + self.repository.link_operation_evidence_with_connection( + connection, + operation_id=operation.operation_id, + attempt=operation.attempt, + claim_token=cast(str, operation.claim_token), + evidence_kind="calibration", + evidence_id=evidence.evidence_id, + evidence_digest=evidence_digest, + ) + self.repository.complete_operation_with_connection( + connection, + operation_kind=TrainingOperationKind.CALIBRATION, + idempotency_key=result.request_id, + claimant_id=cast(str, operation.claimant_id), + claim_token=cast(str, operation.claim_token), + attempt=operation.attempt, + receipt_payload={ + "run_id": run_id, + "lifecycle_version": 2, + "lifecycle_state": "calibrated", + "evidence_id": evidence.evidence_id, + "evidence_digest": evidence_digest, + }, + result_payload={ + "schema_version": "trainer-calibration-result.v1", + "result": result.model_dump(mode="json"), + }, + ) + + self.repository.atomic(publish) + + @_public_guard + def preregister_run(self, run_id: str, *, idempotency_key: str) -> str: + run = self.repository.get_run(run_id) + if run.state != "calibrated" or not idempotency_key: + raise TrainingServiceError("run_preregistration_invalid") + record = self.repository.get_recipe(run.run.recipe_id) + materials = self._training_materials(record) + request_digest = _json_digest( + { + "schema_version": "character-training-request.v1", + "run_id": run_id, + "recipe_id": record.recipe_id, + "materials": materials.model_dump(mode="json"), + "idempotency_key": idempotency_key, + } + ) + + def publish(connection: sqlite3.Connection) -> str: + probe = connection.execute( + "SELECT id FROM training_probe_evidence WHERE run_id=? AND reviewer_id=? " + "AND succeeded=1 ORDER BY created_at DESC LIMIT 1", + (run_id, self.reviewer_id), + ).fetchone() + calibration = connection.execute( + "SELECT id FROM training_calibration_evidence WHERE run_id=? AND reviewer_id=? " + "AND succeeded=1 ORDER BY created_at DESC LIMIT 1", + (run_id, self.reviewer_id), + ).fetchone() + if probe is None or calibration is None: + raise TrainingServiceError("run_prerequisite_missing") + rebound = self.repository.bind_idempotency_with_connection( + connection, + run_id=run_id, + idempotency_key=idempotency_key, + request_digest=request_digest, + probe_evidence_id=str(probe[0]), + calibration_evidence_id=str(calibration[0]), + ) + if rebound != run_id: + return rebound + self.repository.append_event_with_connection( + connection, + run_id, + TrainingRunEvent( + sequence=3, + previous_state="calibrated", + new_state="preregistered", + payload={ + "idempotency_key": idempotency_key, + "request_digest": request_digest, + "probe_evidence_id": str(probe[0]), + "calibration_evidence_id": str(calibration[0]), + }, + ), + expected_version=2, + ) + return run_id + + return str(self.repository.atomic(publish)) + + @_public_guard + def start_training( + self, + run_id: str, + *, + request_id: str, + scratch_root: Path, + progress: Callable[[TrainingProgress], None] | None = None, + ) -> TrainingPublicationReceipt: + with self.repository.database.connect() as connection: + try: + replay = self.repository.reopen_operation_with_connection( + connection, TrainingOperationKind.START, request_id + ) + except TrainingNotFound: + replay = None + if replay is not None: + if replay.run_id != run_id: + raise TrainingServiceError("operation_request_conflict") + if replay.status is TrainingOperationStatus.COMPLETED: + return TrainingPublicationReceipt.model_validate(replay.result_payload) + raise TrainingServiceError("operation_in_progress") + run = self.repository.get_run(run_id) + if run.state != "preregistered": + raise TrainingServiceError("training_state_invalid") + prepared = self._prepare_training_execution( + run_id, request_id=request_id, scratch_root=scratch_root + ) + record = prepared.record + materials = prepared.request.materials + with self.repository.database.connect() as connection: + preregistration = connection.execute( + "SELECT idempotency_key,request_digest FROM training_runs WHERE id=?", + (run_id,), + ).fetchone() + if preregistration is None: + raise TrainingServiceError("run_preregistration_invalid") + artifact_ids = { + "base_model": record.binding.base_model_artifact_id, + "source_snapshot": record.binding.source_snapshot_artifact_id, + **{ + f"lightning:{name}": artifact_id + for name, artifact_id in record.binding.lightning_adapter_artifact_ids.items() + }, + } + request_payload = StartOperationRequest( + request_id=request_id, + run_id=run_id, + recipe_digest=materials.recipe_digest, + dataset_digest=materials.dataset_digest, + backend_digest=materials.backend_digest, + runtime_digest=materials.runtime_digest, + material_digest=materials.material_digest, + source_digest=materials.source_digest, + preregistration_id=str(preregistration["idempotency_key"]), + preregistration_request_digest=str(preregistration["request_digest"]), + checkpoint_steps=checkpoint_steps(record.recipe.total_steps), + checkpoint_fractions=(0.25, 0.5, 0.75, 1.0), + material_artifact_ids=artifact_ids, + ).model_dump(mode="json") + + def begin(connection: sqlite3.Connection) -> TrainingOrchestrationOperation: + operation = self.repository.begin_operation_with_connection( + connection, + operation_id=f"operation_start_{_json_digest(request_payload)[7:]}", + run_id=run_id, + operation_kind=TrainingOperationKind.START, + idempotency_key=request_id, + request_payload=request_payload, + ) + if operation.status is TrainingOperationStatus.COMPLETED: + return operation + claim = self.repository.claim_operation_with_connection( + connection, + operation_kind=TrainingOperationKind.START, + idempotency_key=request_id, + claimant_id=f"training-service:{self.reviewer_id}", + lease_seconds=57600, + ) + if not claim.claimed: + raise TrainingServiceError("operation_in_progress") + return claim.operation + + operation = cast(TrainingOrchestrationOperation, self.repository.atomic(begin)) + if operation.status is TrainingOperationStatus.COMPLETED: + return TrainingPublicationReceipt.model_validate(operation.result_payload) + self.repository.append_event( + run_id, + TrainingRunEvent( + sequence=4, + previous_state="preregistered", + new_state="training", + payload={"request_digest": _json_digest({"request_id": request_id})}, + ), + expected_version=3, + ) + return self._execute_training( + prepared, + progress=progress, + operation=operation, + ) + + @_public_guard + def resume_training( + self, + run_id: str, + *, + request_id: str, + scratch_root: Path, + progress: Callable[[TrainingProgress], None] | None = None, + ) -> TrainingPublicationReceipt: + with self.repository.database.connect() as connection: + try: + replay = self.repository.reopen_operation_with_connection( + connection, TrainingOperationKind.RESUME, request_id + ) + except TrainingNotFound: + replay = None + if replay is not None: + if replay.run_id != run_id: + raise TrainingServiceError("operation_request_conflict") + if replay.status is TrainingOperationStatus.COMPLETED: + return TrainingPublicationReceipt.model_validate(replay.result_payload) + if not ( + replay.status is TrainingOperationStatus.RUNNING + and replay.lease_expires_at is not None + and replay.lease_expires_at <= datetime.now(UTC) + ): + raise TrainingServiceError("operation_in_progress") + run = self.repository.get_run(run_id) + checkpoints = self.repository.list_checkpoints(run_id) + if run.state not in {"training", "cancelled"} or not checkpoints: + raise TrainingServiceError("resume_state_invalid") + prepared = self._prepare_training_execution( + run_id, request_id=request_id, scratch_root=scratch_root + ) + parent_checkpoint_id = prepared.request.parent_checkpoint_id + parent_checkpoint_digest = prepared.request.parent_checkpoint_digest + if parent_checkpoint_id is None or parent_checkpoint_digest is None: + raise TrainingServiceError("resume_state_invalid") + with self.repository.database.connect() as connection: + rows = connection.execute( + "SELECT role,artifact_id,artifact_digest,size_bytes " + "FROM training_checkpoint_inventory_artifacts WHERE checkpoint_id=? " + "ORDER BY ordinal", + (parent_checkpoint_id,), + ).fetchall() + inventory = tuple( + ResumeInventoryArtifact( + role=cast(Any, str(row["role"])), + artifact_id=str(row["artifact_id"]), + digest=str(row["artifact_digest"]), + size_bytes=int(row["size_bytes"]), + ) + for row in rows + ) + request_payload = ResumeOperationRequest( + request_id=request_id, + run_id=run_id, + recipe_digest=prepared.request.materials.recipe_digest, + dataset_digest=prepared.request.materials.dataset_digest, + backend_digest=prepared.request.materials.backend_digest, + runtime_digest=prepared.request.materials.runtime_digest, + material_digest=prepared.request.materials.material_digest, + source_digest=prepared.request.materials.source_digest, + parent_checkpoint_id=parent_checkpoint_id, + parent_checkpoint_digest=parent_checkpoint_digest, + parent_inventory=inventory, + ).model_dump(mode="json") + + def begin(connection: sqlite3.Connection) -> TrainingOrchestrationOperation: + operation = self.repository.begin_operation_with_connection( + connection, + operation_id=f"operation_resume_{_json_digest(request_payload)[7:]}", + run_id=run_id, + operation_kind=TrainingOperationKind.RESUME, + idempotency_key=request_id, + request_payload=request_payload, + ) + if operation.status is TrainingOperationStatus.COMPLETED: + return operation + claim = self.repository.claim_operation_with_connection( + connection, + operation_kind=TrainingOperationKind.RESUME, + idempotency_key=request_id, + claimant_id=f"training-service:{self.reviewer_id}", + lease_seconds=57600, + ) + if not claim.claimed: + raise TrainingServiceError("operation_in_progress") + return claim.operation + + operation = cast(TrainingOrchestrationOperation, self.repository.atomic(begin)) + if operation.status is TrainingOperationStatus.COMPLETED: + return TrainingPublicationReceipt.model_validate(operation.result_payload) + parent_stack = ExitStack() + primary_error: BaseException | None = None + try: + if run.state == "cancelled": + with self.repository.database.connect() as connection: + lifecycle_row = connection.execute( + "SELECT lifecycle_version FROM training_runs WHERE id=?", + (run_id,), + ).fetchone() + if lifecycle_row is None: + raise TrainingServiceError("resume_state_invalid") + lifecycle_version = int(lifecycle_row["lifecycle_version"]) + self.repository.append_event( + run_id, + TrainingRunEvent( + sequence=lifecycle_version + 1, + previous_state="cancelled", + new_state="training", + payload={ + "request_digest": _json_digest({"request_id": request_id}), + "parent_checkpoint_id": parent_checkpoint_id, + "parent_checkpoint_digest": parent_checkpoint_digest, + }, + ), + expected_version=lifecycle_version, + ) + with self.repository.database.connect() as connection: + handles = tuple( + parent_stack.enter_context( + self.artifacts.open_verified_with_connection( + connection, + item.artifact_id, + sha256=item.digest[7:], + byte_size=item.size_bytes, + media_type="application/octet-stream", + ) + ) + for item in inventory + ) + return self._execute_training( + prepared, + progress=progress, + operation=operation, + parent_handles=handles, + close_parent_handles=parent_stack.close, + ) + except TrainingServiceError as error: + primary_error = error + raise + except Exception as error: + primary_error = error + if self._compensate_resume_failure(operation, "training_backend_failed"): + raise TrainingServiceError("training_backend_failed") from None + raise + finally: + try: + parent_stack.close() + except Exception: + if primary_error is None: + raise + + def _prepare_training_execution( + self, + run_id: str, + *, + request_id: str, + scratch_root: Path, + ) -> _PreparedTrainingExecution: + run = self.repository.get_run(run_id) + record = self.repository.get_recipe(run.run.recipe_id) + recipe = record.recipe + materials = self._training_materials(record) + backend = self.trainer_registry.resolve(self.reviewer_id, recipe.backend_id) + prior = self.repository.list_checkpoints(run_id) + parent = prior[-1].checkpoint if prior else None + parent_evidence_digest = ( + self._checkpoint_evidence_digest(parent.checkpoint_id) if parent is not None else None + ) + telemetry_offset = len(self.repository.list_telemetry(run_id)) + schedule = checkpoint_steps(recipe.total_steps) + remaining = tuple(step for step in schedule if parent is None or step > parent.step) + if not remaining: + raise TrainingServiceError("resume_complete") + + raw_optimizer_parameters = recipe.backend_config.get("optimizer_parameters", {}) + if not isinstance(raw_optimizer_parameters, Mapping) or any( + not isinstance(name, str) + or not name + or not isinstance(value, (bool, int, float, str)) + for name, value in raw_optimizer_parameters.items() + ): + raise TrainingServiceError("optimizer_parameters_invalid") + optimizer_parameters: dict[str, bool | int | float | str] = { + str(name): cast(bool | int | float | str, value) + for name, value in raw_optimizer_parameters.items() + } + recipe_owned_optimizer_parameters = { + "gradient_accumulation_steps", + "learning_rate", + } + if recipe_owned_optimizer_parameters.intersection(optimizer_parameters): + raise TrainingServiceError("optimizer_parameters_reserved") + request = TrainingRequest( + request_id=request_id, + run_id=run_id, + backend_id=recipe.backend_id, + materials=materials, + scratch_root=scratch_root, + total_steps=recipe.total_steps, + checkpoint_steps=remaining, + trainable_targets=recipe.trainable_targets, + lora=LoRAConfiguration( + rank=recipe.lora_rank, + alpha=recipe.lora_alpha, + dropout=recipe.lora_dropout, + ), + seed=recipe.seed, + deadline_seconds=recipe.wall_clock_limit_seconds, + optimizer=OptimizerConfiguration( + algorithm=recipe.optimizer, + parameters=tuple( + OptimizerParameter(name=name, value=value) + for name, value in sorted( + { + "gradient_accumulation_steps": recipe.gradient_accumulation_steps, + "learning_rate": recipe.learning_rate, + **optimizer_parameters, + }.items() + ) + ), + ), + start_step=parent.step if parent is not None else 0, + telemetry_sequence_start=telemetry_offset + 1, + parent_checkpoint_id=parent.checkpoint_id if parent is not None else None, + parent_checkpoint_digest=parent_evidence_digest, + optimizer_state_digest=( + str(parent.manifest["optimizer_state_digest"]) if parent is not None else None + ), + ) + resume_evidence = ( + TrainingResumeEvidence( + resume_id=f"resume_{_json_digest({'request_id': request_id})[7:]}", + run_id=run_id, + parent_checkpoint_id=parent.checkpoint_id, + parent_checkpoint_digest=cast(str, parent_evidence_digest), + parent_artifact_digest=parent.artifact_digest, + recipe_digest=record.recipe_id, + dataset_digest=recipe.dataset_digest, + backend_digest=materials.backend_digest, + runtime_digest=materials.runtime_digest, + material_digest=materials.material_digest, + source_digest=materials.source_digest, + optimizer_state_digest=str(parent.manifest["optimizer_state_digest"]), + ) + if parent is not None + else None + ) + return _PreparedTrainingExecution( + record=record, + backend=backend, + request=request, + resume_evidence=resume_evidence, + telemetry_offset=telemetry_offset, + ) + + def _execute_training( + self, + prepared: _PreparedTrainingExecution, + *, + progress: Callable[[TrainingProgress], None] | None, + operation: TrainingOrchestrationOperation | None = None, + parent_handles: tuple[VerifiedArtifactHandle, ...] = (), + close_parent_handles: Callable[[], None] | None = None, + ) -> TrainingPublicationReceipt: + record = prepared.record + recipe = record.recipe + backend = prepared.backend + request = prepared.request + resume_evidence = prepared.resume_evidence + telemetry_offset = prepared.telemetry_offset + run_id = request.run_id + request_id = request.request_id + observed: list[TrainingProgress] = [] + + def observe(item: TrainingProgress) -> None: + if observed and item.step <= observed[-1].step: + raise TrainingServiceError("training_progress_invalid") + observed.append(item) + if progress is not None: + progress(item) + + def cancelled() -> bool: + with self.repository.database.connect() as connection: + return ( + connection.execute( + "SELECT 1 FROM training_cancellation_requests request " + "WHERE request.run_id=? AND request.reviewer_id=? " + "AND NOT EXISTS(SELECT 1 FROM training_cancellation_evidence evidence " + "WHERE evidence.cancellation_request_id=request.id) LIMIT 1", + (run_id, self.reviewer_id), + ).fetchone() + is not None + ) + + try: + with self.repository.database.connect() as connection, ExitStack() as stack: + handles = self._verify_bound_artifacts(connection, recipe, record.binding, stack) + self._provenance_with_connection(connection, recipe, record.binding) + result = backend.train(request, observe, cancelled) + for handle in handles: + self.artifacts.reverify_open_handle(connection, handle) + validate_training_result(request, result) + self._publish_training_result( + record, + result, + observed, + telemetry_offset=telemetry_offset, + resume_evidence=resume_evidence, + operation=operation, + parent_handles=parent_handles, + close_parent_handles=close_parent_handles, + ) + if operation is None: + return self._publication_receipt(result, "evaluating", request_id) + with self.repository.database.connect() as connection: + completed = self.repository.reopen_operation_with_connection( + connection, operation.operation_kind, operation.idempotency_key + ) + return TrainingPublicationReceipt.model_validate(completed.result_payload) + except TrainingCancelled as cancellation: + try: + self._publish_cancellation( + request, + cancellation, + observed, + parent_handles=parent_handles, + ) + except ArtifactError: + self._fail_training(run_id, "training_result_invalid") + raise TrainingServiceError("training_result_invalid") from None + if operation is not None and operation.claimant_id and operation.claim_token: + self.repository.atomic( + lambda connection: self.repository.fail_operation_with_connection( + connection, + operation_kind=operation.operation_kind, + idempotency_key=operation.idempotency_key, + claimant_id=cast(str, operation.claimant_id), + claim_token=cast(str, operation.claim_token), + attempt=operation.attempt, + failure_digest=_json_digest({"reason": "training_cancelled"}), + ) + ) + raise TrainingServiceError("training_cancelled") from None + except TrainingServiceError: + raise + except (ArtifactError, TrainerResultInvalid, TrainingConflict, ValueError): + if self._compensate_resume_failure(operation, "training_result_invalid"): + raise TrainingServiceError("training_result_invalid") from None + self._fail_training(run_id, "training_result_invalid") + raise TrainingServiceError("training_result_invalid") from None + except Exception: + if self._compensate_resume_failure(operation, "training_backend_failed"): + raise TrainingServiceError("training_backend_failed") from None + self._fail_training(run_id, "training_backend_failed") + raise TrainingServiceError("training_backend_failed") from None + + def _compensate_resume_failure( + self, + operation: TrainingOrchestrationOperation | None, + reason: str, + ) -> bool: + if ( + operation is None + or operation.operation_kind is not TrainingOperationKind.RESUME + or operation.lifecycle_state != "cancelled" + or operation.claim_token is None + ): + return False + failure_digest = _json_digest({"reason": reason}) + created_at = datetime.now(UTC).isoformat() + + def compensate(connection: sqlite3.Connection) -> None: + connection.execute( + "INSERT INTO training_resume_attempt_failures " + "(operation_id,attempt,claim_token,failure_digest,created_at) " + "VALUES (?,?,?,?,?)", + ( + operation.operation_id, + operation.attempt, + operation.claim_token, + failure_digest, + created_at, + ), + ) + connection.execute( + "UPDATE training_orchestration_operations SET lease_expires_at=?,updated_at=? " + "WHERE id=? AND status='running' AND attempt=? AND claim_token=?", + ( + "1970-01-01T00:00:00+00:00", + created_at, + operation.operation_id, + operation.attempt, + operation.claim_token, + ), + ) + row = connection.execute( + "SELECT lifecycle_version FROM training_runs WHERE id=?", + (operation.run_id,), + ).fetchone() + if row is None: + raise TrainingConflict("training run missing") + version = int(row["lifecycle_version"]) + self.repository.append_event_with_connection( + connection, + operation.run_id, + TrainingRunEvent( + sequence=version + 1, + previous_state="training", + new_state="cancelled", + payload={ + "resume_operation_id": operation.operation_id, + "resume_attempt": operation.attempt, + "resume_failure_digest": failure_digest, + }, + ), + expected_version=version, + ) + + self.repository.atomic(compensate) + return True + + def _publish_training_result( + self, + record: RecipeRecord, + result: TrainingResult, + observed: list[TrainingProgress], + cancellation: TrainingCancellationEvidence | None = None, + telemetry_offset: int = 0, + resume_evidence: TrainingResumeEvidence | None = None, + operation: TrainingOrchestrationOperation | None = None, + parent_handles: tuple[VerifiedArtifactHandle, ...] = (), + close_parent_handles: Callable[[], None] | None = None, + ) -> None: + files = tuple(item for checkpoint in result.checkpoints for item in checkpoint.files) + staged: list[StagedArtifact] = [] + try: + for item in files: + staged.append( + self.artifacts.stage( + result.output_root / item.relative_path, + "application/octet-stream", + ) + ) + verified_inputs = tuple( + self.artifacts.get(identifier) + for identifier in ( + record.binding.base_model_artifact_id, + record.binding.source_snapshot_artifact_id, + *record.binding.lightning_adapter_artifact_ids.values(), + ) + ) + + def publish( + connection: sqlite3.Connection, artifacts: tuple[Artifact, ...] + ) -> None: + self._provenance_with_connection(connection, record.recipe, record.binding) + if resume_evidence is not None: + self.repository.record_resume_with_connection(connection, resume_evidence) + for sequence, item in enumerate(observed, start=telemetry_offset + 1): + self.repository.append_telemetry_with_connection( + connection, + result.run_id, + TrainingTelemetry( + sequence=sequence, + step=item.step, + payload=item.model_dump(mode="json"), + ), + ) + self._publish_checkpoints( + connection, + record, + result, + cancelled=cancellation is not None, + telemetry_offset=telemetry_offset, + ) + artifact_offset = 0 + for checkpoint in result.checkpoints: + checkpoint_artifacts = artifacts[ + artifact_offset : artifact_offset + len(checkpoint.files) + ] + self.repository.bind_checkpoint_inventory_artifacts_with_connection( + connection, + checkpoint_id=checkpoint.checkpoint_id, + artifact_ids_by_role={ + item.role: artifact.id + for item, artifact in zip( + checkpoint.files, checkpoint_artifacts, strict=True + ) + }, + ) + artifact_offset += len(checkpoint.files) + lifecycle_row = connection.execute( + "SELECT lifecycle_version FROM training_runs WHERE id=?", + (result.run_id,), + ).fetchone() + if lifecycle_row is None: + raise TrainingConflict("training run missing") + lifecycle_version = int(lifecycle_row["lifecycle_version"]) + if cancellation is not None: + self.repository.record_cancellation_with_connection( + connection, cancellation + ) + event = TrainingRunEvent( + sequence=lifecycle_version + 1, + previous_state="training", + new_state="cancelled", + payload={ + "cancellation_id": cancellation.cancellation_id, + "cancellation_request_id": ( + cancellation.cancellation_request_id + ), + "request_digest": cancellation.request_digest, + "checkpoint_digest": cancellation.checkpoint_digest, + "boundary_step": cancellation.boundary_step, + }, + ) + else: + event = TrainingRunEvent( + sequence=lifecycle_version + 1, + previous_state="training", + new_state="evaluating", + payload={"telemetry_digest": result.telemetry_digest}, + ) + self.repository.append_event_with_connection( + connection, + result.run_id, + event, + expected_version=lifecycle_version, + ) + if operation is not None: + for handle in parent_handles: + self.artifacts.reverify_open_handle(connection, handle) + if close_parent_handles is not None: + close_parent_handles() + if operation.operation_kind is TrainingOperationKind.CANCEL: + if cancellation is None: + raise TrainingServiceError("cancellation_evidence_missing") + self._complete_cancel_operation( + connection, operation, cancellation + ) + else: + self._complete_collection_operation( + connection, + operation, + result, + lifecycle_state=cast( + Literal["evaluating", "cancelled"], event.new_state + ), + ) + + self.artifacts.commit_batch(tuple(staged), publish, verify_artifacts=verified_inputs) + except Exception: + for staged_item in staged: + try: + self.artifacts.discard(staged_item) + except ArtifactError: + pass + raise + + @staticmethod + def _publication_receipt( + result: TrainingResult, + lifecycle_state: Literal["evaluating", "cancelled"], + request_id: str, + ) -> TrainingPublicationReceipt: + return TrainingPublicationReceipt( + request_id=request_id, + run_id=result.run_id, + lifecycle_state=lifecycle_state, + checkpoint_ids=tuple(item.checkpoint_id for item in result.checkpoints), + telemetry_digest=result.telemetry_digest, + ) + + def _complete_collection_operation( + self, + connection: sqlite3.Connection, + operation: TrainingOrchestrationOperation, + result: TrainingResult, + *, + lifecycle_state: Literal["evaluating", "cancelled"], + ) -> None: + if operation.claim_token is None or operation.claimant_id is None: + raise TrainingServiceError("operation_claim_missing") + evidence: dict[str, list[dict[str, object]]] = { + "events": [], + "telemetry": [], + "checkpoints": [], + "artifacts": [], + "inventory": [], + } + identifiers: list[tuple[str, str, dict[str, object]]] = [] + operation_started_at = cast(datetime, operation.updated_at).isoformat() + resume_entry: dict[str, object] | None = None + for row in connection.execute( + "SELECT sequence FROM training_run_events WHERE run_id=? " + "AND julianday(created_at)>=julianday(?) ORDER BY sequence", + (result.run_id, operation_started_at), + ): + value = str(row["sequence"]) + identifiers.append(("event", value, {"sequence": int(value)})) + for row in connection.execute( + "SELECT sequence,step FROM training_telemetry WHERE run_id=? " + "AND julianday(created_at)>=julianday(?) ORDER BY sequence", + (result.run_id, operation_started_at), + ): + identifiers.append( + ( + "telemetry", + str(row["sequence"]), + {"sequence": int(row["sequence"]), "step": int(row["step"])}, + ) + ) + for checkpoint in result.checkpoints: + identifiers.append(("checkpoint", checkpoint.checkpoint_id, {"step": checkpoint.step})) + if operation.operation_kind is TrainingOperationKind.RESUME: + row = connection.execute( + "SELECT id FROM training_resume_evidence WHERE run_id=? " + "AND julianday(created_at)>=julianday(?) ORDER BY created_at DESC LIMIT 1", + (result.run_id, operation_started_at), + ).fetchone() + if row is None: + raise TrainingServiceError("resume_evidence_missing") + identifiers.append(("resume", str(row["id"]), {})) + bindings = connection.execute( + "SELECT binding.checkpoint_id,binding.role,binding.artifact_id,artifact.byte_size " + "FROM training_checkpoint_inventory_artifacts binding JOIN artifacts artifact " + "ON artifact.id=binding.artifact_id JOIN training_checkpoints checkpoint " + "ON checkpoint.id=binding.checkpoint_id WHERE checkpoint.run_id=? " + "AND julianday(checkpoint.created_at)>=julianday(?) " + "ORDER BY checkpoint.step,binding.ordinal", + (result.run_id, operation_started_at), + ).fetchall() + for row in bindings: + identifiers.append( + ( + "inventory", + f"{row['checkpoint_id']}:{row['role']}", + {"checkpoint_id": str(row["checkpoint_id"]), "role": str(row["role"])}, + ) + ) + for artifact_id in sorted({str(row["artifact_id"]) for row in bindings}): + size = next( + int(row["byte_size"]) for row in bindings if row["artifact_id"] == artifact_id + ) + identifiers.append(("artifact", artifact_id, {"size_bytes": size})) + names = { + "event": "events", + "telemetry": "telemetry", + "checkpoint": "checkpoints", + "artifact": "artifacts", + "inventory": "inventory", + "resume": "resume", + } + for kind, identifier, metadata in identifiers: + digest = str( + connection.execute( + "SELECT training_operation_evidence_digest(?,?,?)", + (operation.operation_id, kind, identifier), + ).fetchone()[0] + ) + self.repository.link_operation_evidence_with_connection( + connection, + operation_id=operation.operation_id, + attempt=operation.attempt, + claim_token=operation.claim_token, + evidence_kind=kind, + evidence_id=identifier, + evidence_digest=digest, + ) + if kind == "resume": + resume_entry = {"evidence_id": identifier, "digest": digest} + else: + evidence[names[kind]].append( + {"evidence_id": identifier, "digest": digest, **metadata} + ) + lifecycle_version = int( + connection.execute( + "SELECT lifecycle_version FROM training_runs WHERE id=?", (result.run_id,) + ).fetchone()[0] + ) + receipt: dict[str, object] = { + "run_id": result.run_id, + "lifecycle_version": lifecycle_version, + "lifecycle_state": lifecycle_state, + **evidence, + "evaluation": None, + } + if operation.operation_kind is TrainingOperationKind.RESUME: + if resume_entry is None: + raise TrainingServiceError("resume_evidence_missing") + receipt["resume"] = resume_entry + durable = self._publication_receipt( + result, lifecycle_state, str(operation.request_payload["request_id"]) + ) + self.repository.complete_operation_with_connection( + connection, + operation_kind=operation.operation_kind, + idempotency_key=operation.idempotency_key, + claimant_id=operation.claimant_id, + claim_token=operation.claim_token, + attempt=operation.attempt, + receipt_payload=receipt, + result_payload=durable.model_dump(mode="json"), + ) + + def _complete_cancel_operation( + self, + connection: sqlite3.Connection, + operation: TrainingOrchestrationOperation, + cancellation: TrainingCancellationEvidence, + ) -> None: + if operation.claim_token is None or operation.claimant_id is None: + raise TrainingServiceError("operation_claim_missing") + event = connection.execute( + "SELECT sequence FROM training_run_events WHERE run_id=? AND new_state='cancelled' " + "ORDER BY sequence DESC LIMIT 1", + (operation.run_id,), + ).fetchone() + checkpoint = connection.execute( + "SELECT id,step FROM training_checkpoints WHERE run_id=? AND step=? " + "AND publication_state='finalized'", + (operation.run_id, cancellation.boundary_step), + ).fetchone() + if event is None or checkpoint is None: + raise TrainingServiceError("cancellation_evidence_missing") + entries: dict[str, dict[str, object]] = {} + for kind, identifier, metadata in ( + ("cancel", cancellation.cancellation_id, {}), + ("event", str(event["sequence"]), {"sequence": int(event["sequence"])}), + ( + "checkpoint", + str(checkpoint["id"]), + {"step": int(checkpoint["step"])}, + ), + ): + digest = str( + connection.execute( + "SELECT training_operation_evidence_digest(?,?,?)", + (operation.operation_id, kind, identifier), + ).fetchone()[0] + ) + self.repository.link_operation_evidence_with_connection( + connection, + operation_id=operation.operation_id, + attempt=operation.attempt, + claim_token=operation.claim_token, + evidence_kind=kind, + evidence_id=identifier, + evidence_digest=digest, + ) + entries[kind] = {"evidence_id": identifier, "digest": digest, **metadata} + receipt = { + "run_id": operation.run_id, + "lifecycle_version": 5, + "lifecycle_state": "cancelled", + "cancellation": entries["cancel"], + "terminal_event": entries["event"], + "boundary_checkpoint": entries["checkpoint"], + "cancellation_request_id": cancellation.cancellation_request_id, + "request_digest": cancellation.request_digest, + "requested_step": cancellation.requested_step, + "boundary_step": cancellation.boundary_step, + } + self.repository.complete_operation_with_connection( + connection, + operation_kind=TrainingOperationKind.CANCEL, + idempotency_key=operation.idempotency_key, + claimant_id=operation.claimant_id, + claim_token=operation.claim_token, + attempt=operation.attempt, + receipt_payload=receipt, + result_payload=receipt, + ) + + def _publish_checkpoints( + self, + connection: sqlite3.Connection, + record: RecipeRecord, + result: TrainingResult, + *, + cancelled: bool = False, + telemetry_offset: int = 0, + ) -> None: + character_id = self.repository.get_dataset(record.recipe.dataset_digest).character_id + prior = self.repository.list_checkpoints(result.run_id) + parent_digest: str | None = ( + self._checkpoint_evidence_digest(prior[-1].checkpoint.checkpoint_id) if prior else None + ) + for checkpoint in result.checkpoints: + by_role = {item.role: item for item in checkpoint.files} + inventory = tuple( + TrainingCheckpointArtifact.model_validate(item.model_dump(mode="python")) + for item in checkpoint.files + ) + evidence = TrainingCheckpointEvidence( + run_id=result.run_id, + recipe_digest=record.recipe_id, + dataset_digest=record.recipe.dataset_digest, + backend_digest=result.materials.backend_digest, + runtime_digest=result.materials.runtime_digest, + material_digest=result.materials.material_digest, + source_digest=result.materials.source_digest, + step=checkpoint.step, + fraction=checkpoint.fraction, + parent_checkpoint_digest=parent_digest, + safetensors_digest=by_role["weights"].digest, + safetensors_size_bytes=by_role["weights"].size_bytes, + optimizer_state_digest=by_role["optimizer"].digest, + trainer_state_digest=by_role["trainer_state"].digest, + telemetry_digest=checkpoint.telemetry_digest, + telemetry_sequence_start=telemetry_offset + 1, + telemetry_sequence_end=telemetry_offset + + checkpoint.step + - (prior[-1].checkpoint.step if prior else 0), + trainable_targets_digest=checkpoint.trainable_targets_digest, + frozen_parameters_digest=checkpoint.frozen_parameters_digest, + finite_gradient_proof_digest=checkpoint.finite_gradient_proof_digest, + cancellation_boundary=cancelled and checkpoint is result.checkpoints[-1], + inventory=inventory, + ) + manifest = { + "schema_version": "character-training-checkpoint.v1", + "run_id": result.run_id, + "reviewer_id": self.reviewer_id, + "recipe_id": record.recipe_id, + "dataset_digest": record.recipe.dataset_digest, + "character_id": character_id, + "step": checkpoint.step, + "fraction": checkpoint.fraction, + "parent_checkpoint_id": checkpoint.parent_checkpoint_id, + "artifact_digest": by_role["weights"].digest, + "safetensors_size_bytes": by_role["weights"].size_bytes, + "backend_digest": result.materials.backend_digest, + "runtime_digest": result.materials.runtime_digest, + "material_digest": result.materials.material_digest, + "source_digest": result.materials.source_digest, + "optimizer_state_digest": by_role["optimizer"].digest, + "trainer_state_digest": by_role["trainer_state"].digest, + "telemetry_digest": checkpoint.telemetry_digest, + "telemetry_sequence_start": telemetry_offset + 1, + "telemetry_sequence_end": telemetry_offset + + checkpoint.step + - (prior[-1].checkpoint.step if prior else 0), + "trainable_targets_digest": checkpoint.trainable_targets_digest, + "frozen_parameters_digest": checkpoint.frozen_parameters_digest, + "finite_gradient_proof_digest": checkpoint.finite_gradient_proof_digest, + "cancellation_boundary": cancelled and checkpoint is result.checkpoints[-1], + "inventory": [item.model_dump(mode="json") for item in inventory], + } + persisted = TrainingCheckpoint( + checkpoint_id=checkpoint.checkpoint_id, + run_id=result.run_id, + step=checkpoint.step, + fraction=checkpoint.fraction, + parent_checkpoint_id=checkpoint.parent_checkpoint_id, + artifact_digest=by_role["weights"].digest, + manifest_digest=_json_digest(manifest), + manifest=manifest, + ) + self.repository.add_checkpoint_with_connection(connection, persisted, evidence) + parent_digest = _json_digest(evidence.model_dump(mode="json")) + + def _checkpoint_evidence_digest(self, checkpoint_id: str) -> str: + with self.repository.database.connect() as connection: + row = connection.execute( + "SELECT evidence_digest FROM training_checkpoints " + "WHERE id=? AND reviewer_id=? AND publication_state='finalized'", + (checkpoint_id, self.reviewer_id), + ).fetchone() + if row is None: + raise TrainingServiceError("checkpoint_evidence_missing") + return str(row["evidence_digest"]) + + @_public_guard + def request_cancellation( + self, run_id: str, *, request_id: str, requested_step: int + ) -> TrainingOrchestrationOperation: + digest = _json_digest( + {"request_id": request_id, "run_id": run_id, "requested_step": requested_step} + ) + with self.repository.database.connect() as connection: + try: + existing_operation = self.repository.reopen_operation_with_connection( + connection, TrainingOperationKind.CANCEL, request_id + ) + except TrainingNotFound: + existing_operation = None + if existing_operation is not None: + if ( + existing_operation.run_id != run_id + or existing_operation.request_payload.get("request_digest") != digest + or existing_operation.request_payload.get("requested_step") != requested_step + ): + raise TrainingServiceError("operation_request_conflict") + return existing_operation + run = self.repository.get_run(run_id) + if run.state != "training": + raise TrainingServiceError("cancellation_state_invalid") + record = self.repository.get_recipe(run.run.recipe_id) + materials = self._training_materials(record) + request_payload = CancelOperationRequest( + request_id=request_id, + run_id=run_id, + recipe_digest=materials.recipe_digest, + dataset_digest=materials.dataset_digest, + backend_digest=materials.backend_digest, + runtime_digest=materials.runtime_digest, + material_digest=materials.material_digest, + source_digest=materials.source_digest, + cancellation_request_id=request_id, + request_digest=digest, + requested_step=requested_step, + ).model_dump(mode="json") + + def create(connection: sqlite3.Connection) -> TrainingOrchestrationOperation: + existing = connection.execute( + "SELECT 1 FROM training_cancellation_requests WHERE id=?", + (request_id,), + ).fetchone() + if existing is None: + self.repository.request_cancellation_with_connection( + connection, + request_id=request_id, + run_id=run_id, + requested_step=requested_step, + request_digest=digest, + ) + operation = self.repository.begin_operation_with_connection( + connection, + operation_id=f"operation_cancel_{_json_digest(request_payload)[7:]}", + run_id=run_id, + operation_kind=TrainingOperationKind.CANCEL, + idempotency_key=request_id, + request_payload=request_payload, + ) + if operation.status is not TrainingOperationStatus.PENDING: + return operation + claim = self.repository.claim_operation_with_connection( + connection, + operation_kind=TrainingOperationKind.CANCEL, + idempotency_key=request_id, + claimant_id=f"training-service:{self.reviewer_id}", + lease_seconds=57600, + ) + if not claim.claimed: + raise TrainingServiceError("operation_in_progress") + return claim.operation + + return cast(TrainingOrchestrationOperation, self.repository.atomic(create)) + + def _publish_cancellation( + self, + request_model: TrainingRequest, + cancellation: TrainingCancelled, + observed: list[TrainingProgress], + *, + parent_handles: tuple[VerifiedArtifactHandle, ...] = (), + ) -> None: + run_id = request_model.run_id + if ( + cancellation.result is None + or not observed + or observed[-1].step != cancellation.boundary_step + or not observed[-1].checkpoint_boundary + ): + self._fail_training(run_id, "cancellation_evidence_missing") + return + validate_cancelled_training_result( + request_model, cancellation.result, cancellation.boundary_step + ) + with self.repository.database.connect() as connection: + request = connection.execute( + "SELECT id,requested_step,request_digest FROM training_cancellation_requests " + "WHERE run_id=? AND reviewer_id=? ORDER BY created_at,id LIMIT 1", + (run_id, self.reviewer_id), + ).fetchone() + if request is None: + self._fail_training(run_id, "cancellation_evidence_missing") + return + result = cancellation.result + digest = next( + item.digest for item in result.checkpoints[-1].files if item.role == "weights" + ) + evidence = TrainingCancellationEvidence( + cancellation_id=f"cancel_{digest[7:]}", + run_id=run_id, + cancellation_request_id=str(request["id"]), + request_digest=str(request["request_digest"]), + requested_step=int(request["requested_step"]), + boundary_step=cancellation.boundary_step, + checkpoint_digest=digest, + ) + record = self.repository.get_recipe(self.repository.get_run(run_id).run.recipe_id) + with self.repository.database.connect() as connection: + operation = self.repository.reopen_operation_with_connection( + connection, TrainingOperationKind.CANCEL, str(request["id"]) + ) + telemetry_offset = len(self.repository.list_telemetry(run_id)) + self._publish_training_result( + record, + result, + observed, + cancellation=evidence, + telemetry_offset=telemetry_offset, + operation=operation, + parent_handles=parent_handles, + ) + + @_public_guard + def begin_evaluation(self, run_id: str) -> None: + if self.repository.get_run(run_id).state != "training": + raise TrainingServiceError("evaluation_state_invalid") + sequence = len(self.repository.list_events(run_id)) + 1 + self.repository.append_event( + run_id, + TrainingRunEvent( + sequence=sequence, + previous_state="training", + new_state="evaluating", + payload={"checkpoint_count": len(self.repository.list_checkpoints(run_id))}, + ), + expected_version=sequence - 1, + ) + + @_public_guard + def complete_evaluation(self, run_id: str, *, qualified: bool, evidence: object) -> str: + if self.repository.get_run(run_id).state != "evaluating": + raise TrainingServiceError("evaluation_state_invalid") + disposition: Literal["qualified", "rejected"] = "qualified" if qualified else "rejected" + evidence_digest = _json_digest({"evidence": evidence}) + item = TrainingEvaluationEvidence( + evidence_id=f"evaluation_{evidence_digest[7:]}", + run_id=run_id, + disposition=disposition, + evidence_digest=evidence_digest, + ) + sequence = len(self.repository.list_events(run_id)) + 1 + + def publish(connection: sqlite3.Connection) -> None: + self.repository.record_evaluation_with_connection(connection, item) + self.repository.append_event_with_connection( + connection, + run_id, + TrainingRunEvent( + sequence=sequence, + previous_state="evaluating", + new_state=disposition, + payload={ + "evaluation_evidence_id": item.evidence_id, + "evaluation_evidence_digest": evidence_digest, + }, + ), + expected_version=sequence - 1, + ) + + self.repository.atomic(publish) + return disposition + + @_public_guard + def export_terminal_evidence(self, run_id: str) -> bytes: + run = self.repository.get_run(run_id) + if run.state not in {"qualified", "rejected", "failed", "cancelled"}: + raise TrainingServiceError("terminal_evidence_incomplete") + with self.repository.database.connect() as connection, ExitStack() as stack: + operation_rows = connection.execute( + "SELECT operation_kind,idempotency_key,request_digest,request_payload_json,status," + "attempt,receipt_digest,receipt_payload_json,result_digest,result_payload_json," + "failure_digest FROM training_orchestration_operations " + "WHERE run_id=? AND reviewer_id=? ORDER BY created_at,id", + (run_id, self.reviewer_id), + ).fetchall() + evidence_tables = ( + "training_probe_evidence", + "training_calibration_evidence", + "training_resume_evidence", + "training_cancellation_requests", + "training_cancellation_evidence", + "training_evaluation_evidence", + ) + + def normalized_rows(table: str) -> list[dict[str, object]]: + rows = connection.execute( + f"SELECT * FROM {table} WHERE run_id=? AND reviewer_id=? " # noqa: S608 + "ORDER BY created_at,rowid", + (run_id, self.reviewer_id), + ).fetchall() + return [ + { + key: ( + json.loads(str(row[key])) + if key.endswith("_json") + else row[key] + ) + for key in row.keys() + if key not in {"created_at", "reviewer_id"} + } + for row in rows + ] + + telemetry = normalized_rows("training_telemetry") + inventory = connection.execute( + "SELECT inventory.checkpoint_id,inventory.ordinal,inventory.role," + "inventory.artifact_digest,inventory.size_bytes,binding.artifact_id " + "FROM training_checkpoint_inventory inventory " + "JOIN training_checkpoints checkpoint ON checkpoint.id=inventory.checkpoint_id " + "JOIN training_checkpoint_inventory_artifacts binding " + "ON binding.checkpoint_id=inventory.checkpoint_id " + "AND binding.ordinal=inventory.ordinal WHERE checkpoint.run_id=? " + "AND checkpoint.reviewer_id=? ORDER BY checkpoint.step,inventory.ordinal", + (run_id, self.reviewer_id), + ).fetchall() + material_rows = connection.execute( + "SELECT value artifact_id FROM training_orchestration_operations operation," + "json_each(operation.request_payload_json,'$.material_artifact_ids') " + "WHERE operation.run_id=? AND operation.reviewer_id=? " + "AND operation.operation_kind='start' ORDER BY key", + (run_id, self.reviewer_id), + ).fetchall() + artifact_ids = { + *(str(row["artifact_id"]) for row in inventory), + *(str(row["artifact_id"]) for row in material_rows), + } + artifacts: list[dict[str, object]] = [] + for artifact_id in sorted(artifact_ids): + artifact = self.artifacts.get(artifact_id) + handle = stack.enter_context( + self.artifacts.open_verified_with_connection( + connection, + artifact.id, + sha256=artifact.sha256, + byte_size=artifact.byte_size, + media_type=artifact.media_type, + ) + ) + self.artifacts.reverify_open_handle(connection, handle) + artifacts.append( + { + "artifact_id": artifact.id, + "digest": f"sha256:{artifact.sha256}", + "size_bytes": artifact.byte_size, + "media_type": artifact.media_type, + } + ) + operations = [ + { + key: ( + json.loads(str(row[key])) + if key.endswith("_json") and row[key] is not None + else row[key] + ) + for key in row.keys() + } + for row in operation_rows + ] + evidence = { + table.removeprefix("training_"): normalized_rows(table) + for table in evidence_tables + } + payload = { + "schema_version": "character-training-terminal-evidence.v1", + "run": run.run.model_dump(mode="json"), + "state": run.state, + "events": [ + item.event.model_dump(mode="json") for item in self.repository.list_events(run_id) + ], + "checkpoints": [ + item.checkpoint.model_dump(mode="json") + for item in self.repository.list_checkpoints(run_id) + ], + "failures": [ + item.failure.model_dump(mode="json") + for item in self.repository.list_failures(run_id) + ], + "operations": operations, + "evidence": evidence, + "telemetry": telemetry, + "checkpoint_inventory": [dict(row) for row in inventory], + "artifacts": artifacts, + } + return canonical_json(payload).encode() + + def _fail_training(self, run_id: str, code: str) -> None: + run = self.repository.get_run(run_id) + if run.state in {"qualified", "rejected", "failed", "cancelled"}: + return + sequence = len(self.repository.list_events(run_id)) + 1 + failure = TrainingFailure( + failure_id=f"failure_{uuid.uuid4().hex}", run_id=run_id, code=code, message=code + ) + + def publish(connection: sqlite3.Connection) -> None: + self.repository.record_failure_with_connection(connection, failure) + self.repository.append_event_with_connection( + connection, + run_id, + TrainingRunEvent( + sequence=sequence, + previous_state=run.state, + new_state="failed", + payload={"failure_id": failure.failure_id, "code": code}, + ), + expected_version=sequence - 1, + ) + + self.repository.atomic(publish) + + def _training_materials(self, record: RecipeRecord) -> TrainingMaterialHandles: + recipe = record.recipe + return TrainingMaterialHandles( + recipe_digest=training_recipe_id(recipe), + dataset_digest=recipe.dataset_digest, + base_model_digest=recipe.base_model_digest, + backend_digest=_json_digest(recipe.model_dump(mode="json")["backend_config"]), + source_digest=recipe.source_code_digest, + runtime_digest=record.binding.runtime_digest, + material_digest=_json_digest(record.binding.model_dump(mode="json")), + ) + + @_public_guard + def create_dataset(self, request: DatasetCreateInput) -> TrainingDatasetView: + try: + exported = self.character_packs.export_pack(request.character_pack_id) # type: ignore[attr-defined] + if exported.pack.pack_id != request.character_pack_id: + raise ValueError + draft = TrainingDatasetDraft( + character_id=exported.pack.character_id, + character_pack_digest=_json_digest(exported.pack.model_dump(mode="json")), + canonical_outfit_id=request.canonical_outfit_id, + reserved_token=request.reserved_token, + samples=(), + ) + lineage = f"training_lineage_{uuid.uuid4().hex}" + self.repository.create_dataset_for_lineage(lineage, draft.character_id, draft) + return self.dataset_status(lineage) + except (AttributeError, LookupError, PermissionError, TrainingConflict, ValueError): + raise TrainingServiceError("dataset_creation_failed") from None + + @_public_guard + def import_media( + self, + dataset_id: str, + source: Path, + *, + suggestions: TrainingCurationSuggestions | None = None, + ) -> TrainingDatasetView: + head = self._head(dataset_id) + current = self.repository.get_dataset(head.current_dataset_id) + prepared: list[tuple[Path, TrainingMedia, StagedArtifact]] = [] + try: + existing = { + x.artifact_digest for x in self.repository.list_imported_samples(head.lineage_id) + } + with open_training_media_collection_snapshot(source) as collection: + for relative, snapshot in collection.entries: + path = Path(relative) + media = snapshot.media + staged = self.artifacts.stage(snapshot.path, media.media_type) + if (f"sha256:{staged.sha256}", staged.byte_size) != ( + media.sha256, + media.byte_size, + ): + self.artifacts.discard(staged) + raise TrainingServiceError("source_changed") + if media.sha256 in existing or any( + x[1].sha256 == media.sha256 for x in prepared + ): + self.artifacts.discard(staged) + raise TrainingServiceError("duplicate_content") + prepared.append((path, media, staged)) + + def persist(connection: sqlite3.Connection, artifacts: tuple[Artifact, ...]) -> None: + ids = list(current.draft.imported_sample_ids) + for (path, media, _), artifact in zip(prepared, artifacts, strict=True): + self.artifacts.resolve_with_connection( + connection, + artifact.id, + sha256=media.sha256.removeprefix("sha256:"), + byte_size=media.byte_size, + media_type=media.media_type, + ) + imported = _imported( + reviewer_id=self.reviewer_id, + character_id=head.character_id, + lineage_id=head.lineage_id, + artifact=artifact, + media=media, + source_lineage=unicodedata.normalize("NFC", path.stem), + suggestions=suggestions, + ) + self.repository.persist_imported_sample_with_connection(connection, imported) + ids.append(imported.import_id) + changed = current.draft.model_copy(update={"imported_sample_ids": tuple(ids)}) + self.repository.publish_dataset_head_with_connection( + connection, + lineage_id=head.lineage_id, + expected_head_version=head.head_version, + draft=changed, + ) + + self.artifacts.commit_batch(tuple(x[2] for x in prepared), persist) + return self.dataset_status(head.lineage_id) + except TrainingServiceError: + self._discard(prepared) + raise + except (ArtifactError, OSError, TrainingConflict, TrainingMediaInspectionError, ValueError): + self._discard(prepared) + raise TrainingServiceError("media_import_failed") from None + + @_public_guard + def curate_sample( + self, dataset_id: str, import_id: str, decision: CurationInput + ) -> TrainingDatasetView: + head = self._head(dataset_id) + current = self.repository.get_dataset(head.current_dataset_id) + imported = next( + ( + x + for x in self.repository.list_imported_samples(head.lineage_id) + if x.import_id == import_id + ), + None, + ) + if imported is None: + raise TrainingServiceError("sample_not_found") + if self.repository.list_confirmations(import_id): + raise TrainingServiceError("stale_sample") + confirmation = _confirmation(self.reviewer_id, imported.import_id, decision) + samples = current.draft.samples + if confirmation.decision is CurationDecision.ACCEPT: + sample = confirmation.accepted_sample(imported) + if current.draft.reserved_token not in sample.caption: + raise TrainingServiceError("reserved_token_missing") + if sample.annotation.canonical_outfit_id != current.draft.canonical_outfit_id: + raise TrainingServiceError("canonical_outfit_mismatch") + samples += (sample,) + changed = current.draft.model_copy( + update={ + "samples": samples, + "curation_confirmation_ids": current.draft.curation_confirmation_ids + + (confirmation.confirmation_id,), + } + ) + try: + with self.repository.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + self.repository.persist_confirmation_with_connection(connection, confirmation) + self.repository.publish_dataset_head_with_connection( + connection, + lineage_id=head.lineage_id, + expected_head_version=head.head_version, + draft=changed, + ) + connection.commit() + return self.dataset_status(head.lineage_id) + except (TrainingConflict, ValueError): + raise TrainingServiceError("curation_failed") from None + + @_public_guard + def dataset_status(self, dataset_id: str) -> TrainingDatasetView: + head = self._head(dataset_id, allow_sealed=True) + draft = self.repository.get_dataset(head.current_dataset_id).draft + return TrainingDatasetView( + lineage_id=head.lineage_id, + dataset_id=head.current_dataset_id, + head_version=head.head_version, + state=head.state, + character_id=head.character_id, + imported=self.repository.list_imported_samples(head.lineage_id), + coverage=evaluate_training_dataset(draft), + ) + + @_public_guard + def seal_dataset(self, dataset_id: str) -> FrozenTrainingDataset: + head = self._head(dataset_id, allow_sealed=True) + record = self.repository.get_dataset(head.current_dataset_id) + if head.state == "sealed": + if record.frozen_dataset is None: + raise TrainingServiceError("sealed_dataset_invalid") + return record.frozen_dataset + frozen = freeze_training_dataset(record.draft) + with self.repository.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + self.repository.publish_dataset_head_with_connection( + connection, + lineage_id=head.lineage_id, + expected_head_version=head.head_version, + draft=frozen, + seal=True, + ) + connection.commit() + return frozen + + @_public_guard + def preview_recipe(self, dataset_id: str, request: RecipeInput) -> TrainingRecipePreview: + self._frozen(dataset_id) + provenance = self._provenance(dataset_id, campaign_id=request.stage_3_campaign_id) + if provenance[2] != request.stage_3_result_id: + raise TrainingServiceError("prerequisite_drift") + base = self.artifacts.get(request.base_model_artifact_id) + adapters = { + name: self.artifacts.get(value) + for name, value in request.lightning_adapter_artifact_ids.items() + } + snapshot = _source_snapshot_bytes(self.source_root) + digest = "sha256:" + hashlib.sha256(snapshot).hexdigest() + material = { + "dataset_id": dataset_id, + "request": request.model_dump(mode="json"), + "source_snapshot_digest": digest, + "source_snapshot_byte_size": len(snapshot), + "base_model_digest": f"sha256:{base.sha256}", + "lightning_adapter_digests": { + name: f"sha256:{artifact.sha256}" for name, artifact in adapters.items() + }, + "runtime_identity": dict(self.runtime.identity()), + } + return TrainingRecipePreview( + preview_token=_json_digest(material), + source_snapshot_digest=digest, + source_snapshot_byte_size=len(snapshot), + base_model_digest=f"sha256:{base.sha256}", + lightning_adapter_digests=material["lightning_adapter_digests"], # type: ignore[arg-type] + ) + + @_public_guard + def create_recipe( + self, dataset_id: str, request: RecipeInput, *, preview_token: str | None = None + ) -> TrainingRecipeView: + preview = self.preview_recipe(dataset_id, request) + if preview_token is not None and preview.preview_token != preview_token: + raise TrainingServiceError("preview_drift") + snapshot = _source_snapshot_bytes(self.source_root) + if "sha256:" + hashlib.sha256(snapshot).hexdigest() != preview.source_snapshot_digest: + raise TrainingServiceError("source_drift") + with tempfile.TemporaryDirectory(prefix="video-lab-source-") as directory: + path = Path(directory) / "source.snapshot" + path.write_bytes(snapshot) + staged = self.artifacts.stage(path, "application/octet-stream") + + def persist( + connection: sqlite3.Connection, artifacts: tuple[Artifact, ...] + ) -> RecipeRecord: + source = artifacts[0] + self.artifacts.resolve_with_connection( + connection, + source.id, + sha256=preview.source_snapshot_digest[7:], + byte_size=preview.source_snapshot_byte_size, + media_type="application/octet-stream", + ) + recipe = self._resolve_recipe(dataset_id, request, source=source) + binding = self._recipe_binding(request, recipe, source) + return self.repository.create_recipe(recipe, binding, connection=connection) + + verified_inputs = ( + self.artifacts.get(request.base_model_artifact_id), + *tuple( + self.artifacts.get(artifact_id) + for _, artifact_id in sorted(request.lightning_adapter_artifact_ids.items()) + ), + ) + record = self.artifacts.commit_batch( + (staged,), persist, verify_artifacts=verified_inputs + ) + return self._recipe_view(record) + + @_public_guard + def preregister_recipe(self, recipe_id: str) -> TrainingRecipeView: + record = self.repository.get_recipe(recipe_id) + binding = record.binding + before_runtime = dict(self.runtime.identity()) + try: + with self.repository.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + with ExitStack() as verified: + handles = self._verify_bound_artifacts( + connection, record.recipe, binding, verified + ) + self._provenance_with_connection(connection, record.recipe, binding) + if before_runtime != dict(record.recipe.runtime_identity): + raise TrainingServiceError("runtime_drift") + updated = self.repository.preregister_recipe(recipe_id, connection=connection) + for handle in handles: + self.artifacts.reverify_open_handle(connection, handle) + if dict(self.runtime.identity()) != before_runtime: + raise TrainingServiceError("runtime_drift") + for handle in handles: + self.artifacts.reverify_open_handle(connection, handle) + connection.commit() + return self._recipe_view(updated) + except TrainingServiceError: + raise + except (ArtifactError, TrainingConflict, sqlite3.Error, OSError, ValueError): + raise TrainingServiceError("recipe_preregistration_failed") from None + + @_public_guard + def export_dataset_evidence(self, dataset_id: str) -> bytes: + head = self._head(dataset_id, allow_sealed=True) + dataset = self.repository.get_dataset(head.current_dataset_id) + payload = { + "schema_version": "training-dataset-evidence.v1", + "claim_state": head.state, + "head_version": head.head_version, + "dataset": (dataset.frozen_dataset or dataset.draft).model_dump(mode="json"), + } + return canonical_json(payload).encode() + + @_public_guard + def export_recipe_evidence(self, recipe_id: str) -> bytes: + record = self.repository.get_recipe(recipe_id) + payload = { + "schema_version": "training-recipe-evidence.v1", + "claim_state": record.state, + "recipe_id": record.recipe_id, + "recipe": record.recipe.model_dump(mode="json"), + } + return canonical_json(payload).encode() + + def _resolve_recipe( + self, dataset_id: str, request: RecipeInput, *, source: Artifact + ) -> TrainingRecipe: + frozen = self._frozen(dataset_id) + provenance = self._provenance( + frozen.dataset_digest, campaign_id=request.stage_3_campaign_id + ) + base = self.artifacts.get(request.base_model_artifact_id) + adapters = { + name: self.artifacts.get(value) + for name, value in request.lightning_adapter_artifact_ids.items() + } + runtime_identity = self.runtime.identity() + recipe = TrainingRecipe( + schema_version="character-training-recipe.v1", + dataset_digest=frozen.dataset_digest, + character_pack_digest=frozen.character_pack_digest, + stage_2_manifest_digest=provenance[0], + stage_2_evidence_digest=provenance[1], + stage_3_evidence_digest=provenance[2], + base_model_digest=f"sha256:{base.sha256}", + lightning_adapter_digests={ + key: f"sha256:{value.sha256}" for key, value in adapters.items() + }, + source_code_digest=f"sha256:{source.sha256}", + runtime_identity=runtime_identity, + backend_id=request.backend_id, + backend_config=request.backend_config, + trainable_targets=request.trainable_targets, + lora_rank=request.lora_rank, + lora_alpha=request.lora_alpha, + lora_dropout=request.lora_dropout, + optimizer=request.optimizer, + learning_rate=request.learning_rate, + total_steps=request.total_steps, + gradient_accumulation_steps=request.gradient_accumulation_steps, + source_tier_weights=request.source_tier_weights, + checkpoint_plan=request.checkpoint_plan, + calibration_steps=request.calibration_steps, + wall_clock_limit_seconds=request.wall_clock_limit_seconds, + seed=request.seed, + ) + validate_training_recipe_dataset(recipe, frozen) + return recipe + + def _recipe_binding( + self, request: RecipeInput, recipe: TrainingRecipe, source: Artifact + ) -> RecipeMaterialBinding: + return RecipeMaterialBinding( + stage_3_campaign_id=request.stage_3_campaign_id, + stage_3_result_id=request.stage_3_result_id, + base_model_artifact_id=request.base_model_artifact_id, + lightning_adapter_artifact_ids=dict(request.lightning_adapter_artifact_ids), + source_snapshot_artifact_id=source.id, + runtime_digest=_json_digest(dict(recipe.runtime_identity)), + ) + + def _verify_recipe_materials(self, recipe: TrainingRecipe) -> None: + self._artifact_for_digest(recipe.base_model_digest) + for digest in recipe.lightning_adapter_digests.values(): + self._artifact_for_digest(digest) + if _source_digest(self.source_root) != recipe.source_code_digest: + raise TrainingServiceError("source_drift") + if dict(self.runtime.identity()) != dict(recipe.runtime_identity): + raise TrainingServiceError("runtime_drift") + self._provenance( + recipe.dataset_digest, + campaign_id="", + expected=recipe, + ) + + def _verify_bound_artifacts( + self, + connection: sqlite3.Connection, + recipe: TrainingRecipe, + binding: RecipeMaterialBinding, + stack: ExitStack, + ) -> tuple[VerifiedArtifactHandle, ...]: + bindings = ( + ( + binding.base_model_artifact_id, + recipe.base_model_digest, + "application/octet-stream", + ), + ( + binding.source_snapshot_artifact_id, + recipe.source_code_digest, + "application/octet-stream", + ), + *tuple( + ( + artifact_id, + recipe.lightning_adapter_digests[name], + "application/octet-stream", + ) + for name, artifact_id in binding.lightning_adapter_artifact_ids.items() + ), + ) + handles: list[VerifiedArtifactHandle] = [] + for artifact_id, digest, media_type in bindings: + row = connection.execute( + "SELECT byte_size,media_type FROM artifacts WHERE id=?", (artifact_id,) + ).fetchone() + if row is None or str(row["media_type"]) != media_type: + raise TrainingServiceError("material_missing") + handles.append( + stack.enter_context( + self.artifacts.open_verified_with_connection( + connection, + artifact_id, + sha256=digest.removeprefix("sha256:"), + byte_size=int(row["byte_size"]), + media_type=media_type, + ) + ) + ) + return tuple(handles) + + def _provenance_with_connection( + self, + connection: sqlite3.Connection, + recipe: TrainingRecipe, + binding: RecipeMaterialBinding, + ) -> None: + row = connection.execute( + "SELECT result.content_digest," + "json_extract(campaign.definition_json,'$.character_pack_digest')," + "json_extract(campaign.definition_json,'$.stage_2_manifest_digest')," + "json_extract(campaign.definition_json,'$.stage_2_evidence_digest')," + "dataset.dataset_digest,dataset.character_pack_digest,dataset.character_id " + "FROM character_benchmark_campaigns campaign " + "JOIN character_benchmark_results result ON result.campaign_id=campaign.id " + "JOIN training_datasets dataset ON dataset.character_pack_storage_id=" + "campaign.character_pack_storage_id AND dataset.reviewer_id=campaign.reviewer_id " + "WHERE campaign.id=? AND campaign.reviewer_id=? " + "AND dataset.dataset_digest=? AND dataset.state='sealed' " + "AND character_benchmark_provenance_is_valid(campaign.id)=1 " + "AND json_extract(result.result_json,'$.baseline_disposition')=" + "'baseline_established' AND (SELECT new_state FROM character_benchmark_events " + "WHERE campaign_id=campaign.id ORDER BY sequence DESC LIMIT 1)='completed'", + (binding.stage_3_campaign_id, self.reviewer_id, recipe.dataset_digest), + ).fetchone() + if row is None or tuple(str(row[index]) for index in range(6)) != ( + binding.stage_3_result_id, + recipe.character_pack_digest, + recipe.stage_2_manifest_digest, + recipe.stage_2_evidence_digest, + recipe.dataset_digest, + recipe.character_pack_digest, + ): + raise TrainingServiceError("prerequisite_drift") + if binding.runtime_digest != _json_digest(dict(recipe.runtime_identity)): + raise TrainingServiceError("runtime_drift") + + def _artifact_for_digest(self, digest: str) -> Artifact: + with self.repository.database.connect() as connection: + row = connection.execute( + "SELECT id FROM artifacts WHERE sha256=?", (digest[7:],) + ).fetchone() + if row is None: + raise TrainingServiceError("material_missing") + try: + return self.artifacts.get(str(row[0])) + except ArtifactError: + raise TrainingServiceError("material_drift") from None + + def _provenance( + self, + dataset_digest: str, + *, + campaign_id: str, + expected: TrainingRecipe | None = None, + ) -> tuple[str, str, str]: + dataset = self.repository.get_dataset(dataset_digest) + with self.repository.database.connect() as connection: + row = connection.execute( + "SELECT json_extract(b.definition_json,'$.stage_2_manifest_digest')," + "json_extract(b.definition_json,'$.stage_2_evidence_digest'),r.content_digest " + "FROM character_benchmark_campaigns b JOIN character_benchmark_results r " + "ON r.campaign_id=b.id WHERE b.id=? AND b.character_pack_storage_id=(SELECT " + "character_pack_storage_id FROM training_datasets WHERE storage_id=?) " + "AND b.reviewer_id=? AND character_benchmark_provenance_is_valid(b.id)=1 " + "AND json_extract(r.result_json,'$.baseline_disposition')='baseline_established' " + "AND (SELECT new_state FROM character_benchmark_events WHERE campaign_id=b.id " + "ORDER BY sequence DESC LIMIT 1)='completed' LIMIT 1", + (campaign_id, dataset.storage_id, self.reviewer_id), + ).fetchone() + if row is None: + raise TrainingServiceError("prerequisite_missing") + values = str(row[0]), str(row[1]), str(row[2]) + if expected is not None and values != ( + expected.stage_2_manifest_digest, + expected.stage_2_evidence_digest, + expected.stage_3_evidence_digest, + ): + raise TrainingServiceError("prerequisite_drift") + return values + + def _head(self, handle: str, *, allow_sealed: bool = False) -> DatasetLineageRecord: + try: + try: + head = self.repository.get_dataset_lineage(handle) + except TrainingNotFound: + record = self.repository.get_dataset(handle) + heads = [ + x + for x in self.repository.list_dataset_lineages() + if x.current_dataset_id == record.dataset_id + ] + if len(heads) != 1: + raise TrainingServiceError("stale_dataset") from None + head = heads[0] + if handle not in (head.lineage_id, head.current_dataset_id): + raise TrainingServiceError("stale_dataset") + if head.state == "sealed" and not allow_sealed: + raise TrainingServiceError("sealed_dataset") + return head + except TrainingServiceError: + raise + except (TrainingNotFound, TrainingConflict): + raise TrainingServiceError("dataset_not_found") from None + + def _frozen(self, handle: str) -> FrozenTrainingDataset: + head = self._head(handle, allow_sealed=True) + frozen = self.repository.get_dataset(head.current_dataset_id).frozen_dataset + if frozen is None: + raise TrainingServiceError("dataset_not_sealed") + return frozen + + @staticmethod + def _recipe_view(record: RecipeRecord) -> TrainingRecipeView: + return TrainingRecipeView( + recipe_id=record.recipe_id, state=record.state, recipe=record.recipe + ) + + def _discard(self, prepared: list[tuple[Path, TrainingMedia, StagedArtifact]]) -> None: + for _, _, staged in prepared: + try: + self.artifacts.discard(staged) + except ArtifactError: + pass + + +def _imported( + *, + reviewer_id: str, + character_id: str, + lineage_id: str, + artifact: Artifact, + media: TrainingMedia, + source_lineage: str, + suggestions: TrainingCurationSuggestions | None, +) -> ImportedTrainingSample: + provisional = ImportedTrainingSample.model_construct( + import_id="pending", + reviewer_id=reviewer_id, + character_id=character_id, + lineage_id=lineage_id, + artifact_id=artifact.id, + artifact_digest=media.sha256, + artifact_byte_size=media.byte_size, + media=media, + source_lineage=source_lineage, + suggestions=suggestions, + imported_at=datetime.now(UTC), + ) + return ImportedTrainingSample.model_validate( + { + **provisional.model_dump(mode="python"), + "import_id": imported_training_sample_id(provisional), + } + ) + + +def _confirmation( + reviewer_id: str, import_id: str, decision: CurationInput +) -> TrainingCurationConfirmation: + provisional = TrainingCurationConfirmation.model_construct( + confirmation_id="pending", + import_id=import_id, + reviewer_id=reviewer_id, + decision=decision.decision, + confirmed_at=datetime.now(UTC), + version=1, + sample_id=decision.sample_id, + source_tier=decision.source_tier, + source_lineage=decision.source_lineage, + caption=decision.caption, + split=decision.split, + annotation=decision.annotation, + ) + return TrainingCurationConfirmation.model_validate( + { + **provisional.model_dump(mode="python"), + "confirmation_id": training_confirmation_id(provisional), + } + ) + + +def _sort_key(path: Path) -> tuple[str, str]: + value = unicodedata.normalize("NFC", path.as_posix()) + return value.casefold(), value + + +def _json_digest(value: dict[str, object]) -> str: + return "sha256:" + hashlib.sha256(canonical_json(value).encode()).hexdigest() + + +def _calibration_replay_is_equivalent( + operation: TrainingOrchestrationOperation, + current_payload: Mapping[str, object], +) -> bool: + stored = dict(operation.request_payload) + current = dict(current_payload) + if training_operation_payload_digest(stored) == training_operation_payload_digest(current): + return True + if "lora" in stored: + return False + legacy = dict(current) + legacy.pop("lora", None) + return training_operation_payload_digest(stored) == training_operation_payload_digest(legacy) + + +def _source_digest(root: Path) -> str: + return "sha256:" + hashlib.sha256(_source_snapshot_bytes(root)).hexdigest() + + +def _source_snapshot_bytes(root: Path) -> bytes: + """Snapshot a bounded source tree through one held root descriptor.""" + root_descriptor = -1 + try: + root_descriptor = os.open( + root, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + root_identity = _stat_identity(os.fstat(root_descriptor)) + output = bytearray(b"video-lab-source-snapshot.v1\0") + _snapshot_source_directory(root_descriptor, "", output, 0) + current = os.stat(root, follow_symlinks=False) + if _stat_identity(current) != root_identity or not stat.S_ISDIR(current.st_mode): + raise TrainingServiceError("source_drift") + return bytes(output) + except TrainingServiceError: + raise + except OSError: + raise TrainingServiceError("unsafe_source_root") from None + finally: + if root_descriptor >= 0: + os.close(root_descriptor) + + +def _snapshot_source_directory(directory: int, prefix: str, output: bytearray, total: int) -> int: + before = _stat_identity(os.fstat(directory)) + names = sorted(os.listdir(directory), key=lambda name: _source_name_key(prefix, name)) + normalized = [_source_name_key(prefix, name)[1] for name in names] + if len(set(normalized)) != len(normalized): + raise TrainingServiceError("source_normalization_collision") + for name in names: + entry = os.stat(name, dir_fd=directory, follow_symlinks=False) + relative = unicodedata.normalize("NFC", f"{prefix}/{name}" if prefix else name) + if stat.S_ISDIR(entry.st_mode): + child = os.open( + name, + os.O_RDONLY + | os.O_DIRECTORY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + dir_fd=directory, + ) + try: + total = _snapshot_source_directory(child, relative, output, total) + if _stat_identity(os.fstat(child)) != _stat_identity(entry): + raise TrainingServiceError("source_drift") + finally: + os.close(child) + elif stat.S_ISREG(entry.st_mode): + descriptor = os.open( + name, + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + dir_fd=directory, + ) + try: + chunks: list[bytes] = [] + while chunk := os.read(descriptor, 1024 * 1024): + total += len(chunk) + if total > _SOURCE_LIMIT: + raise TrainingServiceError("source_snapshot_too_large") + chunks.append(chunk) + if _stat_identity(os.fstat(descriptor)) != _stat_identity(entry): + raise TrainingServiceError("source_drift") + current = os.stat(name, dir_fd=directory, follow_symlinks=False) + if _stat_identity(current) != _stat_identity(entry): + raise TrainingServiceError("source_drift") + data = b"".join(chunks) + finally: + os.close(descriptor) + encoded = relative.encode() + output.extend(len(encoded).to_bytes(8, "big")) + output.extend(encoded) + output.extend(len(data).to_bytes(8, "big")) + output.extend(data) + else: + raise TrainingServiceError("unsafe_source_root") + if ( + _stat_identity(os.fstat(directory)) != before + or sorted(os.listdir(directory), key=lambda name: _source_name_key(prefix, name)) != names + ): + raise TrainingServiceError("source_drift") + return total + + +def _source_name_key(prefix: str, name: str) -> tuple[str, str]: + value = unicodedata.normalize("NFC", f"{prefix}/{name}" if prefix else name) + return value.casefold(), value + + +def _stat_identity(value: os.stat_result) -> tuple[int, int, int, int]: + return value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_analysis.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_analysis.py new file mode 100644 index 0000000..3d1a89c --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_analysis.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import numpy as np + +from video_lab.validation_models import ( + ArchetypeReversal, + ChallengerPosterior, + OutcomeCounts, + ValidationArchetype, + ValidationDisposition, + ValidationJudgment, + ValidationOutcome, + ValidationReason, + ValidationResult, +) + +_QUALIFICATION_THRESHOLD = 0.90 +_REVERSAL_THRESHOLD = 0.80 +_MINIMUM_REPEAT_AGREEMENTS = 3 + + +def challenger_posterior( + counts: OutcomeCounts, *, draws: int = 65536, seed: int = 73021 +) -> ChallengerPosterior: + if draws <= 0: + raise ValueError("posterior draws must be positive") + samples = np.random.default_rng(seed).dirichlet( + (counts.wins + 1, counts.ties + 1, counts.losses + 1), draws + ) + return ChallengerPosterior( + counts=counts, + superiority_probability=float(np.mean(samples[:, 0] > samples[:, 2])), + rejection_probability=float(np.mean(samples[:, 2] > samples[:, 0])), + ) + + +def analyze_validation( + judgments: tuple[ValidationJudgment, ...], repeat_agreements: int +) -> ValidationResult: + if not 0 <= repeat_agreements <= 4: + raise ValueError("repeat agreements must be between zero and four") + campaign_id = _campaign_id(judgments) + primary = tuple(judgment for judgment in judgments if not judgment.hidden_repeat) + if not _has_complete_primary_evidence(primary): + return ValidationResult( + campaign_id=campaign_id, + disposition=ValidationDisposition.INCONCLUSIVE, + reason=ValidationReason.INCOMPLETE_PRIMARY_EVIDENCE, + challengers=(), + archetype_reversals=(), + repeat_agreements=repeat_agreements, + ) + challengers = tuple( + challenger_posterior(_outcome_counts(group)).model_copy(update={"challenger": challenger}) + for challenger in sorted({judgment.challenger for judgment in primary}) + if (group := tuple(judgment for judgment in primary if judgment.challenger == challenger)) + ) + archetype_reversals = tuple( + _archetype_reversal( + archetype, tuple(judgment for judgment in primary if judgment.archetype is archetype) + ) + for archetype in ValidationArchetype + if any(judgment.archetype is archetype for judgment in primary) + ) + disposition, reason = _decision(challengers, archetype_reversals, repeat_agreements) + return ValidationResult( + campaign_id=campaign_id, + disposition=disposition, + reason=reason, + challengers=challengers, + archetype_reversals=archetype_reversals, + repeat_agreements=repeat_agreements, + ) + + +def _campaign_id(judgments: tuple[ValidationJudgment, ...]) -> str: + campaign_ids = {judgment.campaign_id for judgment in judgments} + if len(campaign_ids) != 1: + raise ValueError("validation analysis requires judgments from exactly one campaign") + return next(iter(campaign_ids)) + + +def _outcome_counts(judgments: tuple[ValidationJudgment, ...]) -> OutcomeCounts: + return OutcomeCounts( + wins=sum(judgment.outcome is ValidationOutcome.WIN for judgment in judgments), + ties=sum(judgment.outcome is ValidationOutcome.TIE for judgment in judgments), + losses=sum(judgment.outcome is ValidationOutcome.LOSS for judgment in judgments), + ) + + +def _has_complete_primary_evidence(judgments: tuple[ValidationJudgment, ...]) -> bool: + if len(judgments) != 36: + return False + challengers = {judgment.challenger for judgment in judgments} + if len(challengers) != 3: + return False + if {judgment.archetype for judgment in judgments} != set(ValidationArchetype): + return False + contexts = {(judgment.challenger, judgment.archetype, judgment.seed) for judgment in judgments} + if len(contexts) != len(judgments): + return False + if not all( + sum( + judgment.challenger == challenger and judgment.archetype is archetype + for judgment in judgments + ) + == 3 + for challenger in challengers + for archetype in ValidationArchetype + ): + return False + return all( + len( + { + frozenset( + judgment.seed + for judgment in judgments + if judgment.challenger == challenger and judgment.archetype is archetype + ) + for challenger in challengers + } + ) + == 1 + for archetype in ValidationArchetype + ) + + +def _archetype_reversal( + archetype: ValidationArchetype, judgments: tuple[ValidationJudgment, ...] +) -> ArchetypeReversal: + posterior = challenger_posterior(_outcome_counts(judgments)) + return ArchetypeReversal( + archetype=archetype, + counts=posterior.counts, + reversal_probability=posterior.rejection_probability, + ) + + +def _decision( + challengers: tuple[ChallengerPosterior, ...], + archetype_reversals: tuple[ArchetypeReversal, ...], + repeat_agreements: int, +) -> tuple[ValidationDisposition, ValidationReason]: + if any(item.rejection_probability >= _QUALIFICATION_THRESHOLD for item in challengers): + return ValidationDisposition.NOT_QUALIFIED, ValidationReason.CHALLENGER_REJECTION + if any(item.reversal_probability >= _REVERSAL_THRESHOLD for item in archetype_reversals): + return ValidationDisposition.NOT_QUALIFIED, ValidationReason.ARCHETYPE_REVERSAL + if repeat_agreements < _MINIMUM_REPEAT_AGREEMENTS: + return ValidationDisposition.INCONCLUSIVE, ValidationReason.REPEAT_RELIABILITY_FAILED + if challengers and all( + item.superiority_probability >= _QUALIFICATION_THRESHOLD for item in challengers + ): + return ValidationDisposition.QUALIFIED, ValidationReason.ALL_GATES_PASSED + return ValidationDisposition.INCONCLUSIVE, ValidationReason.INSUFFICIENT_PRIMARY_EVIDENCE diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_models.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_models.py new file mode 100644 index 0000000..7d06a41 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_models.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +import hashlib +from datetime import datetime +from enum import StrEnum +from typing import Literal, Self + +from pydantic import Field, field_validator, model_validator + +from video_lab.domain import FrozenModel, Manifest, _aware_utc, canonical_json + +VALIDATION_CAMPAIGN_SCHEMA_VERSION: Literal["validation-campaign.v1"] = "validation-campaign.v1" +VALIDATION_PANEL_SCHEMA_VERSION: Literal["validation-panel.v1"] = "validation-panel.v1" +VALIDATION_PROTOCOL_VERSION: Literal["balanced-validation.v1"] = "balanced-validation.v1" +VALIDATION_ANALYSIS_VERSION: Literal["dirichlet-validation.v1"] = "dirichlet-validation.v1" + + +class ValidationIneligible(ValueError): + """Raised when Stage 1 evidence cannot enter the fixed validation protocol.""" + + +class ValidationCampaignState(StrEnum): + DRAFT = "draft" + PREREGISTERED = "preregistered" + GENERATING = "generating" + READY = "ready" + JUDGING = "judging" + ANALYZING = "analyzing" + QUALIFIED = "qualified" + NOT_QUALIFIED = "not_qualified" + INCONCLUSIVE = "inconclusive" + + +class ValidationArchetype(StrEnum): + PORTRAIT_MOTION = "portrait_motion" + FULL_BODY_ACTION = "full_body_action" + CAMERA_MOTION = "camera_motion" + ENVIRONMENT_INTERACTION = "environment_interaction" + + +class ValidationChoice(StrEnum): + A = "a" + B = "b" + TIE = "tie" + + +class ValidationOutcome(StrEnum): + WIN = "win" + TIE = "tie" + LOSS = "loss" + + +class ValidationDisposition(StrEnum): + QUALIFIED = "qualified" + NOT_QUALIFIED = "not_qualified" + INCONCLUSIVE = "inconclusive" + + +class ValidationReason(StrEnum): + ALL_GATES_PASSED = "all_gates_passed" + CHALLENGER_REJECTION = "challenger_rejection" + ARCHETYPE_REVERSAL = "archetype_reversal" + REPEAT_RELIABILITY_FAILED = "repeat_reliability_failed" + INCOMPLETE_PRIMARY_EVIDENCE = "incomplete_primary_evidence" + INSUFFICIENT_PRIMARY_EVIDENCE = "insufficient_primary_evidence" + + +class ValidationCandidates(FrozenModel): + winner: float + lower: float + higher: float + distant: float + + @model_validator(mode="after") + def require_three_distinct_challengers(self) -> Self: + values = (self.winner, self.lower, self.higher, self.distant) + if len(set(values)) != len(values): + raise ValueError("validation candidates must be distinct") + return self + + @property + def challengers(self) -> tuple[float, float, float]: + return (self.lower, self.higher, self.distant) + + +class ValidationPrompt(FrozenModel): + archetype: ValidationArchetype + prompt: str = Field(min_length=1) + negative_prompt: str | None = None + seeds: tuple[int, int, int] + + @field_validator("negative_prompt") + @classmethod + def normalize_negative_prompt(cls, value: str | None) -> str | None: + return value or None + + @model_validator(mode="after") + def require_distinct_seeds(self) -> Self: + if len(set(self.seeds)) != len(self.seeds): + raise ValueError("validation prompt seeds must be distinct") + return self + + +class ValidationPanelInput(FrozenModel): + prompts: tuple[ValidationPrompt, ValidationPrompt, ValidationPrompt, ValidationPrompt] + schedule_seed: int + + @model_validator(mode="after") + def require_each_archetype_once(self) -> Self: + if set(prompt.archetype for prompt in self.prompts) != set(ValidationArchetype): + raise ValueError("validation panel input must contain every archetype exactly once") + return self + + +class ValidationCampaignDefinition(FrozenModel): + id: str = Field(min_length=1) + schema_version: Literal["validation-campaign.v1"] = VALIDATION_CAMPAIGN_SCHEMA_VERSION + stage_1_study_id: str = Field(min_length=1) + stage_1_evidence_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + base_manifest: Manifest + candidates: ValidationCandidates + panel_input: ValidationPanelInput + protocol_version: Literal["balanced-validation.v1"] = VALIDATION_PROTOCOL_VERSION + analysis_version: Literal["dirichlet-validation.v1"] = VALIDATION_ANALYSIS_VERSION + created_at: datetime + + @field_validator("created_at") + @classmethod + def normalize_created_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + +class ValidationTrial(FrozenModel): + id: str = Field(min_length=1) + ordinal: int = Field(gt=0) + archetype: ValidationArchetype + prompt: str = Field(min_length=1) + negative_prompt: str | None = None + seed: int + challenger: float + hidden_repeat: bool = False + reuses_trial_id: str | None = None + winner_display_position: Literal["left", "right"] + + @model_validator(mode="after") + def require_repeat_provenance(self) -> Self: + if self.hidden_repeat != (self.reuses_trial_id is not None): + raise ValueError("only hidden repeats may reuse a primary trial") + if self.reuses_trial_id == self.id: + raise ValueError("a repeat trial cannot reuse itself") + return self + + +class ValidationPanel(FrozenModel): + campaign_id: str = Field(min_length=1) + schema_version: Literal["validation-panel.v1"] = VALIDATION_PANEL_SCHEMA_VERSION + trials: tuple[ValidationTrial, ...] + content_digest: str = Field(pattern=r"^sha256:[0-9a-f]{64}$") + + @property + def challengers(self) -> tuple[float, ...]: + return tuple(sorted({trial.challenger for trial in self.trials})) + + +class ValidationEvent(FrozenModel): + campaign_id: str = Field(min_length=1) + sequence: int = Field(gt=0) + previous_state: ValidationCampaignState + new_state: ValidationCampaignState + created_at: datetime + + @field_validator("created_at") + @classmethod + def normalize_created_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + @model_validator(mode="after") + def require_state_transition(self) -> Self: + if self.previous_state is self.new_state: + raise ValueError("validation event must change state") + return self + + +class ValidationTrialBinding(FrozenModel): + campaign_id: str = Field(min_length=1) + trial_id: str = Field(min_length=1) + experiment_id: str | None = None + review_id: str | None = None + reused_from_trial_id: str | None = None + + +class ValidationExperimentBinding(FrozenModel): + campaign_id: str = Field(min_length=1) + trial_id: str = Field(min_length=1) + experiment_id: str = Field(min_length=1) + + +class ValidationJudgment(FrozenModel): + campaign_id: str = Field(min_length=1) + trial_id: str = Field(min_length=1) + ordinal: int = Field(gt=0) + archetype: ValidationArchetype + challenger: float + seed: int + hidden_repeat: bool + winner_display_position: Literal["left", "right"] + choice: ValidationChoice + created_at: datetime + + @field_validator("created_at") + @classmethod + def normalize_created_at(cls, value: datetime) -> datetime: + return _aware_utc(value) + + @property + def outcome(self) -> ValidationOutcome: + if self.choice is ValidationChoice.TIE: + return ValidationOutcome.TIE + winner_is_left = self.winner_display_position == "left" + if (self.choice is ValidationChoice.A) == winner_is_left: + return ValidationOutcome.WIN + return ValidationOutcome.LOSS + + +class OutcomeCounts(FrozenModel): + wins: int = Field(ge=0) + ties: int = Field(ge=0) + losses: int = Field(ge=0) + + @property + def total(self) -> int: + return self.wins + self.ties + self.losses + + +class ChallengerPosterior(FrozenModel): + challenger: float | None = None + counts: OutcomeCounts + superiority_probability: float = Field(ge=0, le=1) + rejection_probability: float = Field(ge=0, le=1) + + +class ArchetypeReversal(FrozenModel): + archetype: ValidationArchetype + counts: OutcomeCounts + reversal_probability: float = Field(ge=0, le=1) + + +class ValidationResult(FrozenModel): + campaign_id: str = Field(min_length=1) + disposition: ValidationDisposition + reason: ValidationReason + challengers: tuple[ChallengerPosterior, ...] + archetype_reversals: tuple[ArchetypeReversal, ...] + repeat_agreements: int = Field(ge=0, le=4) + + +def validation_panel_digest(campaign_id: str, trials: tuple[ValidationTrial, ...]) -> str: + payload = { + "schema_version": VALIDATION_PANEL_SCHEMA_VERSION, + "campaign_id": campaign_id, + "trials": [trial.model_dump(mode="json") for trial in trials], + } + return f"sha256:{hashlib.sha256(canonical_json(payload).encode()).hexdigest()}" diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_panel.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_panel.py new file mode 100644 index 0000000..c694a72 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_panel.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import hashlib +import random + +from video_lab.adaptive_study_models import AdaptiveOptimizerStep +from video_lab.validation_models import ( + ValidationCampaignDefinition, + ValidationCandidates, + ValidationIneligible, + ValidationPanel, + ValidationTrial, + validation_panel_digest, +) + + +def derive_validation_candidates(step: AdaptiveOptimizerStep) -> ValidationCandidates: + grid = step.posterior.grid + winner = step.posterior.recommendation + try: + index = grid.index(winner) + except ValueError as error: + raise ValidationIneligible( + "Stage 1 winner must appear in the frozen candidate grid" + ) from error + if index == 0 or index == len(grid) - 1: + raise ValidationIneligible("Stage 1 winner must be an interior candidate") + excluded = {winner, grid[index - 1], grid[index + 1]} + distant_candidates = tuple(value for value in grid if value not in excluded) + if not distant_candidates: + raise ValidationIneligible("Stage 1 winner must have a distant challenger") + distant = min( + distant_candidates, + key=lambda value: (-step.posterior.mean[grid.index(value)], value), + ) + return ValidationCandidates( + winner=winner, + lower=grid[index - 1], + higher=grid[index + 1], + distant=distant, + ) + + +def build_validation_panel(definition: ValidationCampaignDefinition) -> ValidationPanel: + scheduler = random.Random(definition.panel_input.schedule_seed) + primary_trials: list[ValidationTrial] = [] + repeat_sources: dict[str, list[ValidationTrial]] = {} + for prompt in definition.panel_input.prompts: + sources: list[ValidationTrial] = [] + for challenger in definition.candidates.challengers: + for seed in prompt.seeds: + trial = ValidationTrial( + id=(f"{definition.id}:primary:{prompt.archetype.value}:{challenger!r}:{seed}"), + ordinal=len(primary_trials) + 1, + archetype=prompt.archetype, + prompt=prompt.prompt, + negative_prompt=prompt.negative_prompt, + seed=seed, + challenger=challenger, + winner_display_position="left", + ) + primary_trials.append(trial) + sources.append(trial) + repeat_sources[prompt.archetype.value] = sorted( + sources, key=lambda trial: (trial.challenger, trial.seed) + ) + + repeat_trials: list[ValidationTrial] = [] + for prompt in definition.panel_input.prompts: + source = repeat_sources[prompt.archetype.value][ + scheduler.randrange(len(repeat_sources[prompt.archetype.value])) + ] + repeat_trials.append( + ValidationTrial( + id=f"{definition.id}:repeat:{prompt.archetype.value}", + ordinal=len(primary_trials) + len(repeat_trials) + 1, + archetype=source.archetype, + prompt=source.prompt, + negative_prompt=source.negative_prompt, + seed=source.seed, + challenger=source.challenger, + hidden_repeat=True, + reuses_trial_id=source.id, + winner_display_position="left", + ) + ) + + presentation = primary_trials + repeat_trials + scheduler.shuffle(presentation) + display_positions = _balanced_display_positions( + presentation, _display_seed(definition.panel_input.schedule_seed) + ) + trials = tuple( + trial.model_copy( + update={ + "ordinal": ordinal, + "winner_display_position": display_positions[trial.id], + } + ) + for ordinal, trial in enumerate(presentation, start=1) + ) + _validate_display_balance(trials) + return ValidationPanel( + campaign_id=definition.id, + trials=trials, + content_digest=validation_panel_digest(definition.id, trials), + ) + + +def _display_seed(schedule_seed: int) -> int: + payload = f"balanced-validation-display.v1:{schedule_seed}".encode() + return int.from_bytes(hashlib.sha256(payload).digest()[:8], "big") + + +def _balanced_display_positions(trials: list[ValidationTrial], seed: int) -> dict[str, str]: + generator = random.Random(seed) + by_challenger: dict[float, list[ValidationTrial]] = {} + for trial in trials: + by_challenger.setdefault(trial.challenger, []).append(trial) + positions: dict[str, str] = {} + odd_challengers: list[float] = [] + for challenger, group in by_challenger.items(): + count = len(group) + labels = ["left"] * (count // 2) + ["right"] * (count // 2) + if count % 2: + odd_challengers.append(challenger) + generator.shuffle(labels) + for trial, position in zip(group, labels, strict=True): + positions[trial.id] = position + required_left = len(trials) // 2 + additional_left = required_left - sum(position == "left" for position in positions.values()) + generator.shuffle(odd_challengers) + for challenger in odd_challengers[:additional_left]: + trial = next(trial for trial in by_challenger[challenger] if trial.id not in positions) + positions[trial.id] = "left" + for challenger in odd_challengers[additional_left:]: + trial = next(trial for trial in by_challenger[challenger] if trial.id not in positions) + positions[trial.id] = "right" + return positions + + +def _validate_display_balance(trials: tuple[ValidationTrial, ...]) -> None: + positions = [trial.winner_display_position for trial in trials] + if abs(positions.count("left") - positions.count("right")) > 1: + raise ValidationIneligible("validation display schedule is not globally balanced") + for challenger in {trial.challenger for trial in trials}: + challenger_positions = [ + trial.winner_display_position for trial in trials if trial.challenger == challenger + ] + if abs(challenger_positions.count("left") - challenger_positions.count("right")) > 1: + raise ValidationIneligible("validation display schedule is not challenger balanced") diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_repository.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_repository.py new file mode 100644 index 0000000..d95b2db --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_repository.py @@ -0,0 +1,641 @@ +from __future__ import annotations + +# ruff: noqa: E501 +import json +import sqlite3 +from typing import cast + +from video_lab.database import Database +from video_lab.domain import canonical_json +from video_lab.validation_models import ( + ValidationCampaignDefinition, + ValidationCampaignState, + ValidationEvent, + ValidationExperimentBinding, + ValidationJudgment, + ValidationPanel, + ValidationResult, + ValidationTrial, + ValidationTrialBinding, + validation_panel_digest, +) + + +class ValidationNotFound(LookupError): + pass + + +class ValidationConflict(RuntimeError): + pass + + +class ValidationRepository: + """Append-only repository for the fixed balanced-validation protocol.""" + + def __init__(self, database: Database) -> None: + self.database = database + + def create_or_get_bound( + self, definition: ValidationCampaignDefinition, *, reviewer_id: str + ) -> ValidationCampaignDefinition: + validated = ValidationCampaignDefinition.model_validate( + definition.model_dump(round_trip=True) + ) + _require_reviewer(reviewer_id) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + row = connection.execute( + "SELECT * FROM validation_campaigns WHERE id = ?", (validated.id,) + ).fetchone() + if row is None: + connection.execute( + "INSERT INTO validation_campaigns(" + "id, schema_version, reviewer_id, stage_1_study_id, " + "stage_1_evidence_digest, base_manifest_json, candidates_json, " + "panel_input_json, protocol_version, analysis_version, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + validated.id, + validated.schema_version, + reviewer_id, + validated.stage_1_study_id, + validated.stage_1_evidence_digest, + canonical_json(validated.base_manifest), + canonical_json(validated.candidates), + canonical_json(validated.panel_input), + validated.protocol_version, + validated.analysis_version, + validated.created_at.isoformat(), + ), + ) + persisted = validated + else: + persisted = _definition_from_row(row) + if not _same_campaign_definition(persisted, validated): + raise ValidationConflict( + "validation campaign id already has a different immutable definition" + ) + if str(row["reviewer_id"]) != reviewer_id: + raise ValidationConflict("validation campaign belongs to another reviewer") + connection.commit() + except Exception: + connection.rollback() + raise + return persisted + + def bind_experiment_or_verify( + self, binding: ValidationExperimentBinding + ) -> ValidationExperimentBinding: + validated = ValidationExperimentBinding.model_validate(binding.model_dump()) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_campaign(connection, validated.campaign_id) + existing = connection.execute( + "SELECT * FROM validation_experiment_bindings " + "WHERE campaign_id = ? AND trial_id = ?", + (validated.campaign_id, validated.trial_id), + ).fetchone() + if existing is None: + connection.execute( + "INSERT INTO validation_experiment_bindings(" + "campaign_id, trial_id, experiment_id) VALUES (?, ?, ?)", + ( + validated.campaign_id, + validated.trial_id, + validated.experiment_id, + ), + ) + persisted = validated + else: + persisted = _experiment_binding_from_row(existing) + if persisted != validated: + raise ValidationConflict( + "validation experiment binding retry does not match evidence" + ) + connection.commit() + except sqlite3.IntegrityError as error: + connection.rollback() + raise ValidationConflict(str(error)) from error + except Exception: + connection.rollback() + raise + return persisted + + def list_experiment_bindings(self, campaign_id: str) -> tuple[ValidationExperimentBinding, ...]: + with self.database.connect() as connection: + self._require_campaign(connection, campaign_id) + rows = connection.execute( + "SELECT binding.* FROM validation_experiment_bindings AS binding " + "JOIN validation_trials AS trial " + "ON trial.campaign_id=binding.campaign_id AND trial.trial_id=binding.trial_id " + "WHERE binding.campaign_id=? ORDER BY trial.ordinal", + (campaign_id,), + ).fetchall() + return tuple(_experiment_binding_from_row(row) for row in rows) + + def preregister_or_verify(self, campaign_id: str, panel: ValidationPanel) -> ValidationEvent: + validated = ValidationPanel.model_validate(panel.model_dump(round_trip=True)) + if validated.campaign_id != campaign_id: + raise ValidationConflict("validation panel belongs to another campaign") + if validation_panel_digest(campaign_id, validated.trials) != validated.content_digest: + raise ValidationConflict("validation panel digest does not match frozen trials") + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + campaign = self._require_campaign(connection, campaign_id) + existing = connection.execute( + "SELECT * FROM validation_events WHERE campaign_id = ? ORDER BY sequence", + (campaign_id,), + ).fetchall() + if not existing: + event = ValidationEvent( + campaign_id=campaign_id, + sequence=1, + previous_state=ValidationCampaignState.DRAFT, + new_state=ValidationCampaignState.PREREGISTERED, + created_at=_definition_from_row(campaign).created_at, + ) + self._insert_event(connection, event) + self._insert_panel(connection, validated) + persisted = event + else: + persisted = _event_from_row(existing[0]) + if ( + persisted.sequence != 1 + or persisted.previous_state is not ValidationCampaignState.DRAFT + or persisted.new_state is not ValidationCampaignState.PREREGISTERED + or self._panel_from_rows(connection, campaign_id) != validated + ): + raise ValidationConflict( + "validation preregistration does not match evidence" + ) + connection.commit() + except Exception: + connection.rollback() + raise + return persisted + + def append_event_or_verify(self, event: ValidationEvent) -> ValidationEvent: + validated = ValidationEvent.model_validate(event.model_dump(round_trip=True)) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_campaign(connection, validated.campaign_id) + existing = connection.execute( + "SELECT * FROM validation_events WHERE campaign_id = ? AND sequence = ?", + (validated.campaign_id, validated.sequence), + ).fetchone() + if existing is not None: + persisted = _event_from_row(existing) + if not _same_event(persisted, validated): + raise ValidationConflict( + "validation event retry does not match persisted evidence" + ) + else: + self._insert_event(connection, validated) + persisted = validated + connection.commit() + except sqlite3.IntegrityError as error: + connection.rollback() + raise ValidationConflict(str(error)) from error + except Exception: + connection.rollback() + raise + return persisted + + def bind_trial_or_verify(self, trial: ValidationTrialBinding) -> ValidationTrialBinding: + validated = ValidationTrialBinding.model_validate(trial) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_campaign(connection, validated.campaign_id) + existing = connection.execute( + "SELECT * FROM validation_trial_bindings WHERE campaign_id = ? AND trial_id = ?", + (validated.campaign_id, validated.trial_id), + ).fetchone() + if existing is None: + trial_row = connection.execute( + "SELECT reuses_trial_id FROM validation_trials " + "WHERE campaign_id = ? AND trial_id = ?", + (validated.campaign_id, validated.trial_id), + ).fetchone() + if trial_row is None: + raise ValidationConflict("validation trial does not belong to campaign") + if trial_row["reuses_trial_id"] != validated.reused_from_trial_id: + raise ValidationConflict( + "validation trial binding has mismatched repeat provenance" + ) + connection.execute( + "INSERT INTO validation_trial_bindings(" + "campaign_id, trial_id, experiment_id, review_id, reused_from_trial_id) " + "VALUES (?, ?, ?, ?, ?)", + ( + validated.campaign_id, + validated.trial_id, + validated.experiment_id, + validated.review_id, + validated.reused_from_trial_id, + ), + ) + persisted = validated + else: + persisted = _binding_from_row(existing) + if persisted != validated: + raise ValidationConflict( + "validation trial binding retry does not match evidence" + ) + connection.commit() + except sqlite3.IntegrityError as error: + connection.rollback() + raise ValidationConflict(str(error)) from error + except Exception: + connection.rollback() + raise + return persisted + + def consume_judgment_or_verify( + self, judgment: ValidationJudgment + ) -> tuple[ValidationJudgment, bool]: + validated = ValidationJudgment.model_validate(judgment.model_dump(round_trip=True)) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_campaign(connection, validated.campaign_id) + existing = connection.execute( + "SELECT * FROM validation_judgments WHERE campaign_id = ? AND trial_id = ?", + (validated.campaign_id, validated.trial_id), + ).fetchone() + if existing is not None: + persisted = _judgment_from_row(existing) + if not _same_judgment(persisted, validated): + raise ValidationConflict( + "validation judgment retry does not match persisted choice" + ) + consumed = False + else: + connection.execute( + "INSERT INTO validation_judgments(" + "campaign_id, trial_id, ordinal, archetype, challenger, seed, hidden_repeat, " + "winner_display_position, choice, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + validated.campaign_id, + validated.trial_id, + validated.ordinal, + validated.archetype.value, + validated.challenger, + validated.seed, + int(validated.hidden_repeat), + validated.winner_display_position, + validated.choice.value, + validated.created_at.isoformat(), + ), + ) + persisted = validated + consumed = True + connection.commit() + except sqlite3.IntegrityError as error: + connection.rollback() + raise ValidationConflict(str(error)) from error + except Exception: + connection.rollback() + raise + return persisted, consumed + + def append_result_or_verify(self, result: ValidationResult) -> ValidationResult: + validated = ValidationResult.model_validate(result.model_dump(round_trip=True)) + with self.database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + self._require_campaign(connection, validated.campaign_id) + existing = connection.execute( + "SELECT * FROM validation_results WHERE campaign_id = ?", + (validated.campaign_id,), + ).fetchone() + if existing is None: + connection.execute( + "INSERT INTO validation_results(" + "campaign_id, disposition, reason, challengers_json, archetype_reversals_json, " + "repeat_agreements, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + validated.campaign_id, + validated.disposition.value, + validated.reason.value, + json.dumps( + [item.model_dump(mode="json") for item in validated.challengers], + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ), + json.dumps( + [ + item.model_dump(mode="json") + for item in validated.archetype_reversals + ], + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ), + validated.repeat_agreements, + self._latest_event_created_at(connection, validated.campaign_id), + ), + ) + persisted = validated + else: + persisted = _result_from_row(existing) + if persisted != validated: + raise ValidationConflict("validation result retry does not match evidence") + connection.commit() + except sqlite3.IntegrityError as error: + connection.rollback() + raise ValidationConflict(str(error)) from error + except Exception: + connection.rollback() + raise + return persisted + + def get(self, campaign_id: str) -> ValidationCampaignDefinition: + with self.database.connect() as connection: + row = self._require_campaign(connection, campaign_id) + return _definition_from_row(row) + + def get_bound(self, campaign_id: str, *, reviewer_id: str) -> ValidationCampaignDefinition: + _require_reviewer(reviewer_id) + with self.database.connect() as connection: + row = self._require_campaign(connection, campaign_id) + if str(row["reviewer_id"]) != reviewer_id: + raise ValidationNotFound(f"validation campaign not found: {campaign_id}") + return _definition_from_row(row) + + def list_bound( + self, *, reviewer_id: str, limit: int = 50 + ) -> tuple[ValidationCampaignDefinition, ...]: + _require_reviewer(reviewer_id) + if not 1 <= limit <= 100: + raise ValueError("validation campaign list limit must be between 1 and 100") + with self.database.connect() as connection: + rows = connection.execute( + "SELECT * FROM validation_campaigns WHERE reviewer_id = ? " + "ORDER BY created_at DESC, id DESC LIMIT ?", + (reviewer_id, limit), + ).fetchall() + return tuple(_definition_from_row(row) for row in rows) + + def get_panel(self, campaign_id: str) -> ValidationPanel: + with self.database.connect() as connection: + self._require_campaign(connection, campaign_id) + return self._panel_from_rows(connection, campaign_id) + + def state(self, campaign_id: str) -> ValidationCampaignState: + events = self.list_events(campaign_id) + return ValidationCampaignState.DRAFT if not events else events[-1].new_state + + def list_events(self, campaign_id: str) -> tuple[ValidationEvent, ...]: + with self.database.connect() as connection: + self._require_campaign(connection, campaign_id) + rows = connection.execute( + "SELECT * FROM validation_events WHERE campaign_id = ? ORDER BY sequence", + (campaign_id,), + ).fetchall() + return tuple(_event_from_row(row) for row in rows) + + def list_trials(self, campaign_id: str) -> tuple[ValidationTrialBinding, ...]: + with self.database.connect() as connection: + self._require_campaign(connection, campaign_id) + rows = connection.execute( + "SELECT trial.campaign_id, trial.trial_id, binding.experiment_id, binding.review_id, " + "COALESCE(binding.reused_from_trial_id, trial.reuses_trial_id) AS reused_from_trial_id " + "FROM validation_trials AS trial LEFT JOIN validation_trial_bindings AS binding " + "ON binding.campaign_id = trial.campaign_id AND binding.trial_id = trial.trial_id " + "WHERE trial.campaign_id = ? ORDER BY trial.ordinal", + (campaign_id,), + ).fetchall() + return tuple(_binding_from_row(row) for row in rows) + + def list_judgments(self, campaign_id: str) -> tuple[ValidationJudgment, ...]: + with self.database.connect() as connection: + self._require_campaign(connection, campaign_id) + rows = connection.execute( + "SELECT * FROM validation_judgments WHERE campaign_id = ? ORDER BY ordinal", + (campaign_id,), + ).fetchall() + return tuple(_judgment_from_row(row) for row in rows) + + def get_result(self, campaign_id: str) -> ValidationResult | None: + with self.database.connect() as connection: + self._require_campaign(connection, campaign_id) + row = connection.execute( + "SELECT * FROM validation_results WHERE campaign_id = ?", (campaign_id,) + ).fetchone() + return None if row is None else _result_from_row(row) + + @staticmethod + def _require_campaign(connection: sqlite3.Connection, campaign_id: str) -> sqlite3.Row: + row = connection.execute( + "SELECT * FROM validation_campaigns WHERE id = ?", (campaign_id,) + ).fetchone() + if row is None: + raise ValidationNotFound(f"validation campaign not found: {campaign_id}") + return cast(sqlite3.Row, row) + + @staticmethod + def _latest_event_created_at(connection: sqlite3.Connection, campaign_id: str) -> str: + row = connection.execute( + "SELECT created_at FROM validation_events WHERE campaign_id = ? " + "ORDER BY sequence DESC LIMIT 1", + (campaign_id,), + ).fetchone() + if row is None: + raise ValidationConflict("validation result has no lifecycle evidence") + return str(row["created_at"]) + + @staticmethod + def _insert_event(connection: sqlite3.Connection, event: ValidationEvent) -> None: + connection.execute( + "INSERT INTO validation_events(" + "campaign_id, sequence, previous_state, new_state, created_at) VALUES (?, ?, ?, ?, ?)", + ( + event.campaign_id, + event.sequence, + event.previous_state.value, + event.new_state.value, + event.created_at.isoformat(), + ), + ) + + @staticmethod + def _insert_panel(connection: sqlite3.Connection, panel: ValidationPanel) -> None: + primary = sorted( + (trial for trial in panel.trials if not trial.hidden_repeat), + key=lambda item: item.ordinal, + ) + repeats = sorted( + (trial for trial in panel.trials if trial.hidden_repeat), key=lambda item: item.ordinal + ) + for trial in (*primary, *repeats): + connection.execute( + "INSERT INTO validation_trials(" + "campaign_id, trial_id, ordinal, archetype, prompt, negative_prompt, seed, challenger, " + "hidden_repeat, reuses_trial_id, winner_display_position, panel_digest) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + panel.campaign_id, + trial.id, + trial.ordinal, + trial.archetype.value, + trial.prompt, + trial.negative_prompt or "", + trial.seed, + trial.challenger, + int(trial.hidden_repeat), + trial.reuses_trial_id, + trial.winner_display_position, + panel.content_digest, + ), + ) + + @staticmethod + def _panel_from_rows(connection: sqlite3.Connection, campaign_id: str) -> ValidationPanel: + rows = connection.execute( + "SELECT * FROM validation_trials WHERE campaign_id = ? ORDER BY ordinal", (campaign_id,) + ).fetchall() + if not rows: + raise ValidationConflict("validation preregistration has no trial evidence") + digests = {str(row["panel_digest"]) for row in rows} + if len(digests) != 1: + raise ValidationConflict("validation preregistration has inconsistent panel digests") + return ValidationPanel( + campaign_id=campaign_id, + trials=tuple(_trial_from_row(row) for row in rows), + content_digest=next(iter(digests)), + ) + + +def _require_reviewer(reviewer_id: str) -> None: + if not reviewer_id or reviewer_id != reviewer_id.strip(): + raise ValidationConflict("validation campaign owner must be a normalized identity") + + +def _definition_from_row(row: sqlite3.Row) -> ValidationCampaignDefinition: + return ValidationCampaignDefinition.model_validate( + { + "id": row["id"], + "schema_version": row["schema_version"], + "stage_1_study_id": row["stage_1_study_id"], + "stage_1_evidence_digest": row["stage_1_evidence_digest"], + "base_manifest": json.loads(str(row["base_manifest_json"])), + "candidates": json.loads(str(row["candidates_json"])), + "panel_input": json.loads(str(row["panel_input_json"])), + "protocol_version": row["protocol_version"], + "analysis_version": row["analysis_version"], + "created_at": row["created_at"], + } + ) + + +def _same_campaign_definition( + left: ValidationCampaignDefinition, right: ValidationCampaignDefinition +) -> bool: + return left.model_dump(mode="json", exclude={"created_at"}) == right.model_dump( + mode="json", exclude={"created_at"} + ) + + +def _experiment_binding_from_row(row: sqlite3.Row) -> ValidationExperimentBinding: + return ValidationExperimentBinding( + campaign_id=str(row["campaign_id"]), + trial_id=str(row["trial_id"]), + experiment_id=str(row["experiment_id"]), + ) + + +def _event_from_row(row: sqlite3.Row) -> ValidationEvent: + return ValidationEvent.model_validate( + { + "campaign_id": row["campaign_id"], + "sequence": row["sequence"], + "previous_state": row["previous_state"], + "new_state": row["new_state"], + "created_at": row["created_at"], + } + ) + + +def _trial_from_row(row: sqlite3.Row) -> ValidationTrial: + return ValidationTrial.model_validate( + { + "id": row["trial_id"], + "ordinal": row["ordinal"], + "archetype": row["archetype"], + "prompt": row["prompt"], + "negative_prompt": row["negative_prompt"] or None, + "seed": row["seed"], + "challenger": row["challenger"], + "hidden_repeat": bool(row["hidden_repeat"]), + "reuses_trial_id": row["reuses_trial_id"], + "winner_display_position": row["winner_display_position"], + } + ) + + +def _binding_from_row(row: sqlite3.Row) -> ValidationTrialBinding: + return ValidationTrialBinding.model_validate( + { + "campaign_id": row["campaign_id"], + "trial_id": row["trial_id"], + "experiment_id": row["experiment_id"], + "review_id": row["review_id"], + "reused_from_trial_id": row["reused_from_trial_id"], + } + ) + + +def _judgment_from_row(row: sqlite3.Row) -> ValidationJudgment: + return ValidationJudgment.model_validate( + { + "campaign_id": row["campaign_id"], + "trial_id": row["trial_id"], + "ordinal": row["ordinal"], + "archetype": row["archetype"], + "challenger": row["challenger"], + "seed": row["seed"], + "hidden_repeat": bool(row["hidden_repeat"]), + "winner_display_position": row["winner_display_position"], + "choice": row["choice"], + "created_at": row["created_at"], + } + ) + + +def _result_from_row(row: sqlite3.Row) -> ValidationResult: + return ValidationResult.model_validate( + { + "campaign_id": row["campaign_id"], + "disposition": row["disposition"], + "reason": row["reason"], + "challengers": json.loads(str(row["challengers_json"])), + "archetype_reversals": json.loads(str(row["archetype_reversals_json"])), + "repeat_agreements": row["repeat_agreements"], + } + ) + + +def _same_event(left: ValidationEvent, right: ValidationEvent) -> bool: + return ( + left.campaign_id, + left.sequence, + left.previous_state, + left.new_state, + ) == ( + right.campaign_id, + right.sequence, + right.previous_state, + right.new_state, + ) + + +def _same_judgment(left: ValidationJudgment, right: ValidationJudgment) -> bool: + return left.model_dump(mode="json", exclude={"created_at"}) == right.model_dump( + mode="json", exclude={"created_at"} + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_service.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_service.py new file mode 100644 index 0000000..3b5e268 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/validation_service.py @@ -0,0 +1,818 @@ +from __future__ import annotations + +import hashlib +from collections.abc import Callable +from enum import StrEnum +from typing import Any, Literal, cast + +from video_lab.adaptive_study_models import AdaptiveStudyState +from video_lab.artifacts import ArtifactError +from video_lab.domain import ( + DisplayPreference, + ExperimentDraft, + FrozenModel, + JobState, + canonical_json, + utc_now, +) +from video_lab.repositories import RecordNotFound, validation_experiment_id +from video_lab.validation_analysis import analyze_validation +from video_lab.validation_models import ( + ValidationCampaignDefinition, + ValidationCampaignState, + ValidationCandidates, + ValidationChoice, + ValidationDisposition, + ValidationEvent, + ValidationIneligible, + ValidationJudgment, + ValidationPanelInput, + ValidationResult, + ValidationTrial, + ValidationTrialBinding, +) +from video_lab.validation_panel import build_validation_panel, derive_validation_candidates +from video_lab.validation_repository import ( + ValidationConflict, + ValidationNotFound, + ValidationRepository, +) + + +class ValidationProgress(FrozenModel): + completed_judgments: int + total_judgments: Literal[40] = 40 + generated_primary: int + total_primary: Literal[36] = 36 + + +class ValidationCampaignPreview(FrozenModel): + primary_count: Literal[36] = 36 + repeat_count: Literal[4] = 4 + candidates: ValidationCandidates + + +class ValidationCampaignView(FrozenModel): + campaign_id: str + state: ValidationCampaignState + next_action: str + progress: ValidationProgress + preview: ValidationCampaignPreview | None = None + review_id: str | None = None + media: tuple[ValidationMediaReference, ...] = () + result: ValidationResult | None = None + + +class ValidationMediaReference(FrozenModel): + position: Literal["left", "right"] + media_ref: str + + +class ValidationCampaignListItem(FrozenModel): + campaign_id: str + state: ValidationCampaignState + next_action: str + progress: ValidationProgress + + +class ValidationNextAction(StrEnum): + PREREGISTER = "preregister" + COMPLETE_GENERATION = "complete_generation" + RECORD_CHOICE = "record_choice" + ANALYZE = "analyze" + NONE = "none" + + +class ValidationCampaignService: + def __init__( + self, + campaigns: ValidationRepository, + stage_1: Any, + experiments: Any, + reviews: Any, + adapters: Any, + *, + reviewer_id: str, + clock: Callable[[], Any] = utc_now, + ) -> None: + self.campaigns = campaigns + self.stage_1 = stage_1 + self.experiments = experiments + self.reviews = reviews + self.adapters = adapters + self.reviewer_id = reviewer_id + self.clock = clock + + def create( + self, *, stage_1_id: str, panel_input: ValidationPanelInput + ) -> ValidationCampaignView: + validated_input = ValidationPanelInput.model_validate( + panel_input.model_dump(round_trip=True) + ) + try: + stage_view = self.stage_1.show(stage_1_id) + except (LookupError, PermissionError, ValueError) as error: + raise ValidationIneligible("Stage 1 evidence is not owned and terminal") from error + if stage_view.state not in { + AdaptiveStudyState.SEARCH_CONVERGED, + AdaptiveStudyState.SEARCH_BUDGET_EXHAUSTED, + AdaptiveStudyState.COMPLETED, + }: + raise ValidationIneligible("Stage 1 evidence is not terminal") + evidence_digest = stage_view.evidence_digest + if not isinstance(evidence_digest, str) or not evidence_digest.startswith("sha256:"): + raise ValidationIneligible("Stage 1 evidence digest is unavailable") + steps = self.stage_1.studies.list_optimizer_steps(stage_1_id) + if not steps: + raise ValidationIneligible("Stage 1 has no persisted terminal optimizer step") + candidates = derive_validation_candidates(steps[-1]) + stage_definition = self.stage_1.studies.get(stage_1_id) + campaign_id = validation_campaign_id( + self.reviewer_id, + evidence_digest.removeprefix("sha256:"), + validation_panel_input_digest(validated_input), + ) + try: + created_at = self.campaigns.get(campaign_id).created_at + except ValidationNotFound: + created_at = self.clock() + definition = ValidationCampaignDefinition( + id=campaign_id, + stage_1_study_id=stage_1_id, + stage_1_evidence_digest=evidence_digest.removeprefix("sha256:"), + base_manifest=stage_definition.base_manifest, + candidates=candidates, + panel_input=validated_input, + created_at=created_at, + ) + persisted = self.campaigns.create_or_get_bound(definition, reviewer_id=self.reviewer_id) + return self._draft_view(persisted) + + def preview(self, campaign_id: str) -> ValidationCampaignPreview: + definition = self.campaigns.get_bound(campaign_id, reviewer_id=self.reviewer_id) + if self.campaigns.state(campaign_id) is not ValidationCampaignState.DRAFT: + raise PermissionError("validation candidates are sealed after preregistration") + return ValidationCampaignPreview(candidates=definition.candidates) + + def preregister(self, campaign_id: str) -> ValidationCampaignView: + definition = self.campaigns.get_bound(campaign_id, reviewer_id=self.reviewer_id) + state = self.campaigns.state(campaign_id) + if state is ValidationCampaignState.DRAFT: + self.campaigns.preregister_or_verify(campaign_id, build_validation_panel(definition)) + state = ValidationCampaignState.PREREGISTERED + if state is ValidationCampaignState.PREREGISTERED: + self.campaigns.append_event_or_verify( + ValidationEvent( + campaign_id=campaign_id, + sequence=2, + previous_state=ValidationCampaignState.PREREGISTERED, + new_state=ValidationCampaignState.GENERATING, + created_at=self.clock(), + ) + ) + state = ValidationCampaignState.GENERATING + if state is not ValidationCampaignState.GENERATING: + return self.show(campaign_id) + if self.experiments is None or self.adapters is None: + raise RuntimeError("validation generation dependencies are unavailable") + capabilities = self.adapters.capabilities(definition.base_manifest.adapter_id) + panel = self.campaigns.get_panel(campaign_id) + for trial in sorted( + (item for item in panel.trials if not item.hidden_repeat), + key=lambda item: item.ordinal, + ): + baseline = definition.base_manifest.model_copy( + update={ + "prompt": trial.prompt, + "negative_prompt": trial.negative_prompt or "", + "seed": trial.seed, + "guidance_scale": definition.candidates.winner, + } + ) + self.experiments.create_validation( + ExperimentDraft( + baseline=baseline, + variable_path="guidance_scale", + alternate_value=trial.challenger, + ), + capabilities, + campaign_id=campaign_id, + trial_id=trial.id, + ) + return self.show(campaign_id) + + def show(self, campaign_id: str) -> ValidationCampaignView: + definition = self.campaigns.get_bound(campaign_id, reviewer_id=self.reviewer_id) + state = self.campaigns.state(campaign_id) + if state is ValidationCampaignState.DRAFT: + return self._draft_view(definition) + generated = self._generated_primary_count(definition) + completed = len(self.campaigns.list_judgments(campaign_id)) + next_action = ( + ValidationNextAction.COMPLETE_GENERATION + if state + in { + ValidationCampaignState.PREREGISTERED, + ValidationCampaignState.GENERATING, + } + else ValidationNextAction.RECORD_CHOICE + if state in {ValidationCampaignState.READY, ValidationCampaignState.JUDGING} + and completed < 40 + else ValidationNextAction.ANALYZE + if state in {ValidationCampaignState.JUDGING, ValidationCampaignState.ANALYZING} + and completed == 40 + else ValidationNextAction.NONE + ) + return ValidationCampaignView( + campaign_id=campaign_id, + state=state, + next_action=next_action, + progress=ValidationProgress( + completed_judgments=completed, + generated_primary=generated, + ), + preview=None, + result=self.campaigns.get_result(campaign_id), + ) + + def list(self, *, limit: int = 50) -> tuple[ValidationCampaignListItem, ...]: + return tuple( + ValidationCampaignListItem( + campaign_id=view.campaign_id, + state=view.state, + next_action=view.next_action, + progress=view.progress, + ) + for definition in self.campaigns.list_bound(reviewer_id=self.reviewer_id, limit=limit) + for view in (self.show(definition.id),) + ) + + def resume(self, campaign_id: str) -> ValidationCampaignView: + definition = self.campaigns.get_bound(campaign_id, reviewer_id=self.reviewer_id) + state = self.campaigns.state(campaign_id) + if state is ValidationCampaignState.DRAFT: + return self._draft_view(definition) + if state in { + ValidationCampaignState.PREREGISTERED, + ValidationCampaignState.GENERATING, + }: + self.preregister(campaign_id) + panel = self.campaigns.get_panel(campaign_id) + all_jobs = tuple( + job + for trial in panel.trials + if not trial.hidden_repeat + for job in self.experiments.get_validation( + validation_experiment_id(campaign_id, trial.id) + ).jobs + ) + if len(all_jobs) != 72 or any(job.state is not JobState.COMPLETED for job in all_jobs): + return self.show(campaign_id) + self.campaigns.append_event_or_verify( + ValidationEvent( + campaign_id=campaign_id, + sequence=3, + previous_state=ValidationCampaignState.GENERATING, + new_state=ValidationCampaignState.READY, + created_at=self.clock(), + ) + ) + state = ValidationCampaignState.READY + if state is ValidationCampaignState.READY: + self.campaigns.append_event_or_verify( + ValidationEvent( + campaign_id=campaign_id, + sequence=4, + previous_state=ValidationCampaignState.READY, + new_state=ValidationCampaignState.JUDGING, + created_at=self.clock(), + ) + ) + state = ValidationCampaignState.JUDGING + if state is ValidationCampaignState.JUDGING: + judgments = self.campaigns.list_judgments(campaign_id) + if len(judgments) == 40: + return self.analyze(campaign_id) + trial = self._next_trial(campaign_id, judgments) + if self.reviews is None: + raise RuntimeError("validation review dependency is unavailable") + review = self.reviews.start_validation( + campaign_id=campaign_id, + trial_id=trial.id, + reviewer_id=self.reviewer_id, + ) + view = self.show(campaign_id) + return view.model_copy( + update={ + "review_id": review.session_id, + "media": tuple( + ValidationMediaReference( + position=item.display_position, + media_ref=item.url, + ) + for item in review.media + ), + } + ) + if state is ValidationCampaignState.ANALYZING: + return self.analyze(campaign_id) + return self.show(campaign_id) + + def record_choice( + self, + campaign_id: str, + choice: ValidationChoice | str, + *, + review_id: str, + ) -> ValidationCampaignView: + self.campaigns.get_bound(campaign_id, reviewer_id=self.reviewer_id) + if self.reviews is None: + raise RuntimeError("validation review dependency is unavailable") + try: + requested_trial_id = self.reviews.validation_trial_for_session( + campaign_id=campaign_id, + session_id=review_id, + reviewer_id=self.reviewer_id, + ) + except LookupError as error: + raise ValidationConflict("validation review does not belong to campaign") from error + validated_choice = ValidationChoice(choice) + judgments = self.campaigns.list_judgments(campaign_id) + existing = next((item for item in judgments if item.trial_id == requested_trial_id), None) + state = self.campaigns.state(campaign_id) + if existing is not None: + if existing.choice is not validated_choice: + raise ValidationConflict( + "validation judgment retry does not match persisted choice" + ) + if state not in { + ValidationCampaignState.JUDGING, + ValidationCampaignState.ANALYZING, + ValidationCampaignState.QUALIFIED, + ValidationCampaignState.NOT_QUALIFIED, + ValidationCampaignState.INCONCLUSIVE, + }: + raise ValidationConflict("validation campaign is not accepting judgments") + trial = next( + item + for item in self.campaigns.get_panel(campaign_id).trials + if item.id == requested_trial_id + ) + else: + if state is not ValidationCampaignState.JUDGING: + raise ValidationConflict("validation campaign is not accepting judgments") + if len(judgments) >= 40: + raise ValidationConflict("validation review has no persisted judgment") + trial = self._next_trial(campaign_id, judgments) + if requested_trial_id != trial.id: + raise ValidationConflict("validation review is not the next scheduled trial") + preference = { + ValidationChoice.A: DisplayPreference.LEFT, + ValidationChoice.B: DisplayPreference.RIGHT, + ValidationChoice.TIE: DisplayPreference.TIE, + }[validated_choice] + try: + revision = self.reviews.submit_validation( + campaign_id=campaign_id, + trial_id=trial.id, + reviewer_id=self.reviewer_id, + preference=preference, + ) + except (LookupError, ValueError) as error: + raise ValidationConflict("validation review retry does not match evidence") from error + if trial.hidden_repeat: + if trial.reuses_trial_id is None: + raise ValidationConflict("validation repeat has no source trial") + experiment_id = validation_experiment_id(campaign_id, trial.reuses_trial_id) + bound_review_id = revision.id + else: + experiment_id = validation_experiment_id(campaign_id, trial.id) + bound_review_id = revision.id + if experiment_id is not None and bound_review_id is not None: + self.campaigns.bind_trial_or_verify( + ValidationTrialBinding( + campaign_id=campaign_id, + trial_id=trial.id, + experiment_id=experiment_id, + review_id=bound_review_id, + reused_from_trial_id=trial.reuses_trial_id, + ) + ) + self.campaigns.consume_judgment_or_verify( + ValidationJudgment( + campaign_id=campaign_id, + trial_id=trial.id, + ordinal=trial.ordinal, + archetype=trial.archetype, + challenger=trial.challenger, + seed=trial.seed, + hidden_repeat=trial.hidden_repeat, + winner_display_position=trial.winner_display_position, + choice=validated_choice, + created_at=self.clock(), + ) + ) + return self.resume(campaign_id) + + def analyze(self, campaign_id: str) -> ValidationCampaignView: + self.campaigns.get_bound(campaign_id, reviewer_id=self.reviewer_id) + state = self.campaigns.state(campaign_id) + if state in { + ValidationCampaignState.QUALIFIED, + ValidationCampaignState.NOT_QUALIFIED, + ValidationCampaignState.INCONCLUSIVE, + }: + return self.show(campaign_id) + judgments = self.campaigns.list_judgments(campaign_id) + if len(judgments) != 40: + raise ValidationIneligible("validation analysis requires exactly 40 judgments") + if state is ValidationCampaignState.JUDGING: + self.campaigns.append_event_or_verify( + ValidationEvent( + campaign_id=campaign_id, + sequence=5, + previous_state=ValidationCampaignState.JUDGING, + new_state=ValidationCampaignState.ANALYZING, + created_at=self.clock(), + ) + ) + elif state is not ValidationCampaignState.ANALYZING: + raise ValidationConflict("validation campaign cannot enter analysis") + result = self.campaigns.get_result(campaign_id) + if result is None: + result = self.campaigns.append_result_or_verify( + analyze_validation( + judgments, + repeat_agreements=self._repeat_agreements(campaign_id, judgments), + ) + ) + terminal = { + ValidationDisposition.QUALIFIED: ValidationCampaignState.QUALIFIED, + ValidationDisposition.NOT_QUALIFIED: ValidationCampaignState.NOT_QUALIFIED, + ValidationDisposition.INCONCLUSIVE: ValidationCampaignState.INCONCLUSIVE, + }[result.disposition] + self.campaigns.append_event_or_verify( + ValidationEvent( + campaign_id=campaign_id, + sequence=6, + previous_state=ValidationCampaignState.ANALYZING, + new_state=terminal, + created_at=self.clock(), + ) + ) + return self.show(campaign_id) + + def export_evidence(self, campaign_id: str) -> dict[str, object]: + definition = self.campaigns.get_bound(campaign_id, reviewer_id=self.reviewer_id) + state = self.campaigns.state(campaign_id) + if state not in { + ValidationCampaignState.QUALIFIED, + ValidationCampaignState.NOT_QUALIFIED, + ValidationCampaignState.INCONCLUSIVE, + }: + raise PermissionError("validation evidence remains sealed before terminal analysis") + panel = self.campaigns.get_panel(campaign_id) + result = self.campaigns.get_result(campaign_id) + if result is None: + raise ValidationConflict("terminal validation campaign has no result") + provenance = self._evidence_provenance(campaign_id) + return { + "schema_version": "balanced-validation-evidence.v2", + "campaign": definition.model_dump(mode="json"), + "panel": panel.model_dump(mode="json"), + "events": [ + event.model_dump(mode="json") for event in self.campaigns.list_events(campaign_id) + ], + "judgments": [ + judgment.model_dump(mode="json") + for judgment in self.campaigns.list_judgments(campaign_id) + ], + **provenance, + "provenance_digest": "sha256:" + + hashlib.sha256(canonical_json(provenance).encode()).hexdigest(), + "result": result.model_dump(mode="json"), + } + + def _evidence_provenance(self, campaign_id: str) -> dict[str, object]: + self.campaigns.get_bound(campaign_id, reviewer_id=self.reviewer_id) + panel = self.campaigns.get_panel(campaign_id) + judgments = self.campaigns.list_judgments(campaign_id) + with self.campaigns.database.connect() as connection: + experiments = [ + dict(row) + for row in connection.execute( + "SELECT binding.campaign_id, binding.trial_id, binding.experiment_id " + "FROM validation_experiment_bindings AS binding " + "JOIN validation_trials AS trial ON trial.campaign_id=binding.campaign_id " + "AND trial.trial_id=binding.trial_id WHERE binding.campaign_id=? " + "ORDER BY trial.ordinal", + (campaign_id,), + ) + ] + rows = connection.execute( + "SELECT binding.campaign_id, binding.trial_id, binding.experiment_id, " + "binding.review_id, binding.reused_from_trial_id, trial.ordinal, " + "trial.winner_display_position, judgment.choice, review.session_id, " + "review.revision, review.display_preference, review.preferred_variant_id, " + "review.supersedes_review_id, session.reviewer_id, " + "session.experiment_id AS session_experiment_id, " + "session.left_variant_id, session.left_job_id, session.left_artifact_id, " + "session.left_artifact_sha256, session.right_variant_id, session.right_job_id, " + "session.right_artifact_id, session.right_artifact_sha256, " + "baseline.id AS baseline_id, left_artifact.sha256 AS left_persisted_sha256, " + "right_artifact.sha256 AS right_persisted_sha256, " + "left_variant.experiment_id AS left_variant_experiment_id, " + "right_variant.experiment_id AS right_variant_experiment_id, " + "left_job.state AS left_job_state, right_job.state AS right_job_state " + "FROM validation_trial_bindings AS binding " + "JOIN validation_trials AS trial ON trial.campaign_id=binding.campaign_id " + "AND trial.trial_id=binding.trial_id " + "JOIN validation_judgments AS judgment ON judgment.campaign_id=binding.campaign_id " + "AND judgment.trial_id=binding.trial_id " + "JOIN review_revisions AS review ON review.id=binding.review_id " + "JOIN review_sessions AS session ON session.id=review.session_id " + "JOIN variants AS left_variant ON left_variant.id=session.left_variant_id " + "JOIN variants AS right_variant ON right_variant.id=session.right_variant_id " + "JOIN variants AS baseline ON baseline.experiment_id=binding.experiment_id " + "AND baseline.label='A' " + "JOIN jobs AS left_job ON left_job.id=session.left_job_id " + "AND left_job.variant_id=session.left_variant_id " + "JOIN jobs AS right_job ON right_job.id=session.right_job_id " + "AND right_job.variant_id=session.right_variant_id " + "JOIN artifacts AS left_artifact ON left_artifact.id=session.left_artifact_id " + "JOIN artifacts AS right_artifact ON right_artifact.id=session.right_artifact_id " + "JOIN job_artifacts AS left_link ON left_link.job_id=session.left_job_id " + "AND left_link.artifact_id=session.left_artifact_id " + "JOIN job_artifacts AS right_link ON right_link.job_id=session.right_job_id " + "AND right_link.artifact_id=session.right_artifact_id " + "WHERE binding.campaign_id=? ORDER BY trial.ordinal", + (campaign_id,), + ).fetchall() + campaign_sessions = connection.execute( + "SELECT DISTINCT session.id FROM review_sessions AS session " + "JOIN validation_trial_bindings AS binding " + "ON binding.experiment_id=session.experiment_id " + "JOIN validation_trials AS trial ON trial.campaign_id=binding.campaign_id " + "AND trial.trial_id=binding.trial_id " + "WHERE binding.campaign_id=? AND session.reviewer_id=" + "validation_session_reviewer(binding.campaign_id, binding.trial_id, ?, " + "trial.hidden_repeat) ORDER BY session.id", + (campaign_id, self.reviewer_id), + ).fetchall() + campaign_revision_ids = connection.execute( + "SELECT review.id FROM review_revisions AS review " + "WHERE review.session_id IN (" + "SELECT DISTINCT session.id FROM review_sessions AS session " + "JOIN validation_trial_bindings AS binding " + "ON binding.experiment_id=session.experiment_id " + "JOIN validation_trials AS trial ON trial.campaign_id=binding.campaign_id " + "AND trial.trial_id=binding.trial_id " + "WHERE binding.campaign_id=? AND session.reviewer_id=" + "validation_session_reviewer(binding.campaign_id, binding.trial_id, ?, " + "trial.hidden_repeat)) ORDER BY review.id", + (campaign_id, self.reviewer_id), + ).fetchall() + trial_by_id = {trial.id: trial for trial in panel.trials} + judgment_by_trial = {judgment.trial_id: judgment for judgment in judgments} + primary_ids = {trial.id for trial in panel.trials if not trial.hidden_repeat} + experiment_trial_ids = {str(item["trial_id"]) for item in experiments} + row_trial_ids = {str(row["trial_id"]) for row in rows} + bound_review_ids = {str(row["review_id"]) for row in rows} + bound_session_ids = {str(row["session_id"]) for row in rows} + if ( + len(panel.trials) != 40 + or len(primary_ids) != 36 + or len(experiments) != 36 + or experiment_trial_ids != primary_ids + or any( + str(item["experiment_id"]) + != validation_experiment_id(campaign_id, str(item["trial_id"])) + for item in experiments + ) + or len(judgments) != 40 + or set(judgment_by_trial) != set(trial_by_id) + or len(rows) != 40 + or len(row_trial_ids) != 40 + or row_trial_ids != set(trial_by_id) + or len(bound_review_ids) != 40 + or len(bound_session_ids) != 40 + or len(campaign_sessions) != 40 + or len(campaign_revision_ids) != 40 + or {str(row["id"]) for row in campaign_revision_ids} != bound_review_ids + ): + raise ValidationConflict("validation evidence provenance is incomplete") + bindings: list[dict[str, object]] = [] + reviews: list[dict[str, object]] = [] + for row in rows: + trial = trial_by_id[str(row["trial_id"])] + source_trial_id = trial.reuses_trial_id or trial.id + expected_experiment = validation_experiment_id(campaign_id, source_trial_id) + expected_reviewer = _validation_session_reviewer_identity( + campaign_id, + trial.id, + self.reviewer_id, + hidden_repeat=trial.hidden_repeat, + ) + if ( + str(row["experiment_id"]) != expected_experiment + or str(row["session_experiment_id"]) != expected_experiment + or str(row["left_variant_experiment_id"]) != expected_experiment + or str(row["right_variant_experiment_id"]) != expected_experiment + or row["reused_from_trial_id"] != trial.reuses_trial_id + or str(row["reviewer_id"]) != expected_reviewer + or int(row["revision"]) != 1 + or row["supersedes_review_id"] is not None + or str(row["left_job_state"]) != "completed" + or str(row["right_job_state"]) != "completed" + or str(row["left_artifact_sha256"]) != str(row["left_persisted_sha256"]) + or str(row["right_artifact_sha256"]) != str(row["right_persisted_sha256"]) + ): + raise ValidationConflict("validation evidence provenance is inconsistent") + bindings.append( + { + "campaign_id": str(row["campaign_id"]), + "trial_id": str(row["trial_id"]), + "experiment_id": str(row["experiment_id"]), + "review_id": str(row["review_id"]), + "reused_from_trial_id": row["reused_from_trial_id"], + } + ) + actual_winner = ( + "left" if str(row["left_variant_id"]) == str(row["baseline_id"]) else "right" + ) + swap = actual_winner != str(row["winner_display_position"]) + display_mapping: dict[str, dict[str, str]] = {} + for position in ("left", "right"): + underlying = ("right" if position == "left" else "left") if swap else position + display_mapping[position] = { + "variant_id": str(row[f"{underlying}_variant_id"]), + "job_id": str(row[f"{underlying}_job_id"]), + "artifact_id": str(row[f"{underlying}_artifact_id"]), + "artifact_sha256": str(row[f"{underlying}_artifact_sha256"]), + } + reviews.append( + { + "trial_id": str(row["trial_id"]), + "review_id": str(row["review_id"]), + "session_id": str(row["session_id"]), + "revision": int(row["revision"]), + "choice": str(row["choice"]), + "persisted_display_preference": str(row["display_preference"]), + "preferred_variant_id": row["preferred_variant_id"], + "display_mapping": display_mapping, + } + ) + public_preference = {"a": "left", "b": "right", "tie": "tie"}[str(row["choice"])] + preferred_variant = row["preferred_variant_id"] + if public_preference == "tie": + preference_valid = preferred_variant is None + else: + preference_valid = ( + str(preferred_variant) == display_mapping[public_preference]["variant_id"] + ) + if not preference_valid: + raise ValidationConflict("validation evidence provenance choice is inconsistent") + persisted_preference = str(row["display_preference"]) + if persisted_preference == "tie": + persisted_preference_valid = preferred_variant is None + elif persisted_preference in {"left", "right"}: + persisted_preference_valid = str(preferred_variant) == str( + row[f"{persisted_preference}_variant_id"] + ) + else: + persisted_preference_valid = False + if not persisted_preference_valid: + raise ValidationConflict("validation review revision provenance is inconsistent") + binding_by_trial = {str(item["trial_id"]): item for item in bindings} + review_by_id = {str(item["review_id"]): item for item in reviews} + for trial in panel.trials: + if not trial.hidden_repeat or trial.reuses_trial_id is None: + continue + repeat_binding = binding_by_trial[trial.id] + source_binding = binding_by_trial[trial.reuses_trial_id] + repeat_review = review_by_id[str(repeat_binding["review_id"])] + source_review = review_by_id[str(source_binding["review_id"])] + repeat_mapping = cast(dict[str, dict[str, str]], repeat_review["display_mapping"]) + source_mapping = cast(dict[str, dict[str, str]], source_review["display_mapping"]) + repeat_artifacts = { + (item["artifact_id"], item["artifact_sha256"]) for item in repeat_mapping.values() + } + source_artifacts = { + (item["artifact_id"], item["artifact_sha256"]) for item in source_mapping.values() + } + if ( + repeat_binding["experiment_id"] != source_binding["experiment_id"] + or repeat_binding["review_id"] == source_binding["review_id"] + or repeat_review["session_id"] == source_review["session_id"] + or repeat_artifacts != source_artifacts + ): + raise ValidationConflict("validation repeat provenance is inconsistent") + if self.reviews is None: + raise ValidationConflict("validation evidence provenance resolver is unavailable") + artifact_ids = { + item["artifact_id"] + for review in reviews + for item in cast(dict[str, dict[str, str]], review["display_mapping"]).values() + } + try: + for artifact_id in sorted(artifact_ids): + self.reviews.artifacts.resolve(artifact_id) + except (ArtifactError, OSError) as error: + raise ValidationConflict( + "validation evidence provenance media is not canonical" + ) from error + return { + "experiment_bindings": experiments, + "trial_bindings": bindings, + "reviews": reviews, + } + + def _generated_primary_count(self, definition: ValidationCampaignDefinition) -> int: + if self.experiments is None: + return 0 + try: + panel = self.campaigns.get_panel(definition.id) + except Exception: + return 0 + count = 0 + for trial in panel.trials: + if trial.hidden_repeat: + continue + try: + self.experiments.experiments.get_validation( + validation_experiment_id(definition.id, trial.id) + ) + except RecordNotFound: + continue + count += 1 + return count + + def _next_trial( + self, campaign_id: str, judgments: tuple[ValidationJudgment, ...] + ) -> ValidationTrial: + consumed = {item.trial_id for item in judgments} + return next( + trial + for trial in self.campaigns.get_panel(campaign_id).trials + if trial.id not in consumed + ) + + def _repeat_agreements( + self, campaign_id: str, judgments: tuple[ValidationJudgment, ...] + ) -> int: + by_trial = {item.trial_id: item for item in judgments} + panel = self.campaigns.get_panel(campaign_id) + agreements = 0 + for trial in panel.trials: + if not trial.hidden_repeat or trial.reuses_trial_id is None: + continue + if by_trial[trial.id].outcome is by_trial[trial.reuses_trial_id].outcome: + agreements += 1 + return agreements + + @staticmethod + def _draft_view(definition: ValidationCampaignDefinition) -> ValidationCampaignView: + return ValidationCampaignView( + campaign_id=definition.id, + state=ValidationCampaignState.DRAFT, + next_action=ValidationNextAction.PREREGISTER, + progress=ValidationProgress(completed_judgments=0, generated_primary=0), + preview=ValidationCampaignPreview(candidates=definition.candidates), + ) + + +def validation_panel_input_digest(panel_input: ValidationPanelInput) -> str: + payload = { + "schema_version": "validation-panel-input.v1", + "panel_input": panel_input.model_dump(mode="json"), + } + return hashlib.sha256(canonical_json(payload).encode()).hexdigest() + + +def _validation_session_reviewer_identity( + campaign_id: str, + trial_id: str, + reviewer_id: str, + *, + hidden_repeat: bool, +) -> str: + if not hidden_repeat: + return reviewer_id + payload = f"validation-review-session.v1\x00{campaign_id}\x00{trial_id}\x00{reviewer_id}" + return "validation_reviewer_" + hashlib.sha256(payload.encode()).hexdigest() + + +def validation_campaign_id( + reviewer_id: str, stage_1_evidence_digest: str, panel_input_digest: str +) -> str: + normalized = reviewer_id.strip() + if not normalized or normalized != reviewer_id: + raise ValidationIneligible("validation reviewer id must be normalized") + payload = { + "schema_version": "validation-campaign-id.v1", + "domain": "video-lab.validation.campaign", + "reviewer_id": normalized, + "stage_1_evidence_digest": stage_1_evidence_digest, + "panel_input_digest": panel_input_digest, + } + return "validation_" + hashlib.sha256(canonical_json(payload).encode()).hexdigest() diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/wan_pending.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/wan_pending.py new file mode 100644 index 0000000..56a5b3f --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/wan_pending.py @@ -0,0 +1,174 @@ +"""Canonical validation for durable Wan pending-gradient records.""" + +from __future__ import annotations + +import base64 +import hashlib +import math +import sys +from collections.abc import Collection, Mapping +from dataclasses import dataclass +from typing import Literal, cast + +_DTYPE_WIDTHS = {"bfloat16": 2, "float16": 2, "float32": 4, "float64": 8} +_DEVICES = {"cpu", "mps"} + + +@dataclass(frozen=True, slots=True) +class TrainableParameterSpec: + physical_name: str + logical_target: str + role: Literal["A", "B"] + shape: tuple[int, ...] + dtype: str + device: str + + +def validate_trainable_parameter_specs( + specs: Collection[TrainableParameterSpec], *, declared_targets: Collection[str] +) -> tuple[TrainableParameterSpec, ...]: + owned = tuple(specs) + declared = tuple(declared_targets) + if ( + not owned + or not declared + or len(set(declared)) != len(declared) + or any(type(target) is not str or not target for target in declared) + or any(type(item) is not TrainableParameterSpec for item in owned) + or len({item.physical_name for item in owned}) != len(owned) + or {item.logical_target for item in owned} != set(declared) + ): + raise ValueError + for item in owned: + if ( + not item.physical_name + or item.role not in {"A", "B"} + or not item.shape + or any(type(dimension) is not int or dimension <= 0 for dimension in item.shape) + or item.dtype not in _DTYPE_WIDTHS + or item.device not in _DEVICES + ): + raise ValueError + for target in declared: + bound = tuple(item for item in owned if item.logical_target == target) + if len(bound) != 2 or {item.role for item in bound} != {"A", "B"}: + raise ValueError + return tuple(sorted(owned, key=lambda item: item.physical_name)) + + +def trainable_parameter_spec_records( + specs: Collection[TrainableParameterSpec], +) -> list[dict[str, object]]: + return [ + { + "physical_name": item.physical_name, + "logical_target": item.logical_target, + "role": item.role, + "shape": list(item.shape), + "dtype": item.dtype, + "device": item.device, + } + for item in sorted(specs, key=lambda value: value.physical_name) + ] + + +def trainable_parameter_specs_from_records( + records: object, +) -> tuple[TrainableParameterSpec, ...]: + if not isinstance(records, list): + raise ValueError + specs: list[TrainableParameterSpec] = [] + for raw in records: + if not isinstance(raw, Mapping) or set(raw) != { + "physical_name", "logical_target", "role", "shape", "dtype", "device" + }: + raise ValueError + shape = raw.get("shape") + role = raw.get("role") + if ( + type(raw.get("physical_name")) is not str + or type(raw.get("logical_target")) is not str + or role not in {"A", "B"} + or not isinstance(shape, list) + or any(type(item) is not int for item in shape) + or type(raw.get("dtype")) is not str + or type(raw.get("device")) is not str + ): + raise ValueError + specs.append( + TrainableParameterSpec( + physical_name=cast(str, raw["physical_name"]), + logical_target=cast(str, raw["logical_target"]), + role=cast(Literal["A", "B"], role), + shape=tuple(cast(list[int], shape)), + dtype=cast(str, raw["dtype"]), + device=cast(str, raw["device"]), + ) + ) + return tuple(specs) + + +def validate_pending_gradients( + value: object, + position: int, + *, + expected_parameters: Collection[str], + expected_specs: Mapping[str, tuple[tuple[int, ...], str, str]] | None = None, +) -> Mapping[str, object]: + if not isinstance(value, Mapping) or set(value) != { + "accumulation_position", + "parameters", + }: + raise ValueError + parameters = value.get("parameters") + expected = set(expected_parameters) + if ( + type(position) is not int + or position < 0 + or value.get("accumulation_position") != position + or not isinstance(parameters, Mapping) + or set(parameters) != (expected if position else set()) + or (expected_specs is not None and set(expected_specs) != expected) + ): + raise ValueError + for name in expected if position else (): + raw = parameters[name] + if not isinstance(raw, Mapping) or set(raw) != { + "digest", + "serialized_b64", + "shape", + "dtype", + "device", + }: + raise ValueError + encoded = raw.get("serialized_b64") + if type(encoded) is not str: + raise ValueError + try: + payload = base64.b64decode(encoded, validate=True) + except (TypeError, ValueError): + raise ValueError from None + shape_value = raw.get("shape") + dtype = raw.get("dtype") + device = raw.get("device") + if ( + not isinstance(shape_value, list) + or not shape_value + or any(type(dimension) is not int or dimension <= 0 for dimension in shape_value) + or type(dtype) is not str + or dtype not in _DTYPE_WIDTHS + or type(device) is not str + or device not in _DEVICES + ): + raise ValueError + shape = tuple(cast(list[int], shape_value)) + elements = math.prod(shape) + if ( + elements > sys.maxsize // _DTYPE_WIDTHS[dtype] + or len(payload) != elements * _DTYPE_WIDTHS[dtype] + or raw.get("digest") != "sha256:" + hashlib.sha256(payload).hexdigest() + ): + raise ValueError + if expected_specs is not None and (shape, dtype, device) != expected_specs[name]: + raise ValueError + return value diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/worker.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/worker.py new file mode 100644 index 0000000..e81d6dc --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/worker.py @@ -0,0 +1,525 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import shutil +import subprocess +from collections.abc import Callable +from dataclasses import dataclass +from fractions import Fraction +from pathlib import Path +from time import monotonic +from typing import Literal + +import imageio_ffmpeg # type: ignore[import-untyped] + +from video_lab.adapter_protocol import ( + AdapterExecutionError, + GenerationAdapter, + GenerationCancelled, + GenerationProgress, + GenerationRequest, + GenerationResult, +) +from video_lab.adapter_registry import AdapterRegistry, UnknownAdapterError +from video_lab.artifacts import ArtifactStore +from video_lab.capabilities import validate_manifest_support +from video_lab.database import Database +from video_lab.domain import Job, JobState, RunAttemptStatus +from video_lab.feasibility_repository import FeasibilityRepository +from video_lab.repositories import JobRepository +from video_lab.telemetry import TelemetrySampler + +WorkerStatus = Literal["idle", "completed", "failed", "cancelled"] + + +@dataclass(frozen=True) +class WorkerOutcome: + status: WorkerStatus + job_id: str | None = None + attempt_id: str | None = None + error_code: str | None = None + + +class InvalidGenerationOutput(ValueError): + """Raised when an adapter result is not the promised MP4.""" + + +class UnsupportedManifest(ValueError): + """Raised when an exact adapter cannot execute the stored manifest.""" + + +class WorkerConfigurationError(RuntimeError): + """Raised when a required local worker dependency is unavailable.""" + + +class Worker: + def __init__( + self, + *, + repository: JobRepository, + artifact_store: ArtifactStore, + adapter_registry: AdapterRegistry, + scratch_root: Path, + worker_id: str, + telemetry_sampler: TelemetrySampler | None = None, + ffprobe_resolver: Callable[[], str | None] | None = None, + ) -> None: + if not worker_id.strip(): + raise ValueError("worker_id must not be blank") + self.repository = repository + self.artifact_store = artifact_store + self.adapter_registry = adapter_registry + self.scratch_root = scratch_root + self.worker_id = worker_id + self.telemetry_sampler = telemetry_sampler or TelemetrySampler() + self.ffprobe_resolver = ffprobe_resolver or (lambda: shutil.which("ffprobe")) + self._ffprobe_executable: str | None = None + self._adapters: dict[str, GenerationAdapter] = {} + + def recover(self) -> tuple[Job, ...]: + return self.repository.recover_interrupted(self.worker_id) + + def run_until_idle(self, max_jobs: int | None = None) -> tuple[WorkerOutcome, ...]: + if max_jobs is not None and max_jobs <= 0: + raise ValueError("max_jobs must be positive") + outcomes: list[WorkerOutcome] = [] + while max_jobs is None or len(outcomes) < max_jobs: + outcome = self.run_once() + if outcome.status == "idle": + return tuple(outcomes) + outcomes.append(outcome) + if outcome.error_code == "worker_configuration_error": + return tuple(outcomes) + return tuple(outcomes) + + def run_once(self, job_id: str | None = None) -> WorkerOutcome: + claimed = self.repository.claim_next(self.worker_id, job_id) + if claimed is None: + return WorkerOutcome(status="idle") + + job_id = claimed.job.id + attempt_id = claimed.attempt.id + started = monotonic() + try: + self.repository.append_event( + job_id, + attempt_id, + "worker_process_started", + {"worker_id": self.worker_id, "process_id": os.getpid()}, + ) + self._sample_best_effort(job_id, attempt_id, started) + self._ffprobe_executable = self.ffprobe_resolver() + if self._ffprobe_executable is None: + raise WorkerConfigurationError("ffprobe is required on PATH") + manifest = self.repository.manifest_for_job(job_id) + adapter = self._adapters.get(manifest.adapter_id) + if adapter is None: + adapter = self.adapter_registry.resolve(manifest.adapter_id) + self._adapters[manifest.adapter_id] = adapter + violations = validate_manifest_support(manifest, adapter.capabilities()) + if violations: + summary = "; ".join( + f"{violation.field}: {violation.reason}" for violation in violations + ) + raise UnsupportedManifest(summary) + adapter_version = getattr(adapter, "adapter_version", None) + if adapter_version != manifest.adapter_version: + raise UnsupportedManifest( + f"adapter version mismatch: expected {manifest.adapter_version}, " + f"got {adapter_version}" + ) + if self._cancelled(job_id): + raise GenerationCancelled("cancellation requested before generation") + + request = GenerationRequest( + job_id=job_id, + attempt_id=attempt_id, + manifest=manifest, + scratch_root=self.scratch_root / attempt_id, + ) + + def progress(update: GenerationProgress) -> None: + if self._cancelled(job_id): + raise GenerationCancelled("cancellation requested") + self._advance_for_progress(job_id, update.phase) + self.repository.append_event( + job_id, + attempt_id, + "generation_progress", + { + "phase": update.phase, + "completed": update.completed, + "total": update.total, + "message": update.message, + "elapsed_seconds": update.elapsed_seconds, + }, + ) + self._sample_best_effort(job_id, attempt_id, started) + + result = adapter.generate(request, progress, lambda: self._cancelled(job_id)) + if result.effective_manifest != manifest: + raise InvalidGenerationOutput("effective manifest differs from requested manifest") + if result.adapter_version != manifest.adapter_version: + raise InvalidGenerationOutput("result adapter version differs from manifest") + self._ensure_generating(job_id) + self.repository.transition(job_id, JobState.ENCODING) + self.repository.append_event( + job_id, + attempt_id, + "encoding_started", + {"video_path": result.video_path.name}, + ) + self._sample_best_effort(job_id, attempt_id, started) + media_evidence = self._verify_result(result, request.scratch_root) + if self._cancelled(job_id): + raise GenerationCancelled("cancellation requested before artifact registration") + staged = self.artifact_store.stage(result.video_path, "video/mp4") + if self._cancelled(job_id): + staged.path.unlink(missing_ok=True) + raise GenerationCancelled("cancellation requested before artifact registration") + artifact = self.artifact_store.commit(staged, None) + self._sample_best_effort(job_id, attempt_id, started) + self.repository.finalize_attempt( + job_id, + attempt_id, + job_state=JobState.COMPLETED, + attempt_status=RunAttemptStatus.COMPLETED, + artifact_id=artifact.id, + error_code=None, + error_message=None, + event_kind="job_completed", + event_payload={ + "artifact_id": artifact.id, + "artifact_sha256": artifact.sha256, + "decoded_frame_sha256": media_evidence["decoded_frame_sha256"], + "decoded_rgb_aggregate_sha256": media_evidence["decoded_rgb_aggregate_sha256"], + "codec": media_evidence["codec"], + "duration_seconds": media_evidence["duration_seconds"], + "frame_count": result.frame_count, + "fps": result.fps, + "width": result.width, + "height": result.height, + "runtime_versions": dict(result.runtime_versions), + "adapter_telemetry": result.model_dump(mode="json")["adapter_telemetry"], + }, + ) + return WorkerOutcome("completed", job_id, attempt_id) + except GenerationCancelled as error: + return self._finish_error( + job_id, + attempt_id, + started, + status="cancelled", + error_code="cancelled", + error=error, + ) + except UnknownAdapterError as error: + return self._finish_error( + job_id, + attempt_id, + started, + status="failed", + error_code="adapter_not_found", + error=error, + ) + except UnsupportedManifest as error: + return self._finish_error( + job_id, + attempt_id, + started, + status="failed", + error_code="unsupported_manifest", + error=error, + ) + except WorkerConfigurationError as error: + return self._finish_error( + job_id, + attempt_id, + started, + status="failed", + error_code="worker_configuration_error", + error=error, + ) + except AdapterExecutionError as error: + return self._finish_error( + job_id, + attempt_id, + started, + status="failed", + error_code="adapter_execution_failed", + error=error, + ) + except InvalidGenerationOutput as error: + return self._finish_error( + job_id, + attempt_id, + started, + status="failed", + error_code="invalid_output", + error=error, + ) + except Exception as error: + return self._finish_error( + job_id, + attempt_id, + started, + status="failed", + error_code="unexpected_error", + error=error, + ) + + def _finish_error( + self, + job_id: str, + attempt_id: str, + started: float, + *, + status: Literal["failed", "cancelled"], + error_code: str, + error: Exception, + ) -> WorkerOutcome: + self._sample_best_effort(job_id, attempt_id, started) + cancelled = status == "cancelled" or self.repository.get(job_id).state is JobState.CANCELLED + terminal_state = JobState.CANCELLED if cancelled else JobState.FAILED + attempt_status = RunAttemptStatus.CANCELLED if cancelled else RunAttemptStatus.FAILED + message = str(error) or error.__class__.__name__ + self.repository.finalize_attempt( + job_id, + attempt_id, + job_state=terminal_state, + attempt_status=attempt_status, + artifact_id=None, + error_code="cancelled" if cancelled else error_code, + error_message=message, + event_kind="job_cancelled" if cancelled else "job_failed", + event_payload={ + "error_code": "cancelled" if cancelled else error_code, + "message": message, + }, + ) + outcome_status: Literal["failed", "cancelled"] = "cancelled" if cancelled else "failed" + return WorkerOutcome( + outcome_status, job_id, attempt_id, "cancelled" if cancelled else error_code + ) + + def _sample_best_effort(self, job_id: str, attempt_id: str, started: float) -> None: + try: + self.repository.append_telemetry( + self.telemetry_sampler.sample(attempt_id, max(0.0, monotonic() - started)) + ) + except Exception as error: + try: + self.repository.append_event( + job_id, + attempt_id, + "telemetry_error", + {"error_type": error.__class__.__name__, "message": str(error)}, + ) + except Exception: + pass + + def _cancelled(self, job_id: str) -> bool: + return self.repository.get(job_id).state is JobState.CANCELLED + + def _advance_for_progress(self, job_id: str, phase: str) -> None: + state = self.repository.get(job_id).state + if phase == "downloading" and state is JobState.PREPARING: + self.repository.transition(job_id, JobState.DOWNLOADING) + elif phase == "loading": + if state is JobState.PREPARING: + self.repository.transition(job_id, JobState.LOADING) + elif state is JobState.DOWNLOADING: + self.repository.transition(job_id, JobState.LOADING) + elif phase == "generating": + self._ensure_generating(job_id) + + def _ensure_generating(self, job_id: str) -> None: + state = self.repository.get(job_id).state + if state is JobState.PREPARING: + state = self.repository.transition(job_id, JobState.LOADING).state + elif state is JobState.DOWNLOADING: + state = self.repository.transition(job_id, JobState.LOADING).state + if state is JobState.LOADING: + self.repository.transition(job_id, JobState.GENERATING) + elif state is not JobState.GENERATING: + raise InvalidGenerationOutput(f"cannot enter generation from {state}") + + def _verify_result(self, result: GenerationResult, scratch_root: Path) -> dict[str, object]: + if result.video_path.suffix.lower() != ".mp4" or result.video_path.is_symlink(): + raise InvalidGenerationOutput("adapter output must be a regular .mp4 file") + try: + root = scratch_root.resolve(strict=True) + path = result.video_path.resolve(strict=True) + except (FileNotFoundError, OSError) as error: + raise InvalidGenerationOutput("adapter output is missing") from error + if not path.is_relative_to(root) or path.suffix.lower() != ".mp4" or not path.is_file(): + raise InvalidGenerationOutput("adapter output escapes attempt scratch root") + if path.stat().st_size == 0: + raise InvalidGenerationOutput("adapter output is missing or empty") + try: + ffprobe = self._ffprobe_executable + if ffprobe is None: + raise WorkerConfigurationError("ffprobe is required on PATH") + probe = subprocess.run( + [ + ffprobe, + "-v", + "error", + "-show_entries", + "format=format_name,duration:format_tags=major_brand:" + "stream=codec_type,codec_name,width,height,avg_frame_rate,nb_frames", + "-of", + "json", + str(path), + ], + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(probe.stdout) + format_names = set(str(payload["format"]["format_name"]).lower().split(",")) + if not format_names.intersection({"mov", "mp4", "m4a", "3gp", "3g2", "mj2"}): + raise InvalidGenerationOutput("adapter output container is not MP4") + duration = float(payload["format"].get("duration", 0.0)) + if not math.isfinite(duration) or duration <= 0: + raise InvalidGenerationOutput("adapter output duration must be positive") + streams = [ + stream + for stream in payload.get("streams", []) + if stream.get("codec_type") == "video" + ] + if len(streams) != 1: + raise InvalidGenerationOutput("adapter output must contain one video stream") + stream = streams[0] + if str(stream.get("codec_name", "")).lower() not in {"h264", "avc1"}: + raise InvalidGenerationOutput("adapter output video codec must be H.264") + if (int(stream["width"]), int(stream["height"])) != (result.width, result.height): + raise InvalidGenerationOutput("output stream dimensions do not match") + reported_frames = str(stream.get("nb_frames", "")) + if not math.isclose( + float(Fraction(stream["avg_frame_rate"])), result.fps, abs_tol=0.01 + ): + raise InvalidGenerationOutput("output stream fps does not match") + reader = imageio_ffmpeg.read_frames(path) + metadata = next(reader) + decoded_hashes = tuple(hashlib.sha256(frame).hexdigest() for frame in reader) + reader.close() + frame_count, _seconds = imageio_ffmpeg.count_frames_and_secs(path) + except InvalidGenerationOutput: + raise + except Exception as error: + raise InvalidGenerationOutput("adapter output is not a readable MP4") from error + codec = str(metadata.get("codec", "")) + size = metadata.get("size") + fps = float(metadata.get("fps", 0.0)) + if not codec or size != (result.width, result.height): + raise InvalidGenerationOutput("output dimensions or codec metadata do not match") + if frame_count != result.frame_count or not math.isclose(fps, result.fps, abs_tol=0.01): + raise InvalidGenerationOutput("output frame count or fps does not match") + if reported_frames not in {"", "N/A"} and int(reported_frames) != len(decoded_hashes): + raise InvalidGenerationOutput("output stream frame count does not match") + if len(decoded_hashes) != result.frame_count: + raise InvalidGenerationOutput("decoded frame count does not match") + return { + "decoded_frame_sha256": decoded_hashes, + "decoded_rgb_aggregate_sha256": hashlib.sha256( + "".join(decoded_hashes).encode("ascii") + ).hexdigest(), + "duration_seconds": duration, + "codec": str(stream.get("codec_name", "")).lower(), + } + + +def _nonblank_worker_id(value: str) -> str: + if not value.strip(): + raise argparse.ArgumentTypeError("worker ID must not be blank") + return value + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="video-lab-worker", + description=( + "Run queued jobs. The default processes one job; --until-idle keeps the worker " + "and adapter alive. --worker-id must remain stable across restarts and be unique " + "among simultaneous workers." + ), + ) + parser.add_argument("--database", type=Path, required=True) + parser.add_argument("--artifacts", type=Path, required=True) + parser.add_argument("--scratch", type=Path, required=True) + parser.add_argument("--animegen-cache", type=Path) + parser.add_argument("--until-idle", action="store_true") + parser.add_argument("--max-jobs", type=int) + parser.add_argument( + "--worker-id", + required=True, + type=_nonblank_worker_id, + help="stable restart identity, unique among simultaneously running workers", + ) + args = parser.parse_args(argv) + if args.max_jobs is not None and not args.until_idle: + parser.error("--max-jobs requires --until-idle") + if args.max_jobs is not None and args.max_jobs <= 0: + parser.error("--max-jobs must be positive") + database = Database(args.database) + database.migrate() + artifact_store = ArtifactStore(database, args.artifacts) + feasibility = FeasibilityRepository(database, artifact_store) + animegen_id = "diffusers.animegen-t2v.wan2.2-a14b.mps.v1" + authorization_bundle = feasibility.generation_authorization_bundle(animegen_id) + runtime_factory = None + if args.animegen_cache is not None and authorization_bundle is not None: + + def runtime_factory(bundle: object) -> object: + from video_lab.adapters.animegen_mps import DiffusersAnimeGenGenerationRuntime + from video_lab.feasibility_repository import GenerationAuthorizationBundle + + if not isinstance(bundle, GenerationAuthorizationBundle): + raise TypeError("AnimeGen authorization bundle is required") + + return DiffusersAnimeGenGenerationRuntime( + cache_root=args.animegen_cache, sources=bundle.resolved_sources + ) + + worker = Worker( + repository=JobRepository(database), + artifact_store=artifact_store, + adapter_registry=AdapterRegistry.with_defaults( + feasibility_repository=feasibility, + animegen_runtime_factory=runtime_factory, + ), + scratch_root=args.scratch, + worker_id=args.worker_id, + ) + worker.recover() + if args.until_idle: + outcomes = worker.run_until_idle(max_jobs=args.max_jobs) + counts = { + status: sum(outcome.status == status for outcome in outcomes) + for status in ("completed", "failed", "cancelled") + } + status = "completed" if counts["failed"] == counts["cancelled"] == 0 else "failed" + print( + json.dumps( + { + "status": status, + "processed": len(outcomes), + **counts, + "outcomes": [outcome.__dict__ for outcome in outcomes], + }, + separators=(",", ":"), + ) + ) + return 0 if status == "completed" else 1 + outcome = worker.run_once() + print(json.dumps(outcome.__dict__, separators=(",", ":"))) + return 0 if outcome.status in {"idle", "completed"} else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/worker_control.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/worker_control.py new file mode 100644 index 0000000..86acafe --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/worker_control.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from typing import Protocol + +from video_lab.domain import Job + + +class SupervisorBusy(RuntimeError): + """Raised when persisted or in-memory worker state blocks a launch.""" + + +class SupervisorShutdownError(RuntimeError): + """Raised when an owned worker cannot be reaped within shutdown bounds.""" + + +class WorkerControl(Protocol): + def status(self) -> dict[str, object]: ... + def run_next(self, worker_id: str) -> dict[str, object]: ... + def cancel(self, job_id: str) -> Job: ... + def recover(self, worker_id: str) -> tuple[Job, ...]: ... + def close(self) -> None: ... diff --git a/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/worker_supervisor.py b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/worker_supervisor.py new file mode 100644 index 0000000..5cd15f9 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/_lab/video_lab/worker_supervisor.py @@ -0,0 +1,327 @@ +from __future__ import annotations + +import errno +import json +import os +import signal +import subprocess +import sys +import threading +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Any, BinaryIO, Protocol + +from video_lab.database import Database +from video_lab.domain import Job +from video_lab.repositories import JobRepository +from video_lab.worker_control import SupervisorBusy, SupervisorShutdownError + +_DEFAULT_OUTPUT_LIMIT = 16 * 1024 +_ALLOWED_ENVIRONMENT = ("PATH", "LANG", "LC_ALL", "TMPDIR", "SYSTEMROOT") +_READ_SIZE = 4096 + + +class _Process(Protocol): + stdout: BinaryIO | None + stderr: BinaryIO | None + returncode: int | None + pid: int + + def poll(self) -> int | None: ... + def wait(self, timeout: float | None = None) -> int: ... + + +Launcher = Callable[..., _Process] +GroupSignaler = Callable[[int, int], None] + + +class _ByteTail: + def __init__(self, limit: int) -> None: + self.limit = limit + self._value = bytearray() + + def append(self, chunk: bytes) -> None: + if len(chunk) >= self.limit: + self._value[:] = chunk[-self.limit :] + return + overflow = len(self._value) + len(chunk) - self.limit + if overflow > 0: + del self._value[:overflow] + self._value.extend(chunk) + + def bytes(self) -> bytes: + return bytes(self._value) + + +class WorkerSupervisor: + """Own at most one isolated worker subprocess for a local web application.""" + + def __init__( + self, + *, + database_path: Path | str, + artifact_root: Path | str, + scratch_root: Path | str, + launcher: Launcher | None = None, + environment: Mapping[str, str] | None = None, + output_limit: int = _DEFAULT_OUTPUT_LIMIT, + group_signaler: GroupSignaler | None = None, + shutdown_timeout: float = 2.0, + ) -> None: + if output_limit < 1: + raise ValueError("output_limit must be positive") + if shutdown_timeout <= 0: + raise ValueError("shutdown_timeout must be positive") + self.database_path = Path(database_path) + self.artifact_root = Path(artifact_root) + self.scratch_root = Path(scratch_root) + self._launcher: Launcher = launcher or subprocess.Popen # type: ignore[assignment] + source_environment = os.environ if environment is None else environment + self._environment = { + key: source_environment[key] + for key in _ALLOWED_ENVIRONMENT + if key in source_environment + } + self._output_limit = output_limit + self._group_signaler = group_signaler or _signal_process_group + self._shutdown_timeout = shutdown_timeout + self._lock = threading.RLock() + self._close_lock = threading.Lock() + self._process: _Process | None = None + self._process_group_id: int | None = None + self._monitor: threading.Thread | None = None + self._closed = False + self._closing = False + self._status: dict[str, object] = {"state": "idle"} + self._refresh_persisted() + + def status(self) -> dict[str, object]: + with self._lock: + if self._process is None and not self._closed: + self._refresh_persisted_locked() + return dict(self._status) + + def run_next(self, worker_id: str) -> dict[str, object]: + worker_id = worker_id.strip() + if not worker_id: + raise ValueError("worker_id must not be blank") + with self._lock: + if self._closed: + raise RuntimeError("worker supervisor is closed") + if self._closing: + raise RuntimeError("worker supervisor is closing") + self._refresh_persisted_locked() + if self._status.get("state") == "recovery_required": + raise SupervisorBusy("persisted worker recovery is required") + if self._process is not None: + raise SupervisorBusy("a worker process is already running") + process = self._launcher( + self._worker_argv(worker_id), + shell=False, + start_new_session=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=False, + env=dict(self._environment), + ) + self._process = process + self._process_group_id = process.pid + self._status = {"state": "running", "worker_id": worker_id} + monitor = threading.Thread( + target=self._collect, + args=(process, worker_id), + name="video-lab-worker-monitor", + daemon=True, + ) + self._monitor = monitor + monitor.start() + return dict(self._status) + + def wait(self, timeout: float | None = None) -> dict[str, object]: + with self._lock: + monitor = self._monitor + if monitor is not None: + monitor.join(timeout) + if monitor.is_alive(): + raise TimeoutError("worker process is still running") + return self.status() + + def cancel(self, job_id: str) -> Job: + return self._repository().request_cancellation(job_id) + + def recover(self, worker_id: str) -> tuple[Job, ...]: + worker_id = worker_id.strip() + if not worker_id: + raise ValueError("worker_id must not be blank") + recovered = self._repository().recover_interrupted(worker_id) + with self._lock: + if self._process is None and not self._closed: + self._refresh_persisted_locked() + return recovered + + def close(self) -> None: + with self._close_lock: + with self._lock: + if self._closed: + return + self._closing = True + process = self._process + process_group_id = self._process_group_id + monitor = self._monitor + self._status = {"state": "shutting_down"} + try: + if ( + process is not None + and process_group_id is not None + and (process.poll() is None or (monitor is not None and monitor.is_alive())) + ): + self._signal_if_present(process_group_id, signal.SIGTERM) + if monitor is not None: + monitor.join(self._shutdown_timeout) + if monitor is not None and monitor.is_alive(): + if process is None: + raise SupervisorShutdownError("worker monitor has no owned process") + if process_group_id is None: + raise SupervisorShutdownError("worker process group ownership is missing") + self._signal_if_present(process_group_id, signal.SIGKILL) + monitor.join(self._shutdown_timeout) + if monitor is not None and monitor.is_alive(): + raise SupervisorShutdownError("worker monitor did not stop after SIGKILL") + except SupervisorShutdownError: + with self._lock: + self._status = {"state": "shutdown_failed"} + raise + except Exception as error: + with self._lock: + self._status = {"state": "shutdown_failed"} + raise SupervisorShutdownError("worker shutdown failed") from error + finally: + with self._lock: + if self._status.get("state") == "shutdown_failed": + self._closing = True + with self._lock: + if ( + self._process is not None + or self._process_group_id is not None + or self._monitor is not None + ): + self._status = {"state": "shutdown_failed"} + raise SupervisorShutdownError("worker ownership remains after shutdown") + self._closed = True + self._closing = False + self._status = {"state": "closed"} + + def _signal_if_present(self, pid: int, signal_number: int) -> None: + try: + self._group_signaler(pid, signal_number) + except ProcessLookupError: + return + except OSError as error: + if error.errno == errno.ESRCH: + return + raise + + def _repository(self) -> JobRepository: + database = Database(self.database_path) + database.migrate() + return JobRepository(database) + + def _refresh_persisted(self) -> None: + with self._lock: + self._refresh_persisted_locked() + + def _refresh_persisted_locked(self) -> None: + if self._process is not None or self._closed: + return + leases = self._repository().active_worker_leases() + if leases: + owners = {lease.worker_id for lease in leases} + self._status = { + "state": "recovery_required", + "worker_id": next(iter(owners)) if len(owners) == 1 else "multiple", + "job_ids": [lease.job_id for lease in leases], + } + elif self._status.get("state") == "recovery_required": + self._status = {"state": "idle"} + + def _worker_argv(self, worker_id: str) -> list[str]: + return [ + sys.executable, + "-m", + "video_lab.worker", + "--database", + str(self.database_path), + "--artifacts", + str(self.artifact_root), + "--scratch", + str(self.scratch_root), + "--worker-id", + worker_id, + ] + + def _collect(self, process: _Process, worker_id: str) -> None: + stdout_tail = _ByteTail(self._output_limit) + stderr_tail = _ByteTail(self._output_limit) + drains = [ + threading.Thread(target=_drain, args=(process.stdout, stdout_tail), daemon=True), + threading.Thread(target=_drain, args=(process.stderr, stderr_tail), daemon=True), + ] + for drain in drains: + drain.start() + completed: dict[str, object] | None = None + reaped = False + try: + exit_code = process.wait() + reaped = True + for drain in drains: + drain.join() + stdout = stdout_tail.bytes().decode("utf-8", errors="replace") + stderr = stderr_tail.bytes().decode("utf-8", errors="replace") + result = _last_json_object(stdout) + state = "completed" if exit_code == 0 and result is not None else "failed" + completed = { + "state": state, + "worker_id": worker_id, + "exit_code": exit_code, + "result": result, + "stdout": stdout, + "stderr": stderr, + } + finally: + current = threading.current_thread() + with self._lock: + if self._process is process and reaped: + if completed is not None and not self._closing and not self._closed: + self._status = completed + self._process = None + self._process_group_id = None + if self._monitor is current: + self._monitor = None + + +def _drain(stream: BinaryIO | None, tail: _ByteTail) -> None: + if stream is None: + return + try: + while chunk := stream.read(_READ_SIZE): + tail.append(chunk) + except ValueError: + pass + finally: + stream.close() + + +def _signal_process_group(pid: int, signal_number: int) -> None: + os.killpg(pid, signal_number) + + +def _last_json_object(output: str) -> dict[str, object] | None: + for line in reversed(output.splitlines()): + try: + value: Any = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and isinstance(value.get("status"), str): + return {str(key): item for key, item in value.items()} + return None diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/adoption.py b/packages/codex/mlx-agent/src/mlx_agent/adoption.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/adoption.py rename to packages/codex/mlx-agent/src/mlx_agent/adoption.py diff --git a/packages/codex/mlx-agent/src/mlx_agent/atomicio.py b/packages/codex/mlx-agent/src/mlx_agent/atomicio.py new file mode 100644 index 0000000..388118b --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/atomicio.py @@ -0,0 +1,11 @@ +"""Atomic queue publication uses the agent's retained-directory writer.""" +from pathlib import Path +from .transactions import _assert_safe_directory, _atomic_in_directory + +def write_text_atomic(path, text, encoding="utf-8"): + location = Path(path) + if location.is_symlink(): + raise OSError("Refusing to write through a symbolic link.") + _assert_safe_directory(location.parent, create=True) + _atomic_in_directory(location.parent, location.name, text.encode(encoding), 0o600) + return location diff --git a/packages/codex/mlx-agent/src/mlx_agent/backend_install.py b/packages/codex/mlx-agent/src/mlx_agent/backend_install.py new file mode 100644 index 0000000..9bb8555 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/backend_install.py @@ -0,0 +1,263 @@ +"""Confirmation-gated install and removal of optional converter backends.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import shutil +import sys +from datetime import datetime, timezone +from pathlib import Path + +from .backends import ( + BACKENDS_DIR, + BackendError, + backend_environment, + backend_target, + backends_root, + load_manifests, + load_registries, + spawn_with_env, +) +from .convert import _write_receipt, receipts_root +from .serve import _pid_alive + +BACKEND_RECEIPT_KIND = "backend" +BACKEND_RECEIPT_SCHEMA_VERSION = "1.0" +BACKEND_RUNNER = Path(__file__).resolve().with_name("backend_runner.py") +REQUIRED_PYTHON = (3, 12) +INSTALL_HOSTS = ("pypi.org", "files.pythonhosted.org") + + +def _utc_now(): + return datetime.now(timezone.utc).isoformat() + + +def _finalize(plan): + canonical = json.dumps(plan, sort_keys=True, separators=(",", ":")) + plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return plan + + +def default_trash_dir(): + if sys.platform == "darwin": + return Path.home() / ".Trash" + return Path.home() / ".local" / "share" / "Trash" / "files" + + +def _optional_manifest(backend_id, manifests): + manifest = manifests.get(backend_id) + if manifest is None: + raise BackendError( + "unknown_backend", "No declared backend named {0}.".format(backend_id), + "Run backend list to see the declared backends.", + ) + if manifest["builtin"]: + raise BackendError( + "builtin_backend", "{0} ships with the main runtime.".format(backend_id), + "Install it with the workbench runtime (make install); backend install manages optional backends only.", + ) + return manifest + + +def _refuse_symlinks(target): + for path in (target, target.parent): + if path.is_symlink(): + raise BackendError( + "symlink_refused", "{0} is a symbolic link.".format(path), + "Point XDG_DATA_HOME at a real directory.", + ) + + +def _check_hash(plan, confirm, preview_hash, verb): + if not preview_hash: + raise BackendError( + "preview_hash_required", + "--confirm requires the hash from a reviewed backend {0} preview.".format(verb), + "Run backend {0} without --confirm, review it, then pass --preview-hash.".format(verb), + ) + if preview_hash != plan["preview_hash"]: + raise BackendError( + "preview_stale", "The supplied preview hash does not match this {0} plan.".format(verb), + "Re-run backend {0} without --confirm and review the fresh plan.".format(verb), + ) + + +def _receipt_file(receipts_dir, backend_id): + return receipts_root(receipts_dir, kind=BACKEND_RECEIPT_KIND) / "{0}.json".format(backend_id) + + +def _read_receipt(path): + try: + value = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if not isinstance(value, dict) or value.get("kind") != BACKEND_RECEIPT_KIND: + return None + if not isinstance(value.get("pid"), int) or isinstance(value.get("pid"), bool): + return None + return value + + +def _running(receipt, pid_alive): + return bool(receipt) and not receipt.get("completed_at") and pid_alive(receipt["pid"]) + + +def plan_install(backend_id, manifests=None, root=None, python=None, version_info=None, trash_dir=None): + manifests = load_manifests() if manifests is None else manifests + manifest = _optional_manifest(backend_id, manifests) + version_info = sys.version_info if version_info is None else version_info + if tuple(version_info[:2]) != REQUIRED_PYTHON: + raise BackendError( + "python_mismatch", + "Backend locks target Python 3.12; this agent runs {0}.{1}.".format(version_info[0], version_info[1]), + "Run mlx-agent with the workbench .venv interpreter (Python 3.12).", + ) + lock = BACKENDS_DIR / manifest["lock"] + if not lock.is_file(): + raise BackendError("lock_missing", "The lock file {0} is missing.".format(lock.name), "Reinstall mlx-agent; the lock ships with it.") + content = lock.read_bytes() + target = backend_target(manifest, root) + python = str(python or sys.executable) + trash = Path(trash_dir) if trash_dir is not None else default_trash_dir() + plan = { + "id": manifest["id"], "version": manifest["version"], "package": manifest["package"], + "target": str(target), "lock": str(lock), + "lock_sha256": hashlib.sha256(content).hexdigest(), + "requirements": sum(1 for line in content.decode("utf-8").splitlines() if line[:1].isalnum()), + "python": python, "network": list(INSTALL_HOSTS), + "argv": [ + python, str(BACKEND_RUNNER), "--id", manifest["id"], "--version", manifest["version"], + "--package", manifest["package"], "--lock", str(lock), "--target", str(target), + "--python", python, "--trash", str(trash), + ], + } + if backend_id == "mlx-audio": + build_lock = BACKENDS_DIR / "mlx-audio-build.lock" + if not build_lock.is_file(): + raise BackendError("lock_missing", "The audio build lock is missing.", "Reinstall mlx-agent; the lock ships with it.") + plan["build_lock_sha256"] = hashlib.sha256(build_lock.read_bytes()).hexdigest() + plan["argv"] += ["--build-lock", str(build_lock)] + plan["network"] += ["github.com", "release-assets.githubusercontent.com"] + plan = _finalize(plan) + plan["argv"] = plan["argv"] + ["--preview-hash", plan["preview_hash"]] + return plan + + +def start_install(plan, receipts_dir=None, confirm=False, preview_hash=None, spawn=None, + pid_alive=_pid_alive, now=_utc_now, env=None): + if not confirm: + return {"status": "preview", "plan": plan, "requires_confirmation": True} + _check_hash(plan, confirm, preview_hash, "install") + target = Path(plan["target"]) + _refuse_symlinks(target) + if target.exists(): + raise BackendError( + "target_exists", "{0} already exists.".format(target), + "Remove it first: mlx-agent backend remove {0}.".format(plan["id"]), + ) + receipt_path = _receipt_file(receipts_dir, plan["id"]) + if _running(_read_receipt(receipt_path), pid_alive): + raise BackendError( + "job_in_progress", "An install of {0} is still running.".format(plan["id"]), + "Wait for it to finish (backend list).", + ) + receipt_path.parent.mkdir(parents=True, exist_ok=True) + target.parent.mkdir(parents=True, exist_ok=True) + log_path = receipt_path.with_suffix(".log") + pid = (spawn or spawn_with_env)(plan["argv"], str(log_path), backend_environment(env)) + receipt = { + "schema_version": BACKEND_RECEIPT_SCHEMA_VERSION, "kind": BACKEND_RECEIPT_KIND, + "id": plan["id"], "version": plan["version"], "target": plan["target"], + "argv": list(plan["argv"]), "pid": pid, "log_path": str(log_path), + "started_at": now(), "preview_hash": plan["preview_hash"], + "completed_at": None, "exit_status": None, "removed_at": None, + } + _write_receipt(receipt_path.parent, receipt, receipt_path.name) + return {"status": "started", "receipt": receipt} + + +def _state(manifest, info, receipt, pid_alive): + if info["installed"]: + return "installed" + if manifest["builtin"] or receipt is None or receipt.get("removed_at"): + return "absent" + if _running(receipt, pid_alive): + return "installing" + if receipt.get("exit_status") == "installed": + return "absent" + return "failed" + + +def list_backends(manifests=None, root=None, receipts_dir=None, pid_alive=_pid_alive, + find_spec=None, now=_utc_now): + manifests = load_manifests() if manifests is None else manifests + registries = load_registries(manifests, root, find_spec or importlib.util.find_spec) + entries = [] + for backend_id in sorted(manifests): + manifest = manifests[backend_id] + info = registries[backend_id] + receipt_path = _receipt_file(receipts_dir, backend_id) + receipt = None if manifest["builtin"] else _read_receipt(receipt_path) + state = _state(manifest, info, receipt, pid_alive) + if receipt and not receipt.get("completed_at") and state in ("installed", "failed"): + receipt["completed_at"] = now() + receipt["exit_status"] = state + _write_receipt(receipt_path.parent, receipt, receipt_path.name) + entries.append({ + "id": backend_id, "version": manifest["version"], "builtin": manifest["builtin"], + "state": state, "categories": sorted(manifest["categories"]), + "model_types": sum(len(entry.get("model_types", ())) for entry in info["registry"].values()), + "registry_source": info["source"], + "target": None if manifest["builtin"] else str(backend_target(manifest, root)), + "log_path": receipt.get("log_path") if receipt else None, + "started_at": receipt.get("started_at") if receipt else None, + "completed_at": receipt.get("completed_at") if receipt else None, + }) + return { + "schema": "backends/1", + "root": str(Path(root) if root is not None else backends_root()), + "backends": entries, + } + + +def plan_remove(backend_id, manifests=None, root=None, trash_dir=None): + manifests = load_manifests() if manifests is None else manifests + manifest = _optional_manifest(backend_id, manifests) + trash = Path(trash_dir) if trash_dir is not None else default_trash_dir() + return _finalize({ + "id": backend_id, "target": str(backend_target(manifest, root)), + "action": "move_to_trash", "trash": str(trash), + }) + + +def remove_backend(plan, receipts_dir=None, confirm=False, preview_hash=None, + pid_alive=_pid_alive, now=_utc_now, move=shutil.move): + if not confirm: + return {"status": "preview", "plan": plan, "requires_confirmation": True} + _check_hash(plan, confirm, preview_hash, "remove") + target = Path(plan["target"]) + _refuse_symlinks(target) + if not target.exists(): + raise BackendError( + "not_installed", "{0} is not installed at {1}.".format(plan["id"], target), + "Nothing to remove; run backend list.", + ) + receipt_path = _receipt_file(receipts_dir, plan["id"]) + receipt = _read_receipt(receipt_path) + if _running(receipt, pid_alive): + raise BackendError( + "job_in_progress", "An install of {0} is still running.".format(plan["id"]), + "Wait for it to finish (backend list).", + ) + trash = Path(plan["trash"]) + trash.mkdir(parents=True, exist_ok=True) + destination = trash / "{0}-backend-{1}".format( + plan["id"], datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + ) + move(str(target), str(destination)) + if receipt: + receipt["removed_at"] = now() + _write_receipt(receipt_path.parent, receipt, receipt_path.name) + return {"status": "removed", "id": plan["id"], "moved_to": str(destination)} diff --git a/packages/codex/mlx-agent/src/mlx_agent/backend_runner.py b/packages/codex/mlx-agent/src/mlx_agent/backend_runner.py new file mode 100644 index 0000000..a555a0c --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/backend_runner.py @@ -0,0 +1,73 @@ +"""Create one isolated backend venv from a hash-locked requirements file.""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +MARKER = ".mlx-agent-backend.json" + + +def _stamp(): + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def _discard(target, trash, backend_id): + if not target.exists(): + return + trash.mkdir(parents=True, exist_ok=True) + destination = trash / "{0}-failed-{1}".format(backend_id, _stamp()) + shutil.move(str(target), str(destination)) + print("moved partial install to {0}".format(destination), flush=True) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("--id", "--version", "--package", "--lock", "--target", "--python", "--trash", "--preview-hash"): + parser.add_argument(name, required=True) + parser.add_argument("--build-lock") + arguments = parser.parse_args(argv) + target = Path(arguments.target) + if target.exists() or target.is_symlink(): + print("target already exists: {0}".format(target), flush=True) + return 3 + venv_python = str(target / "bin" / "python") + steps = [ + [arguments.python, "-m", "venv", str(target)], + [venv_python, "-m", "pip", "install", "--disable-pip-version-check", "--no-input", + "--require-hashes", "--no-deps", "--only-binary=:all:", "-r", arguments.lock], + [venv_python, "-c", "import {0}".format(arguments.package)], + ] + if arguments.id == "mlx-audio": + # num2words depends on the source-only docopt package. Its build uses + # pinned, hashed tools with build isolation disabled; other packages + # and backends retain the binary-only install rule. + build_lock = arguments.build_lock or str(Path(arguments.lock).with_name("mlx-audio-build.lock")) + steps.insert(1, [venv_python, "-m", "pip", "install", "--disable-pip-version-check", "--no-input", + "--require-hashes", "--no-deps", "--only-binary=:all:", "-r", build_lock]) + steps[2] += ["--no-binary=docopt", "--no-build-isolation"] + steps.append([venv_python, str(Path(__file__).with_name("kokoro_runtime.py"))]) + for step in steps: + print("$ " + " ".join(step), flush=True) + completed = subprocess.run(step, stdin=subprocess.DEVNULL, check=False) + if completed.returncode != 0: + print("step failed with exit status {0}".format(completed.returncode), flush=True) + _discard(target, Path(arguments.trash), arguments.id) + return completed.returncode + marker = { + "id": arguments.id, "version": arguments.version, + "preview_hash": arguments.preview_hash, + "installed_at": datetime.now(timezone.utc).isoformat(), + } + (target / MARKER).write_text(json.dumps(marker, sort_keys=True) + "\n", encoding="utf-8") + print("installed {0} {1} at {2}".format(arguments.id, arguments.version, target), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/codex/mlx-agent/src/mlx_agent/backends.py b/packages/codex/mlx-agent/src/mlx_agent/backends.py new file mode 100644 index 0000000..db4f57e --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/backends.py @@ -0,0 +1,676 @@ +"""Declared MLX converter backends: manifests, model-type registries, isolated envs.""" + +from __future__ import annotations + +import ast +import hashlib +import importlib.util +import json +import os +import re +import shutil +import subprocess +import zipfile +import sys +from pathlib import Path + +BACKENDS_DIR = Path(__file__).resolve().parent / "resources" / "backends" +PORTS_DIR = Path(__file__).resolve().parent / "resources" / "ports" +MANIFEST_SCHEMA = "backend/1" +INSTALL_MARKER = ".mlx-agent-backend.json" +PORT_MARKER = ".mlx-agent-port.json" +CATEGORIES = ("text_llm", "vision_language", "speech_to_text", "text_to_speech", "classification", "image_generation", + "video_generation", "music_generation") +MAX_PROBE_FILES = 4000 +MAX_PROBE_FILE_BYTES = 1024 * 1024 +TYPE_PREFERENCE = { + "speech_to_text": ("mlx-audio",), + "text_to_speech": ("mlx-audio",), + "music_generation": ("mlx-audio",), + "vision_language": ("mlx-vlm", "mlx-lm"), + "text_llm": ("mlx-lm", "mlx-vlm"), + "classification": ("mlx-embeddings",), + "image_generation": ("mflux",), + "video_generation": ("mlx-video",), +} +DEFAULT_PREFERENCE = ("mlx-lm", "mlx-vlm", "mlx-audio") +_ID = re.compile(r"[a-z0-9][a-z0-9-]{0,31}") +_PORT = re.compile(r"[a-z][a-z0-9_]{0,63}") +_PORT_FILE = re.compile(r"[A-Za-z0-9_.-]+(/[A-Za-z0-9_.-]+)*") +_PIPELINE = re.compile(r"[A-Z][A-Za-z0-9]{0,95}") +_REPO_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,95}") +_COMMIT = re.compile(r"[0-9a-f]{40}") +_STRIP_SUFFIXES = ("_encoder", "_decoder", "_text", "_vision", "_audio", "_model") +_VISION_FILE = re.compile(r"(vision|visual|image|siglip|clip)", re.IGNORECASE) +_ENV_ALLOWLIST = ( + "PATH", "HOME", "TMPDIR", "LANG", "LC_ALL", "HF_HOME", "HF_HUB_CACHE", + "HF_HUB_OFFLINE", "XDG_CACHE_HOME", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", +) + + +class BackendError(RuntimeError): + """Classified backend failure safe to surface in a result envelope.""" + + def __init__(self, code, message, remediation): + super().__init__(message) + self.code = code + self.remediation = remediation + + +def normalize_type(value): + return str(value).strip().lower().replace("-", "_") + + +def squash_type(value): + return re.sub(r"[^a-z0-9]", "", str(value).lower()) + + +def load_manifests(directory=BACKENDS_DIR): + manifests = {} + for path in sorted(Path(directory).glob("*.json")): + value = json.loads(path.read_text(encoding="utf-8")) + _validate_manifest(value, path) + manifests[value["id"]] = value + return manifests + + +def _invalid(path, detail): + return BackendError( + "manifest_invalid", + "{0}: {1}".format(Path(path).name, detail), + "Fix the backend manifest under resources/backends.", + ) + + +def _validate_manifest(value, path): + if not isinstance(value, dict): + raise _invalid(path, "manifest must be an object") + required = { + "schema": str, "id": str, "version": str, "builtin": bool, "package": str, + "convert": str, "categories": dict, "remap_files": dict, "registry": dict, + } + for key, kind in required.items(): + if not isinstance(value.get(key), kind): + raise _invalid(path, "field {0} must be {1}".format(key, kind.__name__)) + if value["schema"] != MANIFEST_SCHEMA or not _ID.fullmatch(value["id"]) or Path(path).stem != value["id"]: + raise _invalid(path, "schema or id mismatch") + if not set(value["categories"]) <= set(CATEGORIES) or not value["categories"]: + raise _invalid(path, "categories must be a non-empty subset of {0}".format(list(CATEGORIES))) + if not set(value["remap_files"]) <= set(value["categories"]): + raise _invalid(path, "remap_files keys must be declared categories") + if not value["builtin"] and not isinstance(value.get("lock"), str): + raise _invalid(path, "an optional backend needs a lock file") + ports = value.get("ports", {}) + if not isinstance(ports, dict) or not set(ports) <= set(value["categories"]): + raise _invalid(path, "ports keys must be declared categories") + for names in ports.values(): + if not isinstance(names, list) or not all(isinstance(name, str) and _PORT.fullmatch(name) for name in names): + raise _invalid(path, "ports must list module names") + ported = {name for names in ports.values() for name in names} + rules = value.get("port_quantize", {}) + if not isinstance(rules, dict) or not set(rules) <= ported: + raise _invalid(path, "port_quantize keys must be declared ports") + for rule in rules.values(): + if not isinstance(rule, dict) or not set(rule) <= {"include", "exclude", "group_size"} or not all( + isinstance(rule.get(key, []), list) and all(isinstance(item, str) for item in rule.get(key, [])) + for key in ("include", "exclude") + ): + raise _invalid(path, "port_quantize rules list include and exclude name parts") + group = rule.get("group_size", 64) + if not isinstance(group, int) or isinstance(group, bool) or group not in (32, 64, 128): + raise _invalid(path, "port_quantize group_size must be 32, 64, or 128") + bits = value.get("port_bits", {}) + if not isinstance(bits, dict) or not set(bits) <= ported or not all( + isinstance(widths, list) and widths and all(width in (4, 8) and not isinstance(width, bool) for width in widths) + for widths in bits.values() + ): + raise _invalid(path, "port_bits map declared ports to the bit widths (4, 8) their conversions keep accurate") + converters = value.get("port_convert", {}) + if not isinstance(converters, dict) or not set(converters) <= ported or not all( + isinstance(module, str) and _PORT.fullmatch(module) for module in converters.values() + ): + raise _invalid(path, "port_convert maps declared ports to a module inside the port") + pipelines = value.get("port_pipelines", {}) + if not isinstance(pipelines, dict) or not all( + isinstance(name, str) and _PIPELINE.fullmatch(name) and port in ported for name, port in pipelines.items() + ): + raise _invalid(path, "port_pipelines map diffusers pipeline classes to declared ports") + recipes = value.get("port_recipes", {}) + if not isinstance(recipes, dict) or not all(_valid_recipe(repo, recipe, ported) for repo, recipe in recipes.items()): + raise _invalid(path, "port_recipes map repositories to a declared port, a pinned base repository, and a LoRA file") + signatures = value.get("port_signatures", {}) + if not isinstance(signatures, dict) or not set(signatures) <= ported or not all( + isinstance(files, list) and files and all( + isinstance(name, str) and _PORT_FILE.fullmatch(name) and ".." not in name.split("/") for name in files + ) + for files in signatures.values() + ): + raise _invalid(path, "port_signatures map declared ports to the relative files that identify them") + + +def _valid_recipe(repo, recipe, ported): + if not isinstance(repo, str) or not _REPO_ID.fullmatch(repo) or not isinstance(recipe, dict): + return False + if set(recipe) != {"port", "base", "base_revision", "lora", "lora_scale"} or recipe["port"] not in ported: + return False + scale = recipe["lora_scale"] + return ( + isinstance(recipe["base"], str) and bool(_REPO_ID.fullmatch(recipe["base"])) + and isinstance(recipe["base_revision"], str) and bool(_COMMIT.fullmatch(recipe["base_revision"])) + and isinstance(recipe["lora"], str) and bool(_PORT_FILE.fullmatch(recipe["lora"])) and ".." not in recipe["lora"].split("/") + and recipe["lora"].endswith(".safetensors") + and isinstance(scale, (int, float)) and not isinstance(scale, bool) and 0 < scale <= 4 + ) + + +def port_for_pipeline(class_name, manifests): + """The port that converts a diffusers pipeline class (``model_index.json`` ``_class_name``), or None.""" + for backend_id in sorted(manifests): + port = (manifests[backend_id].get("port_pipelines") or {}).get(class_name) + if port: + return port + return None + + +def recipe_for(repo, manifests): + """A curated recipe for a repository whose model is a base checkpoint plus its own LoRA, or None. + + Each recipe was checked against the repository's merged weights; the result + carries the backend and port with the recipe fields. + """ + wanted = (repo or "").casefold() + for backend_id in sorted(manifests): + for name, recipe in (manifests[backend_id].get("port_recipes") or {}).items(): + if name.casefold() == wanted: + return dict(recipe, backend=backend_id, repo=name) + return None + + +def quantize_rule(model_type, backend_id, manifests): + """The tensor-name rule a port declares for what its converter quantizes, or None.""" + manifest = manifests.get(backend_id) or {} + return (manifest.get("port_quantize") or {}).get(normalize_type(model_type or "")) + + +def _port_category(manifest, name): + for category, names in sorted((manifest.get("ports") or {}).items()): + if name in names: + return category + return None + + +def port_converter(model_type, backend_id, manifests): + """The module a port ships for converting its own checkpoints, or None for the backend's converter.""" + manifest = manifests.get(backend_id) or {} + name = normalize_type(model_type or "") + module = (manifest.get("port_convert") or {}).get(name) + category = _port_category(manifest, name) + if module is None or category is None: + return None + return "{0}.{1}".format(_module_path(manifest, category, name), module) + + +def port_bits(model_type, backend_id, manifests): + """The bit widths a port converts to, or None for every supported width.""" + manifest = manifests.get(backend_id) or {} + widths = (manifest.get("port_bits") or {}).get(normalize_type(model_type or "")) + return sorted(widths) if widths else None + + +def port_files(model_type, backend_id, manifests): + """The files that identify a port's checkpoints (its signature), or None.""" + manifest = manifests.get(backend_id) or {} + return (manifest.get("port_signatures") or {}).get(normalize_type(model_type or "")) + + +def snapshot_files(model_type, manifests): + """A ported type's files, the whole of what its converter reads, in whichever backend declares it; or None.""" + for backend_id in sorted(manifests): + files = port_files(model_type, backend_id, manifests) + if files: + return files + return None + + +def port_for_files(names, manifests): + """The ported model type whose identifying files a repository carries (repos without a root config).""" + present = set(names) + for backend_id in sorted(manifests): + for port, files in sorted((manifests[backend_id].get("port_signatures") or {}).items()): + if set(files) <= present: + return port + return None + + +def backends_root(env=None): + env = os.environ if env is None else env + base = env.get("XDG_DATA_HOME") + root = Path(base).expanduser() if base else Path.home() / ".local" / "share" + return root / "mlx-workbench" / "backends" + + +def backend_target(manifest, root=None): + return Path(root if root is not None else backends_root()) / manifest["id"] + + +def backend_python(manifest, root=None): + if manifest["builtin"]: + return Path(sys.executable) + return backend_target(manifest, root) / "bin" / "python" + + +def read_install_marker(manifest, root=None): + location = backend_target(manifest, root) / INSTALL_MARKER + try: + value = json.loads(location.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if not isinstance(value, dict): + return None + if value.get("id") != manifest["id"] or value.get("version") != manifest["version"]: + return None + return value + + +def is_installed(manifest, root=None, find_spec=importlib.util.find_spec): + if manifest["builtin"]: + try: + return find_spec(manifest["package"]) is not None + except (ImportError, ValueError): + return False + return read_install_marker(manifest, root) is not None and backend_python(manifest, root).exists() + + +def package_dir(manifest, root=None, find_spec=importlib.util.find_spec): + if manifest["builtin"]: + try: + spec = find_spec(manifest["package"]) + except (ImportError, ValueError): + return None + if spec is None or not spec.submodule_search_locations: + return None + return Path(list(spec.submodule_search_locations)[0]) + lib = backend_target(manifest, root) / "lib" + for candidate in sorted(lib.glob("python3.*/site-packages/{0}".format(manifest["package"]))): + if candidate.is_dir(): + return candidate + return None + + +def _module_name(relative, prefix): + if not relative.startswith(prefix): + return None + parts = relative[len(prefix):].split("/") + if len(parts) == 2 and parts[1] == "__init__.py": + name = parts[0] + elif len(parts) == 1 and parts[0].endswith(".py"): + name = parts[0][:-3] + else: + return None + if not name or name.startswith("_"): + return None + return name + + +def _vision_marker(relative, prefix): + """The module a `/.py` vision/image file belongs to.""" + if not relative.startswith(prefix): + return None + parts = relative[len(prefix):].split("/") + if len(parts) != 2 or not parts[1].endswith(".py") or parts[1] == "__init__.py": + return None + return parts[0] if _VISION_FILE.search(parts[1][:-3]) else None + + +def _category_prefixes(manifest): + return [directory.strip("/") + "/" for directory in manifest["categories"].values()] + + +def _remap_files(manifest): + return {relative for files in manifest["remap_files"].values() for relative in files} + + +def probe_sources_from_wheel(wheel_path, manifest): + """Read only the files the registry probe needs from a wheel.""" + package_prefix = manifest["package"] + "/" + prefixes = _category_prefixes(manifest) + remap = _remap_files(manifest) + sources = {} + with zipfile.ZipFile(wheel_path) as archive: + for info in archive.infolist(): + if not info.filename.startswith(package_prefix) or info.file_size > MAX_PROBE_FILE_BYTES: + continue + relative = info.filename[len(package_prefix):] + if relative in remap or any(_module_name(relative, prefix) for prefix in prefixes): + sources[relative] = archive.read(info).decode("utf-8", errors="replace") + elif any(_vision_marker(relative, prefix) for prefix in prefixes): + sources[relative] = "" + if len(sources) >= MAX_PROBE_FILES: + break + return sources + + +def probe_sources_from_directory(directory, manifest): + """Read the same files from an installed package directory.""" + directory = Path(directory) + sources = {} + for relative in sorted(_remap_files(manifest)): + _add_source(sources, directory, relative) + for prefix in _category_prefixes(manifest): + base = directory / prefix + if not base.is_dir(): + continue + for entry in sorted(base.iterdir()): + if len(sources) >= MAX_PROBE_FILES: + break + if entry.is_dir(): + _add_source(sources, directory, "{0}{1}/__init__.py".format(prefix, entry.name)) + for child in sorted(entry.iterdir()): + marker = "{0}{1}/{2}".format(prefix, entry.name, child.name) + if child.is_file() and _vision_marker(marker, prefix): + sources[marker] = "" + elif entry.suffix == ".py": + _add_source(sources, directory, "{0}{1}".format(prefix, entry.name)) + return {key: sources[key] for key in sorted(sources) if any( + _module_name(key, prefix) or _vision_marker(key, prefix) + for prefix in _category_prefixes(manifest) + ) or key in _remap_files(manifest)} + + +def _add_source(sources, directory, relative): + location = directory / relative + try: + if not location.is_file() or location.stat().st_size > MAX_PROBE_FILE_BYTES: + return + sources[relative] = location.read_text(encoding="utf-8", errors="replace") + except OSError: + return + + +def _binds_model(text): + try: + tree = ast.parse(text) + except (SyntaxError, ValueError): + return False + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == "Model": + return True + if isinstance(node, ast.ImportFrom) and any( + (alias.asname or alias.name) == "Model" for alias in node.names + ): + return True + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "Model" for target in node.targets + ): + return True + return False + + +def _remapping(text): + try: + tree = ast.parse(text) + except (SyntaxError, ValueError): + return {} + for node in tree.body: + if isinstance(node, ast.Assign): + targets, value = node.targets, node.value + elif isinstance(node, ast.AnnAssign) and node.value is not None: + targets, value = [node.target], node.value + else: + continue + if not any(isinstance(target, ast.Name) and target.id == "MODEL_REMAPPING" for target in targets): + continue + try: + literal = ast.literal_eval(value) + except ValueError: + return {} + if isinstance(literal, dict): + return { + normalize_type(key): normalize_type(item) + for key, item in literal.items() + if isinstance(key, str) and isinstance(item, str) + } + return {} + + +def registry_from_sources(sources, manifest): + """Model types and MODEL_REMAPPING per category from probe sources.""" + registry = {} + for category, directory in sorted(manifest["categories"].items()): + prefix = directory.strip("/") + "/" + names = set() + for relative, text in sources.items(): + module = _module_name(relative, prefix) + if module and _binds_model(text): + names.add(normalize_type(module)) + remapping = {} + for relative in manifest["remap_files"].get(category, ()): + remapping.update(_remapping(sources.get(relative, ""))) + registry[category] = {"model_types": sorted(names), "remapping": dict(sorted(remapping.items()))} + if category == "vision_language": + vision = {normalize_type(_vision_marker(relative, prefix)) for relative in sources + if _vision_marker(relative, prefix)} + registry[category]["vision_types"] = sorted(vision & names) + return registry + + +def load_registries(manifests, root=None, find_spec=importlib.util.find_spec): + """Installed registries win over bundled snapshots when they are non-empty.""" + registries = {} + for backend_id, manifest in manifests.items(): + installed = is_installed(manifest, root, find_spec) + registry, source = manifest["registry"], "snapshot" + if installed: + directory = package_dir(manifest, root, find_spec) + if directory is not None: + live = registry_from_sources(probe_sources_from_directory(directory, manifest), manifest) + if any(entry["model_types"] for entry in live.values()): + registry, source = live, "installed" + registries[backend_id] = {"registry": with_ports(registry, manifest), "source": source, "installed": installed} + return registries + + +def with_ports(registry, manifest): + """The registry plus the model modules mlx-agent ports into this backend.""" + ports = manifest.get("ports") or {} + if not ports: + return registry + merged = {category: dict(entry) for category, entry in registry.items()} + for category, names in ports.items(): + entry = merged.setdefault(category, {"model_types": [], "remapping": {}}) + entry["model_types"] = sorted(set(entry.get("model_types", ())) | {normalize_type(name) for name in names}) + return merged + + +def _port_digest(directory): + digest = hashlib.sha256() + for path in sorted(Path(directory).rglob("*.py")): + if "__pycache__" in path.parts: + continue + digest.update(path.relative_to(directory).as_posix().encode("utf-8") + b"\0") + digest.update(path.read_bytes() + b"\0") + return digest.hexdigest() + + +def _read_port_marker(target): + try: + value = json.loads((target / PORT_MARKER).read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return value if isinstance(value, dict) else None + + +def sync_ports(manifest, root=None, find_spec=importlib.util.find_spec, ports_dir=PORTS_DIR): + """Copy the backend's bundled model ports into its installed package; returns the names written. + + A port directory carries a marker naming the source digest, so an + up-to-date port is left alone and a same-named module the backend + itself ships is never overwritten. + """ + ports = manifest.get("ports") or {} + if not ports: + return [] + directory = package_dir(manifest, root, find_spec) + if directory is None: + raise BackendError( + "backend_not_installed", "The {0} backend is not installed.".format(manifest["id"]), + "Install it first: mlx-agent backend install {0}.".format(manifest["id"]), + ) + written = [] + for category, names in sorted(ports.items()): + base = directory / manifest["categories"][category].strip("/") + for name in names: + source = Path(ports_dir) / manifest["id"] / name + if not (source / "__init__.py").is_file(): + raise BackendError( + "port_missing", "mlx-agent ships no {0} port for {1}.".format(name, manifest["id"]), + "Reinstall mlx-agent; ports ship under resources/ports.", + ) + digest = _port_digest(source) + target = base / name + if target.is_symlink(): + raise BackendError( + "port_target_symlink", "{0} is a symbolic link.".format(target), + "Remove the link; ports are only written to real directories.", + ) + marker = _read_port_marker(target) if target.exists() else None + if target.exists() and (marker is None or marker.get("port") != name): + raise BackendError( + "port_conflict", "{0} already ships {1}; mlx-agent will not replace it.".format(manifest["id"], name), + "Use the backend's own module, or remove the mlx-agent port from the manifest.", + ) + if marker is not None and marker.get("sha256") == digest: + continue + staging = base / ".{0}.mlx-agent-staging".format(name) + if staging.exists(): + shutil.rmtree(staging) + shutil.copytree(source, staging, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + (staging / PORT_MARKER).write_text( + json.dumps({"backend": manifest["id"], "port": name, "sha256": digest}, sort_keys=True) + "\n", + encoding="utf-8", + ) + if target.exists(): + retired = base / ".{0}.mlx-agent-retired".format(name) + if retired.exists(): + shutil.rmtree(retired) + target.rename(retired) + staging.rename(target) + shutil.rmtree(retired) + else: + staging.rename(target) + written.append(name) + return written + + +def _module_path(manifest, category, module): + return "{0}.{1}.{2}".format( + manifest["package"], manifest["categories"][category].strip("/").replace("/", "."), module + ) + + +def _hit(backend_id, manifest, category, module, match): + return { + "backend": backend_id, + "category": category, + "match": match, + "module": _module_path(manifest, category, module), + } + + +def _registry(backend_id, manifest, registries): + entry = registries.get(backend_id, {}) + return entry["registry"] if "registry" in entry else with_ports(manifest["registry"], manifest) + + +def lookup(model_type, manifests, registries): + if not model_type: + return [] + wanted = normalize_type(model_type) + hits = [] + for backend_id in sorted(manifests): + manifest = manifests[backend_id] + registry = _registry(backend_id, manifest, registries) + for category in sorted(registry): + entry = registry[category] + types = set(entry.get("model_types", ())) + remapping = entry.get("remapping", {}) + if wanted in types: + hits.append(_hit(backend_id, manifest, category, wanted, "exact")) + elif remapping.get(wanted) in types: + hits.append(_hit(backend_id, manifest, category, remapping[wanted], "remap")) + return hits + + +def lookup_squashed(architecture, manifests, registries): + wanted = squash_type(architecture or "") + if not wanted: + return [] + hits = [] + for backend_id in sorted(manifests): + manifest = manifests[backend_id] + registry = _registry(backend_id, manifest, registries) + for category in sorted(registry): + for module in registry[category].get("model_types", ()): + if squash_type(module) == wanted: + hits.append(_hit(backend_id, manifest, category, module, "squashed")) + break + return hits + + +def is_vision_type(model_type, manifests, registries): + """True when a vision-language backend's module for this type ships vision files.""" + if not model_type: + return False + wanted = normalize_type(model_type) + for backend_id in sorted(manifests): + entry = _registry(backend_id, manifests[backend_id], registries).get("vision_language") + if not entry: + continue + module = entry.get("remapping", {}).get(wanted, wanted) + if module in entry.get("vision_types", ()): + return True + return False + + +def component_matches(model_type, manifests, registries): + hits = lookup(model_type, manifests, registries) + if hits or not model_type: + return hits + normalized = normalize_type(model_type) + for suffix in _STRIP_SUFFIXES: + if normalized.endswith(suffix) and len(normalized) > len(suffix): + stripped = lookup(normalized[: -len(suffix)], manifests, registries) + return [dict(hit, match="stripped") for hit in stripped] + return [] + + +def choose_backend(hits, task_type, has_vision_config=False): + preference = TYPE_PREFERENCE.get(task_type, DEFAULT_PREFERENCE) + if has_vision_config and task_type == "text_llm": + preference = ("mlx-vlm", "mlx-lm") + ranked = sorted( + (hit for hit in hits if hit["backend"] in preference), + key=lambda hit: (hit["match"] == "remap", preference.index(hit["backend"])), + ) + return ranked[0]["backend"] if ranked else None + + +def backend_environment(env=None): + env = os.environ if env is None else env + return {key: env[key] for key in _ENV_ALLOWLIST if key in env} + + +def spawn_with_env(argv, log_path, env): + handle = open(log_path, "ab", buffering=0) + try: + process = subprocess.Popen( + argv, + stdin=subprocess.DEVNULL, + stdout=handle, + stderr=subprocess.STDOUT, + start_new_session=True, + env=env, + ) + finally: + handle.close() + return process.pid diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/bench.py b/packages/codex/mlx-agent/src/mlx_agent/bench.py similarity index 95% rename from packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/bench.py rename to packages/codex/mlx-agent/src/mlx_agent/bench.py index ba6d8cd..76b344a 100644 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/bench.py +++ b/packages/codex/mlx-agent/src/mlx_agent/bench.py @@ -182,12 +182,20 @@ def measure_runtime(repo, runtime, runs=RUNS_DEFAULT, gen_tokens=GEN_TOKENS_DEFA prefill_values = [ sample["prefill_toks"] for sample in samples if sample["prefill_toks"] is not None ] + # Prefer the server-reported prompt count over the fixed estimate. + prompt_token_values = [ + sample["prompt_tokens"] for sample in samples if sample.get("prompt_tokens") + ] + prompt_tokens = ( + int(round(statistics.median(prompt_token_values))) + if prompt_token_values else BENCH_PROMPT_TOKEN_ESTIMATE + ) measured_at = now() if callable(now) else _utc_now() return BenchMeasurement( repo=repo, runtime=runtime_name, runs=runs, - prompt_tokens=BENCH_PROMPT_TOKEN_ESTIMATE, + prompt_tokens=prompt_tokens, gen_tokens=gen_tokens, ttft_ms=round(statistics.median(ttft_values), 1) if ttft_values else None, decode_toks=round(median_decode, 1), @@ -248,11 +256,17 @@ def _timed_run(stream_generate, repo, gen_tokens, timeout, clock): ) ttft_seconds = first_content_at - started if ttft_seconds > 0: - sample["prefill_toks"] = round(BENCH_PROMPT_TOKEN_ESTIMATE / ttft_seconds, 1) + # A server-reported prompt count beats the fixed estimate. + prompt_tokens = (usage or {}).get("prompt_tokens") or BENCH_PROMPT_TOKEN_ESTIMATE + sample["prefill_toks"] = round(prompt_tokens / ttft_seconds, 1) elif usage and usage.get("completion_tokens") and finished > started: sample["decode_toks"] = round(usage["completion_tokens"] / (finished - started), 1) else: sample["decode_toks"] = round(content_events / max(finished - started, 1e-9), 1) + # A server-reported prompt count always beats the estimate, whatever + # branch produced the decode number. + if usage and usage.get("prompt_tokens"): + sample["prompt_tokens"] = usage["prompt_tokens"] sample.setdefault("prefill_toks", None) return sample diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/blueprint.py b/packages/codex/mlx-agent/src/mlx_agent/blueprint.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/blueprint.py rename to packages/codex/mlx-agent/src/mlx_agent/blueprint.py diff --git a/packages/codex/mlx-agent/src/mlx_agent/capture_platform.py b/packages/codex/mlx-agent/src/mlx_agent/capture_platform.py new file mode 100644 index 0000000..9143d91 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/capture_platform.py @@ -0,0 +1,197 @@ +"""Fail-closed Darwin evidence for an already connected loopback TCP socket.""" + +import ctypes as C +import errno +import os +import signal +import socket +import subprocess +import sys +import time +from typing import Any, Dict, Sequence + +from .compute import process_identity +from .task_contracts import TaskError +from .capture_volume import CaptureVolume + +ProcessIdentity = Dict[str, Any] + + +class _BSDInfo(C.Structure): + _fields_ = [(name, C.c_uint32) for name in ( + "flags", "status", "xstatus", "pid", "ppid", "uid", "gid", "ruid", "rgid", "svuid", "svgid", "reserved")] + _fields_ += [("comm", C.c_char * 16), ("name", C.c_char * 32)] + _fields_ += [(name, C.c_uint32) for name in ("nfiles", "pgid", "jobc", "tdev", "tpgid", "nice")] + _fields_ += [("start_seconds", C.c_uint64), ("start_microseconds", C.c_uint64)] + + +class _FDInfo(C.Structure): + _fields_ = [("fd", C.c_int32), ("kind", C.c_uint32)] + + +class _FileInfo(C.Structure): + _fields_ = [("flags", C.c_uint32), ("status", C.c_uint32), ("offset", C.c_int64), + ("kind", C.c_int32), ("guards", C.c_uint32)] + + +class _VInfoStat(C.Structure): + _fields_ = [("dev", C.c_uint32), ("mode", C.c_uint16), ("nlink", C.c_uint16), + ("ino", C.c_uint64), ("uid", C.c_uint32), ("gid", C.c_uint32)] + _fields_ += [(name, C.c_int64) for name in ( + "atime", "atimensec", "mtime", "mtimensec", "ctime", "ctimensec", "birthtime", "birthtimensec", "size", "blocks")] + _fields_ += [("blksize", C.c_int32), ("flags", C.c_uint32), ("gen", C.c_uint32), + ("rdev", C.c_uint32), ("spare", C.c_int64 * 2)] + + +class _SockBuf(C.Structure): + _fields_ = [(name, C.c_uint32) for name in ("cc", "hiwat", "mbcnt", "mbmax", "lowat")] + _fields_ += [("flags", C.c_int16), ("timeo", C.c_int16)] + + +class _Address(C.Union): + _fields_ = [("bytes", C.c_ubyte * 16), ("words", C.c_uint32 * 4)] + + +class _IPv6Info(C.Structure): + _fields_ = [("hlim", C.c_uint8), ("cksum", C.c_int), ("ifindex", C.c_uint16), ("hops", C.c_int16)] + + +class _InSock(C.Structure): + _fields_ = [("foreign_port", C.c_int), ("local_port", C.c_int), ("generation", C.c_uint64), + ("flags", C.c_uint32), ("flow", C.c_uint32), ("vflag", C.c_uint8), + ("ttl", C.c_uint8), ("reserved", C.c_uint32), ("foreign", _Address), + ("local", _Address), ("tos", C.c_uint8), ("v6", _IPv6Info)] + + +class _TCPSock(C.Structure): + _fields_ = [("ip", _InSock), ("state", C.c_int), ("timers", C.c_int * 4), + ("mss", C.c_int), ("flags", C.c_uint32), ("reserved", C.c_uint32), ("handle", C.c_uint64)] + + +class _Protocol(C.Union): + # Darwin's largest public socket union member is un_sockinfo (528 bytes). + _fields_ = [("tcp", _TCPSock), ("other", C.c_ubyte * 528)] + + +class _SocketInfo(C.Structure): + _fields_ = [("stat", _VInfoStat), ("socket_id", C.c_uint64), ("pcb", C.c_uint64), + ("type", C.c_int), ("protocol", C.c_int), ("family", C.c_int)] + _fields_ += [(name, C.c_int16) for name in ("options", "linger", "state", "qlen", "incqlen", "qlimit", "timeo", "error")] + _fields_ += [("oobmark", C.c_uint32), ("rcv", _SockBuf), ("snd", _SockBuf), + ("kind", C.c_int), ("reserved", C.c_uint32), ("proto", _Protocol)] + + +class _SocketFDInfo(C.Structure): + _fields_ = [("file", _FileInfo), ("socket", _SocketInfo)] + + +def _libproc(): + if sys.platform != "darwin" or C.sizeof(_SocketFDInfo) != 792 or C.sizeof(_BSDInfo) != 136: + raise OSError("The native process/socket ABI is unavailable") + library = C.CDLL("/usr/lib/libproc.dylib", use_errno=True) + library.proc_pidinfo.argtypes = [C.c_int, C.c_int, C.c_uint64, C.c_void_p, C.c_int] + library.proc_pidinfo.restype = C.c_int + library.proc_pidfdinfo.argtypes = [C.c_int, C.c_int, C.c_int, C.c_void_p, C.c_int] + library.proc_pidfdinfo.restype = C.c_int + return library + + +def _started_at(pid): + value = _BSDInfo() + library = _libproc() + if library.proc_pidinfo(pid, 3, 0, C.byref(value), C.sizeof(value)) != C.sizeof(value) or value.pid != pid: + raise OSError(C.get_errno(), "Process start identity is unavailable") + return [value.start_seconds, value.start_microseconds] + + +def capture_process_identity(pid: int) -> ProcessIdentity: + """Bind existing command identity to the kernel's precise start time.""" + if type(pid) is not int or pid <= 0: + raise TaskError("capture_peer_unavailable", "A positive owned process identity is required.") + try: + started = _started_at(pid) + command = process_identity(pid) + if not command or started != _started_at(pid): + raise OSError("The process changed during inspection") + return {"pid": pid, "identity": command, "started_at": started} + except (OSError, ValueError) as error: + raise TaskError("capture_peer_unavailable", "Cannot establish the serving process identity.") from error + + +def stop_owned(process, identity, grace=5): + """Stop an owned process group, signalling only while it still has its recorded identity.""" + if process.poll() is not None: + return + if identity is None or capture_process_identity(process.pid) != identity: + raise TaskError("capture_ownership_lost", "Cannot prove the owned process identity.") + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=grace) + except subprocess.TimeoutExpired: + if capture_process_identity(process.pid) != identity: + raise TaskError("capture_ownership_lost", "Cannot prove the owned process identity.") + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=grace) + + +def _socket_connections(pid): + library = _libproc() + size = library.proc_pidinfo(pid, 1, 0, None, 0) + if size <= 0 or size > 1024 * 1024 - 256: + raise OSError(C.get_errno(), "Owned process descriptor list is unavailable") + descriptors = (_FDInfo * ((size + 256) // C.sizeof(_FDInfo)))() + count = library.proc_pidinfo(pid, 1, 0, descriptors, C.sizeof(descriptors)) + if count <= 0 or count >= C.sizeof(descriptors) or count % C.sizeof(_FDInfo): + raise OSError(C.get_errno(), "Owned process descriptor list changed or is incomplete") + result = [] + for descriptor in descriptors[:count // C.sizeof(_FDInfo)]: + if descriptor.kind != 2: # PROX_FDTYPE_SOCKET + continue + info = _SocketFDInfo() + count = library.proc_pidfdinfo(pid, descriptor.fd, 3, C.byref(info), C.sizeof(info)) + if count != C.sizeof(info): + if count <= 0 and C.get_errno() in (errno.EBADF, errno.ENOENT): + continue # An unrelated descriptor closed; it proves no peer. + raise OSError(C.get_errno(), "Owned socket details are unavailable") + value, tcp = info.socket, info.socket.proto.tcp + if (value.family, value.type, value.protocol, value.kind, tcp.state) != (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, 2, 4): + continue + if not tcp.ip.vflag & 1: + continue + result.append({"fd": descriptor.fd, "socket_id": value.socket_id, "generation": tcp.ip.generation, + "local": [socket.inet_ntop(socket.AF_INET, bytes(tcp.ip.local.bytes[12:])), socket.ntohs(tcp.ip.local_port & 0xffff)], + "peer": [socket.inet_ntop(socket.AF_INET, bytes(tcp.ip.foreign.bytes[12:])), socket.ntohs(tcp.ip.foreign_port & 0xffff)]}) + return result + + +def verify_connected_peer(sock: socket.socket, allowed: Sequence[ProcessIdentity]) -> Dict: + """Inspect only authorized processes; send no HTTP headers or body.""" + if not allowed or len(allowed) > 64 or sock.family != socket.AF_INET: + raise TaskError("capture_peer_unavailable", "Supported owned loopback peer identities are required.") + try: + local, peer = list(sock.getsockname()), list(sock.getpeername()) + if local[0] != "127.0.0.1" or peer[0] != "127.0.0.1": + raise OSError("Capture requires an IPv4 loopback connection") + deadline = time.monotonic() + 2 + while True: + for expected in allowed: + if not isinstance(expected, dict) or set(expected) != {"pid", "identity", "started_at"}: + raise OSError("The allowed process identity is invalid") + if capture_process_identity(expected["pid"]) != expected: + raise TaskError("capture_peer_mismatch", "The pinned serving process changed.") + matches = [entry for entry in _socket_connections(expected["pid"]) + if entry["local"] == peer and entry["peer"] == local] + for match in matches: + if not match["socket_id"]: + raise OSError("The kernel did not expose a socket identity") + if (capture_process_identity(expected["pid"]) == expected + and match in _socket_connections(expected["pid"]) + and list(sock.getsockname()) == local and list(sock.getpeername()) == peer): + return dict(match, schema="connected-peer/1", pid=expected["pid"], identity=dict(expected)) + if time.monotonic() >= deadline: + raise TaskError("capture_peer_mismatch", "The connected socket is not owned by the pinned server.") + time.sleep(.02) + except TaskError: + raise + except (OSError, ValueError, TypeError, KeyError) as error: + raise TaskError("capture_peer_unavailable", "Connected serving-process ownership cannot be inspected.") from error diff --git a/packages/codex/mlx-agent/src/mlx_agent/capture_volume.py b/packages/codex/mlx-agent/src/mlx_agent/capture_volume.py new file mode 100644 index 0000000..5fa5555 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/capture_volume.py @@ -0,0 +1,285 @@ +"""Private, capacity-limited capture volumes with checked mount ownership.""" + +from contextlib import contextmanager +import fcntl +import json +import os +from pathlib import Path +import plistlib +import shutil +import stat +import subprocess +import sys +import time +from uuid import UUID + +from .task_contracts import TaskError, encoded + +HDIUTIL = "/usr/bin/hdiutil" +DISKUTIL = "/usr/sbin/diskutil" +LISTING_DELAYS = (0, 0.05, 0.1, 0.2, 0.4, 0.8) + + +def _run(arguments): + try: + result = subprocess.run(arguments, capture_output=True, timeout=30, umask=0o077) + if result.returncode or len(result.stdout) > 65536 or len(result.stderr) > 65536: + raise OSError("Volume tool failed") + return plistlib.loads(result.stdout) if result.stdout else {} + except (OSError, ValueError, subprocess.SubprocessError, plistlib.InvalidFileException) as error: + raise TaskError("capture_volume_unavailable", "The private capture volume operation failed.", + "Preserve owned recovery data; capture requires an unprivileged private volume.") from error + + +def _identity(path): + info = path.lstat() + return [info.st_dev, info.st_ino] + + +def _private(path, directory=True): + info = path.lstat() + if (stat.S_ISLNK(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077 + or not (stat.S_ISDIR(info.st_mode) if directory else stat.S_ISREG(info.st_mode))): + raise TaskError("unsafe_capture_volume", "Capture storage must remain private and owned.") + + +def _root(path): + path = Path(path) + if not path.is_absolute() or ".." in path.parts or any(parent.is_symlink() for parent in (path, *path.parents)): + raise TaskError("unsafe_capture_volume", "Capture storage cannot traverse links.") + _private(path) + return path + + +def _safe_image(image): + _private(image) + for item in image.rglob("*"): + _private(item, directory=item.is_dir()) + + +def _write_marker(root, record): + payload = encoded(record) + if len(payload) > 4096: + raise TaskError("unsafe_capture_volume", "Capture ownership metadata exceeds its limit.") + temporary = root / "volume.new" + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600) + try: + with os.fdopen(descriptor, "wb") as output: + output.write(payload) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, root / "volume.json") + directory = os.open(root, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + temporary.unlink(missing_ok=True) + + +def _load(root): + root = _root(root) + marker = root / "volume.json" + _private(marker, directory=False) + if marker.stat().st_size > 4096: + raise TaskError("unsafe_capture_volume", "Capture ownership metadata is invalid.") + try: + record = json.loads(marker.read_bytes()) + if (set(record) != {"schema", "task_id", "capacity_bytes", "root_identity", "image_identity", "volume_uuid"} + or record["schema"] != "capture-volume/1" or str(UUID(record["task_id"])) != root.name + or record["root_identity"] != _identity(root) + or record["image_identity"] != _identity(root / "capture.sparsebundle") + or type(record["capacity_bytes"]) is not int or record["capacity_bytes"] <= 0 + or (record["volume_uuid"] is not None and str(UUID(record["volume_uuid"])).upper() != record["volume_uuid"].upper())): + raise ValueError() + _safe_image(root / "capture.sparsebundle") + return record + except (OSError, ValueError, TypeError, KeyError) as error: + raise TaskError("unsafe_capture_volume", "The owned capture volume identity changed.") from error + + +@contextmanager +def _locked(root): + root = _root(root) + descriptor = os.open(root / "volume.lock", os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077: + raise TaskError("unsafe_capture_volume", "Capture lock ownership changed.") + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + raise TaskError("capture_volume_busy", "The owned capture volume is in use.") from error + yield root + finally: + os.close(descriptor) + + +def _listing(root): + """This root's image in one hdiutil listing: None, its one mounted entity, or False when inconsistent.""" + image, mounted = root / "capture.sparsebundle", root / "mounted" + matches = [entry for entry in _run([HDIUTIL, "info", "-plist"]).get("images", []) + if entry.get("image-path") == str(image)] + if not matches: + return None + entities = [entry for image in matches for entry in image.get("system-entities", []) if entry.get("mount-point")] + if len(matches) != 1 or len(entities) != 1 or entities[0]["mount-point"] != str(mounted): + return False + return entities[0] + + +def _attached(root, expect_mount=False): + # The hdiutil listing can lag an attach or half-list an image while other images attach or detach, + # so an inconsistent listing, or an empty one when a mount is expected, is read again before it is trusted. + for delay in LISTING_DELAYS: + time.sleep(delay) + entity = _listing(root) + if entity or (entity is None and not expect_mount): + break + if entity is None: + return None + if entity is False: + raise TaskError("unsafe_capture_volume", "The capture image has an unexpected mount.") + mounted = root / "mounted" + device = entity.get("dev-entry", "") + if not device.startswith("/dev/disk") or "/" in device[len("/dev/"):]: + raise TaskError("unsafe_capture_volume", "The capture device identity is invalid.") + info = _run([DISKUTIL, "info", "-plist", device]) + if info.get("MountPoint") != str(mounted) or not info.get("VolumeUUID"): + raise TaskError("unsafe_capture_volume", "The mounted capture identity is unavailable.") + return {"device": device, "volume_uuid": info["VolumeUUID"], "readonly": info.get("WritableVolume") is False} + + +def _detach(root, expected): + observed = _attached(root, expect_mount=True) + if observed != expected: + raise TaskError("unsafe_capture_volume", "The capture mount changed before detach.") + try: + _run([HDIUTIL, "detach", observed["device"], "-quiet"]) + except TaskError as error: + raise TaskError("capture_volume_unresolved", "The owned capture mount could not be detached.", + "Keep its ownership metadata and retry recovery; do not force-detach another volume.") from error + + +@contextmanager +def _mounted(root, record, readonly): + existing = _attached(root) + if existing: + if record["volume_uuid"] and existing["volume_uuid"].upper() != record["volume_uuid"].upper(): + raise TaskError("unsafe_capture_volume", "The retained capture mount UUID changed.") + _detach(root, existing) + mount = root / "mounted" + mount.mkdir(mode=0o700, exist_ok=True) + _private(mount) + command = [HDIUTIL, "attach", str(root / "capture.sparsebundle"), "-mountpoint", str(mount), + "-nobrowse", "-noautoopen", "-owners", "on", "-plist"] + if readonly: + command.append("-readonly") + try: + _run(command) + except TaskError: + # A tool may fail after committing the mount. Only detach the image + # whose backing path and mountpoint this operation already owns. + attached = _attached(root) + if attached is not None: + _detach(root, attached) + raise + observed = _attached(root, expect_mount=True) + if observed is None: + raise TaskError("capture_volume_unavailable", "The capture image did not mount at its owned path.") + try: + if record["volume_uuid"] and observed["volume_uuid"].upper() != record["volume_uuid"].upper(): + raise TaskError("unsafe_capture_volume", "The capture filesystem UUID changed.") + if readonly and observed["readonly"] is not True: + raise TaskError("unsafe_capture_volume", "A read-only capture mount is required.") + info = mount.stat() + if info.st_uid != os.getuid(): + raise TaskError("capture_volume_unavailable", "The capture filesystem is not owned by this user.") + if not readonly: + mount.chmod(0o700) + _private(mount) + capacity = os.statvfs(mount) + if readonly and not capacity.f_flag & os.ST_RDONLY: + raise TaskError("unsafe_capture_volume", "The kernel did not mount capture data read-only.") + if capacity.f_blocks * capacity.f_frsize > record["capacity_bytes"]: + raise TaskError("unsafe_capture_volume", "The filesystem exceeds the accepted capture capacity.") + yield mount, observed + finally: + _detach(root, observed) + + +class CaptureVolume: + def __init__(self, root): + self.root = root + + @classmethod + def create(cls, root: Path, task_id: str, capacity_bytes: int): + if sys.platform != "darwin": + raise TaskError("capture_volume_unavailable", "Owned capture volumes require macOS.") + root = Path(root) + try: + valid_id = str(UUID(task_id)) == task_id == root.name + except (ValueError, TypeError, AttributeError): + valid_id = False + if (not root.is_absolute() or ".." in root.parts or not valid_id or type(capacity_bytes) is not int + or not 0 < capacity_bytes <= 2 ** 63 - 1 or capacity_bytes % 512): + raise TaskError("unsafe_capture_volume", "Capture identity or capacity is invalid.") + if any(item.is_symlink() for item in (root, *root.parents)): + raise TaskError("unsafe_capture_volume", "Capture storage cannot traverse links.") + root.mkdir(mode=0o700) + _root(root) + created_identity = _identity(root) + image = root / "capture.sparsebundle" + try: + # hdiutil's create -size suffix 'b' means sectors, not bytes. + _run([HDIUTIL, "create", "-sectors", str(capacity_bytes // 512), "-fs", "HFS+", "-layout", "NONE", + "-type", "SPARSEBUNDLE", "-volname", "MLX Capture", "-nospotlight", "-quiet", str(image)]) + _safe_image(image) + record = {"schema": "capture-volume/1", "task_id": task_id, "capacity_bytes": capacity_bytes, + "root_identity": created_identity, "image_identity": _identity(image), "volume_uuid": None} + _write_marker(root, record) + except BaseException: + if not (root / "volume.json").exists(): + try: + _root(root) + if _identity(root) != created_identity: + raise OSError("Created storage was replaced") + for item in root.rglob("*"): + _private(item, directory=item.is_dir()) + if item.lstat().st_dev != created_identity[0]: + raise OSError("Partial storage crosses a filesystem boundary") + if _identity(root) != created_identity: + raise OSError("Created storage was replaced") + shutil.rmtree(root) + except (OSError, TaskError) as error: + raise TaskError("capture_volume_unresolved", "Partial capture storage could not be removed.", + "Inspect {0} for recovery; do not delete a replaced path.".format(root)) from error + raise + with _locked(root): + with _mounted(root, record, readonly=False) as (_, observed): + record["volume_uuid"] = observed["volume_uuid"] + _write_marker(root, record) + return cls(root) + + @staticmethod + @contextmanager + def open_owned(root: Path, readonly: bool): + with _locked(root) as root: + record = _load(root) + if not record["volume_uuid"]: + raise TaskError("capture_volume_unavailable", "Capture creation needs recovery.") + with _mounted(root, record, readonly) as (mounted, _): + yield mounted + + @staticmethod + def remove(root: Path): + with _locked(root) as root: + record = _load(root) + with _mounted(root, record, readonly=bool(record["volume_uuid"])): + pass # Verify the UUID before deleting any retained file. + _load(root) + for item in root.rglob("*"): + if item.is_symlink(): + raise TaskError("unsafe_capture_volume", "Capture removal cannot traverse a link.") + shutil.rmtree(root) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/cli.py b/packages/codex/mlx-agent/src/mlx_agent/cli.py similarity index 64% rename from packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/cli.py rename to packages/codex/mlx-agent/src/mlx_agent/cli.py index bbe2888..463d1a8 100644 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/cli.py +++ b/packages/codex/mlx-agent/src/mlx_agent/cli.py @@ -8,100 +8,9 @@ from datetime import datetime, timezone from pathlib import Path -from .adoption import ADOPTION_SCHEMA_VERSION, AdoptionRequest, AdoptionWorkflow -from .bench import ( - BENCH_PROBE_ID, - GEN_TOKENS_DEFAULT, - RUNS_DEFAULT, - TIMEOUT_DEFAULT, - BenchError, - measure_runtime, -) from .contracts import ErrorDetail, ResultEnvelope -from .convert import ( - Q_BITS_CHOICES, - ConvertError, - plan_convert, - start_convert, - status_convert, -) -from .discovery import DiscoveryRequest, DiscoveryService -from .fleet import ( - FleetConfigAdapter, - FleetError, - assignments_from_adoption, - parse_assignments, - parse_runtime_map, -) -from .fuse import ( - FuseError, - plan_fuse, - start_fuse, - status_fuse, -) -from .host import HostInventory -from .huggingface import HuggingFaceClient -from .installer import Installer, InstallerConflictError -from .interview import build_intent, run_interview -from .lora import ( - BATCH_DEFAULT, - ITERS_DEFAULT, - LAYERS_DEFAULT, - LR_DEFAULT, - LoraError, - plan_lora, - start_lora, - status_lora, -) -from .modality import ALL_FACET_IDS, FOUNDATION_IDS, resolve_facets, resolve_modalities -from .models import DISCOVERY_ROLES, render_md, wire -from .project_blueprint import ( - build_brief, - generate_design_pack, - render_design_pack, - write_design_pack, -) -from .providers import ProviderRegistry -from .research import generate_pack, render_pack, write_pack -from .serve import ( - MAX_TOKENS_DEFAULT, - ServeError, - default_model_present, - load_recipes, - plan_start, - receipts_root, - start_serve, - status_serve, - stop_serve, - wired_port_claim, -) -from .transactions import ( - COOPERATIVE_CONCURRENCY_NOTE, - ConcurrentTransactionError, - Receipt, - Transaction, - _assert_safe_target, - _read_regular, - preview_rollback, - rollback, -) -from .verification import ( - LMStudioRuntimeClient, - MLXVLMRuntimeClient, - OllamaRuntimeClient, - OpenAICompatibleRuntimeClient, - Verifier, -) -from .watch import ( - WatchError, - build_snapshot, - collect_owned, - diff_snapshots, - read_baseline, - snapshot_candidates, - write_snapshot, -) -from .wiring import ConfigAdapter + +# Each command imports its own modules when it runs, so one command never loads every other command. FIXTURE_WARNING = { @@ -135,6 +44,9 @@ def get(url, timeout=10.0): def _discovery_service_from_environment(state_dir=None): + from .discovery import DiscoveryService + from .host import HostInventory + from .huggingface import HuggingFaceClient fixture = os.environ.get("MLX_AGENT_FIXTURE") if not fixture: return DiscoveryService(state_dir=state_dir), None, None @@ -192,6 +104,7 @@ def _validate_fixture(payload): def _add_discovery_arguments(parser): + from .models import DISCOVERY_ROLES parser.add_argument("--role", choices=DISCOVERY_ROLES) parser.add_argument("--limit", type=int, default=6) parser.add_argument("--memory-gb", type=float, help="maximum host memory budget in GB (keeps 20%% runtime headroom)") @@ -215,6 +128,9 @@ def _add_discovery_arguments(parser): def _run_discovery(arguments, legacy): + from .discovery import DiscoveryRequest + from .host import HostInventory + from .models import render_md, wire if arguments.wire: print(wire(arguments.wire, arguments.target, arguments.port)) return 0 @@ -266,6 +182,8 @@ def _run_discovery(arguments, legacy): def _run_inspect_host(arguments): + from .host import HostInventory + from .huggingface import HuggingFaceClient host, warnings = HostInventory.inspect(HuggingFaceClient()._http_get) result = ResultEnvelope.ok("inspect-host", host.to_dict(), warnings=warnings) if arguments.json: @@ -314,6 +232,9 @@ def _emit_adoption_result(result, as_json): def _run_adoption(arguments): + from .adoption import ADOPTION_SCHEMA_VERSION, AdoptionRequest, AdoptionWorkflow + from .discovery import DiscoveryService + from .verification import Verifier operation = "adopt-{0}".format(arguments.adopt_command) state_path = _adoption_state_path(arguments) if not state_path: @@ -388,6 +309,7 @@ def _run_adoption(arguments): def _add_adoption_arguments(parser): + from .models import DISCOVERY_ROLES actions = parser.add_subparsers(dest="adopt_command", required=True) start = actions.add_parser("start", help="start and durably run a model adoption workflow") start.add_argument("--state", help="adoption handoff path") @@ -406,6 +328,8 @@ def _add_adoption_arguments(parser): def _add_research_arguments(parser): + from .modality import ALL_FACET_IDS, FOUNDATION_IDS + from .models import DISCOVERY_ROLES parser.add_argument("--domain", help="one-line domain description (required unless --interview)") parser.add_argument("--role", dest="roles", action="append", choices=DISCOVERY_ROLES, help="model role to research (repeatable)") parser.add_argument("--keyword", dest="keywords", action="append", help="domain keyword to prioritize (repeatable)") @@ -458,6 +382,10 @@ def _emit_research(result, arguments, markdown=None): def _run_research(arguments): + from .huggingface import HuggingFaceClient + from .interview import build_intent, run_interview + from .modality import resolve_facets, resolve_modalities + from .research import generate_pack, render_pack, write_pack operation = "research" try: cli_modalities = tuple(arguments.modalities or ()) @@ -539,6 +467,7 @@ def _run_research(arguments): def _add_blueprint_arguments(parser): + from .modality import FOUNDATION_IDS parser.add_argument("--goal", help="one-line project goal (required)") parser.add_argument( "--modality", @@ -582,6 +511,7 @@ def _emit_blueprint(result, arguments, markdown=None): def _run_blueprint(arguments): + from .project_blueprint import build_brief, generate_design_pack, render_design_pack, write_design_pack operation = "blueprint" try: if not arguments.goal: @@ -642,6 +572,7 @@ def _receipt_data(receipt): def _wire_failure(operation, code, message, remediation, data=None): + from .transactions import COOPERATIVE_CONCURRENCY_NOTE return ResultEnvelope( operation=operation, status="error", data=data or {}, warnings=[{"code": "cooperative_concurrency", "message": COOPERATIVE_CONCURRENCY_NOTE}], @@ -650,12 +581,15 @@ def _wire_failure(operation, code, message, remediation, data=None): def _wire_ok(operation, data): + from .transactions import COOPERATIVE_CONCURRENCY_NOTE return ResultEnvelope.ok(operation, data, warnings=[{ "code": "cooperative_concurrency", "message": COOPERATIVE_CONCURRENCY_NOTE, }]) def _wire_render(arguments): + from .transactions import _assert_safe_target, _read_regular + from .wiring import ConfigAdapter path = _assert_safe_target(arguments.path) existing = _read_regular(path).decode("utf-8") adapter = ConfigAdapter.detect(path, runtime=arguments.target) @@ -665,6 +599,9 @@ def _wire_render(arguments): def _run_wire(arguments): + from .transactions import ( + ConcurrentTransactionError, Receipt, Transaction, _assert_safe_target, _read_regular, preview_rollback, rollback, + ) operation = "wire-{0}".format(arguments.wire_command) try: if arguments.wire_command == "status": @@ -751,6 +688,7 @@ def _run_wire(arguments): def _bench_runtime_client(name): + from .verification import LMStudioRuntimeClient, MLXVLMRuntimeClient, OllamaRuntimeClient, OpenAICompatibleRuntimeClient if name == "ollama": return OllamaRuntimeClient() if name == "lmstudio": @@ -765,6 +703,8 @@ def _bench_runtime_client(name): def _add_bench_arguments(parser): + from .bench import GEN_TOKENS_DEFAULT, RUNS_DEFAULT, TIMEOUT_DEFAULT + from .models import DISCOVERY_ROLES actions = parser.add_subparsers(dest="bench_command", required=True) run = actions.add_parser( "run", @@ -788,6 +728,8 @@ def _add_bench_arguments(parser): def _run_bench(arguments): + from .bench import BenchError, measure_runtime + from .host import HostInventory operation = "bench-{0}".format(arguments.bench_command) try: if arguments.bench_command == "aggregate": @@ -880,16 +822,26 @@ def _run_bench(arguments): def _add_serve_arguments(parser): + from .serve import MAX_TOKENS_DEFAULT actions = parser.add_subparsers(dest="serve_command", required=True) start = actions.add_parser( "start", help="preview, then confirmation-gated launch of a local MLX server", ) - start.add_argument("--repo", required=True, help="publisher/model present in the local Hugging Face cache") - start.add_argument("--runtime", required=True, choices=["mlx_lm", "mlx-vlm"]) + start.add_argument("--repo", default=None, help="publisher/model present in the local Hugging Face cache") + start.add_argument("--path", default=None, help="local model directory to serve instead of a cache repo id") + start.add_argument("--runtime", required=True, choices=["auto", "mlx_lm", "mlx-vlm", "strata"]) + start.add_argument("--strata-speculation", choices=["on", "off"], help="opt-in draft-and-verify decoding for Strata profiles (default: off)") + start.add_argument("--strata-expert-memory-gb", type=float, help="Strata resident expert budget in decimal GB (0.5–64); other experts remain on disk") + start.add_argument("--strata-context-tokens", type=int, help="Strata context capacity (512–131072 tokens)") start.add_argument("--port", type=int, default=None, help="loopback port (defaults per runtime recipe)") start.add_argument("--max-tokens", type=int, default=MAX_TOKENS_DEFAULT) start.add_argument("--adapter-path", default=None, help="LoRA adapter directory (mlx_lm only)") + start.add_argument("--training-receipt", default=None, help="completed LoRA receipt selecting its pinned base and adapter") + start.add_argument("--thinking", choices=["default", "off"], default="default", + help="off asks the chat template to skip reasoning (mlx_lm); other runtimes report unsupported") + start.add_argument("--jit", action="store_true", help="keep endpoint reachable; load local weights on inference requests") + _add_memory_arguments(start) start.add_argument("--launchd", action="store_true", help="install a launchd agent plist instead of spawning now") start.add_argument("--launchd-dir", default=None, help="launchd target directory (defaults to ~/Library/LaunchAgents)") start.add_argument("--confirm", action="store_true", help="authorize this reviewed server launch") @@ -901,12 +853,38 @@ def _add_serve_arguments(parser): stop.add_argument("--port", type=int, required=True) stop.add_argument("--receipts-dir", default=None) stop.add_argument("--json", action="store_true") + unload = actions.add_parser("unload", help="release JIT model weights while keeping the endpoint reachable") + unload.add_argument("--port", type=int, required=True) + unload.add_argument("--expected-pid", type=int, default=None) + unload.add_argument("--receipts-dir", default=None) + unload.add_argument("--json", action="store_true") + policy = actions.add_parser("policy", help="configure owned JIT memory management without restarting") + policy.add_argument("--port", type=int, required=True) + policy.add_argument("--expected-pid", type=int, required=True) + policy.add_argument("--receipts-dir", default=None) + policy.add_argument("--json", action="store_true") + _add_memory_arguments(policy) status = actions.add_parser("status", help="cross-check serve receipts against live processes") status.add_argument("--receipts-dir", default=None) status.add_argument("--json", action="store_true") +def _add_memory_arguments(parser): + parser.add_argument("--idle-timeout", type=int, default=None, help="unload after this many idle seconds (0 disables)") + parser.add_argument("--keep-loaded", action="store_true", help="prevent automatic idle unload after use") + parser.add_argument("--min-headroom-gb", type=float, default=None, help="reserve GB beyond estimated weights and runtime allowance") + + +def _memory_arguments(arguments): + return {"idle_timeout_seconds": arguments.idle_timeout or 0, "keep_loaded": arguments.keep_loaded, + "minimum_headroom_gb": arguments.min_headroom_gb} + + def _run_serve(arguments): + from .serve import ( + ServeError, configure_serve_memory, default_model_present, load_recipes, plan_start, resolve_serve_runtime, + start_serve, status_serve, stop_serve, unload_serve, wired_port_claim, + ) operation = "serve-{0}".format(arguments.serve_command) try: if arguments.serve_command == "status": @@ -923,16 +901,46 @@ def _run_serve(arguments): outcome["status"], outcome["port"], outcome["pid"] ), ) + if arguments.serve_command == "unload": + outcome = unload_serve(arguments.port, arguments.receipts_dir, arguments.expected_pid) + return _emit_serve_result(ResultEnvelope.ok(operation, outcome), arguments.json, + human="Model unloaded; endpoint remains reachable on port {}.".format(arguments.port)) + if arguments.serve_command == "policy": + outcome = configure_serve_memory(arguments.port, _memory_arguments(arguments), arguments.receipts_dir, arguments.expected_pid) + return _emit_serve_result(ResultEnvelope.ok(operation, outcome), arguments.json, + human="JIT memory policy saved and applied on port {}.".format(arguments.port)) recipes = load_recipes() + if arguments.training_receipt and arguments.launchd: + raise ServeError("invalid_arguments", "Receipt-bound adapter serving requires supervised startup.", + "Use serve start without --launchd.") + selected_runtime = "mlx_lm" if arguments.training_receipt and arguments.runtime == "auto" else arguments.runtime + runtime, executable = resolve_serve_runtime( + arguments.repo, arguments.path, selected_runtime, recipes, hf_cache=arguments.hf_cache) + if arguments.strata_speculation is not None and runtime != "strata": + raise ServeError("invalid_arguments", "--strata-speculation requires the Strata runtime.", "Select a prepared local Strata profile.") plan = plan_start( arguments.repo, - arguments.runtime, + runtime, recipes, port=arguments.port, max_tokens=arguments.max_tokens, adapter_path=arguments.adapter_path, + training_receipt=arguments.training_receipt, + path=arguments.path, + jit=arguments.jit, + receipts_dir=arguments.receipts_dir, + hf_cache=arguments.hf_cache, + runtime_executable=executable, + strata_speculation=arguments.strata_speculation == "on", + strata_expert_memory_gb=arguments.strata_expert_memory_gb, + strata_context_tokens=arguments.strata_context_tokens, + memory_policy=_memory_arguments(arguments) if (arguments.idle_timeout is not None or arguments.keep_loaded or arguments.min_headroom_gb is not None) else None, + thinking=arguments.thinking, ) if arguments.launchd: + if arguments.jit: + raise ServeError("invalid_arguments", "JIT endpoints require the supervised serve lifecycle.", + "Use the app's endpoint supervisor instead of --launchd.") return _run_serve_launchd(arguments, plan, operation) if not arguments.confirm: result = ResultEnvelope.ok( @@ -941,7 +949,7 @@ def _run_serve(arguments): if arguments.json: print(json.dumps(result.to_dict(), indent=2)) else: - print("Serve plan: {0} on 127.0.0.1:{1}".format(plan["repo"], plan["port"])) + print("Serve plan: {0} on 127.0.0.1:{1}".format(plan["repo"] or plan["path"], plan["port"])) print(" argv: {0}".format(" ".join(plan["argv"]))) print(" readiness: {0}".format(plan["readiness"])) print(" preview_hash: {0}".format(plan["preview_hash"])) @@ -959,10 +967,11 @@ def _run_serve(arguments): wired_claims=wired_claims, ) receipt = outcome["receipt"] + served_model = receipt["repo"] or receipt.get("path") return _emit_serve_result( ResultEnvelope.ok(operation, outcome), arguments.json, human="serve started: {0} on 127.0.0.1:{1} (pid {2})\n log: {3}".format( - receipt["repo"], receipt["port"], receipt["pid"], receipt["log_path"] + served_model, receipt["port"], receipt["pid"], receipt["log_path"] ), ) except ServeError as error: @@ -986,6 +995,8 @@ def _run_serve(arguments): def _run_serve_launchd(arguments, plan, operation): """Preview-confirm installation of a launchd agent for a serve plan.""" + from .serve import default_model_present, receipts_root + from .transactions import Transaction from .serve import LaunchdPlistAdapter, launchd_label, render_launchd_plist launchd_dir = Path(arguments.launchd_dir) if arguments.launchd_dir else ( @@ -1012,7 +1023,7 @@ def _run_serve_launchd(arguments, plan, operation): "Create it yourself, or pass --launchd-dir.", ) return _emit_serve_result(result, arguments.json) - if not default_model_present(arguments.hf_cache)(plan["repo"]): + if plan["repo"] is not None and not default_model_present(arguments.hf_cache)(plan["repo"]): result = ResultEnvelope.fail( operation, "model_not_local", @@ -1020,6 +1031,14 @@ def _run_serve_launchd(arguments, plan, operation): "Download it with the runtime's own pull command first; serve never downloads.", ) return _emit_serve_result(result, arguments.json) + if plan["path"] is not None and not Path(plan["path"]).is_dir(): + result = ResultEnvelope.fail( + operation, + "model_not_local", + "The model directory is not present: {0}.".format(plan["path"]), + "Point --path at an existing local model directory; serve never downloads models.", + ) + return _emit_serve_result(result, arguments.json) transaction = Transaction(receipts_dir=arguments.receipts_dir) preview = transaction.preview([{ "path": str(target), "content": content, "runtime": "launchd", @@ -1105,6 +1124,8 @@ def _add_fleet_arguments(parser): help="read role assignments from a completed adopt handoff") action.add_argument("--runtime-map", action="append", default=None, metavar="ROLE=RUNTIME", help="per-role runtime override: mlx_lm or mlx-vlm (repeatable)") + action.add_argument("--port-map", action="append", default=None, metavar="ROLE=PORT", + help="per-role api_base port override (repeatable; defaults are mlx_lm 8080, mlx-vlm 8083)") action.add_argument("--allow-missing", action="store_true", help="warn instead of failing when a model is not in a local inventory") action.add_argument("--json", action="store_true") @@ -1116,6 +1137,7 @@ def _add_fleet_arguments(parser): def _fleet_inputs(arguments): + from .fleet import FleetError, assignments_from_adoption, parse_assignments, parse_port_map, parse_runtime_map if arguments.assign and arguments.from_adoption: raise FleetError( "invalid_arguments", @@ -1127,10 +1149,12 @@ def _fleet_inputs(arguments): else: assignments = parse_assignments(arguments.assign) runtime_map = parse_runtime_map(arguments.runtime_map) - return assignments, runtime_map + port_map = parse_port_map(arguments.port_map) + return assignments, runtime_map, port_map def _fleet_missing_models(assignments): + from .verification import LMStudioRuntimeClient, MLXVLMRuntimeClient, OllamaRuntimeClient, OpenAICompatibleRuntimeClient from .model_doctor import ( collect_runtime_inventory, default_hf_cache, @@ -1153,13 +1177,15 @@ def _fleet_missing_models(assignments): def _run_fleet(arguments): + from .fleet import FleetConfigAdapter, FleetError + from .transactions import ConcurrentTransactionError, Transaction, _assert_safe_target, _read_regular operation = "fleet-{0}".format(arguments.fleet_command) try: - assignments, runtime_map = _fleet_inputs(arguments) + assignments, runtime_map, port_map = _fleet_inputs(arguments) path = _assert_safe_target(arguments.path) existing = _read_regular(path).decode("utf-8") adapter = FleetConfigAdapter(path) - content = adapter.render(assignments, runtime_map, existing=existing) + content = adapter.render(assignments, runtime_map, existing=existing, port_map=port_map) missing = _fleet_missing_models(assignments) warnings = [] if missing and not arguments.allow_missing: @@ -1230,6 +1256,7 @@ def _run_fleet(arguments): def _watch_runtime_clients(): + from .verification import LMStudioRuntimeClient, MLXVLMRuntimeClient, OllamaRuntimeClient, OpenAICompatibleRuntimeClient return [ OllamaRuntimeClient(), LMStudioRuntimeClient(), @@ -1254,6 +1281,10 @@ def _add_watch_arguments(parser): def _run_watch(arguments): + from .discovery import DiscoveryRequest + from .watch import ( + WatchError, build_snapshot, collect_owned, diff_snapshots, read_baseline, snapshot_candidates, write_snapshot, + ) operation = "watch-{0}".format(arguments.watch_command) try: service, fixture_warning, fixture_error = _discovery_service_from_environment() @@ -1324,26 +1355,364 @@ def _run_watch(arguments): return 2 +def _add_intake_arguments(parser): + actions = parser.add_subparsers(dest="intake_command", required=True) + resolve = actions.add_parser( + "resolve", help="classify a pasted Hugging Face model and say how it converts (read-only)" + ) + resolve.add_argument("source", help="Hugging Face link or org/name id") + resolve.add_argument("--revision", default=None) + resolve.add_argument("--json", action="store_true") + fetch = actions.add_parser( + "fetch", help="preview, then confirmation-gated download of a model snapshot or one GGUF file" + ) + fetch.add_argument("source") + fetch.add_argument("--file", default=None) + fetch.add_argument("--revision", default=None) + fetch.add_argument("--hf-cache", default=None) + fetch.add_argument("--local-dir", default=None) + fetch.add_argument( + "--model-type", default=None, + help="model_type from intake resolve; a ported type downloads only the port's files", + ) + fetch.add_argument("--confirm", action="store_true") + fetch.add_argument("--preview-hash") + fetch.add_argument("--receipts-dir", default=None) + fetch.add_argument("--json", action="store_true") + status = actions.add_parser("status", help="cross-check download receipts against live processes") + status.add_argument("--receipts-dir", default=None) + status.add_argument("--json", action="store_true") + analysis = actions.add_parser( + "port-analysis", help="deterministic facts for porting an unsupported architecture (no weights)" + ) + analysis.add_argument("source") + analysis.add_argument("--revision", default=None) + analysis.add_argument("--json", action="store_true") + port_plan = actions.add_parser( + "port-plan", help="draft a porting plan with a local loopback model (writes one Markdown file)" + ) + port_plan.add_argument("source") + port_plan.add_argument("--endpoint", required=True, help="loopback OpenAI-compatible base URL") + port_plan.add_argument("--model", required=True, help="model id the endpoint serves") + port_plan.add_argument("--revision", default=None) + port_plan.add_argument("--context-tokens", type=int, default=32768) + port_plan.add_argument("--max-tokens", type=int, default=4096) + port_plan.add_argument("--out-dir", default=None) + port_plan.add_argument("--json", action="store_true") + + +def _intake_resolve_human(payload): + reasons = " ({0})".format(", ".join(payload["reasons"])) if payload["reasons"] else "" + lines = ["{0}: {1}{2}".format(payload["source"]["repo"], payload["verdict"], reasons)] + if payload["backend"]: + lines.append(" backend: {0} ({1})".format( + payload["backend"], "installed" if payload["backend_installed"] else "not installed" + )) + task = payload.get("task") + if task: + lines.append(" task: {0} [{1}] via {2}".format(task["type"], ", ".join(task["use_cases"]), task["source"])) + for component in payload["components"]: + if component["matches"]: + match = component["matches"][0] + target = "{0} {1} ({2})".format(match["backend"], match["module"], match["match"]) + else: + target = "no MLX implementation" + lines.append(" {0}: {1} -> {2}".format(component["role"], component["model_type"], target)) + if payload["mlx_conversions"]: + lines.append(" MLX conversions on the Hub:") + for entry in payload["mlx_conversions"]: + lines.append(" {0} ({1} downloads)".format(entry["repo"], entry["downloads"])) + for warning in payload["warnings"]: + lines.append(" warning: {0}".format(warning)) + return "\n".join(lines) + + +def _run_intake(arguments): + from .intake import resolve as intake_resolve + from .intake_fetch import FetchError, plan_fetch, start_fetch, status_fetch + from .intake_source import IntakeSourceError + from .port_analysis import PortAnalysisError, analyze as port_analyze + from .port_plan import draft_port_plan + operation = "intake-{0}".format(arguments.intake_command) + try: + if arguments.intake_command == "resolve": + payload = intake_resolve(arguments.source, revision=arguments.revision) + return _emit_serve_result( + ResultEnvelope.ok(operation, payload), arguments.json, + human=_intake_resolve_human(payload), + ) + if arguments.intake_command == "status": + jobs = status_fetch(arguments.receipts_dir) + return _emit_serve_result(ResultEnvelope.ok(operation, {"jobs": jobs}), arguments.json) + if arguments.intake_command == "fetch": + plan = plan_fetch(arguments.source, revision=arguments.revision, file=arguments.file, + hf_cache=arguments.hf_cache, local_dir=arguments.local_dir, + model_type=arguments.model_type) + outcome = start_fetch(plan, receipts_dir=arguments.receipts_dir, + confirm=arguments.confirm, preview_hash=arguments.preview_hash) + if outcome["status"] == "preview": + result = ResultEnvelope.ok(operation, {"plan": plan, "requires_confirmation": True}) + if arguments.json: + print(json.dumps(result.to_dict(), indent=2)) + else: + print("Fetch plan: {0} {1}; preview_hash: {2}".format( + plan["repo"], plan["file"] or "(snapshot)", plan["preview_hash"])) + print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") + return 2 + return _emit_serve_result(ResultEnvelope.ok(operation, outcome), arguments.json) + if arguments.intake_command == "port-analysis": + payload = port_analyze(arguments.source, revision=arguments.revision) + return _emit_serve_result(ResultEnvelope.ok(operation, payload), arguments.json) + if arguments.intake_command == "port-plan": + if not 1024 <= arguments.context_tokens <= 1048576 or not 256 <= arguments.max_tokens <= 32768: + raise ValueError("--context-tokens must be 1024..1048576 and --max-tokens 256..32768") + result = draft_port_plan( + arguments.source, arguments.endpoint, arguments.model, revision=arguments.revision, + context_tokens=arguments.context_tokens, max_tokens=arguments.max_tokens, + out_dir=arguments.out_dir, + ) + return _emit_serve_result( + ResultEnvelope.ok(operation, result), arguments.json, + human="port plan draft written: {0}".format(result["path"]), + ) + raise ValueError("unknown intake command") + except IntakeSourceError as error: + result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) + except PortAnalysisError as error: + result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) + except FetchError as error: + result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) + except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: + result = ResultEnvelope.fail( + operation, "intake_failed", str(error), + "Check the pasted link and your network, then retry.", + ) + if arguments.json: + print(json.dumps(result.to_dict(), indent=2)) + else: + payload = result.to_dict()["error"] + print("intake failed [{0}]: {1}\nremediation: {2}".format( + payload["code"], payload["message"], payload["remediation"] + )) + return 2 + + +def _add_backend_arguments(parser): + actions = parser.add_subparsers(dest="backend_command", required=True) + listing = actions.add_parser("list", help="list declared converter backends and their install state") + listing.add_argument("--receipts-dir", default=None) + listing.add_argument("--json", action="store_true") + for name, text in ( + ("install", "preview, then confirmation-gated install of an optional backend into its own venv"), + ("remove", "preview, then move an installed optional backend to the Trash"), + ): + action = actions.add_parser(name, help=text) + action.add_argument("backend") + action.add_argument("--confirm", action="store_true") + action.add_argument("--preview-hash") + action.add_argument("--receipts-dir", default=None) + action.add_argument("--json", action="store_true") + + +def _backend_list_human(report): + lines = ["Backends under {0}:".format(report["root"])] + for entry in report["backends"]: + lines.append(" {0} {1}: {2} ({3} model types, {4})".format( + entry["id"], entry["version"], entry["state"], entry["model_types"], entry["registry_source"] + )) + return "\n".join(lines) + + +def _run_backend(arguments): + from .backend_install import list_backends, plan_install, plan_remove, remove_backend, start_install + from .backends import BackendError + operation = "backend-{0}".format(arguments.backend_command) + try: + if arguments.backend_command == "list": + report = list_backends(receipts_dir=arguments.receipts_dir) + return _emit_serve_result( + ResultEnvelope.ok(operation, report), arguments.json, human=_backend_list_human(report) + ) + if arguments.backend_command == "install": + plan = plan_install(arguments.backend) + outcome = start_install(plan, receipts_dir=arguments.receipts_dir, + confirm=arguments.confirm, preview_hash=arguments.preview_hash) + else: + plan = plan_remove(arguments.backend) + outcome = remove_backend(plan, receipts_dir=arguments.receipts_dir, + confirm=arguments.confirm, preview_hash=arguments.preview_hash) + if outcome["status"] == "preview": + result = ResultEnvelope.ok(operation, {"plan": plan, "requires_confirmation": True}) + if arguments.json: + print(json.dumps(result.to_dict(), indent=2)) + else: + print("Backend {0} plan for {1}; preview_hash: {2}".format( + arguments.backend_command, plan["id"], plan["preview_hash"] + )) + print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") + return 2 + return _emit_serve_result(ResultEnvelope.ok(operation, outcome), arguments.json) + except BackendError as error: + result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) + except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: + result = ResultEnvelope.fail(operation, "backend_failed", str(error), + "Correct the backend arguments or state, then retry.") + if arguments.json: + print(json.dumps(result.to_dict(), indent=2)) + else: + payload = result.to_dict()["error"] + print("backend failed [{0}]: {1}\nremediation: {2}".format( + payload["code"], payload["message"], payload["remediation"] + )) + return 2 + + def _add_convert_arguments(parser): + from .convert import Q_BITS_CHOICES actions = parser.add_subparsers(dest="convert_command", required=True) start = actions.add_parser( "start", help="preview, then confirmation-gated quantization of a cached model", ) - start.add_argument("--repo", required=True, help="publisher/model present in the local Hugging Face cache") + source = start.add_mutually_exclusive_group(required=True) + source.add_argument("--repo", help="publisher/model present in the local Hugging Face cache") + source.add_argument("--gguf", help="path to a local .gguf file to dequantize and convert") + start.add_argument( + "--backend", default=None, + help="optional converter backend for --repo (see backend list); default mlx-lm", + ) + start.add_argument( + "--model-type", default=None, + help="model_type from intake resolve; selects a port's own converter when it ships one", + ) + start.add_argument("--subfolder", default=None, help="checkpoint folder inside the repository (converts from the cache)") start.add_argument("--q-bits", type=int, default=4, choices=Q_BITS_CHOICES) start.add_argument("--out", default=None, help="output directory (default -MLX-bit)") start.add_argument("--confirm", action="store_true", help="authorize this reviewed conversion") start.add_argument("--preview-hash", help="hash returned by the separately reviewed convert preview") start.add_argument("--receipts-dir", default=None) start.add_argument("--hf-cache", default=None) + start.add_argument("--source-path", default=None, help="exact downloaded checkpoint directory for --repo; no second download") start.add_argument("--json", action="store_true") status = actions.add_parser("status", help="cross-check convert receipts against live processes") status.add_argument("--receipts-dir", default=None) status.add_argument("--json", action="store_true") + scan = actions.add_parser( + "scan", + help="inventory local GGUF weights: converted, pending, and redundant copies", + ) + scan.add_argument( + "--gguf-root", action="append", default=None, dest="gguf_roots", + help="directory to scan for .gguf files (repeatable)", + ) + scan.add_argument( + "--mlx-root", action="append", default=None, dest="mlx_roots", + help="directory holding MLX outputs (repeatable; defaults to the GGUF roots)", + ) + scan.add_argument("--receipts-dir", default=None) + scan.add_argument("--limit", type=int, default=None, help="maximum files to inspect") + scan.add_argument( + "--no-signature", action="store_true", + help="skip content signatures (faster, weaker duplicate detection)", + ) + scan.add_argument("--pending-only", action="store_true", help="list only unconverted models") + scan.add_argument("--json", action="store_true") + transcribe = actions.add_parser( + "transcribe", + help="transcribe one audio file with a converted speech model (read-only verification canary)", + ) + transcribe.add_argument("--path", required=True, help="converted speech model directory") + transcribe.add_argument("--audio", required=True, help="audio file (WAV, FLAC, MP3)") + transcribe.add_argument("--language", default=None, help="language code passed to models that take one") + transcribe.add_argument("--timeout", type=int, default=300, help="seconds before the backend run is stopped") + transcribe.add_argument("--json", action="store_true") + decide = actions.add_parser( + "decide", + help="answer typed questions about a state with a converted classification model (read-only verification canary)", + ) + decide.add_argument("--path", required=True, help="converted classification model directory") + decide.add_argument("--request", required=True, help='JSON file: {"state": ..., "questions": {...}}') + decide.add_argument("--timeout", type=int, default=300, help="seconds before the backend run is stopped") + decide.add_argument("--json", action="store_true") + generate = actions.add_parser( + "generate", + help="render one prompt with a converted image-generation model to a new PNG (verification canary)", + ) + generate.add_argument("--path", required=True, help="converted image model directory") + generate.add_argument("--prompt", required=True) + generate.add_argument("--out", required=True, help="new absolute .png path") + generate.add_argument("--width", type=int, default=1024) + generate.add_argument("--height", type=int, default=1024) + generate.add_argument("--steps", type=int, default=40) + generate.add_argument("--seed", type=int, default=42) + generate.add_argument("--timeout", type=int, default=1800, help="seconds before the backend run is stopped") + generate.add_argument("--json", action="store_true") + music = actions.add_parser("music", help="generate music from an existing local music model") + music.add_argument("--path", required=True) + music.add_argument("--caption", required=True) + music.add_argument("--lyrics", default="[instrumental]") + music.add_argument("--out", required=True) + music.add_argument("--duration", type=float, default=15.0) + music.add_argument("--steps", type=int, default=30) + music.add_argument("--seed", type=int, default=42) + music.add_argument("--timeout", type=int, default=3600) + music.add_argument("--json", action="store_true") + speak = actions.add_parser( + "speak", + help="synthesize one text with a converted text-to-speech model to a new WAV (read-only verification canary)", + ) + speak.add_argument("--path", required=True, help="converted text-to-speech model directory") + speak.add_argument("--text", required=True, help="text to speak (1..2000 characters)") + speak.add_argument("--out", required=True, help="new absolute .wav path") + speak.add_argument("--voice", default=None, help="voice name the model lists, such as af_heart") + speak.add_argument("--speed", type=float, default=1.0, help="speech speed multiplier (0.5..2.0)") + speak.add_argument("--lang-code", default=None, help="language code passed to models that take one") + speak.add_argument("--timeout", type=int, default=600, help="seconds before the backend run is stopped") + speak.add_argument("--json", action="store_true") + describe = actions.add_parser( + "describe", + help="answer one question about an image or a video with a converted vision-language model (read-only verification canary)", + ) + describe.add_argument("--path", required=True, help="converted vision-language model directory") + describe.add_argument("--prompt", required=True, help="question about the media (1..4000 characters)") + describe.add_argument("--image", default=None, help="image file (png, jpg, jpeg, webp)") + describe.add_argument("--video", default=None, help="video file (mp4, mov, m4v)") + describe.add_argument("--max-tokens", type=int, default=256, help="answer length limit (1..4096)") + describe.add_argument("--temperature", type=float, default=0.0) + describe.add_argument("--fps", type=float, default=None, help="frames per second sampled from --video (0.1..8, default 1.0)") + describe.add_argument("--max-pixels", type=int, default=None, help="shrink the media to at most this many pixels") + describe.add_argument("--timeout", type=int, default=900, help="seconds before the backend run is stopped") + describe.add_argument("--json", action="store_true") + video = actions.add_parser( + "video", + help="render one prompt with a converted text-to-video model to a new MP4 (verification canary)", + ) + video.add_argument("--path", required=True, help="converted text-to-video model directory") + video.add_argument("--prompt", required=True, help="scene description (1..2000 characters)") + video.add_argument("--out", required=True, help="new absolute .mp4 path") + video.add_argument("--width", type=int, default=832, help="frame width, a multiple of the model's alignment (16 for Wan2.1)") + video.add_argument("--height", type=int, default=480, help="frame height, a multiple of the model's alignment (16 for Wan2.1)") + video.add_argument("--frames", type=int, default=81, help="frames requested, 4n+1 (5..241); the MP4 can hold a few more (Wan2.1 decodes 4 frames per latent step)") + video.add_argument("--fps", type=int, default=None, help="playback rate written to the MP4 (default: the model's, 16 for Wan2.1)") + video.add_argument("--steps", type=int, default=None, help="denoising steps (default: the model's, 50 for Wan2.1)") + video.add_argument("--seed", type=int, default=42) + video.add_argument("--timeout", type=int, default=3600, help="seconds before the backend run is stopped") + video.add_argument("--json", action="store_true") def _run_convert(arguments): + from .compute import ComputeBusy + from .convert import ConvertError, plan_convert, plan_gguf_convert, start_convert, status_convert + from .decide import plan_decide, run_decide + from .describe import plan_describe, run_describe + from .generate import plan_generate, run_generate + from .gguf import GGUFError + from .inference_runner import InferenceError + from .music import plan_music, run_music + from .serve import default_model_present + from .speak import plan_speak, run_speak + from .transcribe import plan_transcribe, run_transcribe + from .video import plan_video, run_video operation = "convert-{0}".format(arguments.convert_command) try: if arguments.convert_command == "status": @@ -1352,7 +1721,88 @@ def _run_convert(arguments): ResultEnvelope.ok(operation, {"jobs": entries}), arguments.json, human=_convert_status_human(entries), ) - plan = plan_convert(arguments.repo, q_bits=arguments.q_bits, out=arguments.out) + if arguments.convert_command == "transcribe": + if not 1 <= arguments.timeout <= 3600: + raise ValueError("--timeout must be 1..3600 seconds") + plan = plan_transcribe(arguments.path, arguments.audio, language=arguments.language) + from .compute import guarded_start + result = guarded_start("transcribe", run_transcribe, plan, timeout=arguments.timeout) + return _emit_serve_result( + ResultEnvelope.ok(operation, result), arguments.json, human=result["text"], + ) + if arguments.convert_command == "decide": + if not 1 <= arguments.timeout <= 3600: + raise ValueError("--timeout must be 1..3600 seconds") + from .compute import guarded_start + result = guarded_start("decide", run_decide, plan_decide(arguments.path, arguments.request), timeout=arguments.timeout) + return _emit_serve_result( + ResultEnvelope.ok(operation, result), arguments.json, human=json.dumps(result["answers"], indent=2), + ) + if arguments.convert_command == "generate": + if not 1 <= arguments.timeout <= 7200: + raise ValueError("--timeout must be 1..7200 seconds") + plan = plan_generate(arguments.path, arguments.prompt, arguments.out, width=arguments.width, + height=arguments.height, steps=arguments.steps, seed=arguments.seed) + from .compute import guarded_start + result = guarded_start("generate", run_generate, plan, timeout=arguments.timeout) + return _emit_serve_result(ResultEnvelope.ok(operation, result), arguments.json, human=result["path"]) + if arguments.convert_command == "music": + if not 1 <= arguments.timeout <= 14400: + raise ValueError("--timeout must be 1..14400 seconds") + plan = plan_music(arguments.path, arguments.caption, arguments.lyrics, arguments.out, + duration=arguments.duration, steps=arguments.steps, seed=arguments.seed) + from .compute import guarded_start + result = guarded_start("music", run_music, plan, timeout=arguments.timeout) + return _emit_serve_result(ResultEnvelope.ok(operation, result), arguments.json, human=result["path"]) + if arguments.convert_command == "speak": + if not 1 <= arguments.timeout <= 7200: + raise ValueError("--timeout must be 1..7200 seconds") + plan = plan_speak(arguments.path, arguments.text, arguments.out, voice=arguments.voice, + speed=arguments.speed, lang_code=arguments.lang_code) + from .compute import guarded_start + result = guarded_start("speak", run_speak, plan, timeout=arguments.timeout) + return _emit_serve_result(ResultEnvelope.ok(operation, result), arguments.json, human=result["path"]) + if arguments.convert_command == "describe": + if not 1 <= arguments.timeout <= 7200: + raise ValueError("--timeout must be 1..7200 seconds") + plan = plan_describe(arguments.path, arguments.prompt, image=arguments.image, video=arguments.video, + max_tokens=arguments.max_tokens, temperature=arguments.temperature, + fps=arguments.fps, max_pixels=arguments.max_pixels) + from .compute import guarded_start + result = guarded_start("describe", run_describe, plan, timeout=arguments.timeout) + return _emit_serve_result(ResultEnvelope.ok(operation, result), arguments.json, human=result["text"]) + if arguments.convert_command == "video": + if not 1 <= arguments.timeout <= 14400: + raise ValueError("--timeout must be 1..14400 seconds") + plan = plan_video(arguments.path, arguments.prompt, arguments.out, width=arguments.width, + height=arguments.height, frames=arguments.frames, fps=arguments.fps, + steps=arguments.steps, seed=arguments.seed) + from .compute import guarded_start + result = guarded_start("video", run_video, plan, timeout=arguments.timeout) + return _emit_serve_result(ResultEnvelope.ok(operation, result), arguments.json, human=result["path"]) + if arguments.convert_command == "scan": + report = _convert_scan(arguments) + return _emit_serve_result( + ResultEnvelope.ok(operation, report), arguments.json, + human=_convert_scan_human(report, arguments.pending_only), + ) + if arguments.gguf: + if arguments.source_path is not None: + raise ConvertError("invalid_arguments", "--source-path is only valid with --repo.", "Pass the GGUF source through --gguf.") + if arguments.backend not in (None, "mlx-lm"): + raise ConvertError( + "invalid_arguments", + "GGUF conversion always uses the built-in mlx-lm backend.", + "Drop --backend when converting a GGUF file.", + ) + plan = plan_gguf_convert( + arguments.gguf, q_bits=arguments.q_bits, out=arguments.out + ) + else: + plan = plan_convert(arguments.repo, q_bits=arguments.q_bits, out=arguments.out, + backend=arguments.backend, model_type=arguments.model_type, + subfolder=arguments.subfolder, hf_cache=arguments.hf_cache, + source_path=arguments.source_path) if not arguments.confirm: result = ResultEnvelope.ok( operation, {"plan": plan, "requires_confirmation": True} @@ -1360,19 +1810,20 @@ def _run_convert(arguments): if arguments.json: print(json.dumps(result.to_dict(), indent=2)) else: - print("Convert plan: {0} -> {1} ({2}bit)".format( - plan["repo"], plan["out"], plan["q_bits"] + print("Convert plan [{0}]: {1} -> {2} ({3}bit)".format( + plan["source"]["kind"], plan["repo"], plan["out"], plan["q_bits"] )) print(" argv: {0}".format(" ".join(plan["argv"]))) print(" preview_hash: {0}".format(plan["preview_hash"])) print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") return 2 - outcome = start_convert( + from .compute import guarded_start + outcome = guarded_start('convert', start_convert, plan, receipts_dir=arguments.receipts_dir, confirm=True, preview_hash=arguments.preview_hash, - model_present=default_model_present(arguments.hf_cache), + model_present=None if arguments.gguf else default_model_present(arguments.hf_cache), ) receipt = outcome["receipt"] return _emit_serve_result( @@ -1381,7 +1832,9 @@ def _run_convert(arguments): receipt["repo"], receipt["out"], receipt["pid"], receipt["log_path"] ), ) - except ConvertError as error: + except ComputeBusy as error: + result = ResultEnvelope.fail(operation, "compute_busy", str(error), "Stop the owning model operation before retrying.") + except (ConvertError, GGUFError, InferenceError) as error: result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: result = ResultEnvelope.fail( @@ -1400,6 +1853,80 @@ def _run_convert(arguments): return 2 +def _convert_scan(arguments): + from .convert import ConvertError, load_receipts + from .gguf import MAX_SCAN_FILES, default_gguf_roots, inventory + from .taxonomy import annotate_inventory + roots = arguments.gguf_roots or default_gguf_roots() + if not roots and not arguments.mlx_roots: + raise ConvertError( + "no_scan_roots", + "No model roots were given and none of the well-known GGUF directories exist.", + "Pass --gguf-root PATH or --mlx-root PATH for each model directory.", + ) + report = inventory( + roots, + mlx_roots=arguments.mlx_roots or (), + receipts=load_receipts(arguments.receipts_dir), + signature=not arguments.no_signature, + limit=arguments.limit or MAX_SCAN_FILES, + ) + if arguments.pending_only: + report = dict(report, models=[ + item for item in report["models"] if item["status"] == "pending" + ]) + return annotate_inventory(report) + + +def _human_bytes(count): + value = float(count) + for unit in ("B", "KB", "MB", "GB", "TB"): + if value < 1024 or unit == "TB": + return "{0:.1f}{1}".format(value, unit) + value /= 1024 + return "{0:.1f}TB".format(value) + + +def _convert_scan_human(report, pending_only=False): + totals = report["totals"] + lines = ["GGUF inventory: {0} files, {1} pending, {2} converted ({3})".format( + totals["gguf"], totals["pending"], totals["converted"], _human_bytes(totals["bytes"]) + )] + for item in report["models"]: + if item["status"] == "shard": + continue + if pending_only and item["status"] != "pending": + continue + lines.append(" [{0}] {1} {2} {3} {4}".format( + item["status"], item["name"], item.get("architecture") or "?", + item.get("quantization") or "?", _human_bytes(item["bytes"]), + )) + if totals["unreadable"]: + lines.append(" {0} file(s) had unreadable headers.".format(totals["unreadable"])) + ollama = report.get("ollama") or [] + if ollama: + lines.append("Ollama models: {0} ({1})".format(len(ollama), _human_bytes(sum(item["bytes"] for item in ollama)))) + for item in ollama: + lines.append(" {0} {1} {2} {3}".format( + item["name"], item["format"], item["quantization"], _human_bytes(item["bytes"]), + )) + duplicates = report["duplicates"] + if duplicates: + lines.append("Duplicates ({0} reclaimable):".format( + _human_bytes(totals["reclaimable_bytes"]) + )) + for group in duplicates: + if group["kind"] == "exact": + lines.append(" same model and quantization, keep {0}".format(group["keep"])) + for path in group["redundant"]: + lines.append(" redundant: {0}".format(path)) + else: + lines.append(" same model at {0}:".format("/".join(group["quantizations"]))) + for path in group["members"]: + lines.append(" variant: {0}".format(path)) + return "\n".join(lines) + + def _convert_status_human(entries): if not entries: return "No convert receipts." @@ -1412,7 +1939,14 @@ def _convert_status_human(entries): def _add_lora_arguments(parser): + from .lora import BATCH_DEFAULT, ITERS_DEFAULT, LAYERS_DEFAULT, LR_DEFAULT actions = parser.add_subparsers(dest="lora_command", required=True) + inspect = actions.add_parser("inspect", help="check local training prerequisites and dataset splits without starting work") + inspect.add_argument("--repo", default=None) + inspect.add_argument("--data", default=None) + inspect.add_argument("--batch-size", type=int, default=BATCH_DEFAULT) + inspect.add_argument("--hf-cache", default=None) + inspect.add_argument("--json", action="store_true") start = actions.add_parser( "start", help="preview, then confirmation-gated LoRA training on a cached model", @@ -1433,10 +1967,28 @@ def _add_lora_arguments(parser): status.add_argument("--receipts-dir", default=None) status.add_argument("--json", action="store_true") + stop = actions.add_parser("stop", help="cancel a matching receipt-owned training process") + stop.add_argument("--pid", type=int, required=True) + stop.add_argument("--receipts-dir", default=None) + stop.add_argument("--json", action="store_true") + def _run_lora(arguments): + from .compute import ComputeBusy + from .lora import LoraError, plan_lora, start_lora, status_lora + from .serve import default_model_present operation = "lora-{0}".format(arguments.lora_command) try: + if arguments.lora_command == "inspect": + from .lora import inspect_lora + report = inspect_lora(arguments.repo, arguments.data, arguments.batch_size, arguments.hf_cache) + return _emit_serve_result(ResultEnvelope.ok(operation, report), arguments.json, + human="\n".join("{0}: {1} — {2}".format(item["id"], item["status"], item["message"]) + for item in report["checks"])) + if arguments.lora_command == "stop": + from .lora import stop_lora + return _emit_serve_result(ResultEnvelope.ok(operation, stop_lora(arguments.pid, arguments.receipts_dir)), + arguments.json, human="Training cancellation requested.") if arguments.lora_command == "status": entries = status_lora(arguments.receipts_dir) return _emit_serve_result( @@ -1451,6 +2003,7 @@ def _run_lora(arguments): learning_rate=arguments.learning_rate, num_layers=arguments.num_layers, out=arguments.out, + hf_cache=arguments.hf_cache, ) if not arguments.confirm: result = ResultEnvelope.ok( @@ -1466,7 +2019,8 @@ def _run_lora(arguments): print(" preview_hash: {0}".format(plan["preview_hash"])) print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") return 2 - outcome = start_lora( + from .compute import guarded_start + outcome = guarded_start('lora', start_lora, plan, receipts_dir=arguments.receipts_dir, confirm=True, @@ -1480,6 +2034,8 @@ def _run_lora(arguments): receipt["repo"], receipt["out"], receipt["pid"], receipt["log_path"] ), ) + except ComputeBusy as error: + result = ResultEnvelope.fail(operation, "compute_busy", str(error), "Stop the owning model operation before retrying.") except LoraError as error: result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: @@ -1530,6 +2086,9 @@ def _add_fuse_arguments(parser): def _run_fuse(arguments): + from .compute import ComputeBusy + from .fuse import FuseError, plan_fuse, start_fuse, status_fuse + from .serve import default_model_present operation = "fuse-{0}".format(arguments.fuse_command) try: if arguments.fuse_command == "status": @@ -1553,7 +2112,8 @@ def _run_fuse(arguments): print(" preview_hash: {0}".format(plan["preview_hash"])) print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") return 2 - outcome = start_fuse( + from .compute import guarded_start + outcome = guarded_start("fuse", start_fuse, plan, receipts_dir=arguments.receipts_dir, confirm=True, @@ -1567,6 +2127,8 @@ def _run_fuse(arguments): receipt["repo"], receipt["out"], receipt["pid"], receipt["log_path"] ), ) + except ComputeBusy as error: + result = ResultEnvelope.fail(operation, "compute_busy", str(error), "Stop the owning model operation before retrying.") except FuseError as error: result = ResultEnvelope.fail(operation, error.code, str(error), error.remediation) except (OSError, TypeError, ValueError, json.JSONDecodeError) as error: @@ -1620,14 +2182,21 @@ def _add_wire_arguments(parser): restore.add_argument("--json", action="store_true") +def _plugin_manifest(): + # The plugin tree exists only in a checkout; an installed package carries the runtime alone. + return Path(__file__).resolve().parents[2] / "plugin.json" + + def _installer_registry(): - root = Path(__file__).resolve().parents[2] + from .providers import ProviderRegistry home = os.environ.get("MLX_AGENT_HOME") config_root = os.environ.get("MLX_AGENT_CONFIG_ROOT") or os.environ.get("XDG_STATE_HOME") xdg_config_home = os.environ.get("XDG_CONFIG_HOME") return ProviderRegistry( - root / "plugin.json", home=home, config_root=config_root, + _plugin_manifest(), home=home, config_root=config_root, xdg_config_home=xdg_config_home, + openclaw_state_dir=os.environ.get("OPENCLAW_STATE_DIR"), + hermes_home=os.environ.get("HERMES_HOME"), ) @@ -1663,6 +2232,7 @@ def _add_doctor_arguments(parser): def _run_model_doctor(arguments): + from .verification import LMStudioRuntimeClient, MLXVLMRuntimeClient, OllamaRuntimeClient, OpenAICompatibleRuntimeClient from .model_doctor import ( PruneError, default_hf_cache, @@ -1787,8 +2357,8 @@ def _installer_result(operation, data, as_json, status="ok", error=None): details = "{0}: {1}".format(item["id"], item["state"]) if item.get("version"): details += " (version {0})".format(item["version"]) - if item.get("error"): - details += " - {0}".format(item["error"]) + if item.get("error") or item.get("warning"): + details += " - {0}".format(item.get("error") or item["warning"]) print(details) print(data["preview"]["diff"]) print("Confirmation required: rerun with --confirm --preview-hash PREVIEW_HASH.") @@ -1800,9 +2370,16 @@ def _installer_result(operation, data, as_json, status="ok", error=None): def _run_installer(arguments): + from .installer import Installer, InstallerConflictError, InstallerPathError operation = arguments.command if operation == "doctor" and "models" in (getattr(arguments, "providers", None) or []): return _run_model_doctor(arguments) + if not _plugin_manifest().is_file(): + return _installer_result(operation, {}, arguments.json, "error", ( + "plugin_files_unavailable", + "Provider installation copies the mlx-agent plugin files, which the Python package does not include.", + "Run this command from a clone of https://github.com/cavi-ai/mlx-agent, or install through the host's plugin marketplace.", + )) try: installer = Installer(_installer_registry(), project_root=arguments.project) detections = [item.to_dict() for item in installer.detected()] @@ -1829,6 +2406,10 @@ def _run_installer(arguments): result = installer.execute(plan, arguments.preview_hash if operation != "doctor" else False) data = result if isinstance(result, dict) else result.to_dict() return _installer_result(operation, {"result": data, "mutated": operation != "doctor"}, arguments.json) + except InstallerPathError as error: + return _installer_result(operation, {}, arguments.json, "error", ( + "destination_not_writable", str(error), error.remediation, + )) except InstallerConflictError as error: lock_code = "legacy_lock_busy" if str(error).startswith("legacy_lock_busy") else ( "legacy_lock_recreated" if str(error).startswith("legacy_lock_recreated") else "artifact_conflict" @@ -1849,52 +2430,70 @@ def legacy_scout_main(argv=None): return _run_discovery(parser.parse_args(argv), legacy=True) -def build_parser(): +def converter_main(argv=None): + """Entry point for the bundled mlx-converter skill script.""" + parser = argparse.ArgumentParser( + description="Inventory local GGUF weights and convert them to MLX.", + ) + _add_convert_arguments(parser) + return _run_convert(parser.parse_args(argv)) + + +def build_parser(command=None): + """Build the parser; given a command name, only that command's arguments are defined.""" + def wanted(name): + return command is None or command == name + parser = argparse.ArgumentParser(description="MLX agent command-line core.") subcommands = parser.add_subparsers(dest="command", required=True) - discover = subcommands.add_parser("discover", help="discover MLX models for this host") - _add_discovery_arguments(discover) - inspect_host = subcommands.add_parser("inspect-host", help="inspect local Apple Silicon and runtime inventory without model discovery") - inspect_host.add_argument("--json", action="store_true") - adopt = subcommands.add_parser("adopt", help="run or inspect resumable model adoption") - _add_adoption_arguments(adopt) - research = subcommands.add_parser("research", help="build a read-only domain research pack (markdown)") - _add_research_arguments(research) - blueprint = subcommands.add_parser( - "blueprint", - help="emit a read-only MLX project design pack (quant/train guidance; no scaffolding)", + task = subcommands.add_parser("task", help="durable local inference tasks") + if wanted("task"): + from . import task_cli + task_cli.add_arguments(task) + subcommands.add_parser("lab", help="experiments, studies, datasets and training") + builders = ( + ("discover", "discover MLX models for this host", _add_discovery_arguments), + ("inspect-host", "inspect local Apple Silicon and runtime inventory without model discovery", + lambda parser: parser.add_argument("--json", action="store_true")), + ("adopt", "run or inspect resumable model adoption", _add_adoption_arguments), + ("research", "build a read-only domain research pack (markdown)", _add_research_arguments), + ("blueprint", "emit a read-only MLX project design pack (quant/train guidance; no scaffolding)", + _add_blueprint_arguments), + ("wire", "render, apply, inspect, or roll back runtime wiring", _add_wire_arguments), + ("bench", "measure a locally served model without downloading it", _add_bench_arguments), + ("serve", "preview and launch a local MLX server (confirmation-gated)", _add_serve_arguments), + ("fleet", "render or apply a one-shot per-role router configuration", _add_fleet_arguments), + ("watch", "snapshot and diff owned models against Hugging Face", _add_watch_arguments), + ("convert", "preview and quantize a cached model (confirmation-gated)", _add_convert_arguments), + ("lora", "preview and run LoRA training on a cached model (confirmation-gated)", _add_lora_arguments), + ("fuse", "preview and fuse a LoRA adapter into its base (confirmation-gated)", _add_fuse_arguments), + ("intake", "resolve, download, and analyze pasted Hugging Face models", _add_intake_arguments), + ("backend", "list, install, or remove optional MLX converter backends (confirmation-gated)", + _add_backend_arguments), + ("providers", "list detected supported provider CLIs", + lambda parser: _add_installer_arguments(parser, include_providers=False)), + ("install", "install declared provider artifacts safely", _add_installer_arguments), + ("update", "update declared provider artifacts safely", _add_installer_arguments), + ("uninstall", "uninstall declared provider artifacts safely", _add_installer_arguments), + ("doctor", "doctor declared provider artifacts safely", _add_doctor_arguments), ) - _add_blueprint_arguments(blueprint) - wire_command = subcommands.add_parser("wire", help="render, apply, inspect, or roll back runtime wiring") - _add_wire_arguments(wire_command) - bench_command = subcommands.add_parser("bench", help="measure a locally served model without downloading it") - _add_bench_arguments(bench_command) - serve_command = subcommands.add_parser("serve", help="preview and launch a local MLX server (confirmation-gated)") - _add_serve_arguments(serve_command) - fleet_command = subcommands.add_parser("fleet", help="render or apply a one-shot per-role router configuration") - _add_fleet_arguments(fleet_command) - watch_command = subcommands.add_parser("watch", help="snapshot and diff owned models against Hugging Face") - _add_watch_arguments(watch_command) - convert_command = subcommands.add_parser("convert", help="preview and quantize a cached model (confirmation-gated)") - _add_convert_arguments(convert_command) - lora_command = subcommands.add_parser("lora", help="preview and run LoRA training on a cached model (confirmation-gated)") - _add_lora_arguments(lora_command) - fuse_command = subcommands.add_parser("fuse", help="preview and fuse a LoRA adapter into its base (confirmation-gated)") - _add_fuse_arguments(fuse_command) - providers_command = subcommands.add_parser("providers", help="list detected supported provider CLIs") - _add_installer_arguments(providers_command, include_providers=False) - for name in ("install", "update", "uninstall", "doctor"): - installer_command = subcommands.add_parser(name, help="{0} declared provider artifacts safely".format(name)) - if name == "doctor": - _add_doctor_arguments(installer_command) - else: - _add_installer_arguments(installer_command) + for name, help_text, add_arguments in builders: + subparser = subcommands.add_parser(name, help=help_text) + if wanted(name): + add_arguments(subparser) return parser def main(argv=None): - parser = build_parser() + raw = list(sys.argv[1:] if argv is None else argv) + if raw and raw[0] == "lab": + from .lab import run + return run(raw[1:]) + parser = build_parser(raw[0] if raw and not raw[0].startswith("-") else None) arguments = parser.parse_args(argv) + if arguments.command == "task": + from .task_cli import run + return run(arguments) if arguments.command == "adopt": return _run_adoption(arguments) if arguments.command == "wire": @@ -1907,6 +2506,10 @@ def main(argv=None): return _run_fleet(arguments) if arguments.command == "watch": return _run_watch(arguments) + if arguments.command == "intake": + return _run_intake(arguments) + if arguments.command == "backend": + return _run_backend(arguments) if arguments.command == "convert": return _run_convert(arguments) if arguments.command == "lora": diff --git a/packages/codex/mlx-agent/src/mlx_agent/coding_capture.py b/packages/codex/mlx-agent/src/mlx_agent/coding_capture.py new file mode 100644 index 0000000..67fa56b --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/coding_capture.py @@ -0,0 +1,140 @@ +"""One owned coding session: extension host, private client server, one prompt and its explicit export.""" + +import base64 +import json +import os +import secrets +import selectors +import subprocess +import time +from pathlib import Path +from urllib.request import Request, urlopen + +from .capture_platform import capture_process_identity, stop_owned +from .coding_capture_contracts import DEFAULT_LIMITS +from .coding_capture_plugins import ExtensionHost, verify_extensions +from .coding_capture_process import export_session, run_client +from .task_contracts import TaskError + +READY_SECONDS = 30 +MAX_API_BYTES = 1024 * 1024 + + +def _unavailable(message): + return TaskError("capture_client_unavailable", message, "Inspect the owned server diagnostics and start a new capture.") + + +class OwnedSession: + """Bootstrap the frozen extensions, start the private server, run one prompt, export it, stop everything. + + Use as a context manager. Every client command rechecks the frozen extensions and the server's process + identity; leaving the context stops the server and the extension host by their recorded identities. + """ + + def __init__(self, manifest, profile_root, configuration, project_path, model, environment=None, + artifact_bytes=DEFAULT_LIMITS["artifact_bytes"], on_process=None, cancelled=None, + host_factory=ExtensionHost): + self.manifest = manifest + self.profile_root, self.project_path = Path(profile_root), Path(project_path) + self.configuration, self.model, self.environment = configuration, model, environment + self.artifact_bytes = artifact_bytes + self.on_process = on_process or (lambda role, identity: None) + self.cancelled = cancelled or (lambda: False) + self._host_factory = host_factory + self.host = self.server = self.server_url = None + self._identity = self._errors = self._client_environment = self._headers = None + + def __enter__(self): + self.host = self._host_factory(Path(self.manifest["client_path"]), self.manifest, self.profile_root, + self.configuration, self.project_path, self.environment) + self.host.__enter__() + try: + self._start_server() + self._await_extensions() + except BaseException: + self.close() + raise + return self + + def __exit__(self, *_): + self.close() + + def _start_server(self): + password = secrets.token_urlsafe(32) + environment = dict(self.host.profile["environment"], OPENCODE_SERVER_PASSWORD=password) + descriptor = os.open(self.profile_root / "server-stderr.log", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + self._errors = os.fdopen(descriptor, "wb") + self.server = subprocess.Popen( + [self.manifest["client_path"], "serve", "--print-logs", "--stdio", "--hostname", "127.0.0.1", "--port", "0"], + env=environment, cwd=self.project_path, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=self._errors, pass_fds=self.host.server_fds, start_new_session=True) + selector = selectors.DefaultSelector() + selector.register(self.server.stdout, selectors.EVENT_READ) + deadline = time.monotonic() + READY_SECONDS + try: + while self.server_url is None and time.monotonic() < deadline and self.server.poll() is None: + if selector.select(min(1, max(0, deadline - time.monotonic()))): + try: + self.server_url = json.loads(self.server.stdout.readline(8192))["url"] + except (ValueError, KeyError, TypeError): + continue + finally: + selector.close() + if not isinstance(self.server_url, str): + raise _unavailable("The owned client server did not report its address.") + # A launcher may exec again before serving; the identity is bound once the server answers. + self._identity = capture_process_identity(self.server.pid) + self.on_process("private_server", self._identity) + self._headers = {"Authorization": "Basic " + base64.b64encode(("opencode:" + password).encode()).decode(), + "x-opencode-directory": str(self.project_path)} + self._client_environment = environment + self._client_environment.pop("MLX_AGENT_CAPTURE_EXTENSION_FD", None) + + def _await_extensions(self): + expected = {bundle["id"] for bundle in self.manifest["bundles"]} + deadline = time.monotonic() + READY_SECONDS + while True: + inventory = self.api("/api/plugin")["data"] + active = {item["id"] for item in inventory if item["state"]["status"] == "active"} + failed = {item.get("id") for item in inventory if item["state"]["status"] == "failed"} + if expected <= active: + return + if expected & failed or time.monotonic() >= deadline: + raise _unavailable("The owned client server did not activate every frozen extension.") + time.sleep(0.05) + + def api(self, path): + """One authenticated read from the owned server, bounded to 1 MiB of JSON.""" + with urlopen(Request(self.server_url + path, headers=self._headers), timeout=READY_SECONDS) as response: + return json.loads(response.read(MAX_API_BYTES)) + + def check_context(self): + verify_extensions(self.manifest) + if capture_process_identity(self.server.pid) != self._identity: + raise TaskError("capture_context_changed", "The owned client server changed.", + "Start a new capture; no prompt was sent to another server.") + + def run(self, prompt, storage, on_event=None): + """Run `prompt` through the owned server and export its session; both retain evidence in `storage`.""" + command = {"client_path": self.manifest["client_path"], "project_path": str(self.project_path), + "environment": dict(self._client_environment), "server_url": self.server_url, "model": self.model, + "storage": storage, "artifact_bytes": self.artifact_bytes, "check_context": self.check_context, + "on_process": self.on_process, "cancelled": self.cancelled} + run = run_client(command, prompt, on_event or (lambda _: None)) + export = export_session(command, run["session_id"], storage) if run["error"] is None else None + return {"run": run, "export": export} + + def close(self): + try: + if self.server is not None: + identity = self._identity + if identity is None and self.server.poll() is None: + # Not yet serving: the unreaped child still holds its PID, so its current identity is ours. + identity = capture_process_identity(self.server.pid) + stop_owned(self.server, identity) + finally: + for stream in (getattr(self.server, "stdin", None), getattr(self.server, "stdout", None), self._errors): + if stream is not None and not stream.closed: + stream.close() + if self.host is not None: + self.host.__exit__(None, None, None) diff --git a/packages/codex/mlx-agent/src/mlx_agent/coding_capture_client.py b/packages/codex/mlx-agent/src/mlx_agent/coding_capture_client.py new file mode 100644 index 0000000..2545026 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/coding_capture_client.py @@ -0,0 +1,128 @@ +"""Observed OpenCode v2 session events and explicit-session transfer evidence.""" + +import json +import math +from typing import Callable, Dict + +from .coding_capture_contracts import DEFAULT_LIMITS +from .task_contracts import TaskError, encoded + +EVENT_TYPES = frozenset(("step_start", "step_finish", "text", "reasoning", "tool_use", "error")) +MESSAGE_TYPES = frozenset(("agent-switched", "model-switched", "location-switched", "user", "synthetic", "system", + "skill", "shell", "assistant", "compaction", "idle")) + + +def _require(condition, message): + if not condition: + raise TaskError("invalid_capture_session", message, "Keep the retained client output; do not substitute another saved session.") + + +def is_identifier(value, prefix): + return isinstance(value, str) and value.startswith(prefix) and len(prefix) < len(value) <= 256 and not any(ord(c) < 33 for c in value) + + +def _milliseconds(value): + # The supported client emits epoch milliseconds, not seconds or ISO strings. + return type(value) in (int, float) and math.isfinite(value) and 946684800000 <= value <= 253402300799999 + + +def _json(raw): + def pairs(items): + result = {} + for key, value in items: + _require(key not in result, "Duplicate keys cannot bind session evidence.") + result[key] = value + return result + try: + value = json.loads(raw, object_pairs_hook=pairs) + encoded(value) # Also reject nested nonfinite numbers and invalid UTF-8. + return value + except (ValueError, TypeError, UnicodeError, RecursionError) as error: + raise TaskError("invalid_capture_session", "Client output is not complete finite UTF-8 JSON.") from error + + +class OpenCodeEvents: + """Incremental NDJSON reader; the observed root identity cannot change.""" + + def __init__(self, on_event: Callable, artifact_bytes: int): + if type(artifact_bytes) is not int or not 1 <= artifact_bytes <= DEFAULT_LIMITS["artifact_bytes"]: + raise TaskError("invalid_capture_limits", "Client output requires an accepted artifact byte limit.") + self.on_event = on_event + self.artifact_bytes = artifact_bytes + self.session_id = None + self.has_error = False + self._buffer = bytearray() + self._bytes = 0 + self._finished = False + + def feed(self, chunk: bytes): + _require(isinstance(chunk, bytes) and not self._finished, "Client event stream has already ended or is not bytes.") + self._bytes += len(chunk) + if self._bytes > self.artifact_bytes: + raise TaskError("capture_artifact_limit", "Client event output exceeds its accepted artifact limit.") + self._buffer.extend(chunk) + while True: + end = self._buffer.find(b"\n") + if end < 0: + return + line = bytes(self._buffer[:end]) + del self._buffer[:end + 1] + value = _json(line) + _require(isinstance(value, dict) and value.get("type") in EVENT_TYPES + and is_identifier(value.get("sessionID"), "ses") and _milliseconds(value.get("timestamp")), + "Client events require a supported type, observed session ID and epoch-millisecond timestamp.") + if self.session_id is not None: + _require(value["sessionID"] == self.session_id, "Client events changed the observed root session.") + self.session_id = value["sessionID"] + self.has_error = self.has_error or value["type"] == "error" + self.on_event(value) + + def finish(self): + _require(not self._buffer, "Client event output ended with a partial JSON line.") + _require(self.session_id is not None, "No client session was observed.") + self._finished = True + + +def validate_session_export(raw: bytes, session_id: str, model: str) -> Dict: + """Validate identity-bearing v2 transfer fields and retain the original body.""" + _require(isinstance(raw, bytes) and len(raw) <= DEFAULT_LIMITS["artifact_bytes"], "Session export exceeds its byte limit.") + _require(is_identifier(session_id, "ses") and isinstance(model, str) and "/" in model, + "An observed session and exact provider/model selection are required.") + provider, selected_model = model.split("/", 1) + _require(bool(provider) and bool(selected_model), "The selected model is incomplete.") + value = _json(raw) + _require(isinstance(value, dict) and set(value) == {"info", "messages"}, "Unsupported session-transfer shape.") + info, messages = value["info"], value["messages"] + _require(isinstance(info, dict) and info.get("id") == session_id and is_identifier(info.get("projectID"), ""), + "Explicit session export does not match the observed session and project.") + if "parentID" in info: + _require(is_identifier(info["parentID"], "ses") and info["parentID"] != session_id, "Invalid exported parent relationship.") + times = info.get("time") + _require(isinstance(times, dict) and all(_milliseconds(times.get(key)) for key in ("created", "updated")), + "Exported session time is not epoch milliseconds.") + + def check_model(reference): + _require(isinstance(reference, dict) and reference.get("providerID") == provider + and reference.get("id") == selected_model, "Exported session model differs from the selected local model.") + + if "model" in info: + check_model(info["model"]) + _require(isinstance(messages, list) and bool(messages), "Export has no retained messages.") + identities, assistant = set(), False + for message in messages: + _require(isinstance(message, dict) and is_identifier(message.get("id"), "msg_"), "Export contains an invalid message identity.") + _require(message["id"] not in identities, "Export contains duplicate message identities.") + identities.add(message["id"]) + _require(message.get("type") in MESSAGE_TYPES, "Export contains an unsupported message type.") + if message["type"] == "idle": + _require(message.get("outcome") in ("succeeded", "failed", "interrupted"), + "Export contains an unsupported idle outcome.") + _require("sessionID" not in message or message["sessionID"] == session_id, "Export contains another session's message.") + if "model" in message: + check_model(message["model"]) + if message["type"] == "assistant": + check_model(message.get("model")) + _require(isinstance(message.get("content"), list), "Exported assistant content is incomplete.") + assistant = True + _require(assistant, "Export has no model-attributable assistant message.") + return value diff --git a/packages/codex/mlx-agent/src/mlx_agent/coding_capture_contracts.py b/packages/codex/mlx-agent/src/mlx_agent/coding_capture_contracts.py new file mode 100644 index 0000000..670d13d --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/coding_capture_contracts.py @@ -0,0 +1,244 @@ +"""Strict, versioned coding-capture contracts, separate from inference tasks.""" + +from datetime import datetime +import json +import math +from pathlib import Path +from typing import Dict +from urllib.parse import urlsplit + +from .task_contracts import MAX_REQUEST_BYTES, STATES, TERMINAL, TaskError, encoded, task_id + +KIND = "coding_capture" +REQUEST_SCHEMA = "coding-capture-request/1" +RUN_SCHEMA = "coding-capture-run/1" +RESULT_SCHEMA = "coding-capture-result/1" +DEFAULT_LIMITS = {"artifact_bytes": 33554432, "total_bytes": 134217728} +METADATA_RESERVE = 65536 +MAX_RECORD_BYTES = 8192 +PROCESS_ROLES = frozenset(("supervisor", "client", "private_server", "relay")) + + +def _require(condition, message, code="invalid_capture"): + if not condition: + raise TaskError(code, message) + + +def _fields(value, required, optional=()): + _require(isinstance(value, dict) and set(required) <= set(value) + and not set(value) - set(required) - set(optional), "Unknown or incomplete capture fields.") + + +def _text(value, maximum=4096): + return isinstance(value, str) and bool(value.strip()) and len(value) <= maximum and "\0" not in value + + +def _path(value): + return (_text(value) and Path(value).is_absolute() and ".." not in Path(value).parts + and not any(ord(char) < 32 for char in value)) + + +def _digest(value): + return isinstance(value, str) and len(value) == 64 and all(char in "0123456789abcdef" for char in value) + + +def _integer(value, minimum=0, maximum=2 ** 63 - 1): + return type(value) is int and minimum <= value <= maximum + + +def _timestamp(value): + try: + return isinstance(value, str) and datetime.fromisoformat(value).tzinfo is not None + except ValueError: + return False + + +def _copy(value): + return json.loads(encoded(value)) + + +def validate_limits(value): + _fields(value, ("artifact_bytes", "total_bytes")) + _require(_integer(value["artifact_bytes"], 1, DEFAULT_LIMITS["artifact_bytes"]) + and _integer(value["total_bytes"], 2 * METADATA_RESERVE, DEFAULT_LIMITS["total_bytes"]) + and value["total_bytes"] % 512 == 0 + and value["artifact_bytes"] <= value["total_bytes"] - METADATA_RESERVE, + "Capture limits require whole bytes within the supported artifact and owned-data limits.") + return _copy(value) + + +def validate_reference(value, expected_name=None): + """References expose relative artifact names, never a mounted filesystem path.""" + _fields(value, ("name", "sha256", "bytes")) + name = value["name"] + _require(_text(name, 128) and not name.startswith(".") and "/" not in name and "\\" not in name + and not any(ord(char) < 32 for char in name) + and (expected_name is None or name == expected_name) + and _digest(value["sha256"]) and _integer(value["bytes"], 0, DEFAULT_LIMITS["artifact_bytes"]), + "Capture artifact identity is invalid.") + return value + + +def _receipt(value): + _fields(value, ("path", "sha256", "bytes")) + _require(_path(value["path"]) and _digest(value["sha256"]) + and _integer(value["bytes"], 1, MAX_REQUEST_BYTES), "Serve receipt identity is invalid.") + + +def _workbench(value): + _fields(value, ("model_signature", "environment_fingerprint", "workload_id")) + _require(_text(value["model_signature"], 512) and _text(value["environment_fingerprint"], 512), + "Original Workbench model and environment identities are required.") + task_id(value["workload_id"]) + + +def validate_process(value): + _fields(value, ("pid", "identity", "started_at")) + _require(_integer(value["pid"], 1) and _text(value["identity"], 2048) + and isinstance(value["started_at"], list) and len(value["started_at"]) == 2 + and _integer(value["started_at"][0], 1) and _integer(value["started_at"][1], 0, 999999), + "Capture processes require PID, command and precise kernel start identity.") + + +def validate_context(value): + _fields(value, ("serve", "client", "project"), ("workbench_context",)) + serve, client, project = value["serve"], value["client"], value["project"] + _fields(serve, ("receipt", "base_url", "model_path", "model_fingerprint", "runtime", "process")) + _receipt(serve["receipt"]) + validate_process(serve["process"]) + try: + url = urlsplit(serve["base_url"]) + local_url = (url.scheme == "http" and url.hostname == "127.0.0.1" and url.port is not None + and 1 <= url.port <= 65535 and not url.username and not url.password + and not url.query and not url.fragment and url.path == "/v1") + except (ValueError, TypeError, AttributeError): + local_url = False + _require(local_url and _path(serve["model_path"]) and _digest(serve["model_fingerprint"]) + and _text(serve["runtime"], 128), "Capture requires a pinned local compatible endpoint and model.") + _fields(client, ("path", "file_identity", "version", "configuration_sha256", "policy_sha256")) + _require(_path(client["path"]) and _text(client["version"], 128) + and _digest(client["configuration_sha256"]) and _digest(client["policy_sha256"]) + and isinstance(client["file_identity"], list) and len(client["file_identity"]) == 4 + and all(_integer(part) for part in client["file_identity"]), "Capture client identity is invalid.") + _fields(project, ("path", "file_identity", "context_sha256")) + _require(_path(project["path"]) and _digest(project["context_sha256"]) + and isinstance(project["file_identity"], list) and len(project["file_identity"]) == 2 + and all(_integer(part) for part in project["file_identity"]), "Capture project identity is invalid.") + if "workbench_context" in value: + _workbench(value["workbench_context"]) + return _copy(value) + + +def validate_capture_request(value: Dict) -> Dict: + _fields(value, ("schema", "kind", "model_path", "serve_receipt", "client_path", "project_path", "prompt", "limits"), + ("workbench_context",)) + _require(value["schema"] == REQUEST_SCHEMA and value["kind"] == KIND, + "Unsupported coding capture request.", "unsupported_capture") + _require(all(_path(value[key]) for key in ("model_path", "client_path", "project_path")) + and _text(value["prompt"], MAX_REQUEST_BYTES), "Capture requires absolute local paths and a coding request.") + _receipt(value["serve_receipt"]) + validate_limits(value["limits"]) + if "workbench_context" in value: + _workbench(value["workbench_context"]) + _require(len(encoded(value)) <= MAX_REQUEST_BYTES, "Capture requests are limited to 1 MiB.", "request_too_large") + return _copy(value) + + +def validate_capture_record(value: Dict) -> Dict: + _fields(value, ("schema", "id", "kind", "state", "created_at", "started_at", "completed_at", "cancel_requested", + "request", "preview_hash", "context", "processes", "client_outcome", "attribution", "result", + "error", "execution_unresolved", "artifacts_removed_at")) + _require(value["schema"] == RUN_SCHEMA and value["kind"] == KIND and value["state"] in STATES, + "Unsupported capture record.") + task_id(value["id"]) + validate_reference(value["request"], "request.json") + _require(_digest(value["preview_hash"]), "Capture preview identity is invalid.") + validate_context(value["context"]) + _require(all(type(value[key]) is bool for key in ("cancel_requested", "execution_unresolved")), + "Capture lifecycle flags must be boolean.") + _require(_timestamp(value["created_at"]) and all(value[key] is None or _timestamp(value[key]) + for key in ("started_at", "completed_at", "artifacts_removed_at")), "Capture timestamps need a timezone.") + state = value["state"] + _require((state in TERMINAL) == (value["completed_at"] is not None), "Capture completion state is inconsistent.") + _require(value["client_outcome"] in ("pending", "succeeded", "failed", "cancelled") + and value["attribution"] in ("pending", "verified", "failed"), "Unknown capture outcome.") + processes = value["processes"] + _require(isinstance(processes, dict) and not set(processes) - PROCESS_ROLES, + "Only capture-owned process roles may be recorded.") + for process in processes.values(): + validate_process(process) + _require(not set(processes) - {"supervisor"} or "supervisor" in processes, + "Owned capture execution requires its supervisor.") + if value["result"] is not None: + validate_reference(value["result"], "result.json") + if value["error"] is not None: + _fields(value["error"], ("code", "message", "remediation")) + _require(all(_text(part) for part in value["error"].values()), "Capture errors require actionable details.") + _require(state != "starting" or value["started_at"] is None, "Starting capture cannot have a start timestamp.") + _require(state != "running" or (value["started_at"] is not None and {"supervisor", "client"} <= set(processes)), + "Running capture requires client and supervisor identities.") + if state == "succeeded": + _require(value["client_outcome"] == "succeeded" and value["attribution"] == "verified" + and value["result"] is not None and value["error"] is None and not value["execution_unresolved"] + and value["started_at"] is not None and {"supervisor", "client"} <= set(processes), + "Successful capture requires durable attributed execution evidence.") + _require(state == "succeeded" or value["result"] is None, "Only successful capture publishes a result reference.") + _require((state == "failed") == (value["error"] is not None), "Failed capture requires an actionable error.") + _require(not value["execution_unresolved"] or state == "failed", "Unresolved execution must remain failed.") + _require(state != "cancelled" or (value["cancel_requested"] and value["client_outcome"] == "cancelled" + and not value["execution_unresolved"]), "Capture cancellation evidence is inconsistent.") + _require(value["artifacts_removed_at"] is None or (state in TERMINAL and not value["execution_unresolved"]), + "Active or unresolved capture artifacts cannot be removed.") + _require(len(encoded(value)) <= MAX_RECORD_BYTES, "Capture task metadata exceeds 8,192 bytes.", "capture_metadata_limit") + return value + + +def validate_capture_result(value: Dict) -> Dict: + _fields(value, ("schema", "task_id", "request_sha256", "preview_hash", "context", "session", "requests", "artifacts", + "client_exit_code", "elapsed_seconds", "completed_at", "attribution")) + _require(value["schema"] == RESULT_SCHEMA, "Unsupported capture result.") + task_id(value["task_id"]) + _require(_digest(value["request_sha256"]) and _digest(value["preview_hash"]), "Capture result hashes are invalid.") + validate_context(value["context"]) + session = value["session"] + _fields(session, ("id", "parent_id", "children")) + _require(_text(session["id"], 128) and (session["parent_id"] is None or _text(session["parent_id"], 128)) + and session["parent_id"] != session["id"] and isinstance(session["children"], list), "Invalid captured session.") + sessions = {session["id"]} + for child in session["children"]: + _fields(child, ("id", "parent_id")) + _require(_text(child["id"], 128) and child["id"] not in sessions + and _text(child["parent_id"], 128) and child["parent_id"] in sessions, + "Capture child sessions require an observed parent relationship.") + sessions.add(child["id"]) + _require(isinstance(value["artifacts"], list) and isinstance(value["requests"], list), "Invalid capture artifact set.") + artifacts = {} + for artifact in value["artifacts"]: + validate_reference(artifact) + _require(artifact["name"] not in artifacts, "Capture artifact names must be unique.") + artifacts[artifact["name"]] = artifact + for request in value["requests"]: + _fields(request, ("artifact", "session_id", "parent_session_id", "project_id", "request_model", "response_model")) + validate_reference(request["artifact"]) + _require(artifacts.get(request["artifact"]["name"]) == request["artifact"], "Capture transport reference is missing.") + _require(_text(request["session_id"], 128) + and (request["parent_session_id"] is None or _text(request["parent_session_id"], 128)) + and (request["project_id"] is None or _text(request["project_id"], 256)) + and _text(request["request_model"], 256) + and (request["response_model"] is None or _text(request["response_model"], 256)), "Invalid observed request identity.") + if value["attribution"] == "verified": + _require(request["session_id"] in sessions and request["response_model"] is not None, + "Verified transport must join an exported session and response model.") + elapsed = value["elapsed_seconds"] + _require(_integer(value["client_exit_code"], -255, 255) and type(elapsed) in (int, float) + and math.isfinite(elapsed) and elapsed >= 0 and _timestamp(value["completed_at"]) + and value["attribution"] in ("pending", "verified", "failed"), "Capture execution outcome is invalid.") + _require(value["attribution"] != "verified" or bool(value["requests"]), "Verified capture requires observed requests.") + _require(len(encoded(value)) <= DEFAULT_LIMITS["artifact_bytes"], "Capture result exceeds its artifact limit.") + return _copy(value) + + +def capture_capabilities() -> Dict: + return {"schema": "coding-capture-capabilities/1", "kind": KIND, "request_schema": REQUEST_SCHEMA, + "run_schema": RUN_SCHEMA, "result_schema": RESULT_SCHEMA, "limits": dict(DEFAULT_LIMITS), + "metadata_reserve_bytes": METADATA_RESERVE, "cancellation": True, "explicit_removal": True} diff --git a/packages/codex/mlx-agent/src/mlx_agent/coding_capture_plugins.py b/packages/codex/mlx-agent/src/mlx_agent/coding_capture_plugins.py new file mode 100644 index 0000000..9d643b3 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/coding_capture_plugins.py @@ -0,0 +1,254 @@ +"""Compile existing local extensions without evaluating them or installing packages.""" + +import hashlib +import json +import os +from pathlib import Path +import stat +import subprocess +import socket + +from .task_contracts import TaskError, encoded + +_COMPILER = Path(__file__).parent / "resources" / "coding_capture" / "freeze.mjs" +_MAX_BYTES = 32 * 1024 * 1024 + + +def _unavailable(message): + return TaskError("capture_client_unavailable", message, + "Use existing absolute local extensions with supported hooks and fully resolved imports.") + + +def file_identity(path): + """Path, inode identity and SHA-256 of one existing regular file.""" + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + raise _unavailable("Extension inputs must be existing regular local files.") + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return {"path": str(path), "identity": [str(value) for value in (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns)], + "sha256": digest.hexdigest()} + + +def verify_extensions(manifest): + """Recheck both original inputs and the immutable owned bundles before bootstrap.""" + try: + if str(Path(manifest["client_alias"]).resolve(strict=True)) != manifest["client_path"]: + raise ValueError("changed client alias") + for item in manifest["inputs"] + manifest["bundles"]: + current = file_identity(Path(item["path"])) + if any(current[key] != item[key] for key in ("identity", "sha256")): + raise ValueError("changed input") + except (OSError, ValueError, KeyError, TaskError) as error: + raise TaskError("capture_context_changed", "An extension input or frozen bundle changed.", + "Create a new preview before starting the coding task.") from error + + +def freeze_extensions(client_path: Path, targets, output_root: Path): + # Reject package operations before even invoking the static compiler. A + # cached npm package is still a package operation in OpenCode's supervisor. + if not isinstance(targets, list) or not 0 <= len(targets) <= 64: + raise _unavailable("Too many or invalid extension targets.") + sources = [] + try: + for target in targets: + if not isinstance(target, str) or not Path(target).is_absolute(): + raise _unavailable("Package and relative extension operations cannot bootstrap a capture.") + source = Path(target) + file_identity(source) + if source.suffix not in (".ts", ".js", ".mjs", ".mts", ".tsx", ".jsx"): + raise _unavailable("Unsupported local extension source format.") + sources.append(str(source)) + except OSError as error: + raise _unavailable("A required local extension is missing.") from error + if len(set(sources)) != len(sources): + raise _unavailable("Repeated extension inputs are ambiguous.") + alias = Path(client_path) + if not alias.is_absolute(): + raise _unavailable("Select an absolute installed client executable.") + try: + client = alias.resolve(strict=True) + file_identity(client) + except OSError as error: + raise _unavailable("The selected client executable is missing.") from error + output_root = Path(output_root) + if not output_root.is_absolute() or any(p.is_symlink() for p in (output_root, *output_root.parents)): + raise _unavailable("Owned extension output cannot traverse symbolic links.") + output_root.mkdir(mode=0o700, parents=False, exist_ok=False) + environment = {key: value for key, value in os.environ.items() + if not key.startswith(("BUN_", "NODE_")) and key not in ("LD_PRELOAD", "DYLD_INSERT_LIBRARIES")} + environment.update(BUN_BE_BUN="1", TMPDIR=str(output_root), XDG_CACHE_HOME=str(output_root / "cache")) + try: + run = subprocess.run([str(client), "--no-install", "--no-env-file", "--no-macros", str(_COMPILER)], + input=encoded({"sources": sources, "output": str(output_root), "maxBytes": _MAX_BYTES}), + cwd=output_root, env=environment, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30) + if run.returncode or len(run.stdout) > 1024 * 1024: + raise _unavailable("Local extension imports could not be frozen without installation.") + result = json.loads(run.stdout) + result.update(client_path=str(client), client_alias=str(alias)) + result["inputs"].append(file_identity(client)) + result["inputs"].append(file_identity(_COMPILER)) + result["inputs"].extend(file_identity(Path(source)) for source in sources) + verify_extensions(result) + return result + except (OSError, subprocess.SubprocessError, ValueError, KeyError) as error: + raise _unavailable("The installed client's static compiler is unavailable or incompatible.") from error + + +def prepare_extension_bootstrap(manifest, profile_root: Path, configuration, project_path: Path, environment=None): + """Prepare an explicit Start's owned host; this function executes no plugin.""" + verify_extensions(manifest) + if any(not bundle["legacy"] for bundle in manifest["bundles"]): + raise _unavailable("Native extension definitions require an adapter that can bind their bootstrap policy.") + if not isinstance(configuration, dict) or configuration.get("plugins") or configuration.get("plugin"): + raise _unavailable("Configured plugin operations must be resolved before preparing an owned profile.") + if len(encoded(configuration)) > 1024 * 1024 or configuration.get("mcp") or configuration.get("providers"): + raise _unavailable("Unsupported configuration sources need a new client context.") + for provider in configuration.get("provider", {}).values(): + if provider.get("npm") != "@ai-sdk/openai-compatible": + raise _unavailable("Capture bootstrap admits only the bundled compatible provider.") + if any(model.get("provider", {}).get("npm") for model in provider.get("models", {}).values()): + raise _unavailable("Model-specific provider packages are not admitted by this adapter.") + profile_root = Path(profile_root) + project_path = Path(project_path) + if not profile_root.is_absolute() or any(p.is_symlink() for p in (profile_root, *profile_root.parents)): + raise _unavailable("Owned profile roots cannot traverse symbolic links.") + if not project_path.is_absolute() or not project_path.is_dir(): + raise _unavailable("An existing explicit project is needed for bootstrap.") + profile_root.mkdir(mode=0o700, exist_ok=True) + if profile_root.stat().st_mode & 0o077 or profile_root.stat().st_uid != os.getuid(): + raise _unavailable("Owned profile permissions changed.") + directories = {name: profile_root / name for name in ("config", "data", "cache", "state", "tmp")} + for path in directories.values(): + path.mkdir(mode=0o700, exist_ok=True) + if path.is_symlink() or path.stat().st_mode & 0o077 or path.stat().st_uid != os.getuid(): + raise _unavailable("Owned profile directory permissions changed.") + plugin_root = directories["config"] / "plugins" + plugin_root.mkdir(mode=0o700, exist_ok=False) + + def write(path, raw): + with path.open("xb") as output: + os.chmod(path, 0o600) + output.write(raw) + output.flush() + os.fsync(output.fileno()) + + def resource(name): + target = profile_root / name + write(target, (_COMPILER.parent / name).read_bytes()) + return target + + for name in ("proxy.mjs", "rpc.mjs", "worker.mjs"): + resource(name) + for index, bundle in enumerate(manifest["bundles"]): + source = "import {proxyPlugin} from " + json.dumps(str(profile_root / "proxy.mjs")) + ";\n" + source += "export default proxyPlugin(" + json.dumps(bundle["id"]) + ");\n" + write(plugin_root / ("extension-" + str(index) + ".ts"), source.encode("utf-8")) + original = profile_root / "original-config.json" + effective = profile_root / "effective-config.json" + receipt = profile_root / "bootstrap-receipt.json" + plan = profile_root / "bootstrap-plan.json" + write(original, encoded(configuration)) + write(plan, encoded({"schema": "coding-capture-bootstrap-plan/1", "bundles": manifest["bundles"], + "original": str(original), "effective": str(effective), "receipt": str(receipt), + "project": str(project_path), "config_root": str(directories["config"])})) + base = os.environ if environment is None else environment + child = {key: value for key, value in base.items() + if not key.startswith(("BUN_", "NODE_", "OPENCODE_")) + and key not in ("LD_PRELOAD", "DYLD_INSERT_LIBRARIES")} + child.update({"XDG_CONFIG_HOME": str(directories["config"]), "XDG_DATA_HOME": str(directories["data"]), + "PWD": str(project_path), + "XDG_CACHE_HOME": str(directories["cache"]), "XDG_STATE_HOME": str(directories["state"]), + "TMPDIR": str(directories["tmp"]), "OPENCODE_CONFIG_DIR": str(directories["config"]), + "OPENCODE_DB": str(directories["data"] / "opencode" / "opencode.db"), + "OPENCODE_DISABLE_PROJECT_CONFIG": "1", "OPENCODE_DISABLE_MODELS_FETCH": "1", + "OPENCODE_DISABLE_AUTOUPDATE": "1", + "OPENCODE_DISABLE_FFF": "1", "MLX_AGENT_CAPTURE_BOOTSTRAP": str(plan), + "OPENCODE_CONFIG": str(effective)}) + return {"environment": child, "bootstrap_plan": str(plan), "receipt_path": str(receipt), + "effective_config_path": str(effective), "config_root": str(directories["config"])} + + +class ExtensionHost: + """One checked Bun child holds the original closures over an inherited socket.""" + + def __init__(self, client_path, manifest, profile_root, configuration, project_path, environment=None): + if str(Path(client_path).resolve()) != manifest["client_path"]: + raise TaskError("capture_context_changed", "The selected client executable changed.") + self.client_path = Path(manifest["client_path"]) + self.manifest, self.root = manifest, Path(profile_root) + self.configuration, self.project_path = configuration, Path(project_path) + self.environment = environment + self.profile = self.process = self.channel = self.identity = None + self.logs = [] + + def __enter__(self): + from .capture_platform import capture_process_identity + self.profile = prepare_extension_bootstrap(self.manifest, self.root, self.configuration, + self.project_path, self.environment) + worker, self.channel = socket.socketpair() + self.channel.settimeout(30) + try: + for name in ("extension-stdout.log", "extension-stderr.log"): + descriptor = os.open(self.root / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + self.logs.append(os.fdopen(descriptor, "wb")) + child = dict(self.profile["environment"], BUN_BE_BUN="1", + MLX_AGENT_CAPTURE_EXTENSION_FD=str(worker.fileno())) + self.process = subprocess.Popen([str(self.client_path), "--no-install", "--no-env-file", "--no-macros", + str(self.root / "worker.mjs")], cwd=self.project_path, env=child, + stdin=subprocess.PIPE, stdout=self.logs[0], stderr=self.logs[1], + pass_fds=(worker.fileno(),), start_new_session=True) + worker.close() + self.identity = capture_process_identity(self.process.pid) + self.channel.sendall(encoded({"id": "engine:1", "method": "bootstrap", "params": {}}) + b"\n") + raw = bytearray() + while b"\n" not in raw: + chunk = self.channel.recv(4096) + if not chunk or len(raw) + len(chunk) > 1024 * 1024: + raise _unavailable("Owned extension bootstrap ended without a bounded reply.") + raw.extend(chunk) + reply = json.loads(raw) + if reply.get("id") != "engine:1" or "error" in reply: + raise _unavailable("A legacy extension has unsupported hooks or could not bootstrap.") + result = reply["result"] + effective = encoded(result["configuration"]) + for path, content in ((self.profile["effective_config_path"], effective), + (self.profile["receipt_path"], encoded({"schema": "coding-capture-bootstrap/1", + "plugins": result["plugins"], "effective_config_sha256": hashlib.sha256(effective).hexdigest()}))): + with Path(path).open("xb") as output: + os.chmod(path, 0o600) + output.write(content) + output.flush() + os.fsync(output.fileno()) + verify_extensions(self.manifest) + # settimeout uses O_NONBLOCK on the shared file description; the + # child's stream reader needs a blocking descriptor after handoff. + self.channel.setblocking(True) + self.profile["environment"]["MLX_AGENT_CAPTURE_EXTENSION_FD"] = str(self.channel.fileno()) + return self + except BaseException: + worker.close() + self.close() + raise + + @property + def server_fds(self): + return (self.channel.fileno(),) + + def close(self): + from .capture_platform import stop_owned + try: + if self.process is not None: + stop_owned(self.process, self.identity) + finally: + if self.process is not None and self.process.stdin is not None: + self.process.stdin.close() + if self.channel is not None: + self.channel.close() + for stream in self.logs: + stream.close() + + def __exit__(self, *_): + self.close() diff --git a/packages/codex/mlx-agent/src/mlx_agent/coding_capture_policy.py b/packages/codex/mlx-agent/src/mlx_agent/coding_capture_policy.py new file mode 100644 index 0000000..ef915ed --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/coding_capture_policy.py @@ -0,0 +1,121 @@ +"""Read-only admission of OpenCode's persisted managed-policy inputs.""" + +import hashlib +import json +from pathlib import Path +import sqlite3 + +from .coding_capture_plugins import file_identity +from .task_contracts import MAX_REQUEST_BYTES, TaskError, encoded + + +def _unavailable(message): + return TaskError("capture_client_unavailable", message, + "Capture requires bound local policy sources and an authoritative snapshot of any remote policy.") + + +def _snapshot(path): + path = Path(path) + resolved = path.resolve(strict=False) + try: + item = file_identity(resolved) + except FileNotFoundError: + return {"path": str(path), "resolved": str(resolved), "exists": False} + return {"path": str(path), "resolved": str(resolved), "exists": True, + "identity": item["identity"], "sha256": item["sha256"]} + + +def verify_policy_sources(manifest): + try: + if any(_snapshot(Path(item["path"])) != item for item in manifest["sources"]): + raise ValueError("changed source") + except (OSError, ValueError, KeyError, TaskError) as error: + raise TaskError("capture_context_changed", "A persisted client policy input changed.", + "Create a new preview before starting the coding task.") from error + + +def _json(raw): + def pairs(items): + value = {} + for key, item in items: + if key in value: + raise ValueError("duplicate key") + value[key] = item + return value + value = json.loads(raw, object_pairs_hook=pairs) + encoded(value) + return value + + +def inspect_policy_sources(data_root: Path, database_paths): + """Static input facts only; the owned instance still needs runtime checks. + + An absent DB is insufficient because startup may import legacy auth.json. + Immutable SQLite reads cannot create or update the original WAL/SHM files. + A nonempty journal requires a separate authoritative snapshot instead. + """ + data_root = Path(data_root) + paths = [Path(path) for path in database_paths] + if (not data_root.is_absolute() or len(paths) > 32 or len(set(paths)) != len(paths) + or any(not path.is_absolute() for path in paths)): + raise _unavailable("Client policy sources need explicit absolute local paths.") + auth = data_root / "auth.json" + inputs = [auth] + for path in paths: + inputs.extend((path, Path(str(path) + "-wal"), Path(str(path) + "-journal"))) + try: + if auth.exists() and auth.stat().st_size > MAX_REQUEST_BYTES: + raise _unavailable("Legacy client authentication metadata exceeds its supported size.") + before = [_snapshot(path) for path in inputs] + if auth.exists(): + entries = _json(auth.read_bytes()) + if not isinstance(entries, dict) or not all(isinstance(value, dict) for value in entries.values()): + raise _unavailable("Legacy client authentication metadata is not understood.") + # The pinned migration trims trailing slashes from integration IDs. + # Reject candidates before a fresh DB could import their secrets. + if any(key.rstrip("/") == "opencode" or value.get("type") == "wellknown" + for key, value in entries.items()): + raise _unavailable("Legacy client state can introduce Console policy or remote configuration.") + for path in paths: + for suffix in ("-wal", "-journal"): + journal = Path(str(path) + suffix) + if journal.exists() and journal.stat().st_size: + raise _unavailable("Client policy has nonempty journal data and needs a stable read-only snapshot.") + if not path.exists() or path.stat().st_size == 0: + continue + connection = sqlite3.connect(path.resolve().as_uri() + "?mode=ro&immutable=1", uri=True) + try: + expected = { + "credential": {"id", "integration_id", "label", "value", "connector_id", "method_id", + "active", "time_created", "time_updated"}, + "kv": {"key", "value", "time_created", "time_updated"}, + } + for table, columns in expected.items(): + kind = connection.execute("SELECT type FROM sqlite_master WHERE name=?", (table,)).fetchone() + actual = {row[1] for row in connection.execute("PRAGMA table_info(" + table + ")")} + if kind != ("table",) or actual != columns: + raise _unavailable("The persisted client policy schema is unsupported.") + # Never select credential.value. Even false/NULL active rows + # can become the selected connection under the pinned resolver. + candidate = connection.execute("SELECT 1 FROM credential WHERE integration_id='opencode' LIMIT 1").fetchone() + if candidate is not None: + raise _unavailable("A Console connection needs its authoritative managed-policy snapshot.") + size = connection.execute("SELECT length(CAST(value AS BLOB)) FROM kv WHERE key='wellknown:sources'").fetchone() + if size is not None and (type(size[0]) is not int or not 0 <= size[0] <= MAX_REQUEST_BYTES): + raise _unavailable("Persisted remote configuration metadata exceeds its supported size.") + source = connection.execute("SELECT value FROM kv WHERE key='wellknown:sources'").fetchone() + if source is not None and _json(source[0]) != []: + raise _unavailable("Persisted remote configuration needs its authoritative policy snapshot.") + finally: + connection.close() + result = {"schema": "coding-capture-policy-sources/1", "basis": "static_inputs", "sources": before, + "managed_policy": "no_console_connection", "remote_configuration": "no_wellknown_sources"} + verify_policy_sources(result) + result["sha256"] = hashlib.sha256(encoded(result)).hexdigest() + return result + except TaskError as error: + if error.code in ("capture_client_unavailable", "capture_context_changed"): + raise + raise _unavailable("Persisted client policy inputs cannot be inspected safely.") from error + except (OSError, ValueError, TypeError, UnicodeError, sqlite3.Error) as error: + raise _unavailable("Persisted client policy inputs cannot be inspected safely.") from error diff --git a/packages/codex/mlx-agent/src/mlx_agent/coding_capture_process.py b/packages/codex/mlx-agent/src/mlx_agent/coding_capture_process.py new file mode 100644 index 0000000..3e4e7a9 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/coding_capture_process.py @@ -0,0 +1,163 @@ +"""Bounded execution of CLI commands attached to one owned client server.""" + +import os +import selectors +import subprocess +import time +from urllib.parse import urlsplit + +from .capture_platform import capture_process_identity, stop_owned +from .coding_capture_client import OpenCodeEvents, is_identifier, validate_session_export +from .coding_capture_contracts import DEFAULT_LIMITS +from .task_contracts import MAX_REQUEST_BYTES, TaskError + + +def _execute(profile, arguments, payload, role, stdout_name, stderr_name, on_stdout=None): + """Retain a bounded prefix on failure; it can never become successful evidence.""" + limit = profile["artifact_bytes"] + if type(limit) is not int or not 1 <= limit <= DEFAULT_LIMITS["artifact_bytes"]: + raise TaskError("invalid_capture_limits", "Client output requires an accepted byte limit.") + try: + url = urlsplit(profile["server_url"]) + valid = (url.scheme == "http" and url.hostname == "127.0.0.1" and url.port is not None + and 0 < url.port < 65536 and url.path in ("", "/") + and not url.username and not url.password and not url.query and not url.fragment) + except (TypeError, ValueError): + valid = False + if not valid: + raise TaskError("capture_client_unavailable", "Select the authenticated owned loopback client server.") + profile["check_context"]() + if profile["cancelled"](): + raise TaskError("capture_cancelled", "Capture was cancelled before the client command started.") + if profile["environment"].get("PWD") != profile["project_path"]: + raise TaskError("capture_context_changed", "Client PWD differs from the accepted project.") + + started = time.monotonic() + process, identity, failure, finished, unresolved = None, None, None, None, False + output, diagnostic = bytearray(), bytearray() + selector = selectors.DefaultSelector() + try: + environment = dict(profile["environment"]) + environment.pop("MLX_AGENT_CAPTURE_EXTENSION_FD", None) + process = subprocess.Popen([profile["client_path"], *arguments], cwd=profile["project_path"], + env=environment, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, start_new_session=True) + identity = capture_process_identity(process.pid) + # The supervisor must persist ownership before any prompt bytes leave. + profile["on_process"](role, identity) + for stream, data in ((process.stdout, output), (process.stderr, diagnostic)): + os.set_blocking(stream.fileno(), False) + selector.register(stream, selectors.EVENT_READ, data) + if payload: + os.set_blocking(process.stdin.fileno(), False) + selector.register(process.stdin, selectors.EVENT_WRITE, None) + else: + process.stdin.close() + offset, drain_deadline = 0, None + while selector.get_map() or process.poll() is None: + if profile["cancelled"](): + raise TaskError("capture_cancelled", "Capture was cancelled during the client command.") + if process.poll() is not None: + drain_deadline = drain_deadline or time.monotonic() + 5 + if time.monotonic() >= drain_deadline: + raise TaskError("capture_execution_unresolved", "A client descendant retained an output stream.") + for key, event in selector.select(0.2): + if event & selectors.EVENT_WRITE: + try: + offset += os.write(key.fd, payload[offset:offset + 16384]) + except BrokenPipeError: + offset = len(payload) + if offset == len(payload): + selector.unregister(key.fileobj) + key.fileobj.close() + continue + chunk = os.read(key.fd, 16384) + if not chunk: + selector.unregister(key.fileobj) + key.fileobj.close() + continue + buffer = key.data + remaining = limit - len(buffer) + buffer.extend(chunk[:remaining]) + if len(chunk) > remaining: + raise TaskError("capture_artifact_limit", "Client command output exceeded its accepted artifact limit.") + if buffer is output and on_stdout is not None: + on_stdout(chunk) + process.wait(timeout=5) + finished = time.monotonic() + profile["check_context"]() + except TaskError as error: + failure = error + unresolved = error.code == "capture_execution_unresolved" + except (OSError, subprocess.SubprocessError) as error: + failure = TaskError("capture_client_failed", "The owned client command could not finish.", + "Inspect its retained diagnostics and start a new capture.") + finally: + try: + if process is not None: + try: + stop_owned(process, identity) + except (TaskError, OSError, subprocess.SubprocessError): + failure = TaskError("capture_ownership_lost", "Owned command termination remains unresolved.", + "Retain the capture and inspect its recorded process identities.") + unresolved = True + finally: + selector.close() + if process is not None: + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + elapsed = (finished or time.monotonic()) - started + artifacts = [profile["storage"].write_artifact(stdout_name, bytes(output)), + profile["storage"].write_artifact(stderr_name, bytes(diagnostic))] + return {"exit_code": process.returncode if process is not None else None, + "elapsed_seconds": elapsed, "process": identity, "artifacts": artifacts, + "stdout": bytes(output), "execution_unresolved": unresolved, + "error": failure.as_dict() if failure else None} + + +def run_client(profile, prompt, on_event): + try: + payload = prompt.encode("utf-8") + except (AttributeError, UnicodeError) as error: + raise TaskError("invalid_capture", "A coding request must be UTF-8 text.") from error + if not payload.strip() or len(payload) > MAX_REQUEST_BYTES or b"\0" in payload: + raise TaskError("invalid_capture", "A coding request must contain at most 1 MiB of text.") + events = OpenCodeEvents(on_event, profile["artifact_bytes"]) + result = _execute(profile, ["run", "--server", profile["server_url"], "--format", "json", "--model", profile["model"]], + payload, "client", "client.ndjson", "client-stderr.log", events.feed) + result.pop("stdout") + if result["error"] is None and result["exit_code"] != 0: + result["error"] = TaskError("capture_client_failed", "The owned client returned a nonzero exit code.", + "Inspect the retained client diagnostics.").as_dict() + if result["error"] is None: + try: + events.finish() + if events.has_error: + raise TaskError("capture_client_failed", "The client reported a session error.") + except TaskError as error: + result["error"] = error.as_dict() + result.update(session_id=events.session_id, has_error=events.has_error) + return result + + +def export_session(profile, session_id, storage): + if not is_identifier(session_id, "ses"): + raise TaskError("invalid_capture_session", "Export requires an observed explicit session identity.") + result = _execute(dict(profile, storage=storage), + ["session", "export", session_id, "--server", profile["server_url"]], + b"", "exporter", "session.json", "export-stderr.log") + raw = result.pop("stdout") + result["session"] = None + if result["error"] is None and result["exit_code"] != 0: + result["error"] = TaskError("capture_export_failed", "The explicit session export returned a nonzero exit code.", + "Keep the bounded profile and diagnostics for recovery.").as_dict() + if result["error"] is None: + try: + session = validate_session_export(raw, session_id, profile["model"]) + if session["info"].get("location", {}).get("directory") != profile["project_path"]: + raise TaskError("invalid_capture_session", "Export does not describe the accepted project directory.") + result["session"] = session + except TaskError as error: + result["error"] = error.as_dict() + return result diff --git a/packages/codex/mlx-agent/src/mlx_agent/coding_capture_storage.py b/packages/codex/mlx-agent/src/mlx_agent/coding_capture_storage.py new file mode 100644 index 0000000..e28e267 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/coding_capture_storage.py @@ -0,0 +1,443 @@ +"""Bounded, private evidence with explicit export and recoverable removal.""" + +import base64 +from contextlib import contextmanager +import errno +import fcntl +import hashlib +import json +import os +from pathlib import Path +import shutil +import stat +import threading +from typing import Dict, Optional +from uuid import uuid4 + +from .capture_platform import CaptureVolume, capture_process_identity +from .coding_capture_contracts import (DEFAULT_LIMITS, KIND, METADATA_RESERVE, validate_capture_request, + validate_capture_result, validate_limits, validate_reference) +from .task_contracts import MAX_REQUEST_BYTES, TERMINAL, TaskError, encoded, task_id as validate_task_id +from .task_store import TaskStore, ensure_private, utc_now + + +def _hash(raw): + return hashlib.sha256(raw).hexdigest() + + +def _private_file(path): + ensure_private(path, directory=False) + if path.stat().st_nlink != 1: + raise TaskError("unsafe_capture_storage", "Capture artifacts cannot share a hard link.") + + +def _failure(error): + code = "capture_budget_exhausted" if getattr(error, "errno", None) in (errno.ENOSPC, errno.EDQUOT) else "capture_storage_failed" + return TaskError(code, "Cannot safely access the owned capture data.", "Keep the capture for recovery; inspect free space and ownership.") + + +class CaptureStorage: + def __init__(self, store: TaskStore, task_id: str): + self.store = store + self.identifier = validate_task_id(task_id) + self.root = store.root / "captures" / self.identifier + self.tombstone = self.root.parent / (self.identifier + ".removed.json") + self._limits = None + self._mounted = None + self._readonly = True + self._lock = threading.RLock() + self._lock_descriptor = None + + @contextmanager + def _exclusive(self): + with self._lock: + if self._lock_descriptor is not None: + yield + return + ensure_private(self.root.parent) + descriptor = os.open(self.root.parent / (self.identifier + ".lock"), + os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077 or info.st_nlink != 1: + raise TaskError("unsafe_capture_storage", "Capture operation lock ownership changed.") + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + raise TaskError("capture_storage_busy", "Another operation owns this capture.") from error + self._lock_descriptor = descriptor + try: + yield + finally: + self._lock_descriptor = None + finally: + os.close(descriptor) + + @classmethod + def create(cls, store: TaskStore, task_id: str, limits: Dict): + limits = validate_limits(limits) + storage = cls(store, task_id) + for path in (store.root, *store.root.parents): + if path.is_symlink(): + raise TaskError("unsafe_capture_storage", "Capture storage cannot traverse a link.") + store.root.mkdir(mode=0o700, parents=True, exist_ok=True) + ensure_private(store.root) + storage.root.parent.mkdir(mode=0o700, exist_ok=True) + ensure_private(storage.root.parent) + if storage.tombstone.exists() or storage.tombstone.is_symlink(): + raise TaskError("task_exists", "Removed capture identities cannot be reused.") + CaptureVolume.create(storage.root, task_id, limits["total_bytes"] - METADATA_RESERVE) + storage._limits = limits + with storage.active() as mounted: + for name in ("artifacts", "profile", "tmp"): + (mounted / name).mkdir(mode=0o700) + storage._capacity(mounted, limits) + return storage + + @contextmanager + def active(self, readonly=False): + """Internal scoped mount. Mounted paths never enter a CLI/MCP result.""" + with self._exclusive(): + ensure_private(self.root.parent) + if self._mounted is not None: + if self._readonly and not readonly: + raise TaskError("capture_readonly", "Capture is mounted for reading only.") + yield self._mounted + return + try: + with CaptureVolume.open_owned(self.root, readonly=readonly) as mounted: + self._mounted, self._readonly = mounted, readonly + try: + yield mounted + finally: + self._mounted = None + except OSError as error: + raise _failure(error) from error + + def _path(self, mounted, name): + validate_reference({"name": name, "sha256": "0" * 64, "bytes": 0}) + directory = mounted / "artifacts" + ensure_private(directory) + path = directory / name + if path.is_symlink(): + raise TaskError("unsafe_capture_storage", "Capture artifacts cannot be links.") + return path + + def _read(self, mounted, name, expected=None, maximum=None): + path = self._path(mounted, name) + _private_file(path) + maximum = maximum if maximum is not None else DEFAULT_LIMITS["artifact_bytes"] + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode) or before.st_size > maximum: + raise TaskError("capture_artifact_limit", "Capture artifact exceeds its accepted size limit.") + with os.fdopen(descriptor, "rb") as stream: + descriptor = None + raw = stream.read(maximum + 1) + after = os.fstat(stream.fileno()) + identity = lambda item: (item.st_dev, item.st_ino, item.st_size, item.st_mtime_ns, item.st_ctime_ns) + if len(raw) > maximum or identity(before) != identity(after) or identity(after) != identity(path.lstat()): + raise TaskError("capture_artifact_changed", "Capture artifact changed during its read.") + reference = {"name": name, "sha256": _hash(raw), "bytes": len(raw)} + if expected is not None and expected != reference: + raise TaskError("capture_artifact_changed", "Retained capture artifact no longer matches its accepted hash.") + return raw, reference + finally: + if descriptor is not None: + os.close(descriptor) + + def _request(self, mounted, record=None): + record = record or self.store.get(self.identifier) + raw, _ = self._read(mounted, "request.json", record["request"], MAX_REQUEST_BYTES) + try: + request = validate_capture_request(json.loads(raw)) + except (ValueError, TypeError) as error: + raise TaskError("capture_request_changed", "Retained capture request is invalid.") from error + context = record["context"] + if (request["model_path"] != context["serve"]["model_path"] + or request["serve_receipt"] != context["serve"]["receipt"] + or request["client_path"] != context["client"]["path"] + or request["project_path"] != context["project"]["path"] + or request.get("workbench_context") != context.get("workbench_context")): + raise TaskError("capture_context_changed", "Original capture request and accepted context disagree.") + self._capacity(mounted, request["limits"]) + return request + + def _capacity(self, mounted, limits): + info = os.statvfs(mounted) + allocated = sum(item.lstat().st_blocks * 512 for item in (self.root / "capture.sparsebundle").rglob("*") + if not item.is_symlink()) + if info.f_blocks * info.f_frsize > limits["total_bytes"] - METADATA_RESERVE or allocated > limits["total_bytes"]: + raise TaskError("capture_budget_exhausted", "Capture filesystem exceeds its accepted owned-data budget.") + + def _atomic(self, mounted, name, raw): + target = self._path(mounted, name) + if target.exists(): + raise TaskError("capture_artifact_exists", "Retained capture evidence cannot be overwritten.") + temporary = target.parent / (".write-" + str(uuid4())) + try: + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(raw) + stream.flush() + os.fsync(stream.fileno()) + # Atomic publication refuses a destination created after the check. + os.link(temporary, target, follow_symlinks=False) + temporary.unlink() + TaskStore.fsync_directory(target.parent) + finally: + temporary.unlink(missing_ok=True) + return {"name": name, "sha256": _hash(raw), "bytes": len(raw)} + + def accept_request(self, value): + value = validate_capture_request(value) + if self._limits != value["limits"]: + raise TaskError("capture_limits_changed", "Capture request limits differ from its allocated volume.") + with self.active() as mounted: + return self._atomic(mounted, "request.json", encoded(value)) + + def write_artifact(self, name, raw): + if not isinstance(raw, bytes): + raise TaskError("invalid_capture_artifact", "Capture artifact content must be bytes.") + with self.active() as mounted: + limits = self._request(mounted)["limits"] + if name in ("request.json", "result.json"): + raise TaskError("invalid_capture_artifact", "Capture request and result have dedicated publication boundaries.") + if len(raw) > limits["artifact_bytes"]: + raise TaskError("capture_artifact_limit", "Capture artifact exceeds its accepted size limit.") + try: + return self._atomic(mounted, name, raw) + except OSError as error: + raise _failure(error) from error + + def _verify(self, mounted, expected_digest=None): + record = self.store.get(self.identifier) + request = self._request(mounted, record) + expected = record["result"] + raw, reference = self._read(mounted, "result.json", expected, request["limits"]["artifact_bytes"]) + if expected_digest is not None and reference["sha256"] != expected_digest: + raise TaskError("capture_result_changed", "Capture result digest changed.") + try: + manifest = validate_capture_result(json.loads(raw)) + except (ValueError, TypeError) as error: + raise TaskError("invalid_capture_result", "Retained capture manifest is invalid.") from error + if (raw != encoded(manifest) or manifest["task_id"] != self.identifier + or manifest["request_sha256"] != record["request"]["sha256"] + or manifest["preview_hash"] != record["preview_hash"] or manifest["context"] != record["context"]): + raise TaskError("capture_context_changed", "Retained capture result is not bound to the accepted request and context.") + references = [] + for artifact in manifest["artifacts"]: + _, observed = self._read(mounted, artifact["name"], artifact, request["limits"]["artifact_bytes"]) + references.append(observed) + return self._verified_reply(raw, reference, references) + + def _verified_reply(self, raw, reference, artifacts): + reply = {"schema": "coding-capture-verified/1", "task_id": self.identifier, "sha256": reference["sha256"], + "manifest_utf8": raw.decode("utf-8"), "artifacts": artifacts} + # Leave space for the shared result envelope within its 12,288-byte cap. + if len(encoded(reply)) > 10240: + raise TaskError("capture_result_limit", "Capture result exceeds the bounded verification response.", + "Retain the session artifacts; use a smaller request cohort for a verified result.") + return reply + + @staticmethod + def _cleanup_profile(mounted): + for name in ("profile", "tmp"): + directory = mounted / name + if directory.exists() or directory.is_symlink(): + ensure_private(directory) + for item in directory.rglob("*"): + if item.is_symlink(): + raise TaskError("unsafe_capture_storage", "Capture profile cleanup cannot traverse a link.") + shutil.rmtree(directory) + TaskStore.fsync_directory(mounted) + + def publish(self, manifest: Dict) -> Dict: + manifest = validate_capture_result(manifest) + with self.active() as mounted: + request = self._request(mounted) + raw = encoded(manifest) + if len(raw) > request["limits"]["artifact_bytes"]: + raise TaskError("capture_artifact_limit", "Capture manifest exceeds its accepted artifact size limit.") + try: + path = self._path(mounted, "result.json") + if not path.exists(): + # Validate referenced evidence and immutable bindings before publication. + record = self.store.get(self.identifier) + if (manifest["task_id"] != self.identifier or manifest["request_sha256"] != record["request"]["sha256"] + or manifest["preview_hash"] != record["preview_hash"] or manifest["context"] != record["context"]): + raise TaskError("capture_context_changed", "Capture result does not match its accepted context.") + for artifact in manifest["artifacts"]: + self._read(mounted, artifact["name"], artifact, request["limits"]["artifact_bytes"]) + self._verified_reply(raw, {"sha256": _hash(raw)}, manifest["artifacts"]) + self._atomic(mounted, "result.json", raw) + verified = self._verify(mounted, _hash(raw)) + if manifest["client_exit_code"] == 0 and manifest["attribution"] == "verified": + self._cleanup_profile(mounted) + return {"name": "result.json", "sha256": verified["sha256"], "bytes": len(raw)} + except OSError as error: + raise _failure(error) from error + + def verify(self, expected_digest: Optional[str] = None) -> Dict: + with self.active(readonly=True) as mounted: + return self._verify(mounted, expected_digest) + + def _evidence(self, mounted, name): + request = self._request(mounted) + if self._path(mounted, "result.json").exists(): + verified = self._verify(mounted) + references = {item["name"]: item for item in verified["artifacts"]} + references["result.json"] = {"name": "result.json", "sha256": verified["sha256"], + "bytes": len(verified["manifest_utf8"].encode("utf-8"))} + references["request.json"] = self.store.get(self.identifier)["request"] + if name not in references: + raise TaskError("capture_artifact_unavailable", "Artifact is not part of the retained result.") + return self._read(mounted, name, references[name], request["limits"]["artifact_bytes"]) + return self._read(mounted, name, maximum=request["limits"]["artifact_bytes"]) + + def read_artifact(self, name: str, offset: int, length: int = 6144) -> Dict: + if type(offset) is not int or offset < 0 or type(length) is not int or not 1 <= length <= 6144: + raise TaskError("invalid_capture_read", "Artifact reads require a nonnegative offset and at most 6,144 bytes.") + with self.active(readonly=True) as mounted: + raw, reference = self._evidence(mounted, name) + chunk = raw[offset:offset + length] + return dict(reference, offset=offset, data_base64=base64.b64encode(chunk).decode("ascii"), + next_offset=offset + len(chunk) if offset + len(chunk) < len(raw) else None) + + def export_artifact(self, name: str, destination: Path, sanitize: bool) -> Dict: + destination = Path(destination) + if type(sanitize) is not bool or not destination.is_absolute() or ".." in destination.parts: + raise TaskError("invalid_capture_export", "Choose a new absolute export file and explicit sharing mode.") + for path in (destination, *destination.parents): + if path.is_symlink(): + raise TaskError("unsafe_capture_export", "Export cannot traverse a symbolic link.") + if destination.exists() or self.store.root == destination or self.store.root in destination.parents: + raise TaskError("unsafe_capture_export", "Export must be a new file outside engine task storage.") + with self.active(readonly=True) as mounted: + raw, source = self._evidence(mounted, name) + if sanitize: + record = self.store.get(self.identifier) + raw = encoded({"schema": "coding-capture-sharing/1", "task_id": self.identifier, + "state": record["state"], "client_outcome": record["client_outcome"], + "attribution": record["attribution"], "content_removed": True, + "omitted": ["prompts", "code", "tool output", "credentials", "project paths"]}) + temporary = destination.parent / (".capture-export-" + str(uuid4())) + try: + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(raw) + stream.flush() + os.fsync(stream.fileno()) + os.link(temporary, destination, follow_symlinks=False) + temporary.unlink() + TaskStore.fsync_directory(destination.parent) + except OSError as error: + raise TaskError("capture_export_failed", "Cannot publish the chosen export file.") from error + finally: + temporary.unlink(missing_ok=True) + return {"source": source, "destination": str(destination), "sha256": _hash(raw), "bytes": len(raw), + "content_removed": sanitize} + + def _removable(self, record): + if record["kind"] != KIND or record["state"] not in TERMINAL or record["execution_unresolved"]: + raise TaskError("capture_removal_unavailable", "Only terminal captures with resolved owned execution can be removed.") + for process in record["processes"].values(): + try: + os.kill(process["pid"], 0) + except ProcessLookupError: + continue + except OSError as error: + raise TaskError("capture_removal_unavailable", "Cannot establish that owned execution has stopped.") from error + if capture_process_identity(process["pid"]) == process: + raise TaskError("capture_removal_unavailable", "A capture-owned process is still running.") + + def _removal_preview(self, record): + self._removable(record) + if record["artifacts_removed_at"] is not None: + raise TaskError("capture_artifacts_removed", "Capture artifacts were already explicitly removed.") + with self.active(readonly=True) as mounted: + request = self._request(mounted, record) + if self._path(mounted, "result.json").exists(): + self._verify(mounted) + tree = hashlib.sha256() + profile_bytes, owned_bytes, count = 0, 0, 0 + artifacts = [] + for path in sorted(mounted.rglob("*")): + if path.is_symlink(): + raise TaskError("unsafe_capture_storage", "Capture removal cannot traverse a symbolic link.") + ensure_private(path, directory=path.is_dir()) + if path.is_file(): + _private_file(path) + info = path.lstat() + relative = str(path.relative_to(mounted)) + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(65536), b""): + digest.update(chunk) + # The volume's device number changes with each attach; the inode identifies a file within it. + tree.update(encoded([relative, info.st_ino, info.st_size, digest.hexdigest()])) + count += 1 + owned_bytes += info.st_size + if path.relative_to(mounted).parts[0] in ("profile", "tmp"): + profile_bytes += info.st_size + elif path.parent == mounted / "artifacts" and not path.name.startswith("."): + artifacts.append({"name": path.name, "sha256": digest.hexdigest(), "bytes": info.st_size}) + marker = self.root / "volume.json" + _private_file(marker) + binding = {"task_id": self.identifier, "request": record["request"], "result": record["result"], + "completed_at": record["completed_at"], "tree_sha256": tree.hexdigest(), + "volume_sha256": _hash(marker.read_bytes())} + return {"schema": "coding-capture-remove-preview/1", "task_id": self.identifier, + "preview_hash": _hash(encoded(binding)), "artifacts": artifacts[:20], + "artifact_count": len(artifacts), "file_count": count, "profile_bytes": profile_bytes, + "owned_bytes": owned_bytes, "limits": request["limits"], "has_more_artifacts": len(artifacts) > 20} + + def remove_preview(self) -> Dict: + return self._removal_preview(self.store.get(self.identifier)) + + def remove(self, preview_hash: str) -> Dict: + with self._exclusive(): + return self._remove_locked(preview_hash) + + def _remove_locked(self, preview_hash): + # The database transaction fences lifecycle changes and competing removals. + # The bounded tombstone bridges volume deletion and the row commit. + with self.store.updating(self.identifier) as record: + self._removable(record) + if not self.root.exists(): + _private_file(self.tombstone) + if self.tombstone.stat().st_size > 2048: + raise TaskError("unsafe_capture_storage", "Removal recovery metadata exceeds its limit.") + saved = json.loads(self.tombstone.read_bytes()) + if (saved.get("task_id") != self.identifier or saved.get("preview_hash") != preview_hash + or saved.get("request_sha256") != record["request"]["sha256"]): + raise TaskError("capture_removal_changed", "Removal recovery identity changed.") + else: + preview = self._removal_preview(record) + if preview["preview_hash"] != preview_hash: + raise TaskError("capture_removal_changed", "Capture contents changed after removal preview.") + saved = {"schema": "coding-capture-removal/1", "task_id": self.identifier, + "preview_hash": preview_hash, "request_sha256": record["request"]["sha256"], + "removed_at": utc_now()} + raw = encoded(saved) + if len(raw) > 2048: + raise TaskError("capture_metadata_limit", "Removal metadata exceeds its limit.") + if self.tombstone.exists() or self.tombstone.is_symlink(): + _private_file(self.tombstone) + if self.tombstone.stat().st_size > 2048: + raise TaskError("unsafe_capture_storage", "Removal recovery metadata exceeds its limit.") + previous = json.loads(self.tombstone.read_bytes()) + if previous != saved and any(previous.get(key) != saved[key] for key in ("schema", "task_id", "preview_hash", "request_sha256")): + raise TaskError("capture_removal_changed", "An earlier removal must be recovered first.") + saved = previous + else: + descriptor = os.open(self.tombstone, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(raw) + stream.flush() + os.fsync(stream.fileno()) + TaskStore.fsync_directory(self.root.parent) + CaptureVolume.remove(self.root) + record["artifacts_removed_at"] = saved["removed_at"] + return {"task_id": self.identifier, "artifacts_removed_at": saved["removed_at"]} diff --git a/packages/codex/mlx-agent/src/mlx_agent/command_args.py b/packages/codex/mlx-agent/src/mlx_agent/command_args.py new file mode 100644 index 0000000..ead458b --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/command_args.py @@ -0,0 +1,508 @@ +"""Strict argument grammar for provider custom-command input. + +Provider commands pass user text to the model, not directly to a process. +This module converts that opaque text into an allowlisted MLX-agent argv list +before a bundled skill may execute the structured CLI. +""" + +from __future__ import annotations + +import argparse +import json +import re +import shlex +import urllib.parse +from pathlib import Path + +from .models import ROLES + + +class CommandArgumentError(ValueError): + """Command input is outside the documented capability grammar.""" + + +_RUNTIMES = {"ollama", "lmstudio", "mlx_lm", "mlx-vlm", "litellm"} +_ROLES = {role for role, _keywords, _label in ROLES} +_MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,191}$") +_SAFE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+:-]{0,127}$") +_SAFE_QUANTIZATION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") +_METACHARACTERS = set(";|&`$<>") + + +def _tokens(raw): + if not isinstance(raw, str): + raise CommandArgumentError("arguments must be text") + if len(raw) > 4096 or any(ord(character) < 32 or ord(character) == 127 for character in raw): + raise CommandArgumentError("arguments must not contain control characters") + if any(character in _METACHARACTERS for character in raw): + raise CommandArgumentError("shell metacharacters are not allowed") + try: + tokens = shlex.split(raw, posix=True) + except ValueError as error: + raise CommandArgumentError("arguments must use balanced shell-style quotes: {0}".format(error)) + if len(tokens) > 32 or any(not token or len(token) > 512 for token in tokens): + raise CommandArgumentError("arguments exceed the supported command grammar") + return tokens + + +def _value(tokens, index, flag): + if index + 1 >= len(tokens) or tokens[index + 1].startswith("--"): + raise CommandArgumentError("{0} requires a value".format(flag)) + return tokens[index + 1] + + +def _path(value, label): + if not isinstance(value, str) or not value or value.startswith("~") or "\\" in value: + raise CommandArgumentError("{0} must be a portable non-empty path".format(label)) + path = Path(value) + if any(part in {"", ".", ".."} for part in path.parts): + raise CommandArgumentError("{0} must not traverse parent directories".format(label)) + return value + + +def _model(value): + if not _MODEL.fullmatch(value): + raise CommandArgumentError("model must be a publisher/model identifier") + return value + + +def _integer(value, label, minimum, maximum): + if not re.fullmatch(r"[0-9]+", value): + raise CommandArgumentError("{0} must be an integer".format(label)) + number = int(value) + if number < minimum or number > maximum: + raise CommandArgumentError("{0} must be between {1} and {2}".format(label, minimum, maximum)) + return str(number) + + +def _decimal(value, label, minimum, maximum): + if not re.fullmatch(r"(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+)", value): + raise CommandArgumentError("{0} must be a decimal number".format(label)) + number = float(value) + if number < minimum or number > maximum: + raise CommandArgumentError("{0} must be between {1} and {2}".format(label, minimum, maximum)) + return value + + +def _endpoint(value): + try: + parsed = urllib.parse.urlsplit(value) + hostname = parsed.hostname + username = parsed.username + password = parsed.password + port = parsed.port + except (UnicodeError, ValueError) as error: + raise CommandArgumentError("endpoint is not a valid local HTTP URL") from error + if ( + parsed.scheme not in {"http", "https"} + or hostname not in {"127.0.0.1", "localhost", "::1"} + or username is not None + or password is not None + or parsed.query + or parsed.fragment + ): + raise CommandArgumentError("endpoint must be a credential-free local HTTP URL") + if port is not None and not 1 <= port <= 65535: + raise CommandArgumentError("endpoint port must be between 1 and 65535") + return value + + +def _parse_scout(tokens): + argv, index = ["discover"], 0 + seen, cache = set(), None + values = { + "--role": (lambda value: value if value in _ROLES else None, "role"), + "--limit": (lambda value: _integer(value, "limit", 1, 50), "limit"), + "--memory-gb": (lambda value: _decimal(value, "memory-gb", 0.25, 1024), "memory-gb"), + "--quantization": (lambda value: value if _SAFE_QUANTIZATION.fullmatch(value) else None, "quantization"), + "--runtime": (lambda value: value if value in _RUNTIMES else None, "runtime"), + "--context": (lambda value: _integer(value, "context", 1024, 1048576), "context"), + "--state-dir": (lambda value: _path(value, "state-dir"), "state-dir"), + "--wire": (lambda value: _model(value), "wire"), + "--target": (lambda value: value if value in _RUNTIMES else None, "target"), + "--port": (lambda value: _integer(value, "port", 1, 65535), "port"), + } + repeatable = {"--license", "--publisher"} + booleans = {"--exclude-gated", "--include-gated", "--new", "--fast", "--json"} + while index < len(tokens): + flag = tokens[index] + if flag in booleans: + if flag in seen: + raise CommandArgumentError("duplicate flag: {0}".format(flag)) + if flag in {"--exclude-gated", "--include-gated"} and {"--exclude-gated", "--include-gated"} & seen: + raise CommandArgumentError("gated-repository flags are mutually exclusive") + seen.add(flag) + argv.append(flag) + index += 1 + elif flag in {"--refresh", "--offline"}: + if cache is not None: + raise CommandArgumentError("--refresh and --offline are mutually exclusive") + cache = flag + argv.append(flag) + index += 1 + elif flag in repeatable: + value = _value(tokens, index, flag) + if not _SAFE_NAME.fullmatch(value): + raise CommandArgumentError("{0} must be a safe identifier".format(flag)) + argv.extend([flag, value]) + index += 2 + elif flag in values: + if flag in seen: + raise CommandArgumentError("duplicate flag: {0}".format(flag)) + value = values[flag][0](_value(tokens, index, flag)) + if value is None: + raise CommandArgumentError("invalid {0} value".format(values[flag][1])) + seen.add(flag) + argv.extend([flag, value]) + index += 2 + else: + raise CommandArgumentError("unsupported Scout argument: {0}".format(flag)) + return argv + + +def _parse_adopt(tokens): + if not tokens or tokens[0] not in {"start", "resume", "status"}: + raise CommandArgumentError("Adopt requires start, resume, or status") + action, argv, index = tokens[0], ["adopt", tokens[0]], 1 + state = None + allowed_booleans = {"--json"} if action != "start" else {"--offline", "--refresh", "--fast", "--no-network", "--measure", "--json"} + while index < len(tokens): + flag = tokens[index] + if flag == "--state": + if state is not None: + raise CommandArgumentError("duplicate flag: --state") + state = _path(_value(tokens, index, flag), "state") + argv.extend([flag, state]) + index += 2 + elif action == "start" and flag == "--role": + role = _value(tokens, index, flag) + if role not in _ROLES: + raise CommandArgumentError("unknown adoption role") + argv.extend([flag, role]) + index += 2 + elif action == "start" and flag == "--shortlist-limit": + argv.extend([flag, _integer(_value(tokens, index, flag), "shortlist-limit", 1, 20)]) + index += 2 + elif flag in allowed_booleans and flag not in argv: + argv.append(flag) + index += 1 + else: + raise CommandArgumentError("unsupported Adopt argument: {0}".format(flag)) + if state is None: + raise CommandArgumentError("Adopt requires an explicit --state path") + return argv + + +def _parse_wire(tokens): + if not tokens or tokens[0] not in {"render", "apply", "status", "rollback"}: + raise CommandArgumentError("Wire requires render, apply, status, or rollback") + action, argv, index = tokens[0], ["wire", tokens[0]], 1 + if action in {"status", "rollback"}: + if index >= len(tokens): + raise CommandArgumentError("Wire {0} requires a receipt path".format(action)) + argv.append(_path(tokens[index], "receipt")) + index += 1 + allowed = ( + {"--json"} + if action == "status" + else {"--confirm", "--preview-hash", "--json"} + ) + else: + if index >= len(tokens): + raise CommandArgumentError("Wire {0} requires a model".format(action)) + argv.append(_model(tokens[index])) + index += 1 + allowed = {"--target", "--path", "--json"} + if action == "apply": + allowed.update({"--confirm", "--preview-hash", "--receipts-dir", "--endpoint"}) + path_seen = False + while index < len(tokens): + flag = tokens[index] + if flag not in allowed: + raise CommandArgumentError("unsupported Wire argument: {0}".format(flag)) + if flag in argv: + raise CommandArgumentError("duplicate flag: {0}".format(flag)) + if flag in {"--json", "--confirm"}: + argv.append(flag) + index += 1 + else: + value = _value(tokens, index, flag) + if flag == "--target": + if value not in _RUNTIMES: + raise CommandArgumentError("unsupported Wire target") + elif flag in {"--path", "--receipts-dir"}: + value = _path(value, flag[2:]) + path_seen = path_seen or flag == "--path" + elif flag == "--preview-hash": + if not re.fullmatch(r"[0-9a-f]{64}", value): + raise CommandArgumentError("preview-hash must be a SHA-256 digest") + elif flag == "--endpoint": + value = _endpoint(value) + argv.extend([flag, value]) + index += 2 + if action in {"render", "apply"} and not path_seen: + raise CommandArgumentError("Wire {0} requires --path".format(action)) + return argv + + +def _parse_bench(tokens): + if not tokens or tokens[0] not in {"run", "aggregate"}: + raise CommandArgumentError("Bench requires the run or aggregate action") + if tokens[0] == "aggregate": + return _parse_bench_aggregate(tokens) + argv, index = ["bench", "run"], 1 + seen = set() + values = { + "--repo": (lambda value: _model(value), "repo"), + "--runtime": (lambda value: value if value in _RUNTIMES else None, "runtime"), + "--role": (lambda value: value if value in _ROLES else None, "role"), + "--runs": (lambda value: _integer(value, "runs", 1, 10), "runs"), + "--gen-tokens": (lambda value: _integer(value, "gen-tokens", 16, 2048), "gen-tokens"), + "--timeout": (lambda value: _decimal(value, "timeout", 1, 600), "timeout"), + "--export": (lambda value: _path(value, "export"), "export"), + } + while index < len(tokens): + flag = tokens[index] + if flag == "--json": + if flag in seen: + raise CommandArgumentError("duplicate flag: --json") + seen.add(flag) + argv.append(flag) + index += 1 + elif flag in values: + if flag in seen: + raise CommandArgumentError("duplicate flag: {0}".format(flag)) + value = values[flag][0](_value(tokens, index, flag)) + if value is None: + raise CommandArgumentError("invalid {0} value".format(values[flag][1])) + seen.add(flag) + argv.extend([flag, value]) + index += 2 + else: + raise CommandArgumentError("unsupported Bench argument: {0}".format(flag)) + for required in ("--repo", "--runtime"): + if required not in seen: + raise CommandArgumentError("Bench run requires {0}".format(required)) + return argv + + +def _parse_bench_aggregate(tokens): + argv, index = ["bench", "aggregate"], 1 + seen = set() + while index < len(tokens): + flag = tokens[index] + if flag == "--json": + if flag in seen: + raise CommandArgumentError("duplicate flag: --json") + seen.add(flag) + argv.append(flag) + index += 1 + elif flag in {"--exports", "--out"}: + if flag in seen: + raise CommandArgumentError("duplicate flag: {0}".format(flag)) + seen.add(flag) + argv.extend([flag, _path(_value(tokens, index, flag), flag[2:])]) + index += 2 + else: + raise CommandArgumentError("unsupported Bench argument: {0}".format(flag)) + if "--exports" not in seen: + raise CommandArgumentError("Bench aggregate requires --exports") + return argv + + +def _parse_doctor(tokens): + if not tokens or tokens[0] != "models": + raise CommandArgumentError("Doctor requires the models action") + argv, index = ["doctor", "models"], 1 + seen = set() + while index < len(tokens): + flag = tokens[index] + if flag == "--json": + if flag in seen: + raise CommandArgumentError("duplicate flag: --json") + seen.add(flag) + argv.append(flag) + index += 1 + elif flag == "--wired-root": + argv.extend([flag, _path(_value(tokens, index, flag), "wired-root")]) + index += 2 + elif flag == "--hf-cache": + if flag in seen: + raise CommandArgumentError("duplicate flag: --hf-cache") + seen.add(flag) + argv.extend([flag, _path(_value(tokens, index, flag), "hf-cache")]) + index += 2 + else: + raise CommandArgumentError("unsupported Doctor argument: {0}".format(flag)) + return argv + + +def _parse_watch(tokens): + if not tokens or tokens[0] not in {"snapshot", "diff"}: + raise CommandArgumentError("Watch requires the snapshot or diff action") + argv, index = ["watch", tokens[0]], 1 + seen = set() + while index < len(tokens): + flag = tokens[index] + if flag == "--json": + if flag in seen: + raise CommandArgumentError("duplicate flag: --json") + seen.add(flag) + argv.append(flag) + index += 1 + elif flag == "--state-dir": + if flag in seen: + raise CommandArgumentError("duplicate flag: --state-dir") + seen.add(flag) + argv.extend([flag, _path(_value(tokens, index, flag), "state-dir")]) + index += 2 + elif flag == "--limit": + if flag in seen: + raise CommandArgumentError("duplicate flag: --limit") + seen.add(flag) + argv.extend([flag, _integer(_value(tokens, index, flag), "limit", 1, 50)]) + index += 2 + else: + raise CommandArgumentError("unsupported Watch argument: {0}".format(flag)) + return argv + + +def _fleet_assignment(value, label): + if "=" not in value: + raise CommandArgumentError("{0} must use role=value form".format(label)) + role, target = value.split("=", 1) + if role not in _ROLES: + raise CommandArgumentError("{0} must name a canonical role".format(label)) + return value + + +def _parse_fleet(tokens): + if not tokens or tokens[0] not in {"render", "apply"}: + raise CommandArgumentError("Fleet requires the render or apply action") + action, argv, index = tokens[0], ["fleet", tokens[0]], 1 + seen = set() + values = { + "--from-adoption": (lambda value: _path(value, "from-adoption"), "from-adoption"), + "--preview-hash": (lambda value: value if re.fullmatch(r"[0-9a-f]{64}", value) else None, "preview-hash"), + "--receipts-dir": (lambda value: _path(value, "receipts-dir"), "receipts-dir"), + "--endpoint": (lambda value: _endpoint(value), "endpoint"), + } + path_seen = False + while index < len(tokens): + flag = tokens[index] + if flag in {"--json", "--confirm", "--allow-missing"}: + if flag in seen: + raise CommandArgumentError("duplicate flag: {0}".format(flag)) + seen.add(flag) + argv.append(flag) + index += 1 + elif flag == "--assign": + value = _value(tokens, index, flag) + _fleet_assignment(value, "--assign") + _model(value.split("=", 1)[1]) + argv.extend([flag, value]) + index += 2 + elif flag == "--runtime-map": + value = _value(tokens, index, flag) + _fleet_assignment(value, "--runtime-map") + if value.split("=", 1)[1] not in ("mlx_lm", "mlx-vlm"): + raise CommandArgumentError("--runtime-map must target mlx_lm or mlx-vlm") + argv.extend([flag, value]) + index += 2 + elif flag == "--path": + if flag in seen: + raise CommandArgumentError("duplicate flag: --path") + seen.add(flag) + argv.extend([flag, _path(_value(tokens, index, flag), "path")]) + path_seen = True + index += 2 + elif flag in values: + if flag in seen: + raise CommandArgumentError("duplicate flag: {0}".format(flag)) + value = values[flag][0](_value(tokens, index, flag)) + if value is None: + raise CommandArgumentError("invalid {0} value".format(values[flag][1])) + seen.add(flag) + argv.extend([flag, value]) + index += 2 + else: + raise CommandArgumentError("unsupported Fleet argument: {0}".format(flag)) + if not path_seen: + raise CommandArgumentError("Fleet {0} requires --path".format(action)) + return argv + + +def _parse_task(tokens): + if not tokens or tokens[0] not in {"capabilities", "start", "status", "list", "result", "cancel", "verify-model"}: + raise CommandArgumentError("Task requires capabilities, start, status, list, result, cancel or verify-model") + action = tokens[0] + argv, seen, index = ["task", action], set(), 1 + required = {"start": "--request", "status": "--id", "result": "--id", "cancel": "--id", "verify-model": "--id"}.get(action) + allowed = {"--state-dir", "--json"} + if required: + allowed.add(required) + if action == "list": + allowed.add("--limit") + while index < len(tokens): + flag = tokens[index] + if flag not in allowed or flag in seen: + raise CommandArgumentError("Unsupported or duplicate Task argument: {0}".format(flag)) + seen.add(flag) + if flag == "--json": + index += 1 + continue + value = _value(tokens, index, flag) + if flag == "--id": + if not re.fullmatch(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", value): + raise CommandArgumentError("Task ID must be a canonical UUID") + elif flag == "--limit": + value = _integer(value, "limit", 1, 100) + else: + value = _path(value, flag) + if not Path(value).is_absolute(): + raise CommandArgumentError("Task paths must be absolute") + argv.extend((flag, value)) + index += 2 + if required and required not in seen: + raise CommandArgumentError("Task {0} requires {1}".format(action, required)) + return argv + ["--json"] + + +def parse_command_arguments(capability, raw): + """Return a validated argv list; this function never runs a process.""" + tokens = _tokens(raw) + if capability == "task": + return _parse_task(tokens) + if capability == "lab": + return ["lab"] + tokens + if capability == "scout": + return _parse_scout(tokens) + if capability == "adopt": + return _parse_adopt(tokens) + if capability == "wire": + return _parse_wire(tokens) + if capability == "bench": + return _parse_bench(tokens) + if capability == "doctor": + return _parse_doctor(tokens) + if capability == "watch": + return _parse_watch(tokens) + if capability == "fleet": + return _parse_fleet(tokens) + raise CommandArgumentError("unknown command capability") + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("capability", choices=("scout", "adopt", "wire", "bench", "doctor", "watch", "fleet", "lab", "task")) + parser.add_argument("raw", help="one opaque custom-command argument string") + arguments = parser.parse_args(argv) + try: + print(json.dumps({"argv": parse_command_arguments(arguments.capability, arguments.raw)})) + except CommandArgumentError as error: + parser.error(str(error)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/command_executor.py b/packages/codex/mlx-agent/src/mlx_agent/command_executor.py similarity index 89% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/command_executor.py rename to packages/codex/mlx-agent/src/mlx_agent/command_executor.py index 737ecaf..b86e3f8 100644 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/command_executor.py +++ b/packages/codex/mlx-agent/src/mlx_agent/command_executor.py @@ -7,12 +7,12 @@ import json import sys -from .gemini_args import GeminiArgumentError, parse_gemini_arguments +from .command_args import CommandArgumentError, parse_command_arguments MAX_ARGUMENT_BYTES = 4096 _PROVIDERS = ("opencode",) -_CAPABILITIES = ("scout", "adopt", "wire", "bench") +_CAPABILITIES = ("scout", "adopt", "wire", "bench", "doctor", "watch", "fleet") class CommandExecutorError(ValueError): @@ -40,8 +40,8 @@ def execute_command(provider, capability, stream, core=None): if provider not in _PROVIDERS or capability not in _CAPABILITIES: raise CommandExecutorError("custom tool request was rejected") try: - arguments = parse_gemini_arguments(capability, read_command_arguments(stream)) - except (GeminiArgumentError, CommandExecutorError) as error: + arguments = parse_command_arguments(capability, read_command_arguments(stream)) + except (CommandArgumentError, CommandExecutorError) as error: raise CommandExecutorError("custom tool request was rejected") from error if core is None: from .cli import main as core diff --git a/packages/codex/mlx-agent/src/mlx_agent/compute.py b/packages/codex/mlx-agent/src/mlx_agent/compute.py new file mode 100644 index 0000000..071002b --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/compute.py @@ -0,0 +1,174 @@ +"""One host compute owner shared by native and agent entrypoints.""" +from __future__ import annotations + +import fcntl +import json +import os +from pathlib import Path +import stat +import subprocess +import sys + + +class ComputeBusy(ValueError): + pass + + +def compute_root(): + if sys.platform == "darwin": + return Path.home() / "Library/Application Support/mlx-agent/compute" + return Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local/state")) / "mlx-agent/compute" + + +def process_identity(pid): + result = subprocess.run(["/bin/ps", "-ww", "-p", str(pid), "-o", "lstart=", "-o", "command="], + capture_output=True, text=True, timeout=5) + return result.stdout.strip() if result.returncode == 0 else None + + +def process_alive(pid): + try: + os.kill(pid, 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True + + +def group_members(pgid): + """Return live members, excluding zombies which no longer own compute.""" + if type(pgid) is not int or pgid <= 0: + raise ValueError("A process group requires a positive ID.") + result = subprocess.run(["/bin/ps", "-axo", "pid=,pgid=,stat="], capture_output=True, text=True, timeout=5) + if result.returncode != 0: + raise OSError("The process table is unavailable.") + members = [] + for line in result.stdout.splitlines(): + fields = line.split() + if len(fields) >= 3 and int(fields[1]) == pgid and not fields[2].startswith("Z"): + members.append(int(fields[0])) + return members + + +class ComputeLease: + def __init__(self, operation, root=None, inspector=process_identity, alive=process_alive, group_probe=group_members): + self.operation = operation + self.root = Path(root) if root is not None else compute_root() + self.inspector, self.alive = inspector, alive + self.descriptor = None + self.detached = False + self.group_probe = group_probe + self.execution_group = None + self.record = None + + def _write(self, value): + encoded = json.dumps(value).encode() + os.lseek(self.descriptor, 0, os.SEEK_SET) + os.write(self.descriptor, encoded) + os.ftruncate(self.descriptor, len(encoded)) + os.fsync(self.descriptor) + self.record = value + + def __enter__(self): + self.root.mkdir(mode=0o700, parents=True, exist_ok=True) + for path in (self.root, *self.root.parents): + if path.is_symlink(): + raise ValueError("Compute state cannot traverse symbolic links.") + info = self.root.stat() + if info.st_uid != os.getuid() or info.st_mode & 0o077: + raise ValueError("Compute state must be a private owned directory.") + descriptor = os.open(self.root / "owner.json", + os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600) + self.descriptor = descriptor + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077: + raise ValueError("Compute ownership must be a private regular file.") + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise ComputeBusy("Another model operation owns this host.") from None + raw = os.read(descriptor, 8193) + if len(raw) > 8192: + raise ValueError("Compute ownership record is invalid.") + record = json.loads(raw) if raw else {} + if record: + fields = {"schema", "pid", "identity", "operation"} + if isinstance(record, dict) and record.get("schema") == 2: + fields.add("execution_group") + if not isinstance(record, dict) or set(record) != fields: + raise ValueError("Compute ownership record is invalid.") + pid = record["pid"] + if type(pid) is not int or pid <= 0 or record["schema"] not in (1, 2): + raise ValueError("Compute ownership record is invalid.") + if record["schema"] == 2: + group = record["execution_group"] + if (not isinstance(group, dict) or set(group) != {"pgid", "leader_identity"} + or type(group["pgid"]) is not int or group["pgid"] <= 0 + or not isinstance(group["leader_identity"], str) or not group["leader_identity"]): + raise ValueError("Compute execution group is invalid.") + try: + members = self.group_probe(group["pgid"]) + except (OSError, ValueError, subprocess.SubprocessError) as error: + raise ComputeBusy("The recorded execution group cannot be inspected; compute ownership is retained.") from error + if members: + raise ComputeBusy("A recorded task execution group still owns host compute (group {0}).".format(group["pgid"])) + observed = self.inspector(pid) + if self.alive(pid) and (record["identity"] is None or observed is None or observed == record["identity"]): + raise ComputeBusy("Host compute is owned by " + str(record["operation"]) + " (pid " + str(pid) + ").") + identity = self.inspector(os.getpid()) + if identity is None: + raise ComputeBusy("The current process identity could not be established.") + self._write({"schema": 1, "pid": os.getpid(), "identity": identity, "operation": self.operation}) + return self + except BaseException: + os.close(descriptor) + self.descriptor = None + raise + + def bind_process_group(self, pid): + if self.descriptor is None or type(pid) is not int or pid <= 0: + raise ValueError("An active compute lease and positive group leader are required.") + identity = self.inspector(pid) + if not identity or os.getpgid(pid) != pid: + raise ComputeBusy("The task execution group identity could not be established.") + self.execution_group = {"pgid": pid, "leader_identity": identity} + self._write(dict(self.record, schema=2, execution_group=self.execution_group)) + + def handoff(self, pid): + if type(pid) is not int or pid <= 0: + raise ValueError("Detached compute owner requires a positive PID.") + identity = self.inspector(pid) + if identity is None: + if not self.alive(pid): + return # A process that already exited owns no compute. + self._write({"schema": 1, "pid": pid, "identity": None, "operation": self.operation}) + self.detached = True + raise ComputeBusy("The launched process identity could not be established; its ownership is retained.") + self._write({"schema": 1, "pid": pid, "identity": identity, "operation": self.operation}) + self.detached = True + + def __exit__(self, *_): + if self.descriptor is not None: + try: + group_stopped = True + if self.execution_group is not None: + try: + group_stopped = not self.group_probe(self.execution_group["pgid"]) + except (OSError, ValueError, subprocess.SubprocessError): + group_stopped = False + if not self.detached and group_stopped: + self._write({}) + finally: + os.close(self.descriptor) + self.descriptor = None + + +def guarded_start(operation, launch, *args, **kwargs): + with ComputeLease(operation) as lease: + result = launch(*args, **kwargs) + pid = result.get("receipt", {}).get("pid") + if pid is not None: + lease.handoff(pid) + return result diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/contextfit.py b/packages/codex/mlx-agent/src/mlx_agent/contextfit.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/contextfit.py rename to packages/codex/mlx-agent/src/mlx_agent/contextfit.py diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/contracts.py b/packages/codex/mlx-agent/src/mlx_agent/contracts.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/contracts.py rename to packages/codex/mlx-agent/src/mlx_agent/contracts.py diff --git a/packages/codex/mlx-agent/src/mlx_agent/convert.py b/packages/codex/mlx-agent/src/mlx_agent/convert.py new file mode 100644 index 0000000..887fc91 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/convert.py @@ -0,0 +1,549 @@ +"""Confirmation-gated local MLX quantization jobs.""" + +from __future__ import annotations + +import hashlib +import json +import re +import sys +from datetime import datetime, timezone +from pathlib import Path + +from .backends import ( + BackendError, + backend_environment, + backend_python, + load_manifests, + port_bits, + port_converter, + recipe_for, + read_install_marker, + spawn_with_env, + sync_ports, +) +from .gguf import GGUFError, describe_gguf +from .intake_source import validate_subfolder +from .serve import ( + _argv_matches, + _pid_alive, + _pid_command, + _terminate_pid, +) +from .transactions import _atomic_in_directory +from .wiring import _MODEL + + +CONVERT_RECEIPT_SCHEMA_VERSION = "1.0" +CONVERT_RECEIPT_KIND = "convert" +Q_BITS_CHOICES = (4, 8) +EXECUTABLE = "mlx_lm.convert" +GGUF_RUNNER = Path(__file__).resolve().with_name("gguf_runner.py") +MUSIC_REQUANTIZE_RUNNER = Path(__file__).resolve().with_name("music_requantize_runner.py") +GGUF_REQUIRED_MODULES = ("torch", "transformers", "gguf") +GGUF_PORT_REQUIRED_MODULES = ("gguf", "mlx") +MAX_LOG_TAIL_BYTES = 64 * 1024 +_UNSAFE_SLUG = re.compile(r"[^A-Za-z0-9._-]+") +_MODEL_TYPE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}") + + +class ConvertError(RuntimeError): + """Classified convert failure safe to surface in a result envelope.""" + + def __init__(self, code, message, remediation): + super().__init__(message) + self.code = code + self.remediation = remediation + + +def _utc_now(): + return datetime.now(timezone.utc).isoformat() + + +def receipts_root(root=None, kind="convert"): + base = Path(root) if root is not None else Path.cwd() + return base / ".mlx-agent-receipts" / kind + + +def cached_snapshot(repo, hf_cache=None, revision=None): + """``/models--org--name/snapshots/`` when that snapshot is cached, else None.""" + from .model_doctor import default_hf_cache + + entry = (Path(hf_cache) if hf_cache else default_hf_cache()) / "models--{0}".format(repo.replace("/", "--")) + if revision is None: + try: + revision = (entry / "refs" / "main").read_text(encoding="utf-8").strip() + except OSError: + return None + if not revision or "/" in revision or revision in (".", ".."): + return None + snapshot = entry / "snapshots" / revision + return snapshot if snapshot.is_dir() else None + + +def cached_subfolder(repo, subfolder, hf_cache=None): + """``/`` when that folder is cached, else None.""" + snapshot = cached_snapshot(repo, hf_cache) + folder = snapshot / subfolder if snapshot is not None else None + return folder if folder is not None and folder.is_dir() else None + + +def plan_convert(repo, q_bits=4, out=None, backend=None, manifests=None, backends_root_dir=None, model_type=None, + subfolder=None, hf_cache=None, source_path=None): + """Render the exact conversion plan; side-effect free (a subfolder is looked up in the cache). + + ``model_type`` (from intake) selects a port's own converter when the port + ships one; otherwise the backend's converter runs. A ``subfolder`` + checkpoint converts from its cached folder, so every converter reads it + as a local directory. + """ + if not isinstance(repo, str) or not _MODEL.fullmatch(repo): + raise ConvertError( + "invalid_repo", + "convert requires a safe publisher/model identifier.", + "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", + ) + if model_type is not None and (not isinstance(model_type, str) or not _MODEL_TYPE.fullmatch(model_type)): + raise ConvertError( + "invalid_arguments", "model_type must be a config model_type such as qwen2 or laya.", + "Pass --model-type exactly as intake resolve reports it.", + ) + _validate_q_bits(q_bits) + if subfolder is not None: + try: + validate_subfolder(subfolder) + except ValueError as error: + raise ConvertError("invalid_arguments", str(error), "Pass --subfolder as the folder path inside the repository.") from error + name = repo.split("/", 1)[1] + ("-" + subfolder.replace("/", "-") if subfolder else "") + if out is None: + out = "{0}-MLX-{1}bit".format(name, q_bits) + local_source = Path(source_path).expanduser().absolute() if source_path is not None else cached_snapshot(repo, hf_cache) + if source_path is not None and not local_source.is_dir(): + raise ConvertError("source_not_found", "The downloaded source directory is missing.", + "Download or select the source again before previewing.") + hf_path = str(local_source) if local_source is not None else repo + if subfolder and source_path is None: + folder = cached_subfolder(repo, subfolder, hf_cache) + if folder is None: + raise ConvertError( + "subfolder_not_cached", "{0}/{1} is not in the Hugging Face cache.".format(repo, subfolder), + "Download it first: mlx-agent intake fetch {0}/{1}.".format(repo, subfolder), + ) + hf_path = str(folder) + flags = ["--hf-path", hf_path, "--mlx-path", str(out), "--quantize", "--q-bits", str(q_bits)] + plan = { + "repo": repo, + "source": dict({"kind": "hf-cache", "repo": repo}, **({"subfolder": subfolder} if subfolder else {})), + "slug": "{0}-{1}bit".format(name, q_bits), + "q_bits": q_bits, + "out": str(out), + } + if hf_path != repo: + source = Path(hf_path) + # Bind the reviewed plan to the local source, including config contents + # and weight identities. Snapshot links may point to cache-owned blobs. + entries = [] + for item in sorted(source.rglob("*")): + if item.is_file(): + stat = item.stat() + content = hashlib.sha256(item.read_bytes()).hexdigest() if item.name.endswith(".json") and stat.st_size <= 1024 * 1024 else "" + entries.append([str(item.relative_to(source)), stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, content]) + plan["source"]["path"] = str(source) + plan["source"]["fingerprint"] = hashlib.sha256(json.dumps(entries, separators=(",", ":")).encode()).hexdigest() + if backend in (None, "mlx-lm"): + plan["argv"] = [EXECUTABLE] + flags + return _finalize_plan(plan) + try: + manifests = load_manifests() if manifests is None else manifests + except BackendError as error: + raise ConvertError(error.code, str(error), error.remediation) from error + manifest = manifests.get(backend) + if manifest is None or manifest["builtin"]: + raise ConvertError( + "unknown_backend", + "No optional backend named {0}.".format(backend), + "Run mlx-agent backend list to see the declared backends.", + ) + widths = port_bits(model_type, backend, manifests) + if widths and q_bits not in widths: + raise ConvertError( + "invalid_arguments", + "The {0} port converts at {1} bits only.".format(model_type, " or ".join(str(width) for width in widths)), + "Pass --q-bits {0}.".format(widths[-1]), + ) + python = backend_python(manifest, backends_root_dir) + plan["backend"] = backend + plan["backends_root"] = str(python.parent.parent.parent) + converter = port_converter(model_type, backend, manifests) + recipe = recipe_for(repo, manifests) if converter else None + if recipe and recipe["backend"] == backend and recipe["port"] == model_type: + flags = _recipe_flags(recipe, flags, hf_cache) + plan["recipe"] = {key: recipe[key] for key in ("base", "base_revision", "lora", "lora_scale")} + if backend == "mlx-audio" and _is_quantized_music_source(hf_path): + plan["argv"] = [str(python), str(MUSIC_REQUANTIZE_RUNNER)] + flags + else: + plan["argv"] = [str(python), "-m", converter or manifest["convert"]] + flags + ports = sorted(name for names in (manifest.get("ports") or {}).values() for name in names) + if ports: + plan["ports"] = ports + if converter: + plan["port_converter"] = converter + return _finalize_plan(plan) + + +def _is_quantized_music_source(path): + """Exported MLX music weights need the music loader, not the raw converter.""" + try: + config = json.loads((Path(path) / "config.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + return False + return (isinstance(config, dict) and config.get("model_type") == "minimax_music3" + and isinstance(config.get("quantization"), dict) and bool(config["quantization"])) + + +def _recipe_flags(recipe, flags, hf_cache): + """Converter flags for a recipe: the cached base snapshot and the cached LoRA, both local.""" + base = cached_snapshot(recipe["base"], hf_cache, revision=recipe["base_revision"]) + snapshot = cached_snapshot(recipe["repo"], hf_cache) + lora = snapshot / recipe["lora"] if snapshot is not None else None + if base is None or lora is None or not lora.is_file(): + raise ConvertError( + "recipe_not_cached", + "{0} needs {1}@{2} and its {3} in the Hugging Face cache.".format( + recipe["repo"], recipe["base"], recipe["base_revision"][:7], recipe["lora"]), + "Download it first: mlx-agent intake fetch {0} --model-type {1}.".format(recipe["repo"], recipe["port"]), + ) + flags = list(flags) + flags[flags.index("--hf-path") + 1] = str(base) + return flags + ["--lora", str(lora), "--lora-scale", repr(float(recipe["lora_scale"]))] + + +def _finalize_plan(plan): + canonical = json.dumps(plan, sort_keys=True, separators=(",", ":")) + plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return plan + + +def _validate_q_bits(q_bits): + if not isinstance(q_bits, int) or isinstance(q_bits, bool): + raise ConvertError( + "invalid_arguments", "q_bits must be an integer.", + "Pass --q-bits 4 or --q-bits 8.", + ) + if q_bits not in Q_BITS_CHOICES: + raise ConvertError( + "invalid_arguments", + "q_bits must be one of {0}.".format(list(Q_BITS_CHOICES)), + "Only bounded 4bit and 8bit recipes are supported in v1.", + ) + + +def _slugify(value, q_bits): + slug = _UNSAFE_SLUG.sub("-", str(value)).strip("-.") or "model" + return "{0}-{1}bit".format(slug[:96], q_bits) + + +def plan_gguf_convert(gguf_path, q_bits=4, out=None, runner=None, describe=describe_gguf): + """Render the exact GGUF to MLX conversion plan; pure and side-effect free.""" + _validate_q_bits(q_bits) + location = Path(str(gguf_path)).expanduser() + if location.suffix.lower() != ".gguf": + raise ConvertError( + "invalid_source", + "convert --gguf requires a path ending in .gguf.", + "Pass the GGUF weights file itself, not its directory.", + ) + if not location.is_file(): + raise ConvertError( + "source_not_found", + "No GGUF file at {0}.".format(location), + "Run convert scan to list the GGUF files under your configured roots.", + ) + try: + entry = describe(location) + except GGUFError as error: + raise ConvertError(error.code, str(error), error.remediation) from error + if not entry.get("readable"): + detail = entry.get("error") or {} + raise ConvertError( + detail.get("code", "gguf_header_unreadable"), + detail.get("message", "The GGUF header could not be parsed."), + "Re-download the file; a truncated GGUF cannot be converted.", + ) + shard = entry.get("shard") + if shard is not None and shard["index"] != 1: + raise ConvertError( + "shard_not_first", + "{0} is shard {1} of {2}.".format(location.name, shard["index"], shard["total"]), + "Point convert at the -00001-of-{0:05d} shard; the loader reads the rest itself.".format( + shard["total"] + ), + ) + draft = entry.get("draft") + if draft is not None and draft.get("port") is None: + raise ConvertError( + "unsupported_draft", + "{0} is a {1} speculative-decoding drafter for {2}; it runs only beside that model and no port converts it.".format( + location.name, entry.get("architecture"), draft.get("target") or "its target model"), + "Convert the target model itself; DeepSeek-V4 DSpark is the only drafter family mlx-agent converts.", + ) + if out is None: + out = "{0}-MLX-{1}bit".format(location.stem, q_bits) + resolved_runner = Path(runner) if runner is not None else GGUF_RUNNER + plan = { + "repo": str(location), + "source": { + "kind": "gguf", + "path": str(location), + "name": location.name, + "bytes": entry["bytes"], + "signature": entry.get("signature"), + "architecture": entry.get("architecture"), + "quantization": entry.get("quantization"), + "model_key": entry.get("model_key"), + }, + "slug": _slugify(location.stem, q_bits), + "q_bits": q_bits, + "out": str(out), + "argv": [ + sys.executable, + str(resolved_runner), + "--gguf", str(location), + "--out", str(out), + "--q-bits", str(q_bits), + ], + } + if entry.get("signature"): + plan["argv"].extend(["--signature", entry["signature"]]) + if draft is not None: + plan["draft"] = dict(draft) + plan["port_converter"] = draft["port"] + plan["argv"].extend(["--port", draft["port"]]) + return _finalize_plan(plan) + + +def _slug(value): + slug = value.get("slug") + if isinstance(slug, str) and slug: + return slug + return "{0}-{1}bit".format(value["repo"].split("/", 1)[1], value["q_bits"]) + + +def _receipt_path(root, plan): + return root / "{0}.json".format(_slug(plan)) + + +def _read_receipt(path, kind=CONVERT_RECEIPT_KIND): + try: + value = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError, json.JSONDecodeError): + return None + if not isinstance(value, dict) or value.get("kind") != kind: + return None + if not isinstance(value.get("pid"), int) or isinstance(value.get("pid"), bool): + return None + if not isinstance(value.get("argv"), list) or not isinstance(value.get("repo"), str): + return None + return value + + +def _write_receipt(root, receipt, filename): + content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + _atomic_in_directory(root, filename, content, 0o600) + + +def _source_kind(value): + source = value.get("source") + if isinstance(source, dict) and isinstance(source.get("kind"), str): + return source["kind"] + return "hf-cache" + + +def _default_module_present(names): + import importlib.util + + return [name for name in names if importlib.util.find_spec(name) is None] + + +def start_convert(plan, receipts_dir=None, confirm=False, preview_hash=None, + which=None, model_present=None, spawn=None, now=_utc_now, + pid_alive=None, module_present=None): + """Execute a reviewed conversion plan; the only mutating entry point.""" + from .serve import _default_spawn, _default_which + + which = which or _default_which + pid_alive = pid_alive or _pid_alive + root = receipts_root(receipts_dir) + + if not confirm: + return {"status": "preview", "plan": plan, "requires_confirmation": True} + if not preview_hash: + raise ConvertError( + "preview_hash_required", + "--confirm requires the hash from a reviewed convert preview.", + "Run convert start without --confirm, inspect the plan, then pass --preview-hash.", + ) + if preview_hash != plan["preview_hash"]: + raise ConvertError( + "preview_stale", + "The supplied preview hash does not match this convert plan.", + "Re-run convert start without --confirm and review the fresh plan.", + ) + backend = plan.get("backend") + if backend: + manifest = load_manifests().get(backend) + root_dir = plan.get("backends_root") + if ( + manifest is None + or read_install_marker(manifest, root_dir) is None + or not Path(plan["argv"][0]).exists() + ): + raise ConvertError( + "backend_not_installed", + "The {0} backend is not installed.".format(backend), + "Install it first: mlx-agent backend install {0}; convert never installs runtimes.".format(backend), + ) + elif which(EXECUTABLE) is None: + raise ConvertError( + "runtime_not_installed", + "The {0} executable is not installed.".format(EXECUTABLE), + "Install mlx-lm yourself (pip install mlx-lm); convert never installs runtimes.", + ) + if _source_kind(plan) == "gguf": + required = GGUF_PORT_REQUIRED_MODULES if plan.get("port_converter") else GGUF_REQUIRED_MODULES + missing = (module_present or _default_module_present)(required) + if missing: + raise ConvertError( + "runtime_not_installed", + "GGUF conversion needs these modules in this interpreter: {0}.".format( + ", ".join(missing) + ), + "Install them yourself (for example: uv pip install {0}); convert never installs runtimes.".format(" ".join(required)), + ) + if not Path(plan["source"]["path"]).is_file(): + raise ConvertError( + "source_not_found", + "The source GGUF is no longer at {0}.".format(plan["source"]["path"]), + "Re-run convert scan and preview a fresh plan.", + ) + elif plan.get("source", {}).get("path") is not None: + if not Path(plan["source"]["path"]).is_dir(): + raise ConvertError("source_not_found", "The reviewed local source is no longer present.", "Preview the conversion again.") + elif model_present is not None and not model_present(plan["repo"]): + raise ConvertError( + "model_not_local", + "The source model is not present in the local Hugging Face cache.", + "Download it with the runtime's own pull command first; convert never downloads.", + ) + if Path(plan["out"]).exists(): + raise ConvertError( + "output_exists", + "The output path already exists: {0}".format(plan["out"]), + "Pick a fresh --out path; convert never overwrites.", + ) + for receipt_path in sorted(root.glob("*.json")) if root.is_dir() else []: + existing = _read_receipt(receipt_path) + if existing is None or existing.get("completed_at"): + continue + if pid_alive(existing["pid"]): + raise ConvertError( + "job_in_progress", + "A convert job is still running (pid {0}).".format(existing["pid"]), + "Wait for it to finish (convert status) before starting another.", + ) + + if backend: + try: + sync_ports(manifest, plan.get("backends_root")) + except BackendError as error: + raise ConvertError(error.code, str(error), error.remediation) from error + root.mkdir(parents=True, exist_ok=True) + slug = _slug(plan) + log_path = root / "{0}.log".format(slug) + if spawn is None: + environment = backend_environment() + environment["HF_HUB_OFFLINE"] = "1" + environment["TRANSFORMERS_OFFLINE"] = "1" + + def spawn(argv, log_path): + return spawn_with_env(argv, log_path, environment) + + spawn = spawn or _default_spawn + pid = spawn(plan["argv"], str(log_path)) + receipt = { + "schema_version": CONVERT_RECEIPT_SCHEMA_VERSION, + "kind": CONVERT_RECEIPT_KIND, + "repo": plan["repo"], + "source": dict(plan.get("source") or {"kind": "hf-cache", "repo": plan["repo"]}), + "slug": slug, + "q_bits": plan["q_bits"], + "out": plan["out"], + "argv": list(plan["argv"]), + "pid": pid, + "log_path": str(log_path), + "started_at": now(), + "preview_hash": plan["preview_hash"], + "completed_at": None, + "exit_status": None, + } + if backend: + receipt["backend"] = backend + _write_receipt(root, receipt, "{0}.json".format(slug)) + return {"status": "started", "receipt": receipt} + + +def load_receipts(receipts_dir=None): + """Read every convert receipt; used to pair GGUF sources with their outputs.""" + root = receipts_root(receipts_dir) + receipts = [] + if not root.is_dir(): + return receipts + for path in sorted(root.glob("*.json")): + receipt = _read_receipt(path) + if receipt is not None: + receipts.append(receipt) + return receipts + + +def status_convert(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, + now=_utc_now): + """Cross-check convert receipts against live processes; marks exits once.""" + root = receipts_root(receipts_dir) + entries = [] + if not root.is_dir(): + return entries + for path in sorted(root.glob("*.json")): + receipt = _read_receipt(path) + if receipt is None: + continue + entry = { + "receipt": str(path), + "repo": receipt["repo"], + "source": _source_kind(receipt), + "q_bits": receipt["q_bits"], + "out": receipt["out"], + "pid": receipt["pid"], + "log_path": receipt.get("log_path"), + "started_at": receipt.get("started_at"), + "completed_at": receipt.get("completed_at"), + } + if receipt.get("completed_at"): + entry["state"] = receipt.get("exit_status") or "done" + entries.append(entry) + continue + if pid_alive(receipt["pid"]): + command = pid_command(receipt["pid"]) + entry["state"] = "running" if _argv_matches( + {"argv": receipt["argv"], "port": None, "repo": receipt["repo"]}, + command, + require_port=False, + ) else "unknown" + entries.append(entry) + continue + exit_status = "done" if Path(receipt["out"]).exists() else "failed" + receipt["completed_at"] = now() + receipt["exit_status"] = exit_status + _write_receipt(root, receipt, "{0}.json".format(_slug(receipt))) + entry["state"] = exit_status + entry["completed_at"] = receipt["completed_at"] + entries.append(entry) + return entries diff --git a/packages/codex/mlx-agent/src/mlx_agent/convert_queue.py b/packages/codex/mlx-agent/src/mlx_agent/convert_queue.py new file mode 100644 index 0000000..e63ab17 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/convert_queue.py @@ -0,0 +1,532 @@ +"""Durable convert queue. Serializes jobs under mlx-agent's one-job rule.""" + +from __future__ import annotations + +import json +import itertools +import os +import threading +from pathlib import Path + +from . import queue_bridge as bridge +from .atomicio import write_text_atomic + + +QUEUE_SCHEMA_VERSION = "1.1" +LEGACY_QUEUE_SCHEMA_VERSION = "1.0" +QUEUE_FILENAME = "convert-queue.json" +ITEM_STATES = ("queued", "starting", "failed") +QUEUE_ITEM_KEYS = { + "id", "kind", "preview_hash", "q_bits", "out", "path", "repo", + "hf_cache", "label", "state", "failure", +} +LEGACY_QUEUE_ITEM_KEYS = QUEUE_ITEM_KEYS - {"failure"} +FAILURE_KEYS = {"code", "message", "remediation"} + + +def queue_path(config_path=None): + """Return the durable queue path for a config profile or default state.""" + if config_path is not None: + return Path(config_path).expanduser().with_name(QUEUE_FILENAME) + state_home = os.environ.get("XDG_STATE_HOME") + root = Path(state_home).expanduser() if state_home else Path.home() / ".local" / "state" + return root / "mlx-workbench" / QUEUE_FILENAME + + +class QueuePersistenceError(RuntimeError): + """A classified failure to load or persist durable queue state.""" + + def __init__(self, code, message, remediation, path=None, preserved_path=None): + super().__init__(message) + self.code = code + self.remediation = remediation + self.path = str(path) if path is not None else None + self.preserved_path = ( + str(preserved_path) if preserved_path is not None else None + ) + + def to_dict(self): + payload = { + "code": self.code, + "message": str(self), + "remediation": self.remediation, + } + if self.path is not None: + payload["path"] = self.path + if self.preserved_path is not None: + payload["preserved_path"] = self.preserved_path + return payload + + +class QueueOperationError(RuntimeError): + """A classified invalid conversion queue operation.""" + + def __init__(self, code, message, remediation): + super().__init__(message) + self.code = code + self.remediation = remediation + + def to_dict(self): + return { + "code": self.code, + "message": str(self), + "remediation": self.remediation, + } + + +def _optional_string(value): + return value is None or (isinstance(value, str) and bool(value)) + + +def _validate_item(item, legacy=False): + expected_keys = LEGACY_QUEUE_ITEM_KEYS if legacy else QUEUE_ITEM_KEYS + if not isinstance(item, dict) or set(item) != expected_keys: + raise ValueError("queue item fields do not match schema") + item_id = item["id"] + if ( + not isinstance(item_id, str) + or not item_id.startswith("cq-") + or not item_id[3:].isdigit() + ): + raise ValueError("queue item id is invalid") + if item["kind"] not in ("gguf", "repo"): + raise ValueError("queue item kind is invalid") + if not isinstance(item["preview_hash"], str) or not item["preview_hash"]: + raise ValueError("queue item preview_hash is invalid") + if isinstance(item["q_bits"], bool) or item["q_bits"] not in (4, 8): + raise ValueError("queue item q_bits is invalid") + for field in ("out", "path", "repo", "hf_cache"): + if not _optional_string(item[field]): + raise ValueError("queue item {0} is invalid".format(field)) + if not isinstance(item["label"], str) or not item["label"]: + raise ValueError("queue item label is invalid") + allowed_states = ("queued", "starting") if legacy else ITEM_STATES + if item["state"] not in allowed_states: + raise ValueError("queue item state is invalid") + if item["kind"] == "gguf": + if not item["path"] or item["repo"] is not None: + raise ValueError("GGUF queue item requires only path") + elif not item["repo"] or item["path"] is not None: + raise ValueError("repo queue item requires only repo") + if legacy: + return + failure = item["failure"] + if item["state"] == "failed": + if not isinstance(failure, dict) or set(failure) != FAILURE_KEYS: + raise ValueError("failed queue item requires failure details") + if any(not isinstance(failure[key], str) or not failure[key] + for key in FAILURE_KEYS): + raise ValueError("queue item failure details are invalid") + elif failure is not None: + raise ValueError("active queue item cannot include failure details") + + +def _validated_items(payload): + if not isinstance(payload, dict): + raise ValueError("queue state must be an object") + if set(payload) != {"schema_version", "items"}: + raise ValueError("queue state fields do not match schema") + version = payload["schema_version"] + if version not in (LEGACY_QUEUE_SCHEMA_VERSION, QUEUE_SCHEMA_VERSION): + raise ValueError("queue schema version is unsupported") + if not isinstance(payload["items"], list): + raise ValueError("queue items must be a list") + legacy = version == LEGACY_QUEUE_SCHEMA_VERSION + items = [] + for item in payload["items"]: + _validate_item(item, legacy=legacy) + items.append(dict(item, failure=None) if legacy else dict(item)) + return items + + +class QueueStore: + """Atomically persist schema-versioned conversion queue snapshots.""" + + def __init__(self, path): + self.path = Path(path) + + def load(self): + if not self.path.exists(): + return [] + try: + payload = json.loads(self.path.read_text(encoding="utf-8")) + return _validated_items(payload) + except (OSError, UnicodeError, ValueError, TypeError, KeyError) as error: + preserved = self._preserve_corrupt() + raise QueuePersistenceError( + "queue_state_invalid", + "The saved conversion queue is invalid: {0}".format(error), + "Inspect the preserved corrupt file, then submit conversions again.", + path=self.path, + preserved_path=preserved, + ) from error + + def save(self, items): + items = _validated_items({ + "schema_version": QUEUE_SCHEMA_VERSION, + "items": items, + }) + payload = { + "schema_version": QUEUE_SCHEMA_VERSION, + "items": items, + } + if self.path.is_symlink(): + raise QueuePersistenceError( + "queue_write_failed", + "{0} is a symbolic link; refusing to write through it.".format( + self.path + ), + "Remove the symbolic link and let the queue recreate the file.", + path=self.path, + ) + temporary = self.path.with_name(self.path.name + ".tmp") + if temporary.is_symlink(): + raise QueuePersistenceError( + "queue_write_failed", + "{0} is a symbolic link; refusing to write through it.".format( + temporary + ), + "Remove the symbolic link and let the queue recreate the file.", + path=self.path, + ) + try: + write_text_atomic( + self.path, json.dumps(payload, indent=2, sort_keys=True) + "\n" + ) + except (OSError, UnicodeError, ValueError) as error: + raise QueuePersistenceError( + "queue_write_failed", + "The conversion queue could not be saved: {0}".format(error), + "Check the state directory permissions and available disk space, then retry.", + path=self.path, + ) from error + + def _preserve_corrupt(self): + suffix = 1 + while True: + preserved = self.path.with_name( + "{0}.{1}.corrupt".format(self.path.name, suffix) + ) + if not preserved.exists(): + os.replace(self.path, preserved) + return preserved + suffix += 1 + + +class _MemoryQueueStore: + """Non-persistent store used by explicitly in-memory queue instances.""" + + def __init__(self): + self.items = [] + + def load(self): + return [dict(item) for item in self.items] + + def save(self, items): + self.items = [dict(item) for item in items] + + +class ConvertQueue: + """FIFO of reviewed convert plans with optional durable persistence.""" + + def __init__(self, path=None, store=None): + self._lock = threading.Lock() + self._drain_lock = threading.Lock() + self.store = store or (QueueStore(path) if path is not None else _MemoryQueueStore()) + self.load_error = None + try: + self._items = self.store.load() + except QueuePersistenceError as error: + self._items = [] + self.load_error = error.to_dict() + next_id = max( + (int(item["id"][3:]) for item in self._items), + default=0, + ) + 1 + self._ids = itertools.count(next_id) + self.last_error = None + + def enqueue(self, kind, preview_hash, q_bits, out=None, path=None, repo=None, + label=None, hf_cache=None): + if kind not in ("gguf", "repo"): + raise ValueError("kind must be gguf or repo") + if not isinstance(preview_hash, str) or not preview_hash: + raise ValueError("preview_hash is required") + item = { + "id": "cq-{0}".format(next(self._ids)), + "kind": kind, + "preview_hash": preview_hash, + "q_bits": q_bits, + "out": out, + "path": path, + "repo": repo, + "hf_cache": hf_cache, + "label": label or path or repo or "convert", + "state": "queued", + "failure": None, + } + with self._lock: + self._publish_locked(self._items + [item]) + return dict(item) + + def _publish_locked(self, candidate): + self.store.save(candidate) + self._items = candidate + + def snapshot(self): + with self._lock: + return [dict(item) for item in self._items] + + def _find_index_locked(self, item_id): + return next( + (index for index, item in enumerate(self._items) + if item["id"] == item_id), + None, + ) + + def _actionable_index_locked(self): + starting = next( + (index for index, item in enumerate(self._items) + if item["state"] == "starting"), + None, + ) + if starting is not None: + return starting + return next( + (index for index, item in enumerate(self._items) + if item["state"] == "queued"), + None, + ) + + def _replace_locked(self, item_id, replacement): + index = self._find_index_locked(item_id) + if index is None: + return False + candidate = list(self._items) + candidate[index] = replacement + self._publish_locked(candidate) + return True + + def _remove_locked(self, item_id): + index = self._find_index_locked(item_id) + if index is None: + return False + self._publish_locked(self._items[:index] + self._items[index + 1:]) + return True + + def cancel(self, item_id): + with self._lock: + index = self._find_index_locked(item_id) + if index is None: + return False + if self._items[index]["state"] == "starting": + raise QueueOperationError( + "invalid_queue_state", + "A starting conversion cannot be removed.", + "Wait for receipt reconciliation, then retry the operation.", + ) + self._remove_locked(item_id) + return True + + def clear(self): + with self._lock: + candidate = [ + item for item in self._items if item["state"] == "starting" + ] + count = len(self._items) - len(candidate) + if count: + self._publish_locked(candidate) + return count + + def retry(self, item_id): + with self._drain_lock: + with self._lock: + index = self._find_index_locked(item_id) + if index is None: + raise QueueOperationError( + "queue_item_not_found", + "The conversion queue item no longer exists.", + "Refresh Jobs and retry the operation.", + ) + item = self._items[index] + if item["state"] != "failed": + raise QueueOperationError( + "invalid_queue_state", + "Only a failed conversion can be retried.", + "Wait for the current operation or choose a failed item.", + ) + retried = dict(item, state="queued", failure=None) + candidate = self._items[:index] + self._items[index + 1:] + [retried] + self._publish_locked(candidate) + return dict(retried) + + def move(self, item_id, direction): + if direction not in ("up", "down"): + raise QueueOperationError( + "invalid_queue_direction", + "Queue direction must be up or down.", + "Refresh Jobs and use one of the available move controls.", + ) + with self._drain_lock: + with self._lock: + index = self._find_index_locked(item_id) + if index is None: + raise QueueOperationError( + "queue_item_not_found", + "The conversion queue item no longer exists.", + "Refresh Jobs and retry the operation.", + ) + if self._items[index]["state"] != "queued": + raise QueueOperationError( + "invalid_queue_state", + "Only a queued conversion can be moved.", + "Wait for the current operation or choose a pending item.", + ) + queued_indices = [ + position for position, item in enumerate(self._items) + if item["state"] == "queued" + ] + queued_position = queued_indices.index(index) + adjacent_position = queued_position + (-1 if direction == "up" else 1) + if adjacent_position < 0 or adjacent_position >= len(queued_indices): + return False + adjacent_index = queued_indices[adjacent_position] + candidate = list(self._items) + candidate[index], candidate[adjacent_index] = ( + candidate[adjacent_index], candidate[index] + ) + self._publish_locked(candidate) + return True + + def try_start_next(self, agent_path, runner=None): + """Start the head item if no convert is live. Returns a status dict or None.""" + with self._drain_lock: + return self._try_start_next(agent_path, runner=runner) + + def _try_start_next(self, agent_path, runner=None): + with self._lock: + actionable_index = self._actionable_index_locked() + if actionable_index is None: + return None + item = dict(self._items[actionable_index]) + status = bridge.jobs(agent_path, runner=runner) + raw_jobs = status.get("jobs") + if raw_jobs is None: + raw_jobs = [] + elif not isinstance(raw_jobs, list): + raise bridge.BridgeError( + "job_status_invalid", + "convert status did not return a job list.", + "Retry conversion and upgrade mlx-agent if the issue persists.", + ) + entries = [entry for entry in raw_jobs if isinstance(entry, dict)] + + if item["state"] == "starting": + try: + receipts = bridge.read_convert_receipts(entries) + except bridge.BridgeError as error: + payload = { + "status": "waiting_recovery", + "item": item, + "error": error.to_dict(), + } + self.last_error = payload + return payload + if any(self._receipt_matches(item, receipt) for receipt in receipts): + with self._lock: + self._remove_locked(item["id"]) + self.last_error = None + return {"status": "recovered", "item": item} + queued = dict(item, state="queued", failure=None) + with self._lock: + if not self._replace_locked(item["id"], queued): + return None + item = queued + + if any(entry.get("state") == "running" for entry in entries): + return None + + starting = dict(item, state="starting", failure=None) + with self._lock: + if not self._replace_locked(item["id"], starting): + return None + try: + if starting["kind"] == "gguf": + result = bridge.start( + agent_path, + starting["path"], + starting["preview_hash"], + q_bits=starting["q_bits"], + out=starting.get("out"), + runner=runner, + ) + else: + result = bridge.start_repo( + agent_path, + starting["repo"], + starting["preview_hash"], + q_bits=starting["q_bits"], + out=starting.get("out"), + hf_cache=starting.get("hf_cache"), + runner=runner, + ) + try: + with self._lock: + self._remove_locked(starting["id"]) + except QueuePersistenceError as error: + persistence_error = error.to_dict() + self.last_error = { + "status": "waiting_recovery", + "item": starting, + "error": persistence_error, + } + return { + "status": "started", + "item": starting, + "result": result, + "persistence_error": persistence_error, + } + self.last_error = None + return {"status": "started", "item": starting, "result": result} + except bridge.BridgeError as error: + if error.code in ("job_in_progress", "compute_busy"): + with self._lock: + queued = dict(starting, state="queued", failure=None) + self._replace_locked(starting["id"], queued) + return None + failure = error.to_dict() + failed = dict(starting, state="failed", failure=failure) + try: + with self._lock: + self._replace_locked(starting["id"], failed) + except QueuePersistenceError as persistence_error: + self.last_error = { + "status": "waiting_recovery", + "item": starting, + "error": persistence_error.to_dict(), + } + raise + payload = { + "status": "failed", + "item": failed, + "error": failure, + } + self.last_error = None + return payload + + @staticmethod + def _receipt_matches(item, receipt): + if not isinstance(receipt, dict): + return False + if receipt.get("preview_hash") != item["preview_hash"]: + return False + if item.get("out") is not None and receipt.get("out") != item.get("out"): + return False + if receipt.get("q_bits") != item["q_bits"]: + return False + source = receipt.get("source") + if not isinstance(source, dict): + return False + if item["kind"] == "gguf": + return source.get("path") == item["path"] + return source.get("repo") == item["repo"] diff --git a/packages/codex/mlx-agent/src/mlx_agent/decide.py b/packages/codex/mlx-agent/src/mlx_agent/decide.py new file mode 100644 index 0000000..1d986fe --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/decide.py @@ -0,0 +1,117 @@ +"""Answer typed questions about a state with a converted classification model through its backend (read-only).""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +from .backends import ( + backend_python, + is_installed, + load_manifests, + load_registries, + lookup, +) +from .inference_runner import InferenceError, RunnerContract, invoke + +DECIDE_SCHEMA = "decide/1" +DECIDE_RUNNER = Path(__file__).resolve().with_name("decide_runner.py") +DEFAULT_TIMEOUT_SECONDS = 300 +MAX_REQUEST_BYTES = 1024 * 1024 +QUESTION_TYPES = ("choice", "score", "noul") + + +class DecideError(InferenceError): + """Classified decision failure safe to surface in a result envelope.""" + + +def _invalid(message): + return DecideError( + "invalid_request", message, + 'Pass a JSON object {"state": ..., "questions": {"id": {"type": "choice|score|noul", "instructions": ...}}}.', + ) + + +def read_request(request_path): + path = Path(request_path).expanduser() + if not path.is_file(): + raise DecideError("request_not_found", "No request file at {0}.".format(path), "Pass a JSON request file.") + if path.stat().st_size > MAX_REQUEST_BYTES: + raise _invalid("The request is larger than {0} bytes.".format(MAX_REQUEST_BYTES)) + try: + request = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise _invalid("The request is not readable JSON: {0}".format(error)) from error + validate_request(request) + return path + + +def validate_request(request): + """Validate decision content without reading or running a model.""" + if not isinstance(request, dict) or "state" not in request: + raise _invalid("The request needs a state.") + if not isinstance(request["state"], (str, dict, list)): + raise _invalid("The state is text or a JSON object.") + questions = request.get("questions") + if not isinstance(questions, dict) or not questions: + raise _invalid("The request needs at least one question.") + for name, question in questions.items(): + if not isinstance(question, dict) or question.get("type") not in QUESTION_TYPES: + raise _invalid("Question {0!r} needs a type of {1}.".format(name, ", ".join(QUESTION_TYPES))) + if "instructions" not in question: + raise _invalid("Question {0!r} needs instructions.".format(name)) + return request + + +def plan_decide(model_path, request_path, manifests=None, registries=None, root=None): + """Pick the classification backend for a converted model directory; pure apart from reading inputs.""" + model = Path(model_path).expanduser() + try: + config = json.loads((model / "config.json").read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise DecideError( + "model_unreadable", "No readable config.json in {0}: {1}".format(model, error), + "Pass the converted model directory.", + ) from error + model_type = config.get("model_type") if isinstance(config, dict) else None + request = read_request(request_path) + manifests = load_manifests() if manifests is None else manifests + registries = load_registries(manifests, root) if registries is None else registries + hits = sorted( + (hit for hit in lookup(model_type, manifests, registries) if hit["category"] == "classification"), + key=lambda hit: hit["backend"], + ) + if not hits: + raise DecideError( + "not_classification_model", "No classification backend implements model_type {0!r}.".format(model_type), + "Decisions apply to converted classification models such as Laya.", + ) + manifest = manifests[hits[0]["backend"]] + if not is_installed(manifest, root): + raise DecideError( + "backend_not_installed", "The {0} backend is not installed.".format(manifest["id"]), + "Install it first: mlx-agent backend install {0}.".format(manifest["id"]), + ) + argv = [ + str(backend_python(manifest, root)), str(DECIDE_RUNNER), + "--module", hits[0]["module"], "--model", str(model), "--request", str(request), + ] + return {"backend": manifest["id"], "model": str(model), "request": str(request), "argv": argv} + + +CONTRACT = RunnerContract( + verb="decide", error=DecideError, accept=lambda value, plan: isinstance(value.get("answers"), dict), + action="answer the request", + remediation="Check the request's questions and that the directory is a complete converted model.", + activity="The decision", timeout_remediation="Use a shorter state or a larger --timeout.", +) + + +def run_decide(plan, manifests=None, root=None, timeout=DEFAULT_TIMEOUT_SECONDS, runner=subprocess.run): + result = invoke(plan, CONTRACT, manifests, root, timeout, runner) + return { + "schema": DECIDE_SCHEMA, "backend": plan["backend"], "model": plan["model"], "request": plan["request"], + "answers": result["answers"], "usage": result.get("usage"), + "seconds": result.get("seconds"), "load_seconds": result.get("load_seconds"), + } diff --git a/packages/codex/mlx-agent/src/mlx_agent/decide_runner.py b/packages/codex/mlx-agent/src/mlx_agent/decide_runner.py new file mode 100644 index 0000000..08fda0a --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/decide_runner.py @@ -0,0 +1,31 @@ +"""Run inside a classification backend's environment: load one converted model, answer one request, print JSON.""" + +import argparse +import importlib +import json +import sys +import time + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--module", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request", required=True) + arguments = parser.parse_args(argv) + + with open(arguments.request, encoding="utf-8") as handle: + request = json.load(handle) + port = importlib.import_module(arguments.module) + started = time.time() + model, tokenizer = port.load(arguments.model) + loaded = time.time() + result = port.predict(model, tokenizer, request["state"], request["questions"]) + result["seconds"] = round(time.time() - loaded, 3) + result["load_seconds"] = round(loaded - started, 3) + print(json.dumps(result), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/codex/mlx-agent/src/mlx_agent/describe.py b/packages/codex/mlx-agent/src/mlx_agent/describe.py new file mode 100644 index 0000000..ac8126d --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/describe.py @@ -0,0 +1,147 @@ +"""Answer one question about an image or a video with a converted vision-language model through its backend (read-only).""" + +from __future__ import annotations + +import json +import math +import subprocess +from pathlib import Path + +from .backends import ( + backend_python, + is_installed, + is_vision_type, + load_manifests, + load_registries, + lookup, +) +from .inference_runner import InferenceError, RunnerContract, invoke + +DESCRIBE_SCHEMA = "describe/1" +DESCRIBE_RUNNER = Path(__file__).resolve().with_name("describe_runner.py") +DEFAULT_TIMEOUT_SECONDS = 900 +MAX_PROMPT_CHARS = 4000 +MAX_TOKENS_RANGE = range(1, 4097) +MAX_TEMPERATURE = 2.0 +MIN_FPS, MAX_FPS = 0.1, 8.0 +DEFAULT_FPS = 1.0 +MAX_PIXELS_LIMIT = 100_000_000 +IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".webp") +VIDEO_SUFFIXES = (".mp4", ".mov", ".m4v") + + +class DescribeError(InferenceError): + """Classified vision-language failure safe to surface in a result envelope.""" + + +def _invalid(message, remediation): + return DescribeError("invalid_arguments", message, remediation) + + +def _number(value): + return not isinstance(value, bool) and isinstance(value, (int, float)) and math.isfinite(value) + + +def _media(kind, value, suffixes): + path = Path(value).expanduser() + if not path.is_absolute(): + path = path.resolve() + if path.suffix.lower() not in suffixes: + raise _invalid( + "--{0} must be a {1} file.".format(kind, "/".join(suffix[1:] for suffix in suffixes)), + "Pass --{0} with one of: {1}.".format(kind, ", ".join(suffixes)), + ) + if not path.is_file(): + raise _invalid("No {0} file at {1}.".format(kind, path), "Pass the path of an existing {0} file.".format(kind)) + return path + + +def plan_describe(model_path, prompt, image=None, video=None, max_tokens=256, temperature=0.0, fps=None, + max_pixels=None, manifests=None, registries=None, root=None): + """Pick the vision-language backend for a converted model directory; pure apart from reading files.""" + model = Path(model_path).expanduser() + try: + config = json.loads((model / "config.json").read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise DescribeError( + "model_unreadable", "No readable config.json in {0}: {1}".format(model, error), + "Pass the converted model directory.", + ) from error + if not isinstance(config, dict): + raise DescribeError("model_unreadable", "config.json in {0} is not an object.".format(model), "Pass the converted model directory.") + if not isinstance(prompt, str) or not prompt.strip() or len(prompt) > MAX_PROMPT_CHARS or any( + ord(character) < 32 and character not in "\n\t" for character in prompt + ): + raise _invalid("The prompt must be 1..{0} characters of plain text.".format(MAX_PROMPT_CHARS), "Pass --prompt with a question about the media.") + if (image is None) == (video is None): + raise _invalid("Pass exactly one of --image or --video.", "Pass --image FILE for a picture or --video FILE for a clip.") + if not isinstance(max_tokens, int) or isinstance(max_tokens, bool) or max_tokens not in MAX_TOKENS_RANGE: + raise _invalid("--max-tokens must be 1..4096.", "Pass --max-tokens 256, for example.") + if not _number(temperature) or not 0 <= temperature <= MAX_TEMPERATURE: + raise _invalid("--temperature must be 0..{0}.".format(MAX_TEMPERATURE), "Pass --temperature 0 for a repeatable answer.") + if fps is not None and video is None: + raise _invalid("--fps applies to --video only.", "Drop --fps when describing an image.") + if fps is not None and (not _number(fps) or not MIN_FPS <= fps <= MAX_FPS): + raise _invalid("--fps must be {0}..{1}.".format(MIN_FPS, MAX_FPS), "Pass --fps 1.0, for example.") + if max_pixels is not None and (not isinstance(max_pixels, int) or isinstance(max_pixels, bool) or not 1 <= max_pixels <= MAX_PIXELS_LIMIT): + raise _invalid("--max-pixels must be 1..{0}.".format(MAX_PIXELS_LIMIT), "Pass --max-pixels 1000000, for example.") + kind = "image" if image is not None else "video" + media = _media(kind, image if image is not None else video, IMAGE_SUFFIXES if image is not None else VIDEO_SUFFIXES) + model_type = config.get("model_type") + manifests = load_manifests() if manifests is None else manifests + registries = load_registries(manifests, root) if registries is None else registries + hits = sorted( + (hit for hit in lookup(model_type, manifests, registries) if hit["category"] == "vision_language"), + key=lambda hit: hit["backend"], + ) + if not hits or not is_vision_type(model_type, manifests, registries): + raise DescribeError( + "not_vision_model", "No vision-language backend implements model_type {0!r} with a vision tower.".format(model_type), + "Describing media applies to converted vision-language models such as Qwen2.5-VL.", + ) + manifest = manifests[hits[0]["backend"]] + if not is_installed(manifest, root): + raise DescribeError( + "backend_not_installed", "The {0} backend is not installed.".format(manifest["id"]), + "Install it first: mlx-agent backend install {0}.".format(manifest["id"]), + ) + used_fps = (DEFAULT_FPS if fps is None else float(fps)) if kind == "video" else None + argv = [ + str(backend_python(manifest, root)), str(DESCRIBE_RUNNER), "--model", str(model), "--prompt=" + prompt, + "--{0}".format(kind), str(media), "--max-tokens", str(max_tokens), "--temperature", repr(float(temperature)), + ] + if used_fps is not None: + argv += ["--fps", repr(used_fps)] + if max_pixels is not None: + argv += ["--max-pixels", str(max_pixels)] + return { + "backend": manifest["id"], "model": str(model), "model_type": model_type, + "input": {"kind": kind, "path": str(media)}, "argv": argv, + } + + +def _video_unsupported(plan, detail): + return ("The {0} backend cannot take video for model_type {1!r}: {2}".format(plan["backend"], plan["model_type"], detail), + "Describe a still image with --image, or use a model type that reads video.") + + +def _not_vision_model(plan, detail): + return ("The {0} backend has no image path for model_type {1!r}.".format(plan["backend"], plan["model_type"]), + "Describing media applies to converted vision-language models such as Qwen2.5-VL.") + + +CONTRACT = RunnerContract( + verb="describe", error=DescribeError, accept=lambda value, plan: isinstance(value.get("text"), str), + action="describe the {input[kind]}", + remediation="Check that the directory is a complete converted vision-language model and that the media file decodes.", + activity="Description", timeout_remediation="Use a shorter clip, fewer tokens, or a larger --timeout.", + refusals={"video_unsupported": _video_unsupported, "not_vision_model": _not_vision_model}, +) + + +def run_describe(plan, manifests=None, root=None, timeout=DEFAULT_TIMEOUT_SECONDS, runner=subprocess.run): + result = invoke(plan, CONTRACT, manifests, root, timeout, runner) + keys = ("text", "prompt_tokens", "generation_tokens", "prompt_tps", "generation_tps", + "peak_memory_gb", "seconds", "load_seconds") + return dict({"schema": DESCRIBE_SCHEMA, "backend": plan["backend"], "model": plan["model"]}, + **{key: result.get(key) for key in keys}, input=dict(plan["input"])) diff --git a/packages/codex/mlx-agent/src/mlx_agent/describe_runner.py b/packages/codex/mlx-agent/src/mlx_agent/describe_runner.py new file mode 100644 index 0000000..ef3ee5b --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/describe_runner.py @@ -0,0 +1,125 @@ +"""Run inside a vision-language backend's environment: load one converted model, answer one question about an image or video, print JSON.""" + +import argparse +import json +import sys +import time + + +def _fit(image, max_pixels): + """Shrink a PIL image, keeping its aspect ratio, until it holds at most max_pixels pixels.""" + width, height = image.size + if max_pixels is None or width * height <= max_pixels: + return image + scale = (max_pixels / float(width * height)) ** 0.5 + return image.resize((max(1, int(width * scale)), max(1, int(height * scale)))) + + +def _answer(result, processor): + """The generated text decoded by the tokenizer with its special tokens skipped. + + mlx-vlm's streamed text drops only `all_special_ids`, so other special added tokens leak into it + (moondream3 opens every answer with `<|md_reserved_4|>`). A stop token ends the ids but never + reaches the text, and the processor's own output cleanup still applies. + """ + token_ids = list(getattr(result, "token_ids", None) or []) + if getattr(result, "finish_reason", None) == "stop": + token_ids = token_ids[:-1] + if not token_ids: + return str(result.text).strip() + tokenizer = getattr(processor, "tokenizer", processor) + text = tokenizer.decode(token_ids, skip_special_tokens=True) + clean_output = getattr(processor, "clean_output", None) + if callable(clean_output): + text = clean_output(text) + return str(text).strip() + + +def _refuse(error, model_type, detail): + print(json.dumps({"error": error, "model_type": model_type, "detail": detail}), flush=True) + return 3 + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--prompt", required=True) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--image") + source.add_argument("--video") + parser.add_argument("--max-tokens", type=int, required=True) + parser.add_argument("--temperature", type=float, required=True) + parser.add_argument("--fps", type=float, default=1.0) + parser.add_argument("--max-pixels", type=int, default=None) + arguments = parser.parse_args(argv) + + import mlx.core as mx + from mlx_vlm import apply_chat_template, generate, load + from mlx_vlm.prompt_utils import MODEL_CONFIG + + started = time.time() + model, processor = load(arguments.model) + load_seconds = time.time() - started + config = model.config + model_type = str(getattr(config, "model_type", None) or "") + + kind = "image" if arguments.image else "video" + if model_type.lower() not in MODEL_CONFIG: + error = "not_vision_model" if kind == "image" else "video_unsupported" + return _refuse(error, model_type, "mlx_vlm has no chat format for images or frames of this model type") + + images, videos, template = [], None, {} + if kind == "image": + images = [arguments.image] + else: + from mlx_vlm.generate.video import processor_handles_video, resolve_video_inputs + from mlx_vlm.utils import VideoSampling + + if processor_handles_video(processor): + videos = [arguments.video] + template = {"video": videos, "fps": arguments.fps} + if arguments.max_pixels is not None: + template["max_pixels"] = arguments.max_pixels + else: + try: + resolution = resolve_video_inputs( + processor, [arguments.video], fps=arguments.fps, max_frames=16, + sampling=VideoSampling(fps=arguments.fps), + ) + except ImportError as error: + return _refuse("video_unsupported", model_type, "frame sampling needs a video decoder: {0}".format(error)) + images = list(resolution.images) + if arguments.max_pixels is not None and images: + from PIL import Image + + images = [_fit(Image.open(item) if isinstance(item, str) else item, arguments.max_pixels) for item in images] + + prompt = apply_chat_template(processor, config, arguments.prompt, num_images=len(images), **template) + + kwargs = {"max_tokens": arguments.max_tokens, "temperature": arguments.temperature, "verbose": False} + if videos: + kwargs["video"] = videos + kwargs["fps"] = arguments.fps + if images: + kwargs["image"] = images + + mx.reset_peak_memory() + started = time.time() + result = generate(model, processor, prompt, **kwargs) + seconds = time.time() - started + + print(json.dumps({ + "text": _answer(result, processor), + "prompt_tokens": int(result.prompt_tokens), + "generation_tokens": int(result.generation_tokens), + "prompt_tps": round(float(result.prompt_tps), 3), + "generation_tps": round(float(result.generation_tps), 3), + "peak_memory_gb": round(float(result.peak_memory), 3), + "seconds": round(seconds, 3), + "load_seconds": round(load_seconds, 3), + }), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/discovery.py b/packages/codex/mlx-agent/src/mlx_agent/discovery.py similarity index 96% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/discovery.py rename to packages/codex/mlx-agent/src/mlx_agent/discovery.py index 889e34d..d49675e 100644 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/discovery.py +++ b/packages/codex/mlx-agent/src/mlx_agent/discovery.py @@ -6,6 +6,7 @@ import json import os import tempfile +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path @@ -28,6 +29,8 @@ CACHE_SCHEMA_VERSION = "2.0" +# Concurrent Hugging Face model inspections during one discovery. +INSPECTION_WORKERS = 6 CACHE_TTL_SECONDS = 24 * 60 * 60 @@ -196,11 +199,19 @@ def discover(self, request): self._write_cache(request, data) return ResultEnvelope.ok("discover", self._response_data(request, data, "refreshed" if request.refresh else "miss")) + def _inspect_all(self, repos): + """Inspect listed repositories concurrently; results keep the listing order.""" + if not repos: + return {} + with ThreadPoolExecutor(max_workers=min(INSPECTION_WORKERS, len(repos))) as pool: + return dict(zip(repos, pool.map(self._huggingface.inspect_model, repos))) + def _build_response(self, request, raw, host_data, list_url=None): buckets = {role: [] for role in DISCOVERY_ROLES} rejected = {} seen_repo, seen_base = set(), {} rejection_limit = max(1, int(request.limit or 1)) * 3 + listed = [] for model in raw: repo = model.get("id") or model.get("modelId") if not repo or repo in seen_repo: @@ -212,7 +223,10 @@ def _build_response(self, request, raw, host_data, list_url=None): and classify(repo) != request.role ): continue - enrichment = {} if request.fast else self._huggingface.inspect_model(repo) + listed.append((repo, model)) + enrichments = {} if request.fast else self._inspect_all([repo for repo, _model in listed]) + for repo, model in listed: + enrichment = {} if request.fast else enrichments[repo] roles, tool_use = classify_roles(repo, enrichment) selected_roles = ( (request.role,) if request.role in roles else () diff --git a/packages/codex/mlx-agent/src/mlx_agent/fetch_runner.py b/packages/codex/mlx-agent/src/mlx_agent/fetch_runner.py new file mode 100644 index 0000000..819aa88 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/fetch_runner.py @@ -0,0 +1,64 @@ +"""Download a Hugging Face snapshot (or one file) and record the outcome.""" + +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def _snapshot_download(**kwargs): + from huggingface_hub import snapshot_download + + return snapshot_download(**kwargs) + + +def main(argv=None, download=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", required=True) + parser.add_argument("--revision", default="main") + parser.add_argument("--file", default=None) + parser.add_argument("--ignore", action="append", default=None) + parser.add_argument("--allow", action="append", default=None) + parser.add_argument("--base-repo", default=None, help="also download this pinned base snapshot first") + parser.add_argument("--base-revision", default=None) + parser.add_argument("--base-ignore", action="append", default=None) + parser.add_argument("--cache-dir", default=None) + parser.add_argument("--local-dir", default=None) + parser.add_argument("--marker", required=True) + arguments = parser.parse_args(argv) + kwargs = {"repo_id": arguments.repo, "revision": arguments.revision} + if arguments.file: + kwargs["allow_patterns"] = [arguments.file] + if arguments.allow: + kwargs["allow_patterns"] = arguments.allow + if arguments.ignore: + kwargs["ignore_patterns"] = arguments.ignore + if arguments.cache_dir: + kwargs["cache_dir"] = arguments.cache_dir + if arguments.local_dir: + kwargs["local_dir"] = arguments.local_dir + try: + if arguments.base_repo: + base = {"repo_id": arguments.base_repo, "revision": arguments.base_revision or "main"} + if arguments.base_ignore: + base["ignore_patterns"] = arguments.base_ignore + if arguments.cache_dir: + base["cache_dir"] = arguments.cache_dir + print("downloading base {0}".format(arguments.base_repo), flush=True) + (download or _snapshot_download)(**base) + path = (download or _snapshot_download)(**kwargs) + result, code = {"exit_status": "done", "path": str(path)}, 0 + print("downloaded {0} to {1}".format(arguments.repo, path), flush=True) + except Exception as error: # noqa: BLE001 - recorded in the marker and the log + result, code = {"exit_status": "failed", "error": str(error)}, 1 + print("download failed: {0}".format(error), flush=True) + result["finished_at"] = datetime.now(timezone.utc).isoformat() + Path(arguments.marker).write_text(json.dumps(result, sort_keys=True) + "\n", encoding="utf-8") + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/fleet.py b/packages/codex/mlx-agent/src/mlx_agent/fleet.py similarity index 75% rename from packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/fleet.py rename to packages/codex/mlx-agent/src/mlx_agent/fleet.py index 71de98f..e2058dc 100644 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/fleet.py +++ b/packages/codex/mlx-agent/src/mlx_agent/fleet.py @@ -138,6 +138,54 @@ def parse_runtime_map(values): return runtime_map +def parse_port_map(values): + """Parse repeatable role=port api_base overrides. + + The defaults (mlx_lm 8080, mlx-vlm 8083) stay the bounded targets; + --port-map lets a supervisor with its own port discipline (for example + per-role always-on endpoints) point roles at its real loopback ports. + """ + port_map = {} + for value in values or []: + if not isinstance(value, str) or "=" not in value: + raise FleetError( + "invalid_port_map", + "Port overrides must use role=port form: {0}".format(value), + "Pass --port-map coding=8766.", + ) + role, port_text = value.split("=", 1) + role = role.strip() + port_text = port_text.strip() + if role not in ROLE_DEFAULT_RUNTIME: + raise FleetError( + "invalid_role", + "Unknown fleet role: {0}".format(role), + "Use one of: {0}.".format(", ".join(sorted(ROLE_DEFAULT_RUNTIME))), + ) + if role in port_map: + raise FleetError( + "duplicate_role", + "Role {0} has two port overrides.".format(role), + "Override each role's port at most once.", + ) + try: + port = int(port_text) + except ValueError: + raise FleetError( + "invalid_port", + "Port must be a number: {0}.".format(port_text), + "Pass a loopback port between 1 and 65535.", + ) + if not 1 <= port <= 65535: + raise FleetError( + "invalid_port", + "Port {0} is outside 1-65535.".format(port), + "Pass a loopback port between 1 and 65535.", + ) + port_map[role] = port + return port_map + + class FleetConfigAdapter: """Validate the exact bounded multi-model router subset fleet renders.""" @@ -147,14 +195,22 @@ class FleetConfigAdapter: def __init__(self, path=None): self.path = Path(path) if path is not None else None - def render(self, assignments, runtime_map=None, existing=""): + def render(self, assignments, runtime_map=None, existing="", port_map=None): runtime_map = runtime_map or {} + port_map = port_map or {} if not assignments: raise FleetError( "missing_assignments", "Fleet requires at least one role assignment.", "Pass --assign role=repo or --from-adoption .", ) + stray_ports = sorted(set(port_map) - set(assignments)) + if stray_ports: + raise FleetError( + "unassigned_port_map", + "Port overrides reference unassigned roles: {0}.".format(", ".join(stray_ports)), + "Assign the role first, or drop its --port-map override.", + ) if not isinstance(existing, str): raise TypeError("existing config content must be text") require_secret_free_config(existing) @@ -173,7 +229,7 @@ def render(self, assignments, runtime_map=None, existing=""): "Fleet routes only through: {0}.".format(", ".join(FLEET_RUNTIMES)), "Use mlx_lm or mlx-vlm for every role.", ) - port = RUNTIME_PORTS[runtime] + port = port_map.get(role, RUNTIME_PORTS[runtime]) lines.extend([ " - model_name: {0}".format(role), " litellm_params:", @@ -182,13 +238,19 @@ def render(self, assignments, runtime_map=None, existing=""): " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY", ]) content = "\n".join(lines) + "\n" - self.validate(content) + self.validate(content, port_map=port_map) return content - def validate(self, content): + def validate(self, content, port_map=None): if not isinstance(content, str): raise TypeError("configuration content must be text") require_secret_free_config(content) + allowed_ports = {"8080", "8083"} + allowed_ports.update(str(port) for port in (port_map or {}).values()) + allowed_api_bases = tuple( + " api_base: http://127.0.0.1:{0}/v1".format(port) + for port in sorted(allowed_ports) + ) lines = content.splitlines() if len(lines) < 7 or (len(lines) - 2) % 5 != 0: raise ValueError("fleet configuration must contain whole five-line entries") @@ -205,10 +267,7 @@ def validate(self, content): model_match = re.fullmatch(r" model: openai/(" + _MODEL.pattern[1:-1] + r")", model) if model_match is None: raise ValueError("fleet entry has an invalid model identifier") - if api_base not in ( - " api_base: http://127.0.0.1:8080/v1", - " api_base: http://127.0.0.1:8083/v1", - ): + if api_base not in allowed_api_bases: raise ValueError("fleet entry api_base must be a bounded loopback port") if api_key != " api_key: os.environ/MLX_AGENT_LOCAL_API_KEY": raise ValueError("fleet entry must reference the managed API key env var") diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/fuse.py b/packages/codex/mlx-agent/src/mlx_agent/fuse.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/fuse.py rename to packages/codex/mlx-agent/src/mlx_agent/fuse.py diff --git a/packages/codex/mlx-agent/src/mlx_agent/generate.py b/packages/codex/mlx-agent/src/mlx_agent/generate.py new file mode 100644 index 0000000..52d33f0 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/generate.py @@ -0,0 +1,95 @@ +"""Render one prompt with a converted image-generation model through its backend (writes only the PNG).""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +from .backends import ( + backend_python, + is_installed, + load_manifests, + load_registries, + lookup, +) +from .inference_runner import InferenceError, RunnerContract, invoke + +GENERATE_SCHEMA = "generate/1" +GENERATE_RUNNER = Path(__file__).resolve().with_name("generate_runner.py") +DEFAULT_TIMEOUT_SECONDS = 1800 +MAX_PROMPT_CHARS = 2000 +SIDES = range(256, 2049) + + +class GenerateError(InferenceError): + """Classified image-generation failure safe to surface in a result envelope.""" + + +def _invalid(message, remediation): + return GenerateError("invalid_arguments", message, remediation) + + +def plan_generate(model_path, prompt, out, width=1024, height=1024, steps=40, seed=42, + manifests=None, registries=None, root=None): + """Pick the image backend for a converted model directory; pure apart from reading config.json.""" + model = Path(model_path).expanduser() + try: + config = json.loads((model / "config.json").read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise GenerateError( + "model_unreadable", "No readable config.json in {0}: {1}".format(model, error), + "Pass the converted model directory.", + ) from error + if not isinstance(prompt, str) or not prompt.strip() or len(prompt) > MAX_PROMPT_CHARS or any( + ord(character) < 32 and character not in "\n\t" for character in prompt + ): + raise _invalid("The prompt must be 1..{0} characters of text.".format(MAX_PROMPT_CHARS), "Pass --prompt with plain text.") + target = Path(out).expanduser() + if not target.is_absolute() or target.suffix.lower() != ".png" or target.exists(): + raise _invalid("--out must be a new absolute .png path.", "Pass a path such as /Users/me/Pictures/render.png.") + for name, value in (("width", width), ("height", height)): + if not isinstance(value, int) or isinstance(value, bool) or value not in SIDES or value % 16: + raise _invalid("--{0} must be a multiple of 16 from 256 to 2048.".format(name), "Pass --{0} 1024, for example.".format(name)) + if not isinstance(steps, int) or isinstance(steps, bool) or not 1 <= steps <= 100: + raise _invalid("--steps must be 1..100.", "Pass --steps 40, for example.") + if not isinstance(seed, int) or isinstance(seed, bool) or not 0 <= seed < 2 ** 32: + raise _invalid("--seed must be 0..4294967295.", "Pass --seed 42, for example.") + model_type = config.get("model_type") if isinstance(config, dict) else None + manifests = load_manifests() if manifests is None else manifests + registries = load_registries(manifests, root) if registries is None else registries + hits = sorted( + (hit for hit in lookup(model_type, manifests, registries) if hit["category"] == "image_generation"), + key=lambda hit: hit["backend"], + ) + if not hits: + raise GenerateError( + "not_image_model", "No image-generation backend implements model_type {0!r}.".format(model_type), + "Image generation applies to converted image models such as Qwen-Image 2.1.", + ) + manifest = manifests[hits[0]["backend"]] + if not is_installed(manifest, root): + raise GenerateError( + "backend_not_installed", "The {0} backend is not installed.".format(manifest["id"]), + "Install it first: mlx-agent backend install {0}.".format(manifest["id"]), + ) + argv = [ + str(backend_python(manifest, root)), str(GENERATE_RUNNER), "--module", hits[0]["module"], "--model", str(model), + "--prompt", prompt, "--out", str(target), "--width", str(width), "--height", str(height), + "--steps", str(steps), "--seed", str(seed), + ] + return {"backend": manifest["id"], "model": str(model), "out": str(target), "argv": argv} + + +CONTRACT = RunnerContract( + verb="generate", error=GenerateError, accept=lambda value, plan: isinstance(value.get("path"), str), + action="render the prompt", remediation="Check that the directory is a complete converted image model.", + activity="Image generation", timeout_remediation="Use fewer steps, a smaller size, or a larger --timeout.", +) + + +def run_generate(plan, manifests=None, root=None, timeout=DEFAULT_TIMEOUT_SECONDS, runner=subprocess.run): + result = invoke(plan, CONTRACT, manifests, root, timeout, runner) + keys = ("path", "width", "height", "steps", "seed", "seconds", "load_seconds", "pixel_std") + return dict({"schema": GENERATE_SCHEMA, "backend": plan["backend"], "model": plan["model"]}, + **{key: result.get(key) for key in keys}) diff --git a/packages/codex/mlx-agent/src/mlx_agent/generate_runner.py b/packages/codex/mlx-agent/src/mlx_agent/generate_runner.py new file mode 100644 index 0000000..5bc4d9e --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/generate_runner.py @@ -0,0 +1,36 @@ +"""Run inside an image backend's environment: load one converted model, render one prompt to a PNG, print JSON.""" + +import argparse +import importlib +import json +import sys +import time + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--module", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--prompt", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--width", type=int, required=True) + parser.add_argument("--height", type=int, required=True) + parser.add_argument("--steps", type=int, required=True) + parser.add_argument("--seed", type=int, required=True) + arguments = parser.parse_args(argv) + + port = importlib.import_module(arguments.module) + started = time.time() + model = port.load(arguments.model) + load_seconds = time.time() - started + result = port.generate( + model, arguments.prompt, arguments.out, width=arguments.width, height=arguments.height, + steps=arguments.steps, seed=arguments.seed, + ) + result["load_seconds"] = round(load_seconds, 3) + print(json.dumps(result), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/codex/mlx-agent/src/mlx_agent/gguf.py b/packages/codex/mlx-agent/src/mlx_agent/gguf.py new file mode 100644 index 0000000..c7db1d5 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/gguf.py @@ -0,0 +1,1038 @@ +"""Read-only GGUF inventory: header parsing, duplicate grouping, conversion pairing. + +Nothing here downloads, deletes, moves, or converts. It reads bounded GGUF +headers, groups files that describe the same model, and reports which of them +already have an MLX output. Mutating conversion lives in :mod:`mlx_agent.convert`. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import struct +from pathlib import Path + + +MAX_SCAN_FILES = 4000 +MAX_METADATA_PAIRS = 512 +MAX_STRING_BYTES = 8192 +MAX_ARRAY_ITEMS = 1024 +MAX_CONFIG_BYTES = 1024 * 1024 +SIGNATURE_CHUNK_BYTES = 1024 * 1024 +PROVENANCE_NAME = "mlx-converter.json" +GGUF_MAGIC = b"GGUF" +GGUF_SUFFIX = ".gguf" +SUPPORTED_GGUF_VERSIONS = (2, 3) + +_SKIP_DIRECTORIES = { + ".git", ".hg", ".svn", "node_modules", "__pycache__", ".venv", "venv", + ".tox", ".mypy_cache", ".pytest_cache", ".quarantine", ".Trash", +} +_SHARD = re.compile(r"^(?P.+?)-(?P\d{5})-of-(?P\d{5})$") +_QUANT_IN_NAME = re.compile( + r"(? MAX_STRING_BYTES * MAX_ARRAY_ITEMS: + raise _Truncated("declared length is out of bounds") + chunk = handle.read(count) + if len(chunk) != count: + raise _Truncated("header ended early") + return chunk + + +def _read_scalar(handle, value_type): + layout, size = _SCALARS[value_type] + return struct.unpack(layout, _read_exactly(handle, size))[0] + + +def _read_string(handle): + length = struct.unpack(" MAX_STRING_BYTES: + raise _Truncated("string exceeds the bounded header budget") + return _read_exactly(handle, length).decode("utf-8", "replace") + + +def _read_value(handle, value_type, depth=0): + if value_type in _SCALARS: + return _read_scalar(handle, value_type) + if value_type == _STRING: + return _read_string(handle) + if value_type == _ARRAY and depth == 0: + item_type = struct.unpack(" MAX_ARRAY_ITEMS: + handle.seek(count * size, os.SEEK_CUR) + return {"array": item_type, "count": count, "skipped": True} + return [_read_scalar(handle, item_type) for _ in range(count)] + raise _Truncated("unsupported metadata value type") + + +def _skip_value(handle, value_type): + """Consume a value we do not keep, without materializing it.""" + if value_type in _SCALARS: + handle.seek(_SCALARS[value_type][1], os.SEEK_CUR) + return + if value_type == _STRING: + length = struct.unpack(" MAX_STRING_BYTES * MAX_ARRAY_ITEMS: + raise _Truncated("string exceeds the bounded header budget") + handle.seek(length, os.SEEK_CUR) + return + if value_type != _ARRAY: + raise _Truncated("unsupported metadata value type") + item_type = struct.unpack(" MAX_METADATA_PAIRS or not 0 < tensor_count <= 100000: + raise _Truncated("unbounded tensor header") + for _ in range(tensor_count): + name = _read_string(handle) + rank = struct.unpack(" 64 * 1024 * 1024: + raise _Truncated("oversized tensor header") + alignment = metadata.get("general.alignment", 32) + if type(alignment) is not int or not 0 < alignment <= 4096 or alignment & (alignment - 1): + raise _Truncated("invalid tensor alignment") + tensor_data_offset = (handle.tell() + alignment - 1) // alignment * alignment + except (_Truncated, struct.error) as error: + raise GGUFError( + "gguf_header_unreadable", + "{0} has a truncated or malformed GGUF header.".format(location.name), + "Re-download the file; a truncated shard cannot be converted.", + ) from error + except OSError as error: + raise GGUFError( + "gguf_unreadable", + "{0} could not be read.".format(location.name), + "Check the path and its permissions, then rerun the scan.", + ) from error + result = { + "version": version, + "tensor_count": tensor_count, + "metadata_pairs": pair_count, + "metadata": metadata, + } + if include_tensors: + result.update(tensors=tensors, tensor_data_offset=tensor_data_offset) + return result + + +def file_signature(path, chunk_bytes=SIGNATURE_CHUNK_BYTES): + """Cheap content signature: size plus the head and tail of the file. + + Full digests of multi-gigabyte weights are not worth their I/O. Two files + that agree on size, head and tail are treated as the same bytes. + """ + location = Path(path) + size = location.stat().st_size + digest = hashlib.sha256() + digest.update(str(size).encode("ascii")) + with location.open("rb") as handle: + digest.update(handle.read(chunk_bytes)) + if size > chunk_bytes * 2: + handle.seek(-chunk_bytes, os.SEEK_END) + digest.update(handle.read(chunk_bytes)) + return digest.hexdigest() + + +def _normalize_stem(stem): + shard = _SHARD.match(stem) + if shard: + stem = shard.group("stem") + without_quant = _QUANT_IN_NAME.sub("", stem) + cleaned = re.sub(r"[^a-z0-9]+", "-", without_quant.lower()).strip("-") + return cleaned or stem.lower() + + +def _quantization(metadata, stem): + file_type = metadata.get("general.file_type") + if isinstance(file_type, int) and file_type in _FILE_TYPES: + return _FILE_TYPES[file_type] + match = _QUANT_IN_NAME.search(stem) + if match: + return match.group(1).upper() + return "unknown" + + +def _structure(metadata, tensor_count): + architecture = metadata.get("general.architecture") + fields = [str(architecture), str(tensor_count)] + for suffix in _STRUCTURE_SUFFIXES: + key = "{0}{1}".format(architecture, suffix) + fields.append(str(metadata.get(key))) + return hashlib.sha256("|".join(fields).encode("utf-8")).hexdigest()[:16] + + +def _draft(metadata, architecture): + """Drafter facts for a speculative-decoding GGUF, else None; ``port`` is set when one converts it.""" + if str(architecture).lower() not in DRAFT_ARCHITECTURES: + return None + port = next((name for name, arch, key in _DRAFT_PORTS if arch == str(architecture).lower() and key in metadata), None) + block_size = metadata.get("{0}.block_size".format(architecture)) + return { + "port": port, + "target": metadata.get("general.name") or metadata.get("general.basename"), + "block_size": block_size if isinstance(block_size, int) and not isinstance(block_size, bool) else None, + } + + +def describe_gguf(path, signature=True): + """Build one inventory entry for a GGUF file. Never raises for bad headers.""" + location = Path(path) + try: + stat = location.stat() + except OSError as error: + raise GGUFError( + "gguf_unreadable", + "{0} could not be read.".format(location.name), + "Check the path and its permissions, then rerun the scan.", + ) from error + stem = location.stem + shard = _SHARD.match(stem) + entry = { + "path": str(location), + "name": location.name, + "bytes": stat.st_size, + "modified_at": int(stat.st_mtime), + "shard": None if shard is None else { + "index": int(shard.group("index")), + "total": int(shard.group("total")), + }, + "model_key": _normalize_stem(stem), + "architecture": None, + "quantization": "unknown", + "parameters": None, + "structure": None, + "signature": None, + "companion": any(token in stem.lower() for token in _COMPANION_NAMES), + "draft": None, + "readable": True, + "error": None, + } + try: + header = read_gguf_header(location) + except GGUFError as error: + entry["readable"] = False + entry["error"] = {"code": error.code, "message": str(error)} + return entry + metadata = header["metadata"] + name = metadata.get("general.name") or metadata.get("general.basename") + entry["architecture"] = metadata.get("general.architecture") + entry["companion"] = ( + any(token in stem.lower() for token in _COMPANION_NAMES) + or str(entry["architecture"]).lower() in _COMPANION_ARCHITECTURES + ) + entry["quantization"] = _quantization(metadata, stem) + entry["structure"] = _structure(metadata, header["tensor_count"]) + entry["tensor_count"] = header["tensor_count"] + parameters = metadata.get("general.parameter_count") + if isinstance(parameters, int) and not isinstance(parameters, bool): + entry["parameters"] = parameters + if isinstance(name, str) and name: + entry["model_key"] = _normalize_stem(name) + entry["draft"] = _draft(metadata, entry["architecture"]) + if entry["draft"] is not None: + # A drafter carries its target's name; keep it out of the target's duplicate group. + entry["model_key"] += "-draft" + if signature: + try: + entry["signature"] = file_signature(location) + except OSError: + entry["signature"] = None + return entry + + +def _iter_files(roots, suffix, limit=MAX_SCAN_FILES): + seen = set() + for root in roots: + base = Path(root).expanduser() + if not base.is_dir(): + continue + for dirpath, dirnames, filenames in os.walk(base, followlinks=False): + if suffix == ".gguf" and "strata-model.json" in filenames: + from .strata_runtime import StrataError, read_model + try: + read_model(Path(dirpath)) + except StrataError: + pass + else: + dirnames[:] = [] + continue + dirnames[:] = sorted( + name for name in dirnames + if name not in _SKIP_DIRECTORIES and not name.startswith(".") + ) + for filename in sorted(filenames): + if not filename.lower().endswith(suffix): + continue + location = Path(dirpath) / filename + try: + resolved = location.resolve() + except OSError: + continue + if resolved in seen: + continue + seen.add(resolved) + yield location + if len(seen) >= limit: + return + + +def scan_gguf(roots, signature=True, limit=MAX_SCAN_FILES): + """Inventory every GGUF file beneath the configured roots.""" + entries = [] + for location in _iter_files(roots, GGUF_SUFFIX, limit): + try: + entries.append(describe_gguf(location, signature=signature)) + except GGUFError: + continue + entries.sort(key=lambda entry: entry["path"]) + return entries + + +def read_provenance(directory): + """Read an MLX output's converter provenance marker, if it has one.""" + location = Path(directory) / PROVENANCE_NAME + try: + value = json.loads(location.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if not isinstance(value, dict) or not isinstance(value.get("source"), dict): + return None + return value + + +def _mlx_config(directory): + """Return the MLX marker for a model directory, or None if it is not one. + + A directory only counts as an MLX output when its ``config.json`` carries a + ``quantization`` block (what ``mlx_lm.convert -q`` writes) or when this tool + left a provenance marker. Plain PyTorch checkpoints in the Hugging Face + cache also hold ``config.json`` and safetensors, and must not be mistaken + for conversions. + """ + location = Path(directory) / "config.json" + try: + if location.stat().st_size > MAX_CONFIG_BYTES: + return None + value = json.loads(location.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if not isinstance(value, dict): + return None + quantization = value.get("quantization") + if not isinstance(quantization, dict): + return None + return { + "bits": quantization.get("bits"), + "group_size": quantization.get("group_size"), + "model_type": value.get("model_type"), + } + + +def _mlx_community_snapshot(directory): + """A Hugging Face cache snapshot of an mlx-community repository: MLX weights even when unquantized (bf16).""" + parts = Path(directory).parts + return len(parts) >= 3 and parts[-2] == "snapshots" and parts[-3].startswith("models--mlx-community--") + + +def _component_weights(directory): + """A converter that saves per-component folders (mflux) lists them in config.json ``components``.""" + try: + value = json.loads((Path(directory) / "config.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + return False + names = value.get("components") if isinstance(value, dict) else None + if not isinstance(names, list) or not names or not all( + isinstance(name, str) and name and "/" not in name and name not in (".", "..") for name in names + ): + return False + return all(any(Path(directory, name).glob("*.safetensors")) for name in names) + + +def scan_mlx_outputs(roots, limit=MAX_SCAN_FILES): + """Inventory MLX model directories beneath the configured roots.""" + outputs = [] + seen = set() + for root in roots: + base = Path(root).expanduser() + if not base.is_dir(): + continue + for dirpath, dirnames, filenames in os.walk(base, followlinks=False): + dirnames[:] = sorted( + name for name in dirnames + if name not in _SKIP_DIRECTORIES and not name.startswith(".") + ) + if "config.json" not in filenames: + continue + directory = Path(dirpath) + if "strata-model.json" in filenames: + from .strata_runtime import StrataError, read_model + try: + profile = read_model(directory) + except StrataError: + continue + resolved = directory.resolve() + if resolved not in seen: + seen.add(resolved) + outputs.append({"path": str(directory), "name": directory.name, + "model_key": _normalize_stem(directory.name), "quantization": None, + "provenance": None, "runtime": "strata", "format": "gguf", + "weight_bytes": profile["weight_bytes"]}) + dirnames[:] = [] + if len(seen) >= limit: + return sorted(outputs, key=lambda item: item["path"]) + continue + # is_file follows cache symlinks: a snapshot whose weight blobs are missing is not a model. + if not any(name.endswith(".safetensors") and (directory / name).is_file() for name in filenames) \ + and not _component_weights(directory): + continue + provenance = read_provenance(directory) + marker = _mlx_config(directory) + if provenance is None and marker is None and not _mlx_community_snapshot(directory): + continue + try: + resolved = directory.resolve() + except OSError: + continue + if resolved in seen: + continue + seen.add(resolved) + dirnames[:] = [] + outputs.append({ + "path": str(directory), + "name": directory.name, + "model_key": _normalize_stem(directory.name), + "quantization": marker, + "provenance": provenance, + }) + if len(seen) >= limit: + return sorted(outputs, key=lambda item: item["path"]) + return sorted(outputs, key=lambda item: item["path"]) + + +def _output_matches(entry, output): + provenance = output.get("provenance") or {} + source = provenance.get("source") or {} + if entry.get("signature") and source.get("signature") == entry["signature"]: + return "provenance" + if source.get("path") == entry["path"]: + return "provenance" + key = entry["model_key"] + if len(key) < _MIN_NAME_MATCH: + return None + candidate = output["model_key"] + if candidate == key or candidate.startswith(key + "-"): + return "name" + return None + + +def pair_conversions(entries, outputs, receipts=()): + """Mark each GGUF as converted or pending, by provenance then by name.""" + receipt_sources = {} + for receipt in receipts: + source = receipt.get("source") or {} + for key in ("signature", "path"): + value = source.get(key) + if isinstance(value, str) and value: + receipt_sources.setdefault(value, receipt) + paired = [] + for entry in entries: + shard = entry.get("shard") + if shard is not None and shard["index"] != 1: + paired.append(dict(entry, status="shard", outputs=[], evidence="shard")) + continue + if entry.get("companion"): + paired.append(dict(entry, status="companion", outputs=[], evidence="companion")) + continue + matches = [] + evidence = None + for output in outputs: + match = _output_matches(entry, output) + if match is None: + continue + matches.append(output["path"]) + evidence = "provenance" if match == "provenance" else evidence or "name" + receipt = receipt_sources.get(entry.get("signature")) or receipt_sources.get(entry["path"]) + if receipt is not None and not matches: + out = receipt.get("out") + if isinstance(out, str) and Path(out).is_dir(): + matches.append(out) + evidence = "receipt" + paired.append(dict( + entry, + status="converted" if matches else "pending", + outputs=sorted(set(matches)), + evidence=evidence, + )) + return paired + + +def group_duplicates(entries): + """Group GGUFs that describe the same model. + + ``exact`` groups are byte-identical or same-structure-same-quantization + files: every member but the keeper is redundant. ``variant`` groups are the + same model at different quantization levels — reported, never recommended + for removal, because the choice between them is the user's. + """ + exact = {} + variant = {} + for entry in entries: + if entry.get("shard") is not None and entry["shard"]["index"] != 1: + continue + if not entry.get("readable") or entry.get("companion"): + continue + structure = entry.get("structure") + identity = entry.get("signature") or ( + "{0}:{1}:{2}".format(entry["model_key"], structure, entry["quantization"]) + ) + exact.setdefault(identity, []).append(entry) + if structure: + variant.setdefault("{0}:{1}".format(entry["model_key"], structure), []).append(entry) + + def _keeper(members): + # Keep the already-converted copy, then an Ollama copy (only Ollama removes its blobs), else the oldest path. + converted = [item for item in members if item.get("status") == "converted"] + pool = converted or [item for item in members if item.get("status") == "ollama"] or members + return sorted(pool, key=lambda item: (item["modified_at"], item["path"]))[0] + + groups = [] + for identity, members in sorted(exact.items()): + if len(members) < 2: + continue + keep = _keeper(members) + groups.append({ + "kind": "exact", + "identity": identity, + "model_key": keep["model_key"], + "quantization": keep["quantization"], + "keep": keep["path"], + "redundant": sorted(item["path"] for item in members if item["path"] != keep["path"]), + "reclaimable_bytes": sum( + item["bytes"] for item in members if item["path"] != keep["path"] + ), + }) + for identity, members in sorted(variant.items()): + quantizations = {item["quantization"] for item in members} + if len(members) < 2 or len(quantizations) < 2: + continue + groups.append({ + "kind": "variant", + "identity": identity, + "model_key": members[0]["model_key"], + "quantizations": sorted(quantizations), + "members": sorted(item["path"] for item in members), + "redundant": [], + "reclaimable_bytes": 0, + }) + return groups + + +def _hf_snapshot(directory): + parts = Path(directory).parts + return len(parts) >= 3 and parts[-2] == "snapshots" and parts[-3].startswith("models--") + + +def _mlx_name(directory): + """Folder name of an MLX model; a cache snapshot uses its repository name.""" + return Path(directory).parts[-3].split("--")[-1] if _hf_snapshot(directory) else Path(directory).name + + +def _mlx_name_key(name, sizes=False): + """A model name without trailing MLX quantization labels, and without its parameter count when ``sizes``.""" + stem = name + while True: + trimmed = _MLX_QUANT_SUFFIX.sub("", stem) + if sizes: + trimmed = _SIZE_SUFFIX.sub("", trimmed) + if trimmed == stem or not trimmed: + break + stem = trimmed + return _normalize_stem(stem) + + +def _mlx_model_key(directory): + return _mlx_name_key(_mlx_name(directory)) + + +def _read_json_object(location, limit): + try: + if Path(location).stat().st_size > limit: + return None + value = json.loads(Path(location).read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return value if isinstance(value, dict) else None + + +def _mlx_structure(directory): + return _config_structure(_read_json_object(Path(directory) / "config.json", MAX_CONFIG_BYTES)) + + +def _config_structure(value): + """Architecture dimensions from a config.json object (or its text_config), or None when it names none.""" + if not isinstance(value, dict): + return None + nested = value.get("text_config") if isinstance(value.get("text_config"), dict) else {} + fields = [str(value.get(key, nested.get(key))) for key in _MLX_STRUCTURE_KEYS] + if all(field == "None" for field in fields[1:]): + return None + return hashlib.sha256("|".join(fields).encode("utf-8")).hexdigest()[:16] + + +def _mlx_files(directory): + """(relative path, size, file identity) of every file in an MLX folder, cache symlinks followed. + + None when a file cannot be read, such as a cache symlink whose blob is gone. + """ + base = Path(directory) + files = [] + for dirpath, dirnames, filenames in os.walk(base, followlinks=False): + dirnames[:] = sorted(name for name in dirnames if name not in _SKIP_DIRECTORIES and not name.startswith(".")) + for filename in filenames: + location = Path(dirpath) / filename + try: + stat = location.stat() + except OSError: + return None + files.append((location.relative_to(base).as_posix(), stat.st_size, (stat.st_dev, stat.st_ino))) + return sorted(files) + + +def _mlx_quantization(output): + marker = output.get("quantization") + bits = marker.get("bits") if isinstance(marker, dict) else None + return "{0}-bit".format(bits) if isinstance(bits, int) and not isinstance(bits, bool) else "unknown" + + +def _ollama_stores(roots): + """Ollama model stores at or below the roots: folders holding ``blobs`` beside a manifests tree.""" + stores = [] + seen = set() + for root in roots: + base = Path(root).expanduser() + if not base.is_dir(): + continue + for dirpath, dirnames, _filenames in os.walk(base, followlinks=False): + if "blobs" in dirnames and any(tree in dirnames for tree in _OLLAMA_MANIFEST_TREES): + resolved = Path(dirpath).resolve() + if resolved not in seen: + seen.add(resolved) + stores.append(Path(dirpath)) + dirnames[:] = [] + continue + dirnames[:] = sorted( + name for name in dirnames + if name not in _SKIP_DIRECTORIES and not name.startswith(".") + ) + return stores + + +def _ollama_blob(store, digest): + if not isinstance(digest, str) or not _OLLAMA_DIGEST.match(digest): + return None + location = store / "blobs" / digest.replace(":", "-") + return location if location.is_file() else None + + +def _describe_ollama(store, tree, manifest, signature): + """One Ollama model and the set of its layer digests, or None when it is not a local model.""" + parts = manifest.relative_to(tree).parts + value = _read_json_object(manifest, MAX_MANIFEST_BYTES) + layers = value.get("layers") if value else None + if len(parts) < 3 or not isinstance(layers, list) or not layers: + return None + described = [] + for layer in layers: + media = layer.get("mediaType") if isinstance(layer, dict) else None + blob = _ollama_blob(store, layer.get("digest")) if isinstance(media, str) else None + if blob is None: + return None + described.append((media[len(_OLLAMA_MEDIA):] if media.startswith(_OLLAMA_MEDIA) else media, layer, blob)) + config = value.get("config") if isinstance(value.get("config"), dict) else {} + config_blob = _ollama_blob(store, config.get("digest")) + config = (_read_json_object(config_blob, MAX_CONFIG_BYTES) if config_blob else None) or {} + model, tag = parts[-2], parts[-1] + name = "{0}:{1}".format(model if parts[1:-2] == ("library",) else "/".join(parts[:-1]), tag) + sizes = {layer["digest"]: layer.get("size") for _kind, layer, _blob in described} + record = { + "name": name, + "path": str(manifest), + "blob": None, + "bytes": sum(size for size in sizes.values() if isinstance(size, int) and not isinstance(size, bool)), + "modified_at": int(manifest.stat().st_mtime), + } + gguf = next((blob for kind, _layer, blob in described if kind == "model"), None) + if gguf is not None: + try: + entry = describe_gguf(gguf, signature=signature) + except GGUFError: + return None + record.update( + format="gguf", blob=str(gguf), architecture=entry["architecture"], model_key=entry["model_key"], + quantization=entry["quantization"], structure=entry["structure"], signature=entry["signature"], + readable=entry["readable"], draft=entry["draft"], + ) + elif any(kind == "tensor" for kind, _layer, _blob in described): + settings = next((_read_json_object(blob, MAX_CONFIG_BYTES) for kind, layer, blob in described + if kind == "json" and layer.get("name") == "config.json"), None) or {} + marker = settings.get("quantization") if isinstance(settings.get("quantization"), dict) else {} + label = _mlx_quantization({"quantization": marker}) + if isinstance(config.get("file_type"), str) and config["file_type"]: + label = config["file_type"] + record.update( + format="safetensors", architecture=settings.get("model_type"), + model_key=_mlx_name_key("{0}-{1}".format(model, tag)), quantization=label, + structure=_config_structure(settings), signature=None, readable=True, draft=None, + ) + else: + return None + return record, frozenset(layer["digest"] for _kind, layer, _blob in described) + + +def scan_ollama(roots, signature=True, limit=MAX_SCAN_FILES): + """Inventory the models in Ollama stores at or below the roots. Read-only. + + A GGUF model is described from its model blob, a safetensors (MLX) model + from its ``config.json`` layer. A manifest naming a missing blob is skipped. + A model listed in both manifest trees, by name or by identical layers, + counts once, from the current (v2) tree. + """ + models = [] + seen = set() + for store in _ollama_stores(roots): + for name in _OLLAMA_MANIFEST_TREES: + tree = store / name + for dirpath, dirnames, filenames in os.walk(tree, followlinks=False): + dirnames[:] = sorted(item for item in dirnames if not item.startswith(".")) + for filename in sorted(item for item in filenames if not item.startswith(".")): + found = _describe_ollama(store, tree, Path(dirpath) / filename, signature) + if found is None: + continue + keys = {(store, found[0]["name"]), (store, found[1])} + if keys & seen: + continue + seen.update(keys) + models.append(found[0]) + if len(models) >= limit: + return sorted(models, key=lambda item: item["name"]) + return sorted(models, key=lambda item: item["name"]) + + +def _ollama_gguf_entries(models): + """Ollama GGUF models as duplicate-grouping entries; models sharing one blob count once.""" + entries = {} + for model in models: + if model["format"] != "gguf" or model["blob"] in entries: + continue + try: + size = Path(model["blob"]).stat().st_size + except OSError: + continue + entries[model["blob"]] = dict(model, bytes=size, shard=None, companion=False, status="ollama") + return list(entries.values()) + + +def group_mlx_duplicates(outputs, signature=True, ollama=()): + """Group MLX folders that hold the same model, with the same rules as :func:`group_duplicates`. + + ``exact`` folders hold identical weights: the same weight files at the same + sizes, and with ``signature`` the same head and tail bytes; without it, the + same model name and bit width too. Folders whose weights are the same files + on disk (cache revisions sharing blobs) count once. ``variant`` folders and + Ollama safetensors models share a model name (without its parameter count) + and architecture at different bit widths. + """ + described = [] + for output in outputs: + if output.get("runtime") == "strata": + continue + files = _mlx_files(output["path"]) + if files is None: + continue + weights = [item for item in files if item[0].endswith(".safetensors")] + if not weights: + continue + try: + modified = (Path(output["path"]) / "config.json").stat().st_mtime + except OSError: + continue + described.append(dict( + output, + model_key=_mlx_model_key(output["path"]), + variant_key=_mlx_name_key(_mlx_name(output["path"]), sizes=True), + label=_mlx_quantization(output), + structure=_mlx_structure(output["path"]), + weights=tuple((name, size) for name, size, _identity in weights), + identities=frozenset(identity for _name, _size, identity in weights), + bytes=sum(size for _name, size, _identity in files), + modified_at=modified, + )) + + candidates = {} + for item in described: + key = item["weights"] if signature else (item["model_key"], item["label"], item["weights"]) + candidates.setdefault(key, {}).setdefault(item["identities"], item) + exact = {} + for key, distinct in candidates.items(): + if len(distinct) < 2: + continue + for item in distinct.values(): + if not signature: + exact.setdefault(hashlib.sha256(repr(key).encode("utf-8")).hexdigest(), []).append(item) + continue + try: + digest = hashlib.sha256("|".join( + "{0}:{1}".format(name, file_signature(Path(item["path"]) / name)) + for name, _size in item["weights"] + ).encode("utf-8")).hexdigest() + except OSError: + continue + exact.setdefault(digest, []).append(item) + + def _keeper(members): + # Keep a converter output, then a shared Hugging Face cache snapshot, then the oldest copy. + return sorted(members, key=lambda item: ( + item.get("provenance") is None, not _hf_snapshot(item["path"]), item["modified_at"], item["path"], + ))[0] + + groups = [] + for identity, members in sorted(exact.items(), key=lambda pair: sorted(item["path"] for item in pair[1])): + if len(members) < 2: + continue + keep = _keeper(members) + groups.append({ + "kind": "exact", + "format": "mlx", + "identity": "mlx:" + identity, + "model_key": keep["model_key"], + "quantization": keep["label"], + "keep": keep["path"], + "redundant": sorted(item["path"] for item in members if item["path"] != keep["path"]), + "reclaimable_bytes": sum(item["bytes"] for item in members if item["path"] != keep["path"]), + }) + stored = [ + dict(model, label=model["quantization"], status="ollama", + variant_key=_mlx_name_key(model["name"].rsplit("/", 1)[-1].replace(":", "-"), sizes=True)) + for model in ollama if model["format"] == "safetensors" + ] + variant = {} + for item in described + stored: + if item["structure"] and len(item["variant_key"]) >= _MIN_NAME_MATCH: + variant.setdefault("{0}:{1}".format(item["variant_key"], item["structure"]), []).append(item) + for identity, members in sorted(variant.items()): + labels = {item["label"] for item in members} + if len(members) < 2 or len(labels) < 2: + continue + named = sorted(members, key=lambda item: (item.get("status") == "ollama", item["path"]))[0] + groups.append({ + "kind": "variant", + "format": "mlx", + "identity": "mlx:" + identity, + "model_key": named["model_key"], + "quantizations": sorted(labels), + "members": sorted(item["path"] for item in members), + "redundant": [], + "reclaimable_bytes": 0, + }) + return groups + + +def inventory(gguf_roots, mlx_roots=(), receipts=(), signature=True, limit=MAX_SCAN_FILES): + """One read-only pass: what exists, what is converted, what is redundant.""" + entries = scan_gguf(gguf_roots, signature=signature, limit=limit) + outputs = scan_mlx_outputs(mlx_roots or gguf_roots, limit=limit) + ollama = scan_ollama(list(gguf_roots) + list(mlx_roots), signature=signature, limit=limit) + paired = pair_conversions(entries, outputs, receipts) + duplicates = group_duplicates(paired + _ollama_gguf_entries(ollama)) + group_mlx_duplicates( + outputs, signature=signature, ollama=ollama) + for group in duplicates: + group["group_id"] = "{0}:{1}".format(group["kind"], group["identity"]) + pending = [item for item in paired if item["status"] == "pending"] + return { + "roots": { + "gguf": [str(Path(root).expanduser()) for root in gguf_roots], + "mlx": [str(Path(root).expanduser()) for root in (mlx_roots or gguf_roots)], + }, + "models": paired, + "outputs": outputs, + "ollama": ollama, + "pending": [item["path"] for item in pending], + "duplicates": duplicates, + "totals": { + "gguf": len(paired), + "pending": len(pending), + "converted": len([item for item in paired if item["status"] == "converted"]), + "unreadable": len([item for item in paired if not item["readable"]]), + "bytes": sum(item["bytes"] for item in paired), + "reclaimable_bytes": sum(group["reclaimable_bytes"] for group in duplicates), + }, + } diff --git a/packages/codex/mlx-agent/src/mlx_agent/gguf_runner.py b/packages/codex/mlx-agent/src/mlx_agent/gguf_runner.py new file mode 100644 index 0000000..b06da89 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/gguf_runner.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Standalone GGUF to MLX conversion worker. + +Spawned by ``mlx_agent.convert`` with an absolute path, so it must import +nothing from :mod:`mlx_agent`. Two bounded steps, in order: + +1. dequantize the GGUF back to Hugging Face weights with ``transformers``; +2. quantize those weights into MLX with the ``mlx_lm.convert`` executable. + +The intermediate fp16 checkpoint is large and temporary; it is removed unless +``--keep-intermediate`` is passed. ``--port`` instead hands the GGUF to a +bundled port that writes the MLX weights itself, for architectures +``transformers`` cannot load (llama.cpp's DSpark drafters). The worker writes a +provenance marker into the output so a later scan can tell which GGUF produced +it. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def _paths_without_script_dir(entries, script_dir, cwd): + """Drop the runner directory so ``import gguf`` hits the installed package. + + Script launch puts this package directory on ``sys.path[0]``, which shadows + the installed ``gguf`` package with sibling ``gguf.py`` (no ``GGUFReader``). + """ + script_dir = Path(script_dir).resolve() + cwd = Path(cwd).resolve() + + def _is_within_script_dir(candidate): + try: + candidate.resolve().relative_to(script_dir) + return True + except (ValueError, OSError): + return False + + cleaned = [] + for entry in entries: + if entry in ("", "."): + if cwd != script_dir: + cleaned.append(entry) + continue + try: + entry_path = Path(entry) + if not entry_path.is_absolute(): + entry_path = cwd / entry_path + if _is_within_script_dir(entry_path): + continue + except OSError: + cleaned.append(entry) + continue + cleaned.append(entry) + return cleaned + + +sys.path[:] = _paths_without_script_dir(sys.path, Path(__file__).resolve().parent, Path.cwd()) + + +PROVENANCE_NAME = "mlx-converter.json" +SIGNATURE_CHUNK_BYTES = 1024 * 1024 +EXECUTABLE = "mlx_lm.convert" +REQUIRED_MODULES = ("torch", "transformers", "gguf") +PORT_REQUIRED_MODULES = ("gguf", "mlx") +PORTS_DIR = Path(__file__).resolve().parent / "resources" / "ports" +# --port name to its converter under resources/ports///convert.py. +GGUF_PORTS = {"deepseek_v4_dspark": "mlx-lm"} + + +def _log(message): + print("[mlx-converter] {0}".format(message), flush=True) + + +def _signature(path, chunk_bytes=SIGNATURE_CHUNK_BYTES): + size = path.stat().st_size + digest = hashlib.sha256() + digest.update(str(size).encode("ascii")) + with path.open("rb") as handle: + digest.update(handle.read(chunk_bytes)) + if size > chunk_bytes * 2: + handle.seek(-chunk_bytes, os.SEEK_END) + digest.update(handle.read(chunk_bytes)) + return digest.hexdigest() + + +def _missing_modules(names=REQUIRED_MODULES): + missing = [] + for name in names: + if importlib.util.find_spec(name) is None: + missing.append(name) + return missing + + +def dequantize(gguf_path, work_dir): + """Materialize Hugging Face weights from a GGUF file.""" + from transformers import AutoModelForCausalLM, AutoTokenizer + + directory = str(gguf_path.parent) + filename = gguf_path.name + _log("dequantizing {0}".format(filename)) + model = AutoModelForCausalLM.from_pretrained(directory, gguf_file=filename) + model.save_pretrained(str(work_dir)) + del model + tokenizer = AutoTokenizer.from_pretrained(directory, gguf_file=filename) + tokenizer.save_pretrained(str(work_dir)) + _log("dequantized to {0}".format(work_dir)) + + +def quantize(work_dir, out_dir, q_bits): + """Hand the dequantized checkpoint to mlx_lm.convert.""" + executable = shutil.which(EXECUTABLE) + if executable is None: + raise RuntimeError( + "{0} is not on PATH; install mlx-lm into this interpreter".format(EXECUTABLE) + ) + argv = [ + executable, + "--hf-path", str(work_dir), + "--mlx-path", str(out_dir), + "-q", + "--q-bits", str(q_bits), + ] + _log("running {0}".format(" ".join(argv))) + completed = subprocess.run(argv, stdin=subprocess.DEVNULL, check=False) + if completed.returncode != 0: + raise RuntimeError("{0} exited {1}".format(EXECUTABLE, completed.returncode)) + + +def load_port(name): + """The bundled port's ``convert`` module, loaded by path (this runner imports nothing from mlx_agent).""" + location = PORTS_DIR / GGUF_PORTS[name] / name / "convert.py" + spec = importlib.util.spec_from_file_location("mlx_agent_gguf_port_{0}".format(name), location) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def convert_with_port(name, gguf_path, out_dir, q_bits): + """Run a bundled port into a fresh ``out_dir``; returns the port's summary.""" + out_dir.mkdir(parents=True) + _log("converting {0} with the {1} port".format(gguf_path.name, name)) + return load_port(name).convert(gguf_path, out_dir, q_bits, log=_log) + + +def write_provenance(out_dir, gguf_path, q_bits, signature, port=None): + marker = { + "schema_version": "1.0", + "tool": "mlx-agent.gguf", + "created_at": datetime.now(timezone.utc).isoformat(), + "q_bits": q_bits, + "source": { + "kind": "gguf", + "path": str(gguf_path), + "name": gguf_path.name, + "bytes": gguf_path.stat().st_size, + "signature": signature, + }, + } + if port is not None: + marker["port"] = port + target = Path(out_dir) / PROVENANCE_NAME + target.write_text( + json.dumps(marker, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--gguf", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--q-bits", type=int, required=True, choices=(4, 8)) + parser.add_argument("--work-dir", default=None) + parser.add_argument("--signature", default=None) + parser.add_argument("--keep-intermediate", action="store_true") + parser.add_argument("--port", choices=sorted(GGUF_PORTS), default=None) + arguments = parser.parse_args(argv) + + gguf_path = Path(arguments.gguf).expanduser() + out_dir = Path(arguments.out).expanduser() + if not gguf_path.is_file(): + _log("source GGUF not found: {0}".format(gguf_path)) + return 2 + if out_dir.exists(): + _log("output already exists: {0}".format(out_dir)) + return 2 + missing = _missing_modules(PORT_REQUIRED_MODULES if arguments.port else REQUIRED_MODULES) + if missing: + _log("missing required modules: {0}".format(", ".join(missing))) + return 2 + if arguments.port: + try: + convert_with_port(arguments.port, gguf_path, out_dir, arguments.q_bits) + write_provenance( + out_dir, gguf_path, arguments.q_bits, + arguments.signature or _signature(gguf_path), port=arguments.port, + ) + except (OSError, RuntimeError, ValueError, ImportError, KeyError) as error: + _log("conversion failed: {0}".format(error)) + if out_dir.is_dir(): + shutil.rmtree(out_dir, ignore_errors=True) + _log("removed partial output {0}".format(out_dir)) + return 1 + _log("converted {0} -> {1}".format(gguf_path.name, out_dir)) + return 0 + + work_dir = Path(arguments.work_dir) if arguments.work_dir else out_dir.with_name( + out_dir.name + ".hf-intermediate" + ) + if work_dir.exists(): + _log("intermediate directory already exists: {0}".format(work_dir)) + return 2 + work_dir.mkdir(parents=True) + try: + dequantize(gguf_path, work_dir) + quantize(work_dir, out_dir, arguments.q_bits) + write_provenance( + out_dir, gguf_path, arguments.q_bits, + arguments.signature or _signature(gguf_path), + ) + except (OSError, RuntimeError, ValueError, ImportError) as error: + _log("conversion failed: {0}".format(error)) + return 1 + finally: + if not arguments.keep_intermediate and work_dir.is_dir(): + shutil.rmtree(work_dir, ignore_errors=True) + _log("removed intermediate {0}".format(work_dir)) + _log("converted {0} -> {1}".format(gguf_path.name, out_dir)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/host.py b/packages/codex/mlx-agent/src/mlx_agent/host.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/host.py rename to packages/codex/mlx-agent/src/mlx_agent/host.py diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/huggingface.py b/packages/codex/mlx-agent/src/mlx_agent/huggingface.py similarity index 60% rename from packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/huggingface.py rename to packages/codex/mlx-agent/src/mlx_agent/huggingface.py index 3a84ed5..20b0f2e 100644 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/huggingface.py +++ b/packages/codex/mlx-agent/src/mlx_agent/huggingface.py @@ -4,6 +4,7 @@ import json import queue import re +import struct import threading import time import urllib.parse @@ -27,6 +28,52 @@ HF_CARD_HOST = "huggingface.co" MODEL_CARD_MAX_BYTES = 512 * 1024 _CARD_PATH_SUFFIX = "/raw/main/README.md" +RAW_TEXT_MAX_BYTES = 8 * 1024 * 1024 +_RAW_JSON_FILES = frozenset({"config.json", "model.safetensors.index.json", "model_index.json"}) +_RAW_PY_FILE = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}\.py") +_RAW_REVISION = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}") +_SAFETENSORS_FILE = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}\.safetensors") +_FOLDER = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,254}") +MAX_FOLDER_DEPTH = 8 +SAFETENSORS_HEADER_MAX_BYTES = 16 * 1024 * 1024 +_SAFETENSORS_PROBE_BYTES = 1024 * 1024 + + +class HuggingFaceHTTPError(http.client.HTTPException): + """A non-2xx Hugging Face response; carries the HTTP status.""" + + def __init__(self, status, message): + super().__init__(message) + self.status = status + + +def _is_hub_file_path(parts, kind): + """////[/...]/: folders are plain names, never . or ..""" + if len(parts) < 6 or len(parts) > 6 + MAX_FOLDER_DEPTH or parts[0] != "" or parts[3] != kind: + return False + if not parts[1] or not parts[2] or not _RAW_REVISION.fullmatch(parts[4]): + return False + return all(_FOLDER.fullmatch(folder) for folder in parts[5:-1]) + + +def _is_valid_raw_path(path): + """Accept ///raw/[/...]/ for intake files only.""" + parts = path.split("/") + if not _is_hub_file_path(parts, "raw"): + return False + return parts[-1] in _RAW_JSON_FILES or bool(_RAW_PY_FILE.fullmatch(parts[-1])) + + +def _is_valid_resolve_path(path): + """Accept ///resolve/[/...]/.safetensors for weight files.""" + parts = path.split("/") + return _is_hub_file_path(parts, "resolve") and bool(_SAFETENSORS_FILE.fullmatch(parts[-1])) + + +def _is_hub_storage_host(host): + """Hosts the Hub redirects weight downloads to (its CDN and Xet storage).""" + host = (host or "").lower() + return host == "hf.co" or host.endswith(".hf.co") or host.endswith(".huggingface.co") def _is_allowed_api_path(path): @@ -161,8 +208,9 @@ def _http_json_operation(connection, target, deadline, clock): "redirect responses are not allowed for Hugging Face requests" ) if not 200 <= response.status < 300: - raise http.client.HTTPException( - "Hugging Face returned HTTP status {0}".format(response.status) + raise HuggingFaceHTTPError( + response.status, + "Hugging Face returned HTTP status {0}".format(response.status), ) content_length = response.headers.get("Content-Length") if content_length is not None: @@ -226,7 +274,44 @@ def http_card_text( ) -def _http_text_operation(connection, target, deadline, clock): +def http_raw_text( + url, + timeout=8.0, + connection_factory=None, + clock=time.monotonic, + completion_wait=None, +): + """Read one bounded intake file (config, index, or top-level .py) as text.""" + parsed = urllib.parse.urlsplit(url) + if parsed.scheme != "https" or parsed.hostname != HF_CARD_HOST: + raise ValueError("raw URL must use the fixed HTTPS host") + if parsed.port not in (None, 443): + raise ValueError("raw URL must use the default HTTPS port") + if parsed.username is not None or parsed.password is not None: + raise ValueError("raw URL must not contain credentials") + if parsed.query or parsed.fragment: + raise ValueError("raw URL must not contain a query or fragment") + if not _is_valid_raw_path(parsed.path): + raise ValueError("raw URL must target config.json, the safetensors index, or a .py file") + + deadline = clock() + timeout + remaining = _deadline_remaining(deadline, clock) + if connection_factory is None: + connection = http.client.HTTPSConnection(HF_CARD_HOST, 443, timeout=remaining) + else: + connection = connection_factory(HF_CARD_HOST, 443, remaining) + return _run_http_worker( + connection, + lambda: _http_text_operation( + connection, parsed.path, deadline, clock, max_bytes=RAW_TEXT_MAX_BYTES + ), + deadline, + clock, + completion_wait, + ) + + +def _http_text_operation(connection, target, deadline, clock, max_bytes=MODEL_CARD_MAX_BYTES): try: connection.request("GET", target, headers=UA) _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) @@ -237,8 +322,9 @@ def _http_text_operation(connection, target, deadline, clock): "redirect responses are not allowed for card requests" ) if not 200 <= response.status < 300: - raise http.client.HTTPException( - "card host returned HTTP status {0}".format(response.status) + raise HuggingFaceHTTPError( + response.status, + "card host returned HTTP status {0}".format(response.status), ) content_length = response.headers.get("Content-Length") if content_length is not None: @@ -246,20 +332,115 @@ def _http_text_operation(connection, target, deadline, clock): declared_length = int(content_length) except (TypeError, ValueError) as error: raise ValueError("invalid card Content-Length") from error - if declared_length < 0 or declared_length > MODEL_CARD_MAX_BYTES: + if declared_length < 0 or declared_length > max_bytes: raise ValueError("card response exceeds size limit") body = _read_bounded_body( response, connection, deadline, clock, - max_bytes=MODEL_CARD_MAX_BYTES, + max_bytes=max_bytes, ) return body.decode("utf-8", errors="replace") finally: connection.close() +def http_safetensors_header( + url, + timeout=8.0, + connection_factory=None, + clock=time.monotonic, + completion_wait=None, +): + """Read only the JSON header of one safetensors file through ranged GETs. + + The Hub answers with one redirect to its storage host; that redirect is + followed only to a Hub storage host over HTTPS, and every body is bounded + by the requested range, so no weights are downloaded. + """ + parsed = urllib.parse.urlsplit(url) + if parsed.scheme != "https" or parsed.hostname != HF_CARD_HOST: + raise ValueError("safetensors URL must use the fixed HTTPS host") + if parsed.port not in (None, 443): + raise ValueError("safetensors URL must use the default HTTPS port") + if parsed.username is not None or parsed.password is not None: + raise ValueError("safetensors URL must not contain credentials") + if parsed.query or parsed.fragment: + raise ValueError("safetensors URL must not contain a query or fragment") + if not _is_valid_resolve_path(parsed.path): + raise ValueError("safetensors URL must target a .safetensors file") + + deadline = clock() + timeout + + def ranged(host, target, start, end): + remaining = _deadline_remaining(deadline, clock) + if connection_factory is None: + connection = http.client.HTTPSConnection(host, 443, timeout=remaining) + else: + connection = connection_factory(host, 443, remaining) + return _run_http_worker( + connection, + lambda: _http_range_operation(connection, target, start, end, deadline, clock), + deadline, + clock, + completion_wait, + ) + + host, target = HF_CARD_HOST, parsed.path + status, location, body = ranged(host, target, 0, _SAFETENSORS_PROBE_BYTES - 1) + if location is not None: + redirected = urllib.parse.urlsplit(location) + if redirected.scheme != "https" or not _is_hub_storage_host(redirected.hostname): + raise ValueError("safetensors redirect must stay on a Hub storage host") + if redirected.port not in (None, 443) or redirected.username is not None or redirected.password is not None: + raise ValueError("safetensors redirect must use the default HTTPS port without credentials") + host = redirected.hostname + target = redirected.path + ("?" + redirected.query if redirected.query else "") + status, location, body = ranged(host, target, 0, _SAFETENSORS_PROBE_BYTES - 1) + if location is not None: + raise http.client.HTTPException("safetensors requests follow at most one redirect") + if len(body) < 8: + raise ValueError("safetensors file is shorter than its header length") + (length,) = struct.unpack(" SAFETENSORS_HEADER_MAX_BYTES: + raise ValueError("safetensors header exceeds size limit") + if len(body) < 8 + length: + status, location, rest = ranged(host, target, len(body), 8 + length - 1) + if location is not None: + raise http.client.HTTPException("safetensors requests follow at most one redirect") + body += rest + header = json.loads(body[8: 8 + length].decode("utf-8")) + if not isinstance(header, dict): + raise ValueError("safetensors header is not a JSON object") + return header + + +def _http_range_operation(connection, target, start, end, deadline, clock): + """One ranged GET: (status, redirect location or None, bounded body).""" + try: + headers = dict(UA) + headers["Range"] = "bytes={0}-{1}".format(start, end) + connection.request("GET", target, headers=headers) + _set_connection_timeout(connection, _deadline_remaining(deadline, clock)) + response = connection.getresponse() + _deadline_remaining(deadline, clock) + if 300 <= response.status < 400: + location = response.getheader("Location") + if not location: + raise http.client.HTTPException("redirect without a Location header") + return response.status, location, b"" + if response.status not in (200, 206): + raise HuggingFaceHTTPError( + response.status, + "safetensors host returned HTTP status {0}".format(response.status), + ) + body = _read_bounded_body(response, connection, deadline, clock, max_bytes=end - start + 1) + return response.status, None, body + finally: + connection.close() + + def _read_bounded_body(response, connection, deadline, clock, max_bytes=HF_RESPONSE_MAX_BYTES): chunks = [] total = 0 @@ -295,9 +476,12 @@ def _set_connection_timeout(connection, timeout): class HuggingFaceClient: - def __init__(self, http_get=http_json, card_get=http_card_text): + def __init__(self, http_get=http_json, card_get=http_card_text, raw_get=http_raw_text, + header_get=http_safetensors_header): self._http_get = http_get self._card_get = card_get + self._raw_get = raw_get + self._header_get = header_get @property def http_get(self): @@ -321,6 +505,33 @@ def fetch_dataset_card(self, repo, timeout=8): except Exception: return None + def fetch_model_info(self, repo, revision="main", timeout=8): + """Model API document with per-file sizes; raises on any failure.""" + quoted = "/".join(urllib.parse.quote(part) for part in repo.split("/")) + if revision == "main": + url = "{0}/{1}?blobs=true".format(HF_API, quoted) + else: + url = "{0}/{1}/revision/{2}?blobs=true".format( + HF_API, quoted, urllib.parse.quote(revision) + ) + return self._http_get(url, timeout=timeout) + + def fetch_raw_text(self, repo, revision, filename, timeout=8): + """One intake file as text; raises on any failure.""" + quoted = "/".join(urllib.parse.quote(part) for part in repo.split("/")) + url = "https://{0}/{1}/raw/{2}/{3}".format( + HF_CARD_HOST, quoted, urllib.parse.quote(revision), urllib.parse.quote(filename) + ) + return self._raw_get(url, timeout=timeout) + + def fetch_safetensors_header(self, repo, revision, filename, timeout=8): + """One weight file's tensor header (names, dtypes, shapes); raises on any failure.""" + quoted = "/".join(urllib.parse.quote(part) for part in repo.split("/")) + url = "https://{0}/{1}/resolve/{2}/{3}".format( + HF_CARD_HOST, quoted, urllib.parse.quote(revision), urllib.parse.quote(filename) + ) + return self._header_get(url, timeout=timeout) + @staticmethod def list_models_url(sort="trendingScore", limit_fetch=300): query = urllib.parse.urlencode({"filter": "mlx", "sort": sort, "direction": "-1", "limit": limit_fetch}) @@ -338,6 +549,17 @@ def list_adapters_url(search="", limit_fetch=20): params["search"] = search return "{0}?{1}".format(HF_API, urllib.parse.urlencode(params)) + @staticmethod + def list_mlx_conversions_url(repo, limit_fetch=8): + params = [ + ("filter", "base_model:quantized:{0}".format(repo)), + ("filter", "mlx"), + ("sort", "downloads"), + ("direction", "-1"), + ("limit", limit_fetch), + ] + return "{0}?{1}".format(HF_API, urllib.parse.urlencode(params)) + @staticmethod def list_datasets_url(search="", limit_fetch=20): params = { @@ -360,6 +582,10 @@ def list_adapters(self, search="", limit_fetch=20, timeout=10): ) return rows if isinstance(rows, list) else [] + def list_mlx_conversions(self, repo, limit_fetch=8, timeout=8): + """List MLX models quantized from a repository (read-only).""" + return self._http_get(self.list_mlx_conversions_url(repo, limit_fetch=limit_fetch), timeout=timeout) + def list_datasets(self, search="", limit_fetch=20, timeout=10): """List dataset rows from the Hub (read-only).""" rows = self._http_get( diff --git a/packages/codex/mlx-agent/src/mlx_agent/inference_runner.py b/packages/codex/mlx-agent/src/mlx_agent/inference_runner.py new file mode 100644 index 0000000..3e8d519 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/inference_runner.py @@ -0,0 +1,90 @@ +"""Run one inference runner inside its backend's environment and read the result it prints.""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import dataclass, field +from typing import Callable, Mapping, Tuple + +from .backends import BackendError, backend_environment, load_manifests, sync_ports +from .wiring import redact_secrets + +MAX_FAILURE_LINES = 6 +MAX_FAILURE_CHARS = 800 + + +class InferenceError(RuntimeError): + """Classified inference failure safe to surface in a result envelope.""" + + def __init__(self, code, message, remediation): + super().__init__(message) + self.code = code + self.remediation = remediation + + +@dataclass(frozen=True) +class RunnerContract: + """How one verb's runner reports a result, a refusal and a failure. + + Failures raise `error` with `_timeout` or `_failed`; `action` and refusal messages are + formatted with the plan, and every runner detail is redacted before it reaches a message. + """ + + verb: str + error: type + accept: Callable[[dict, dict], bool] + action: str + remediation: str + activity: str + timeout_remediation: str + refusals: Mapping[str, Callable[[dict, str], Tuple[str, str]]] = field(default_factory=dict) + environment: Mapping[str, str] = field(default_factory=dict) + + +def redacted(text): + """Runner text with credentials removed, cut to the last MAX_FAILURE_CHARS characters.""" + return redact_secrets(text)[-MAX_FAILURE_CHARS:] + + +def invoke(plan, contract, manifests=None, root=None, timeout=None, runner=subprocess.run): + """Run plan["argv"] for plan["backend"]; return the last printed object the contract accepts. + + A printed refusal wins over the exit status; an accepted object counts only when the runner exits 0. + """ + manifests = load_manifests() if manifests is None else manifests + try: + sync_ports(manifests[plan["backend"]], root) + except BackendError as error: + raise contract.error(error.code, str(error), error.remediation) from error + environment = backend_environment() + environment.update(contract.environment) + try: + completed = runner(plan["argv"], stdin=subprocess.DEVNULL, capture_output=True, text=True, + timeout=timeout, env=environment, check=False) + except subprocess.TimeoutExpired as error: + raise contract.error( + "{0}_timeout".format(contract.verb), "{0} exceeded {1} s.".format(contract.activity, timeout), + contract.timeout_remediation, + ) from error + for line in reversed((completed.stdout or "").splitlines()): + try: + value = json.loads(line) + except ValueError: + continue + if not isinstance(value, dict): + continue + refusal = contract.refusals.get(value.get("error")) + if refusal is not None: + message, remediation = refusal(plan, redacted(str(value.get("detail") or "no detail"))) + raise contract.error(value["error"], message, remediation) + if contract.accept(value, plan): + if completed.returncode == 0: + return value + break + tail = "\n".join((completed.stderr or "").strip().splitlines()[-MAX_FAILURE_LINES:]) or "no output" + raise contract.error( + "{0}_failed".format(contract.verb), + "The {0} backend could not {1}: {2}".format(plan["backend"], contract.action.format(**plan), redacted(tail)), + contract.remediation, + ) diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/installer.py b/packages/codex/mlx-agent/src/mlx_agent/installer.py similarity index 81% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/installer.py rename to packages/codex/mlx-agent/src/mlx_agent/installer.py index 06819e5..15bcdec 100644 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/installer.py +++ b/packages/codex/mlx-agent/src/mlx_agent/installer.py @@ -12,28 +12,59 @@ from .providers import detect_providers from .transactions import ( LegacyLockError, Receipt, Transaction, _assert_safe_directory, _assert_safe_target, - _atomic_in_directory, _read_regular, _read_target, _walk_directory, - _rollback_receipt_hash, legacy_lock_problem, rollback, + _atomic_in_directory, _physical_absolute, _read_regular, _read_target, _walk_directory, + _rollback_receipt_hash, legacy_lock_problem, load_receipt, rollback, ) -from .wiring import redact_secrets +from .wiring import contains_resolved_secrets, redact_secrets class InstallerConflictError(ValueError): """A requested operation would touch an unowned or modified artifact.""" +class InstallerPathError(ValueError): + """A destination root cannot be written through safely on this host.""" + + def __init__(self, message, remediation): + super().__init__(message) + self.remediation = remediation + + +def _first_symlinked_component(path): + """Return the first symlinked ancestor of an absolute path, or None. + + Writes walk every component with ``O_NOFOLLOW``, so a symlinked directory + anywhere above an artifact makes the whole install fail. Provider roots are + resolved before they reach here, so anything this finds is a root that + became a symlink afterwards — and finding it turns an ``[Errno 20]`` from + the descriptor walk into a diagnosis. + + The path is normalized through the same fixed ``/var`` alias handling the + descriptor walk uses, so the platform's own compatibility symlink is not + mistaken for one of the user's. + """ + try: + normalized = _physical_absolute(path) + except ValueError: + return None + cursor = Path(normalized.anchor or "/") + for component in normalized.parts[1:]: + cursor = cursor / component + try: + if cursor.is_symlink(): + return cursor, os.readlink(str(cursor)) + except OSError: + return None + return None + + class _ArtifactAdapter: version = "installer-1.0" def validate(self, content): if not isinstance(content, str): raise TypeError("installer artifacts must be UTF-8 text") - redacted = redact_secrets(content) - try: - contains_persisted_secret = json.loads(redacted) != json.loads(content) - except (TypeError, ValueError, json.JSONDecodeError): - contains_persisted_secret = redacted != content - if contains_persisted_secret: + if contains_resolved_secrets(content): raise ValueError("installer artifacts must not contain persisted secrets") return True @@ -132,8 +163,15 @@ def _read_optional_target(path): def _adapter_for_artifact(artifact): - is_runtime_source = "src" in artifact.source.parts and "mlx_agent" in artifact.source.parts - return _SourceCodeArtifactAdapter() if is_runtime_source else _ArtifactAdapter() + return _adapter_for_parts(artifact.source.parts) + + +def _adapter_for_target(path): + return _adapter_for_parts(Path(path).parts) + + +def _adapter_for_parts(parts): + return _SourceCodeArtifactAdapter() if "src" in parts and "mlx_agent" in parts else _ArtifactAdapter() class Installer: @@ -182,7 +220,10 @@ def plan(self, action, providers, scope="user", project_root=None): if len(selected) != len(set(selected)) or any(item not in definitions for item in selected): raise ValueError("providers must be unique supported provider IDs") if action == "doctor": + # Doctor's job is to report unhealthy layouts, including symlinked + # ones. Only mutating actions refuse them up front. return self._doctor_plan(selected, definitions, scope, project) + self._assert_writable_roots(selected, definitions, scope, project) if action == "uninstall": return self._uninstall_plan(selected, definitions, scope, project) return self._install_plan(action, selected, definitions, scope, project) @@ -269,10 +310,13 @@ def _install_plan(self, action, selected, definitions, scope, project): for provider_id in selected: definition = definitions[provider_id] changes = [] + owned = self._provider_receipts(definition, definitions, scope, project) + declared = set() for artifact in definition.artifacts: if not definition.applies_to(scope, artifact): continue target = definition.artifact_destination(scope, project, artifact) + declared.add(str(target)) desired_bytes, source_exists, _source_mode = _read_target(artifact.source) if not source_exists: raise ValueError("provider artifact source disappeared: {0}".format(artifact.source)) @@ -282,13 +326,14 @@ def _install_plan(self, action, selected, definitions, scope, project): raise ValueError("provider artifact source must be UTF-8") from error current = _read_optional_target(target) if current is not None: - history = self._artifact_history(definition, target, scope, project) + history = self._artifact_history(owned, target) if not history or _sha256(current) != history[-1].after_hashes[str(target)]: raise InstallerConflictError("refusing to overwrite unowned or modified artifact: {0}".format(target)) if current == desired.encode("utf-8"): continue adapter = _adapter_for_artifact(artifact) changes.append({"path": str(target), "content": desired, "adapter": adapter}) + changes.extend(self._stale_removals(owned, declared)) if changes: transaction = self._new_transaction(scope, project) preview = transaction.preview(changes) @@ -302,17 +347,32 @@ def _uninstall_plan(self, selected, definitions, scope, project): for provider_id in selected: definition = definitions[provider_id] provider_steps = [] - for artifact in definition.artifacts: - if not definition.applies_to(scope, artifact): - continue - target = definition.artifact_destination(scope, project, artifact) - history = self._artifact_history(definition, target, scope, project) + owned = self._provider_receipts(definition, definitions, scope, project) + targets = { + str(definition.artifact_destination(scope, project, artifact)) + for artifact in definition.artifacts if definition.applies_to(scope, artifact) + } + # A layout an earlier release installed and the current manifest no + # longer declares is still receipt-owned, so uninstall removes it. + targets.update(name for receipt in owned for name in receipt.targets) + for target_name in sorted(targets): + target = Path(target_name) + history = self._artifact_history(owned, target) if not history: continue oldest, latest = history[0], history[-1] - target_name = str(target) if oldest.backup_paths.get(target_name) is not None: raise InstallerConflictError("artifact was not receipt-owned at installation: {0}".format(target)) + if latest.after_modes[target_name] is None: + # Pruned by an update: absent by receipt. Rolling back the + # chain restores it, so compensation must remove it again. + if _read_optional_target(target) is not None: + raise InstallerConflictError("refusing to uninstall over a file recreated at a pruned path: {0}".format(target)) + restore_changes.setdefault(provider_id, []).append({ + "path": target_name, "remove": True, "adapter": _adapter_for_target(target), + }) + provider_steps.extend(history) + continue location = _assert_safe_target(target) content = _read_regular(location) if _sha256(content) != latest.after_hashes[target_name]: @@ -321,7 +381,7 @@ def _uninstall_plan(self, selected, definitions, scope, project): restore_changes.setdefault(provider_id, []).append({ "path": target_name, "content": content.decode("utf-8"), - "adapter": _adapter_for_artifact(artifact), + "adapter": _adapter_for_target(target), }) provider_steps.extend(history) for receipt in sorted({item.receipt_path: item for item in provider_steps}.values(), key=lambda item: item.timestamp, reverse=True): @@ -333,6 +393,25 @@ def _uninstall_plan(self, selected, definitions, scope, project): uninstall_hashes=uninstall_hashes, restore_changes=tuple((provider_id, tuple(changes)) for provider_id, changes in restore_changes.items()), summary=summary, definitions=definitions) + def _assert_writable_roots(self, selected, definitions, scope, project): + """Refuse early, and by name, when a destination root is a symlink.""" + for provider_id in selected: + root = definitions[provider_id].destination(scope, project) + found = _first_symlinked_component(root) + if found is None: + continue + location, target = found + raise InstallerPathError( + "{0} installs under {1}, but {2} is a symlink to {3}.".format( + provider_id, root, location, target + ), + "Artifacts are written through a no-follow descriptor walk, which " + "cannot traverse a symlinked directory. Point the install at the real " + "location instead — for user scope, export XDG_CONFIG_HOME (or " + "MLX_AGENT_HOME) to the directory the symlink resolves to — or replace " + "the symlink with a real directory.", + ) + def _doctor_plan(self, selected, definitions, scope, project): return self._make_plan("doctor", selected, scope, project, definitions=definitions) @@ -342,15 +421,17 @@ def _cleanup_empty_artifact_dirs(self, plan): for provider_id in plan.provider_ids: definition = definitions[provider_id] provider_root = definition.destination(plan.scope, plan.project_root) - for artifact in definition.artifacts: - if not definition.applies_to(plan.scope, artifact): - continue - destination = definition.artifact_destination(plan.scope, plan.project_root, artifact) + destinations = { + definition.artifact_destination(plan.scope, plan.project_root, artifact) + for artifact in definition.artifacts if definition.applies_to(plan.scope, artifact) + } + destinations.update(Path(name) for receipt in plan.rollback_receipts for name in receipt.targets) + for destination in sorted(destinations): try: relative = destination.relative_to(provider_root) except ValueError: # Some providers own companion artifacts (for example, - # Gemini command files) beneath a host-wide directory + # provider command files) beneath a host-wide directory # rather than their package directory. Removing empty # parents there could remove host-owned structure. continue @@ -369,7 +450,7 @@ def _cleanup_empty_artifact_dirs(self, plan): current = current.parent def _doctor(self, plan): - problems, checked, provider_states = [], [], [] + problems, warnings, checked, provider_states = [], [], [], [] definitions = self.registry.definitions() detections = { item.id: item for item in detect_providers( @@ -382,6 +463,7 @@ def _doctor(self, plan): for provider_id in plan.provider_ids: definition = definitions[provider_id] artifact_problems = [] + owned = self._provider_receipts(definition, definitions, plan.scope, plan.project_root) for artifact in definition.artifacts: if not definition.applies_to(plan.scope, artifact): continue @@ -391,7 +473,7 @@ def _doctor(self, plan): safe_root = _assert_safe_directory(plan.project_root if plan.scope == "project" else definition.user_root) location = _assert_safe_target(target) location.relative_to(safe_root) - history = self._artifact_history(definition, target, plan.scope, plan.project_root) + history = self._artifact_history(owned, target) if not history or _sha256(_read_regular(location)) != history[-1].after_hashes[str(target)]: raise ValueError("artifact hash is not receipt-owned") except (OSError, ValueError): @@ -410,6 +492,8 @@ def _doctor(self, plan): }) elif detection.state == "native-visible": integration_state = "native-visible" + if detection.warning: + warnings.append({"provider": provider_id, "code": "provider_untested", "message": detection.warning}) else: integration_state = "staged" code = "provider_unsupported" if detection.state == "unsupported" else "provider_absent" @@ -439,7 +523,7 @@ def _doctor(self, plan): problems.append({"provider": provider_id, "code": "legacy_lock_invalid", "message": str(error)}) continue problems.extend({"provider": provider_id, **item} for item in legacy_problem) - return {"healthy": not problems, "problems": problems, "checked": checked, + return {"healthy": not problems, "problems": problems, "warnings": warnings, "checked": checked, "providers": provider_states, "cooperative_concurrency": "Transaction advisory locks protect cooperative writers."} @@ -524,10 +608,57 @@ def _new_transaction(self, scope, project, transaction_id=None): def _operation_transaction_id(batch_id, child_index, provider_id): return str(uuid.uuid5(uuid.UUID(batch_id), "{0}:{1}".format(child_index, provider_id))) - def _artifact_history(self, definition, target, scope, project): - target_name = str(target) + def _provider_receipts(self, definition, definitions, scope, project): + """Applied receipts one provider owns, oldest first. + + A receipt is owned when every target is a declared artifact, or when the + provider's destination root is the most specific root that contains all + of its targets. The second rule keeps the paths of a layout the manifest + has since dropped under the receipts that installed them. + """ allowed = {str(definition.artifact_destination(scope, project, artifact)) for artifact in definition.artifacts if definition.applies_to(scope, artifact)} - return [item for item in self._all_receipts(scope, project) if target_name in item.targets and set(item.targets).issubset(allowed) and item.status == "applied"] + return [ + item for item in self._all_receipts(scope, project) + if item.status == "applied" and ( + set(item.targets).issubset(allowed) + or self._receipt_provider(item, definitions, scope, project) == definition.id + ) + ] + + @staticmethod + def _receipt_provider(receipt, definitions, scope, project): + targets = [Path(name) for name in receipt.targets] + best, depth = None, -1 + for provider_id, definition in definitions.items(): + root = definition.destination(scope, project) + if len(root.parts) > depth and all(root == target or root in target.parents for target in targets): + best, depth = provider_id, len(root.parts) + return best + + @staticmethod + def _artifact_history(owned, target): + target_name = str(target) + return [item for item in owned if target_name in item.targets] + + @staticmethod + def _stale_removals(owned, declared): + """Plan removal of receipt-owned files the current manifest no longer declares.""" + latest = {} + for receipt in owned: + for name in receipt.targets: + latest[name] = receipt + removals = [] + for name in sorted(latest): + receipt = latest[name] + if name in declared or receipt.after_modes[name] is None: + continue + current = _read_optional_target(Path(name)) + if current is None: + continue + if _sha256(current) != receipt.after_hashes[name]: + raise InstallerConflictError("refusing to remove user-modified artifact: {0}".format(name)) + removals.append({"path": name, "remove": True, "adapter": _adapter_for_target(name)}) + return removals def _all_receipts(self, scope, project): receipts_dir = self._receipts_dir(scope, project) @@ -548,7 +679,7 @@ def _load_receipt(self, path, scope, project): location = _assert_safe_target(path) if location.parent.parent != receipts_dir: raise ValueError("installer receipt escapes its scoped receipt directory") - return Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) + return load_receipt(location) @staticmethod def _receipt_identity(receipt): @@ -598,7 +729,11 @@ def _compensate_uninstall(self, changes, completed, scope, project): try: affected = {target for receipt in completed for target in receipt.targets} for _provider_id, provider_changes in changes: - provider_changes = [change for change in provider_changes if change["path"] in affected] + # A pruned path the completed rollbacks left absent is already in its pre-uninstall state. + provider_changes = [ + change for change in provider_changes + if change["path"] in affected and not (change.get("remove") and not expected[change["path"]]["exists"]) + ] if not provider_changes: continue expected_current = {change["path"]: expected[change["path"]] for change in provider_changes} diff --git a/packages/codex/mlx-agent/src/mlx_agent/intake.py b/packages/codex/mlx-agent/src/mlx_agent/intake.py new file mode 100644 index 0000000..9144374 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/intake.py @@ -0,0 +1,387 @@ +"""Resolve a pasted Hugging Face model into a conversion verdict (read-only).""" + +from __future__ import annotations + +import fnmatch +import http.client +import json +import math + +from .backends import ( + choose_backend, component_matches, load_manifests, load_registries, lookup, port_bits, port_files, + port_for_files, port_for_pipeline, quantize_rule, recipe_for, snapshot_files, +) +from .huggingface import HuggingFaceClient, HuggingFaceHTTPError +from .intake_fetch import FETCH_IGNORE_PATTERNS +from .intake_source import parse_hf_source, validate_revision +from .taxonomy import classify + +INTAKE_SCHEMA = "intake/1" +TRANSPORT_ERRORS = (OSError, TimeoutError, http.client.HTTPException, ValueError) +_BLOCKING_STATUSES = (401, 403, 404, 410) +ESTIMATE_BITS = (4, 8) +ESTIMATE_MAX_SHARDS = 16 +QUANT_GROUP_SIZE = 64 +_DTYPE_BYTES = { + "F64": 8, "F32": 4, "F16": 2, "BF16": 2, "F8_E4M3": 1, "F8_E5M2": 1, + "I64": 8, "I32": 4, "I16": 2, "I8": 1, "U8": 1, "BOOL": 1, +} +_SUPPORT_SUFFIXES = (".json", ".jinja", ".model", ".tiktoken", ".txt") + + +def _text(value): + return value if isinstance(value, str) else None + + +def summarize_files(info): + siblings = info.get("siblings") if isinstance(info.get("siblings"), list) else [] + names, gguf, python = set(), [], [] + total, safetensors, sizes = 0, 0, {} + for sibling in siblings: + if not isinstance(sibling, dict) or not isinstance(sibling.get("rfilename"), str): + continue + name = sibling["rfilename"] + size = sibling.get("size") + if not isinstance(size, int) or isinstance(size, bool) or size < 0: + size = 0 + names.add(name) + sizes[name] = size + total += size + lowered = name.lower() + if lowered.endswith(".gguf"): + gguf.append({"name": name, "bytes": size}) + elif lowered.endswith(".safetensors"): + safetensors += 1 + elif lowered.endswith(".py") and "/" not in name: + python.append(name) + return { + "names": names, + "sizes": sizes, + "bytes": total, + "summary": { + "safetensors": safetensors, + "gguf": sorted(gguf, key=lambda item: item["name"]), + "python": sorted(python), + }, + } + + +def fetch_config(client, repo, revision, warnings, prefix=""): + try: + value = json.loads(client.fetch_raw_text(repo, revision, prefix + "config.json")) + except TRANSPORT_ERRORS as error: + warnings.append("config.json unreadable: {0}".format(error)) + return {} + if not isinstance(value, dict): + warnings.append("config.json is not a JSON object") + return {} + return value + + +def _group_size(rule): + return (rule or {}).get("group_size", QUANT_GROUP_SIZE) + + +def _quantizes(name, shape, rule): + if len(shape) < 2 or shape[-1] % _group_size(rule): + return False + if rule is None: + return name.endswith(".weight") + return any(name.startswith(prefix) for prefix in rule.get("include", ())) and not any( + part in name for part in rule.get("exclude", ()) + ) + + +def estimate_output_bytes(client, repo, revision, files, rule, warnings, support_files=None, prefix=""): + """Converted size per bit width from the weight files' headers (no weights read). + + Quantized tensors cost bits/8 bytes per weight plus an affine scale and + bias per group (64, or the port's group size) in the weight's dtype; + everything else keeps its source size. Supporting files (config, + tokenizer) are added as listed: root-level files, or the non-weight + files a port's signature names. + """ + shards = sorted(name for name in files["names"] if name.endswith(".safetensors") and "/" not in name) + if not shards or len(shards) > ESTIMATE_MAX_SHARDS: + return None + if any(name.endswith(".pth") and "/" not in name for name in files["names"]): + # Root-level PyTorch pickles (a Wan checkpoint's text encoder and VAE) hold weights the headers do not describe. + return None + totals = {bits: 0 for bits in ESTIMATE_BITS} + for shard in shards: + try: + header = client.fetch_safetensors_header(repo, revision, prefix + shard) + except TRANSPORT_ERRORS as error: + warnings.append("output size estimate unavailable: {0} header unreadable: {1}".format(shard, error)) + return None + for name, tensor in header.items(): + if name == "__metadata__": + continue + shape = tensor.get("shape") if isinstance(tensor, dict) else None + width = _DTYPE_BYTES.get(tensor.get("dtype")) if isinstance(tensor, dict) else None + if width is None or not isinstance(shape, list) or not all( + isinstance(dim, int) and not isinstance(dim, bool) and dim >= 0 for dim in shape + ): + warnings.append("output size estimate unavailable: {0} has an unreadable tensor {1}".format(shard, name)) + return None + count = math.prod(shape) + for bits in totals: + if _quantizes(name, shape, rule): + totals[bits] += -(-count * bits // 8) + (count // _group_size(rule)) * 2 * width + else: + totals[bits] += count * width + if support_files is not None: + support = sum(files["sizes"].get(name, 0) for name in support_files if not name.endswith(".safetensors")) + else: + support = sum( + size for name, size in files["sizes"].items() + if "/" not in name and name.endswith(_SUPPORT_SUFFIXES) and name != "model.safetensors.index.json" + ) + return {str(bits): total + support for bits, total in totals.items()} + + +def components(model_type, config, manifests, registries): + result = [] + if model_type: + result.append({ + "role": "model", "config_key": None, "model_type": model_type, + "matches": component_matches(model_type, manifests, registries), + }) + for key in sorted(config): + value = config[key] + if key.endswith("_config") and isinstance(value, dict) and isinstance(value.get("model_type"), str): + result.append({ + "role": key[: -len("_config")], "config_key": key, "model_type": value["model_type"], + "matches": component_matches(value["model_type"], manifests, registries), + }) + return result + + +def scope_files(info, subfolder): + """The repository's file summary, or one subfolder's with names relative to it (GGUF names stay full).""" + if not subfolder: + return summarize_files(info) + prefix = subfolder + "/" + siblings = [ + dict(sibling, rfilename=sibling["rfilename"][len(prefix):]) + for sibling in (info.get("siblings") if isinstance(info.get("siblings"), list) else []) + if isinstance(sibling, dict) and isinstance(sibling.get("rfilename"), str) and sibling["rfilename"].startswith(prefix) + ] + files = summarize_files({"siblings": siblings}) + for item in files["summary"]["gguf"]: + item["name"] = prefix + item["name"] + return files + + +def _source_url(source): + url = "https://huggingface.co/{0}".format(source["repo"]) + if source.get("subfolder"): + url += "/tree/{0}/{1}".format(source["revision"], source["subfolder"]) + return url + + +def _empty_payload(text, source): + return { + "schema": INTAKE_SCHEMA, + "source": { + "input": text.strip(), "repo": source["repo"], "revision": source["revision"], + "file": source["file"], "subfolder": source.get("subfolder"), "url": _source_url(source), + }, + "verdict": "unknown", "reasons": [], "backend": None, "backend_installed": False, + "model_type": None, "components": [], "task": None, "custom_code": False, "gated": False, + "library_name": None, "pipeline_tag": None, "transformers_version": None, "bytes": 0, + "download_bytes": 0, "cached": False, "estimated_output_bytes": None, "q_bits": list(ESTIMATE_BITS), "recipe": None, + "files": {"safetensors": 0, "gguf": [], "python": []}, "mlx_conversions": [], "warnings": [], + } + + +def fetch_pipeline_class(client, repo, revision, warnings, prefix=""): + """``_class_name`` of a diffusers ``model_index.json``, or None.""" + try: + value = json.loads(client.fetch_raw_text(repo, revision, prefix + "model_index.json")) + except TRANSPORT_ERRORS as error: + warnings.append("model_index.json unreadable: {0}".format(error)) + return None + return _text(value.get("_class_name")) if isinstance(value, dict) else None + + +def recipe_download_bytes(client, recipe, files, warnings): + """The base snapshot (less ignored formats) plus the recipe's LoRA: what fetch downloads for a recipe.""" + lora = files["sizes"].get(recipe["lora"], 0) + try: + info = client.fetch_model_info(recipe["base"], revision=recipe["base_revision"]) + except (HuggingFaceHTTPError,) + TRANSPORT_ERRORS as error: + warnings.append("{0} size unavailable: {1}".format(recipe["base"], error)) + return lora + base = summarize_files(info if isinstance(info, dict) else {}) + return lora + download_bytes(base, None, {}) + + +def download_bytes(files, model_type, manifests): + """What ``intake fetch`` of the snapshot takes: a ported type's files, else every file but ignored formats.""" + ported = snapshot_files(model_type, manifests) + if ported: + return sum(files["sizes"].get(name, 0) for name in ported) + return sum( + size for name, size in files["sizes"].items() + if not any(fnmatch.fnmatch(name, pattern) for pattern in FETCH_IGNORE_PATTERNS) + ) + + +def _verdict(payload, tags, files, config, manifests, registries): + if payload["gated"]: + return "blocked", ["gated"], None + if payload["library_name"] == "mlx" or "mlx" in tags: + hits = lookup(payload["model_type"], manifests, registries) if payload["model_type"] else [] + backend = choose_backend(hits, payload["task"]["type"], "vision_config" in config) + return "already_mlx", [], backend + if files["summary"]["gguf"] and not files["summary"]["safetensors"]: + return "gguf", [], None + if not payload["model_type"]: + return "unsupported", ["no_config"], None + hits = lookup(payload["model_type"], manifests, registries) + backend = choose_backend(hits, payload["task"]["type"], "vision_config" in config) + if backend: + installed = registries.get(backend, {}).get("installed", False) + return ("convertible" if installed else "convertible_after_install"), [], backend + reasons = ["arch_not_in_registry"] + if payload["custom_code"]: + reasons.append("custom_code") + return "unsupported", reasons, None + + +MLX_CONVERSIONS_LIMIT = 8 + + +def list_mlx_conversions(client, repo, warnings): + """Existing MLX conversions of ``repo`` on the Hub, in the API's download order.""" + try: + rows = client.list_mlx_conversions(repo, limit_fetch=MLX_CONVERSIONS_LIMIT) + except (HuggingFaceHTTPError,) + TRANSPORT_ERRORS as error: + warnings.append("mlx conversions lookup failed: {0}".format(error)) + return [] + if not isinstance(rows, list): + warnings.append("mlx conversions lookup returned a non-list document") + return [] + found = [] + for row in rows: + name = row.get("id") if isinstance(row, dict) else None + if not isinstance(name, str) or not name or name == repo: + continue + downloads = row.get("downloads") + if not isinstance(downloads, int) or isinstance(downloads, bool) or downloads < 0: + downloads = 0 + found.append({ + "repo": name, "downloads": downloads, + "library_name": _text(row.get("library_name")), + }) + return found[:MLX_CONVERSIONS_LIMIT] + + +def snapshot_cached(repo, revision, names, prefix, model_type, manifests, hf_cache=None): + """True when the Hugging Face cache already holds every file ``intake fetch`` would download.""" + from fnmatch import fnmatch + + from .backends import snapshot_files + from .convert import cached_snapshot + from .intake_fetch import FETCH_IGNORE_PATTERNS + + snapshot = cached_snapshot(repo, hf_cache, revision=revision) if revision else None + if snapshot is None or (model_type and snapshot_files(model_type, manifests)): + return False + wanted = [name for name in names if not any(fnmatch(prefix + name, pattern) for pattern in FETCH_IGNORE_PATTERNS)] + return bool(wanted) and all((snapshot / (prefix + name)).is_file() for name in wanted) + + +def resolve(text, revision=None, client=None, manifests=None, registries=None): + source = parse_hf_source(text) + if revision: + source["revision"] = validate_revision(revision) + client = client or HuggingFaceClient() + manifests = load_manifests() if manifests is None else manifests + registries = load_registries(manifests) if registries is None else registries + payload = _empty_payload(text, source) + try: + info = client.fetch_model_info(source["repo"], revision=source["revision"]) + except HuggingFaceHTTPError as error: + if error.status in _BLOCKING_STATUSES: + payload.update(verdict="blocked", reasons=["not_found_or_private"]) + else: + payload.update(verdict="unknown", reasons=["hub_unreachable"]) + payload["warnings"].append(str(error)) + return payload + except TRANSPORT_ERRORS as error: + payload.update(verdict="unknown", reasons=["hub_unreachable"]) + payload["warnings"].append(str(error)) + return payload + if not isinstance(info, dict): + payload.update(verdict="unknown", reasons=["hub_unreachable"]) + payload["warnings"].append("the model API returned a non-object document") + return payload + + subfolder = source.get("subfolder") + prefix = subfolder + "/" if subfolder else "" + files = scope_files(info, subfolder) + tags = [tag for tag in info.get("tags") or [] if isinstance(tag, str)] + # The model card's config describes the repository root, not a subfolder checkpoint. + api_config = info.get("config") if isinstance(info.get("config"), dict) and not subfolder else {} + payload.update( + files=files["summary"], bytes=files["bytes"], gated=bool(info.get("gated")), + library_name=_text(info.get("library_name")), pipeline_tag=_text(info.get("pipeline_tag")), + ) + if subfolder and not files["names"]: + payload.update(verdict="unsupported", reasons=["subfolder_not_found"]) + return payload + config = {} + if "config.json" in files["names"] and not payload["gated"]: + config = fetch_config(client, source["repo"], source["revision"], payload["warnings"], prefix) + recipe = None if subfolder or payload["gated"] else recipe_for(source["repo"], manifests) + if recipe is not None and recipe["lora"] not in files["names"]: + payload["warnings"].append("{0} no longer ships {1}; its recipe does not apply".format(source["repo"], recipe["lora"])) + recipe = None + pipeline = None + if recipe is None and "model_index.json" in files["names"] and not payload["gated"]: + pipeline = port_for_pipeline( + fetch_pipeline_class(client, source["repo"], source["revision"], payload["warnings"], prefix), manifests, + ) + payload["model_type"] = ( + (recipe["port"] if recipe else None) or pipeline + or _text(config.get("model_type")) or _text(api_config.get("model_type")) + or (None if payload["gated"] else port_for_files(files["names"], manifests)) + ) + if recipe is not None: + payload["recipe"] = {key: recipe[key] for key in ("base", "base_revision", "lora", "lora_scale")} + payload["transformers_version"] = _text(config.get("transformers_version")) + payload["custom_code"] = ( + "custom_code" in tags or bool(config.get("auto_map")) or bool(api_config.get("auto_map")) + ) + payload["download_bytes"] = ( + recipe_download_bytes(client, recipe, files, payload["warnings"]) if recipe + else download_bytes(files, payload["model_type"], manifests) + ) + payload["cached"] = recipe is None and snapshot_cached( + source["repo"], _text(info.get("sha")), files["names"], prefix, payload["model_type"], manifests) + payload["components"] = components(payload["model_type"], config, manifests, registries) + payload["task"] = classify( + source["repo"], tags=tags, pipeline_tag=payload["pipeline_tag"], + model_type=payload["model_type"], config_keys=tuple(config), + manifests=manifests, registries=registries, local=False, + ) + verdict, reasons, backend = _verdict(payload, tags, files, config, manifests, registries) + payload.update( + verdict=verdict, reasons=reasons, backend=backend, + backend_installed=bool(backend and registries.get(backend, {}).get("installed")), + ) + if verdict != "already_mlx" and not subfolder: + payload["mlx_conversions"] = list_mlx_conversions(client, source["repo"], payload["warnings"]) + if verdict in ("convertible", "convertible_after_install"): + payload["q_bits"] = port_bits(payload["model_type"], backend, manifests) or list(ESTIMATE_BITS) + # A recipe's weights come from its base repository and a pipeline's from component folders, + # whose quantization the header estimate does not model: no estimate rather than a wrong one. + if verdict in ("convertible", "convertible_after_install") and not recipe and not pipeline: + payload["estimated_output_bytes"] = estimate_output_bytes( + client, source["repo"], source["revision"], files, + quantize_rule(payload["model_type"], backend, manifests), payload["warnings"], + support_files=port_files(payload["model_type"], backend, manifests), prefix=prefix, + ) + return payload diff --git a/packages/codex/mlx-agent/src/mlx_agent/intake_fetch.py b/packages/codex/mlx-agent/src/mlx_agent/intake_fetch.py new file mode 100644 index 0000000..7149d6f --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/intake_fetch.py @@ -0,0 +1,179 @@ +"""Confirmation-gated downloads of Hugging Face snapshots for intake.""" + +from __future__ import annotations + +import hashlib +import json +import re +import sys +from datetime import datetime, timezone +from pathlib import Path + +from .backends import backend_environment, load_manifests, recipe_for, snapshot_files, spawn_with_env +from .convert import _default_module_present, _write_receipt, receipts_root +from .intake_source import parse_hf_source, validate_file, validate_revision +from .serve import _pid_alive + +FETCH_RECEIPT_KIND = "fetch" +FETCH_RUNNER = Path(__file__).resolve().with_name("fetch_runner.py") +FETCH_MODULES = ("huggingface_hub",) +FETCH_IGNORE_PATTERNS = ( + "*.h5", "*.msgpack", "*.onnx", "*.onnx_data", "*.tflite", "*.ot", "*.mlmodel", "*.gguf", "*.mp4", + "onnx/*", "coreml/*", +) +_UNSAFE = re.compile(r"[^A-Za-z0-9._-]+") + + +class FetchError(RuntimeError): + def __init__(self, code, message, remediation): + super().__init__(message) + self.code = code + self.remediation = remediation + + +def _utc_now(): + return datetime.now(timezone.utc).isoformat() + + +def plan_fetch(text, revision=None, file=None, hf_cache=None, local_dir=None, python=None, model_type=None, + manifests=None): + """A download plan. A snapshot of a ported model type (``model_type`` from intake) takes only the + port's files; any other snapshot skips formats MLX never reads. A subfolder source takes only that folder.""" + source = parse_hf_source(text) + if revision: + source["revision"] = validate_revision(revision) + if file is not None: + source["file"] = validate_file(file) + local = None + if local_dir is not None: + local = Path(str(local_dir)).expanduser() + if not local.is_absolute(): + raise FetchError("invalid_arguments", "--local-dir must be an absolute path.", + "Pass the full destination directory.") + python = str(python or sys.executable) + argv = [python, str(FETCH_RUNNER), "--repo", source["repo"], "--revision", source["revision"]] + ignore_patterns, allow_patterns = [], [] + prefix = source["subfolder"] + "/" if source.get("subfolder") else "" + ported = recipe = None + if not source["file"] and model_type: + manifests = load_manifests() if manifests is None else manifests + ported = snapshot_files(model_type, manifests) + recipe = None if prefix else recipe_for(source["repo"], manifests) + recipe = recipe if recipe and recipe["port"] == model_type else None + if source["file"]: + argv += ["--file", source["file"]] + elif recipe: + # The recipe's model is its pinned base snapshot plus the LoRA this repository ships. + allow_patterns = [recipe["lora"]] + ignore_patterns = list(FETCH_IGNORE_PATTERNS) + argv += ["--allow", recipe["lora"], "--base-repo", recipe["base"], "--base-revision", recipe["base_revision"]] + for pattern in ignore_patterns: + argv += ["--base-ignore", pattern] + else: + if ported: + allow_patterns = [prefix + name for name in ported] + elif prefix: + allow_patterns = [prefix + "*"] + if not ported: + ignore_patterns = list(FETCH_IGNORE_PATTERNS) + for name in allow_patterns: + argv += ["--allow", name] + for pattern in ignore_patterns: + argv += ["--ignore", pattern] + if hf_cache: + argv += ["--cache-dir", str(hf_cache)] + if local is not None: + argv += ["--local-dir", str(local)] + slug = _UNSAFE.sub("-", "{0}--{1}".format( + source["repo"].replace("/", "--"), source["file"] or (source.get("subfolder") or "snapshot").replace("/", "--") + )).strip("-.")[:160] + plan = { + "repo": source["repo"], "revision": source["revision"], "file": source["file"], + "subfolder": source.get("subfolder"), + "base": {"repo": recipe["base"], "revision": recipe["base_revision"]} if recipe else None, + "cache_dir": str(hf_cache) if hf_cache else None, + "local_dir": str(local) if local is not None else None, + "ignore_patterns": ignore_patterns, + "allow_patterns": allow_patterns, + "slug": slug, "argv": argv, "network": ["huggingface.co"], + } + canonical = json.dumps(plan, sort_keys=True, separators=(",", ":")) + plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return plan + + +def _read_json(path): + try: + value = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError, TypeError): + return None + return value if isinstance(value, dict) else None + + +def start_fetch(plan, receipts_dir=None, confirm=False, preview_hash=None, spawn=None, + pid_alive=_pid_alive, module_present=None, now=_utc_now, env=None): + if not confirm: + return {"status": "preview", "plan": plan, "requires_confirmation": True} + if not preview_hash: + raise FetchError("preview_hash_required", "--confirm requires the hash from a reviewed fetch preview.", + "Run intake fetch without --confirm, review it, then pass --preview-hash.") + if preview_hash != plan["preview_hash"]: + raise FetchError("preview_stale", "The supplied preview hash does not match this fetch plan.", + "Re-run intake fetch without --confirm and review the fresh plan.") + missing = (module_present or _default_module_present)(FETCH_MODULES) + if missing: + raise FetchError("runtime_not_installed", + "Downloads need {0} in this interpreter.".format(", ".join(missing)), + "Run mlx-agent from the workbench .venv; fetch never installs runtimes.") + root = receipts_root(receipts_dir, kind=FETCH_RECEIPT_KIND) + receipt_path = root / "{0}.json".format(plan["slug"]) + existing = _read_json(receipt_path) + if existing and not existing.get("completed_at") and isinstance(existing.get("pid"), int) \ + and pid_alive(existing["pid"]) and not Path(existing.get("marker", "")).exists(): + raise FetchError("job_in_progress", "A download of {0} is still running.".format(plan["repo"]), + "Wait for it to finish (intake status).") + root.mkdir(parents=True, exist_ok=True) + marker = root / "{0}.done.json".format(plan["slug"]) + if marker.exists(): + marker.unlink() + log_path = root / "{0}.log".format(plan["slug"]) + argv = list(plan["argv"]) + ["--marker", str(marker)] + pid = (spawn or spawn_with_env)(argv, str(log_path), backend_environment(env)) + receipt = { + "schema_version": "1.0", "kind": FETCH_RECEIPT_KIND, "repo": plan["repo"], + "revision": plan["revision"], "file": plan["file"], "subfolder": plan.get("subfolder"), + "local_dir": plan["local_dir"], "cache_dir": plan["cache_dir"], "slug": plan["slug"], "argv": argv, "pid": pid, + "log_path": str(log_path), "marker": str(marker), "started_at": now(), + "preview_hash": plan["preview_hash"], "completed_at": None, "exit_status": None, + } + _write_receipt(root, receipt, receipt_path.name) + return {"status": "started", "receipt": receipt} + + +def status_fetch(receipts_dir=None, pid_alive=_pid_alive): + root = receipts_root(receipts_dir, kind=FETCH_RECEIPT_KIND) + entries = [] + if not root.is_dir(): + return entries + for path in sorted(root.glob("*.json")): + if path.name.endswith(".done.json"): + continue + receipt = _read_json(path) + if not receipt or receipt.get("kind") != FETCH_RECEIPT_KIND or not isinstance(receipt.get("pid"), int): + continue + entry = { + "receipt": str(path), "repo": receipt.get("repo"), "revision": receipt.get("revision"), + "file": receipt.get("file"), "subfolder": receipt.get("subfolder"), "local_dir": receipt.get("local_dir"), + "state": "failed", + "path": None, "log_path": receipt.get("log_path"), "started_at": receipt.get("started_at"), + "completed_at": None, + } + marker = _read_json(receipt.get("marker", "")) + if marker: + entry["state"] = "done" if marker.get("exit_status") == "done" else "failed" + entry["path"] = marker.get("path") + entry["completed_at"] = marker.get("finished_at") + elif pid_alive(receipt["pid"]): + entry["state"] = "running" + entries.append(entry) + return entries diff --git a/packages/codex/mlx-agent/src/mlx_agent/intake_source.py b/packages/codex/mlx-agent/src/mlx_agent/intake_source.py new file mode 100644 index 0000000..a89e393 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/intake_source.py @@ -0,0 +1,106 @@ +"""Parse a pasted Hugging Face model link or bare org/name id.""" + +from __future__ import annotations + +import re +import urllib.parse + +_HOSTS = frozenset({"huggingface.co", "www.huggingface.co", "hf.co"}) +_SEGMENT = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}") +_REVISION = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}") +_FILE_SEGMENT = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,254}") +_NON_MODEL_OWNERS = frozenset({ + "api", "blog", "collections", "datasets", "docs", "models", + "organizations", "papers", "settings", "spaces", +}) +_MAX_INPUT = 2048 + + +class IntakeSourceError(ValueError): + """A pasted source that is not a Hugging Face model reference.""" + + code = "invalid_source" + remediation = ( + "Paste a model link such as https://huggingface.co/org/name " + "(tree, blob, and resolve links work) or a bare org/name id; " + "org/name/folder or a tree link names a checkpoint in a subfolder." + ) + + +def validate_revision(text): + if not isinstance(text, str) or not _REVISION.fullmatch(text): + raise IntakeSourceError("Unsupported revision: {0!r}".format(text)) + return text + + +def validate_file(text): + if not isinstance(text, str) or not text.lower().endswith(".gguf"): + raise IntakeSourceError("Only .gguf files can be picked: {0!r}".format(text)) + parts = text.split("/") + if not all(_FILE_SEGMENT.fullmatch(part) for part in parts): + raise IntakeSourceError("Invalid file path: {0!r}".format(text)) + return text + + +def validate_subfolder(text): + parts = text.split("/") if isinstance(text, str) else [] + if not parts or not all(_FILE_SEGMENT.fullmatch(part) for part in parts): + raise IntakeSourceError("Invalid subfolder: {0!r}".format(text)) + return text + + +def parse_hf_source(text): + """Return {"repo", "revision", "file", "subfolder"}; file is set only for a .gguf link, subfolder for + a checkpoint below the repository root (bare org/name/folder, or a tree link into a folder).""" + if not isinstance(text, str): + raise IntakeSourceError("The source must be text.") + value = text.strip() + if ( + not value + or len(value) > _MAX_INPUT + or any(ord(character) < 32 or ord(character) == 127 for character in value) + ): + raise IntakeSourceError("The source is empty, too long, or contains control characters.") + lowered = value.lower() + host_prefixes = tuple(host + "/" for host in _HOSTS) + if "://" not in value and not lowered.startswith(host_prefixes): + return _from_segments(value.split("/"), bare=True, original=value) + if "://" not in value: + value = "https://" + value + parsed = urllib.parse.urlsplit(value) + host = (parsed.hostname or "").lower() + if parsed.scheme.lower() not in ("https", "http") or host not in _HOSTS: + raise IntakeSourceError("Not a Hugging Face model link: {0}".format(text.strip())) + if parsed.username is not None or parsed.password is not None: + raise IntakeSourceError("Links with credentials are refused.") + try: + port = parsed.port + except ValueError as error: + raise IntakeSourceError("The link has an invalid port.") from error + if port not in (None, 80, 443): + raise IntakeSourceError("The link has an unexpected port.") + segments = [urllib.parse.unquote(part) for part in parsed.path.split("/") if part] + return _from_segments(segments, bare=False, original=text.strip()) + + +def _from_segments(segments, bare, original): + if bare and len(segments) < 2: + raise IntakeSourceError("A bare id must be org/name or org/name/folder: {0}".format(original)) + if len(segments) < 2: + raise IntakeSourceError("The link does not name a model repository: {0}".format(original)) + owner, name = segments[0], segments[1] + if owner.lower() in _NON_MODEL_OWNERS: + raise IntakeSourceError("Datasets, Spaces, and site pages are not models: {0}".format(original)) + if not _SEGMENT.fullmatch(owner) or not _SEGMENT.fullmatch(name): + raise IntakeSourceError("Invalid repository id: {0}/{1}".format(owner, name)) + result = {"repo": "{0}/{1}".format(owner, name), "revision": "main", "file": None, "subfolder": None} + if len(segments) >= 4 and segments[2] in ("tree", "blob", "resolve"): + result["revision"] = validate_revision(segments[3]) + rest = segments[4:] + if segments[2] != "tree" and rest and rest[-1].lower().endswith(".gguf"): + result["file"] = validate_file("/".join(rest)) + elif segments[2] == "tree" and rest: + result["subfolder"] = validate_subfolder("/".join(rest)) + elif bare and len(segments) > 2: + result["subfolder"] = validate_subfolder("/".join(segments[2:])) + return result diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/interview.py b/packages/codex/mlx-agent/src/mlx_agent/interview.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/interview.py rename to packages/codex/mlx-agent/src/mlx_agent/interview.py diff --git a/packages/codex/mlx-agent/src/mlx_agent/jit_serve.py b/packages/codex/mlx-agent/src/mlx_agent/jit_serve.py new file mode 100644 index 0000000..5768e56 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/jit_serve.py @@ -0,0 +1,564 @@ +"""Loopback OpenAI gateway with one owned, lazily started local-model worker. + +No runtime imports or downloads. Inference goes to the existing recipe's +server process; a request lease lasts through the final streamed byte. +""" +from __future__ import annotations + +import argparse +import hashlib +import hmac +import http.client +import json +import math +import os +import re +import signal +import socket +import subprocess +import sys +import threading +import time +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from socketserver import TCPServer +from pathlib import Path + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from mlx_agent.transactions import _atomic_in_directory + +MAX_BODY = 16 * 1024 * 1024 +MAX_REQUESTS = 32 + + +def validate_memory_policy(policy=None): + defaults = {"idle_timeout_seconds": 0, "keep_loaded": False, "minimum_headroom_gb": None} + if policy is None: + return defaults + if not isinstance(policy, dict) or set(policy) - set(defaults): + raise ValueError("Unknown memory policy fields.") + result = dict(defaults, **policy) + idle, keep, reserve = (result[key] for key in defaults) + if type(idle) is not int or not 0 <= idle <= 86400: + raise ValueError("Idle timeout must be an integer between 0 and 86400 seconds.") + if type(keep) is not bool: + raise ValueError("Keep loaded must be a boolean.") + if reserve is not None and (type(reserve) not in (int, float) or not math.isfinite(reserve) or not 0 <= reserve <= 1024): + raise ValueError("Headroom reserve must be a finite value between 0 and 1024 GB.") + return result + + +def available_memory_bytes(): + """Fresh OS headroom, or unknown. Runs on the worker-loading thread.""" + try: + if sys.platform == "darwin": + output = subprocess.run(["/usr/bin/vm_stat"], capture_output=True, text=True, timeout=2, check=True).stdout + page_size = int(re.search(r"page size of (\d+) bytes", output)[1]) + pages = [int(re.search(r"^Pages {}:\s+(\d+)".format(name), output, re.M)[1]) + for name in ("free", "inactive")] + return sum(pages) * page_size + if sys.platform.startswith("linux"): + return int(re.search(r"^MemAvailable:\s+(\d+) kB", Path("/proc/meminfo").read_text(), re.M)[1]) * 1024 + except (OSError, ValueError, TypeError, subprocess.SubprocessError): + pass + return None + + +class GatewayError(RuntimeError): + def __init__(self, status, code, message): + super().__init__(message) + self.status, self.code = status, code + + +def model_fingerprint(path): + """File identity manifest, deliberately not a model-quality assessment.""" + root = Path(path) + rows = [] + for item in sorted(root.rglob("*")): + if any(part.startswith(".") for part in item.relative_to(root).parts): + continue + if item.is_file(): + stat = item.stat() + rows.append([str(item.relative_to(root)), str(item.resolve()), stat.st_dev, + stat.st_ino, stat.st_size, stat.st_mtime_ns]) + if len(rows) > 20000: + raise ValueError("Model directory has too many files.") + if not root.is_dir() or not rows: + raise ValueError("Local model files are unavailable.") + return hashlib.sha256(json.dumps(rows, separators=(",", ":")).encode()).hexdigest() + + +class ModelWorker: + def __init__(self, config): + self.config = config + self.condition = threading.Condition() + self.process = None + self.worker_port = None + self.worker_argv = None + self.model_state = "unloaded" + self.active_requests = 0 + self.last_error = None + self.closed = False + self.log = None + self.memory_policy = validate_memory_policy(config.get("memory_policy")) + self.memory_check = None + self.load_blocked_reason = None + self.last_unload_reason = None + self.last_activity = None + self.clock = time.monotonic + self.memory_probe = available_memory_bytes + + def _persist(self): + state_path = self.config.get("state_path") + if state_path: + path = Path(state_path) + if path.is_symlink(): + raise OSError("Refusing a symbolic-link worker state file.") + state = {"gateway_pid": os.getpid(), "worker_pid": self.process.pid if self.process else None, + "worker_argv": self.worker_argv, "model_state": self.model_state} + _atomic_in_directory(path.parent, path.name, json.dumps(state).encode(), 0o600) + + def _refresh(self): + if self.process and self.process.poll() is not None and self.model_state == "loaded": + self.process = None + self.model_state = "unloaded" + self._persist() + + def status(self): + with self.condition: + self._refresh() + return {"model_state": self.model_state, "active_requests": self.active_requests, + "worker_pid": self.process.pid if self.process and self.process.poll() is None else None, + "last_error": self.last_error, "memory_policy": dict(self.memory_policy), + "memory_check": self.memory_check, "load_blocked_reason": self.load_blocked_reason, + "last_unload_reason": self.last_unload_reason} + + def configure(self, changes): + with self.condition: + try: + policy = validate_memory_policy(dict(self.memory_policy, **changes)) + except (ValueError, TypeError) as error: + raise GatewayError(400, "invalid_policy", str(error)) from error + config = dict(self.config, memory_policy=policy) + if config.get("config_path"): + path = Path(config["config_path"]) + try: + if path.is_symlink(): + raise OSError("Refusing a symbolic-link JIT configuration.") + _atomic_in_directory(path.parent, path.name, json.dumps(config).encode(), 0o600) + except OSError as error: + raise GatewayError(503, "policy_not_saved", str(error)) from error + self.config.update(config) + self.memory_policy = policy + return self.status() + + def _check_memory(self): + with self.condition: + reserve = self.memory_policy["minimum_headroom_gb"] + self.load_blocked_reason = None + if reserve is None: + return + weights = sum(item.stat().st_size for item in Path(self.config["local_path"]).rglob("*") + if item.is_file() and item.suffix in (".safetensors", ".npz", ".bin", ".gguf")) + if weights and self.config.get("adapter_path") is not None: + weights += self.config["adapter_identity"]["adapters.safetensors"]["bytes"] + available = self.memory_probe() + required = math.ceil(weights * 1.10 + 1.5e9 + reserve * 1e9) if weights else None + if self.config.get("runtime") == "strata": + from mlx_agent.strata_runtime import read_model, memory_estimate + required = math.ceil(memory_estimate(read_model(self.config["local_path"]), + self.config.get("strata_expert_memory_gb"), self.config.get("strata_context_tokens", 131072)) + reserve * 1e9) + with self.condition: + self.memory_check = {"available_bytes": available, "required_available_bytes": required, + "captured_at": time.time(), "estimated": True} + if type(available) is not int or available < 0 or required is None: + self.load_blocked_reason = "Memory headroom or model weight size is unavailable." + raise GatewayError(503, "memory_unknown", self.load_blocked_reason) + if available < required: + self.load_blocked_reason = "Insufficient memory headroom for estimated weights, runtime allowance and reserve." + raise GatewayError(503, "insufficient_headroom", self.load_blocked_reason) + + def acquire(self): + with self.condition: + if not self.condition.wait_for(lambda: self.model_state != "unloading" or self.closed, timeout=10): + raise GatewayError(503, "model_unloading", "Model unload has not finished.") + if self.closed: + raise GatewayError(503, "endpoint_stopping", "Endpoint is stopping.") + self._refresh() + if self.active_requests >= MAX_REQUESTS: + raise GatewayError(503, "endpoint_busy", "Too many inference requests.") + self.active_requests += 1 + starter = self.model_state not in ("loaded", "loading") + if starter: + self.model_state = "loading" + self.last_error = None + try: + if starter: + self._start() + else: + with self.condition: + self.condition.wait_for(lambda: self.model_state != "loading" or self.closed, + timeout=self.config.get("worker_readiness_seconds", 60) + 5) + with self.condition: + if self.closed or self.model_state != "loaded": + raise GatewayError(503, "model_load_failed", self.last_error or "Model is not ready.") + return self.worker_port + except BaseException: + self.release() + raise + + def _check_adapter(self): + argv = self.config["worker_argv"] + path = self.config.get("adapter_path") + if path is None and "--adapter-path" not in argv: + return + from mlx_agent.lora import adapter_identity + if (not isinstance(path, str) or "--adapter-path" not in argv + or argv.index("--adapter-path") + 1 >= len(argv) + or argv[argv.index("--adapter-path") + 1] != path + or not self.config.get("adapter_identity") + or adapter_identity(path) != self.config["adapter_identity"]): + raise GatewayError(503, "adapter_changed", "Adapter files changed or have no reviewed identity. Stop and review a fresh serve plan.") + + def _check_training_receipt(self): + binding = self.config.get("training_binding") + if binding is None: + return + try: + from mlx_agent.training_artifacts import read_trained_adapter + _, current = read_trained_adapter(binding["training_receipt"]) + if current == binding: + return + except (OSError, ValueError, TypeError, KeyError): + pass + raise GatewayError(503, "training_receipt_changed", "The training receipt or its pinned artifacts changed. Stop and review a fresh serve plan.") + + def _start(self): + try: + if model_fingerprint(self.config["local_path"]) != self.config["fingerprint"]: + raise GatewayError(503, "model_changed", "Model files changed. Stop and review a fresh serve plan.") + self._check_training_receipt() + self._check_adapter() + self._check_memory() + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + port = reservation.getsockname()[1] + argv = [str(port) if value == "{worker_port}" else value for value in self.config["worker_argv"]] + environment = dict(os.environ, HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", HF_DATASETS_OFFLINE="1") + log = open(self.config["log_path"], "ab", buffering=0) if self.config.get("log_path") else subprocess.DEVNULL + with self.condition: + if self.closed: + raise GatewayError(503, "endpoint_stopping", "Endpoint is stopping.") + # Inherit the gateway's session: serve stop signals the whole + # owned group, including a worker when the gateway is killed. + self.process = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=log, + stderr=subprocess.STDOUT, env=environment) + self.log = log + self.worker_port, self.worker_argv = port, argv + self._persist() + process = self.process + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + deadline = time.monotonic() + self.config.get("worker_readiness_seconds", 60) + while time.monotonic() < deadline: + if process.poll() is not None or self.closed: + raise GatewayError(503, "model_load_failed", "Model worker exited before readiness.") + try: + with opener.open("http://127.0.0.1:{}/v1/models".format(port), timeout=1) as response: + if response.status == 200: + break + except OSError: + time.sleep(0.1) + else: + raise GatewayError(503, "model_load_failed", "Model worker did not become ready in 60 seconds.") + if model_fingerprint(self.config["local_path"]) != self.config["fingerprint"]: + raise GatewayError(503, "model_changed", "Model files changed while loading.") + self._check_training_receipt() + self._check_adapter() + with self.condition: + if self.closed: + raise GatewayError(503, "endpoint_stopping", "Endpoint is stopping.") + self.model_state = "loaded" + self._persist() + self.condition.notify_all() + except BaseException as error: + self._stop_worker() + with self.condition: + self.model_state = "failed" + self.last_error = str(error) + self._persist() + self.condition.notify_all() + if isinstance(error, GatewayError): + raise + raise GatewayError(503, "model_load_failed", str(error)) from error + + def release(self): + with self.condition: + self.active_requests -= 1 + if self.active_requests == 0: + self.last_activity = self.clock() + self.condition.notify_all() + + def _stop_worker(self): + with self.condition: + process = self.process + if process and process.poll() is None: + process.terminate() + try: + process.wait(timeout=3) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=3) + with self.condition: + self.process = None + if self.log not in (None, subprocess.DEVNULL): + self.log.close() + self.log = None + + def unload_if_idle(self): + with self.condition: + self._refresh() + policy = self.memory_policy + if (self.closed or self.model_state != "loaded" or self.active_requests or policy["keep_loaded"] + or not policy["idle_timeout_seconds"] or self.last_activity is None + or self.clock() - self.last_activity < policy["idle_timeout_seconds"]): + return False + self.model_state = "unloading" + self._finish_unload("idle") + return True + + def unload(self): + with self.condition: + if self.closed: + raise GatewayError(503, "endpoint_stopping", "Endpoint is stopping.") + if self.active_requests or self.model_state == "unloading": + raise GatewayError(409, "model_busy", "Finish active requests before unloading.") + self.model_state = "unloading" + self._finish_unload("manual") + return self.status() + + def _finish_unload(self, reason): + try: + self._stop_worker() + with self.condition: + self.model_state = "unloaded" + self.last_error = None + self.load_blocked_reason = None + self.last_unload_reason = reason + self._persist() + self.condition.notify_all() + except BaseException: + with self.condition: + self.model_state = "failed" + self.condition.notify_all() + raise + + def close(self): + with self.condition: + self.closed = True + self.condition.notify_all() + self._stop_worker() + with self.condition: + self.model_state = "unloaded" + self._persist() + + +class GatewayServer(ThreadingHTTPServer): + daemon_threads = True + block_on_close = False + + def __init__(self, address, config): + self.controller = ModelWorker(config) + self.config = config + self.slots = threading.BoundedSemaphore(MAX_REQUESTS + 4) + super().__init__(address, GatewayHandler) + + def server_bind(self): + # HTTPServer normally performs reverse DNS here. This gateway binds + # a numeric loopback address and must start without network discovery. + TCPServer.server_bind(self) + self.server_name, self.server_port = self.server_address[:2] + + def serve_forever(self, poll_interval=0.5): + stopped = threading.Event() + + def reclaim_idle(): + while not stopped.wait(0.5): + try: + self.controller.unload_if_idle() + except OSError: + # Persist/stop failures remain visible in controller state. + pass + + timer = threading.Thread(target=reclaim_idle, daemon=True) + timer.start() + try: + super().serve_forever(poll_interval) + finally: + stopped.set() + timer.join(timeout=7) + + def process_request(self, request, client_address): + if not self.slots.acquire(blocking=False): + request.close() + return + try: + super().process_request(request, client_address) + except BaseException: + self.slots.release() + raise + + def process_request_thread(self, request, client_address): + try: + super().process_request_thread(request, client_address) + finally: + self.slots.release() + + +class GatewayHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server_version = "MLX-JIT" + sys_version = "" + + def setup(self): + self.request.settimeout(120) + super().setup() + + def log_message(self, *args): + pass + + def _guard(self, control=False): + port = self.server.server_port + if self.headers.get("Host") not in ("127.0.0.1:{}".format(port), "localhost:{}".format(port)): + raise GatewayError(403, "invalid_host", "Use the loopback endpoint address.") + origin = self.headers.get("Origin") + if origin and origin not in ("http://127.0.0.1:{}".format(port), "http://localhost:{}".format(port)): + raise GatewayError(403, "invalid_origin", "Cross-origin access is refused.") + if control and not hmac.compare_digest(self.headers.get("Authorization", "").encode(), + ("Bearer " + self.server.config["control_token"]).encode()): + raise GatewayError(403, "control_denied", "Endpoint control authorization required.") + + def _json(self, status, value): + encoded = json.dumps(value).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.send_header("Connection", "close") + self.end_headers() + self.close_connection = True + self.wfile.write(encoded) + + def _error(self, error): + self._json(error.status, {"error": {"code": error.code, "message": str(error)}}) + + def do_GET(self): + try: + self._guard(control=self.path == "/_mlx/status") + if self.path == "/v1/models": + self._json(200, {"object": "list", "data": [{"id": self.server.config["model"], "object": "model"}]}) + elif self.path == "/health": + self._json(200, {"status": "ok"}) + elif self.path == "/_mlx/status": + self._json(200, self.server.controller.status()) + else: + raise GatewayError(404, "not_found", "Unknown endpoint.") + except GatewayError as error: + self._error(error) + + def do_POST(self): + connection = None + leased = False + response_started = False + try: + self._guard(control=self.path in ("/_mlx/unload", "/_mlx/policy")) + if self.headers.get("Transfer-Encoding"): + raise GatewayError(400, "invalid_body", "Use a bounded Content-Length request.") + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError: + raise GatewayError(400, "invalid_body", "Invalid Content-Length.") + if not 0 <= length <= MAX_BODY: + raise GatewayError(413, "body_too_large", "Request exceeds 16 MiB.") + try: + body = json.loads(self.rfile.read(length)) if length else {} + except (ValueError, UnicodeError): + raise GatewayError(400, "invalid_body", "Expected a JSON object.") + if not isinstance(body, dict): + raise GatewayError(400, "invalid_body", "Expected a JSON object.") + if self.path == "/_mlx/unload": + self._json(200, self.server.controller.unload()) + return + if self.path == "/_mlx/policy": + self._json(200, self.server.controller.configure(body)) + return + if self.path not in ("/v1/chat/completions", "/v1/completions"): + raise GatewayError(404, "not_found", "Unknown inference endpoint.") + aliases = {self.server.config["model"], self.server.config["local_path"], "default_model"} + local = Path(self.server.config["local_path"]) + aliases.add(local.name) + if local.parent.name == "snapshots" and local.parent.parent.name.startswith("models--"): + aliases.add(local.parent.parent.name[len("models--"):].replace("--", "/")) + requested = body.get("model", "default_model") + if not isinstance(requested, str) or requested not in aliases: + raise GatewayError(400, "model_not_allowed", "This endpoint serves only its configured local model.") + if any(body.get(key) not in (None, "default_model") for key in ("adapter_path", "adapter", "draft_model", "draft_adapter")): + raise GatewayError(400, "model_not_allowed", "Request-selected adapters and draft models are refused.") + tokens = body.get("max_tokens", self.server.config["max_tokens"]) + if not isinstance(tokens, int) or isinstance(tokens, bool) or tokens <= 0: + raise GatewayError(400, "invalid_body", "max_tokens must be a positive integer.") + body["max_tokens"] = min(tokens, self.server.config["max_tokens"]) + body["model"] = self.server.config["local_path"] + port = self.server.controller.acquire() + leased = True + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=120) + connection.request("POST", self.path, json.dumps(body), {"Content-Type": "application/json"}) + response = connection.getresponse() + self.send_response(response.status) + self.send_header("Content-Type", response.getheader("Content-Type", "application/json")) + self.send_header("Connection", "close") + self.end_headers() + response_started = True + self.close_connection = True + while chunk := response.read1(65536): + self.wfile.write(chunk) + self.wfile.flush() + except GatewayError as error: + self._error(error) + except (OSError, http.client.HTTPException): + if not response_started: + self._error(GatewayError(503, "worker_unavailable", "Model worker connection failed.")) + finally: + if connection: + connection.close() + if leased: + self.server.controller.release() + + +def make_server(config, port): + return GatewayServer(("127.0.0.1", port), config) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--port", required=True, type=int) + parser.add_argument("--config", required=True) + args = parser.parse_args() + path = Path(args.config) + if path.is_symlink(): + raise ValueError("Refusing a symbolic-link JIT configuration.") + config = json.loads(path.read_text()) + if args.model != config["model"]: + raise ValueError("JIT configuration identity changed.") + server = make_server(config, args.port) + for sig in (signal.SIGINT, signal.SIGTERM): + signal.signal(sig, lambda *_: threading.Thread(target=server.shutdown, daemon=True).start()) + try: + server.serve_forever() + finally: + server.controller.close() + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/packages/codex/mlx-agent/src/mlx_agent/kokoro_runtime.py b/packages/codex/mlx-agent/src/mlx_agent/kokoro_runtime.py new file mode 100644 index 0000000..c6b4a14 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/kokoro_runtime.py @@ -0,0 +1,41 @@ +"""Keep packaged Kokoro phonemizer data below eSpeak's path-length limit.""" + +from contextlib import contextmanager +import os +from pathlib import Path +from tempfile import TemporaryDirectory + + +@contextmanager +def kokoro_data_directory(): + import espeakng_loader + + data = Path(espeakng_loader.get_data_path()) + previous = os.environ.get("ESPEAK_DATA_PATH") + # eSpeak's native path buffer is 160 bytes. A private directory under the + # system's short temporary root avoids resolving a long venv path in C. + with TemporaryDirectory(prefix="mlx-espeak-", dir="/tmp") as directory: + (Path(directory) / "espeak-ng-data").symlink_to(data, target_is_directory=True) + os.environ["ESPEAK_DATA_PATH"] = directory + try: + yield + finally: + if previous is None: + os.environ.pop("ESPEAK_DATA_PATH", None) + else: + os.environ["ESPEAK_DATA_PATH"] = previous + + +def check_english(): + with kokoro_data_directory(): + import en_core_web_sm # noqa: F401 -- require the installed pipeline before Misaki can download it + from misaki import en, espeak + + g2p = en.G2P(trf=False, british=False, fallback=espeak.EspeakFallback(british=False)) + phonemes, _ = g2p("A local speech test.") + if not phonemes: + raise RuntimeError("The English phonemizer produced no phonemes") + + +if __name__ == "__main__": + check_english() diff --git a/packages/codex/mlx-agent/src/mlx_agent/lab.py b/packages/codex/mlx-agent/src/mlx_agent/lab.py new file mode 100644 index 0000000..877a0e8 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/lab.py @@ -0,0 +1,146 @@ +"""Optional experiment engine; the core CLI never imports its dependencies.""" +from __future__ import annotations + +from contextlib import nullcontext +import json +import os +from pathlib import Path +import selectors +import subprocess +import time +import sys + +ENGINE_ROOT = Path(__file__).resolve().parent / "_lab" +MAX_OUTPUT_BYTES = 8 * 1024 * 1024 + +def state_root(): + if sys.platform == "darwin": + return Path.home() / "Library/Application Support/mlx-agent/lab" + return Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local/state")) / "mlx-agent/lab" + +def _error(code, message, remediation="Install requirements-lab.txt in the selected Python environment."): + print(json.dumps({"schema": "lab/1", "status": "error", + "error": {"code": code, "message": message, + "remediation": remediation}})) + return 2 + +def _operation(arguments): + value_options = {"--database", "--artifacts", "--reviewer-id"} + iterator = iter(arguments) + for token in iterator: + if token in value_options: + next(iterator, None) + elif any(token.startswith(option + "=") for option in value_options): + continue + elif not token.startswith("-"): + return token + return None + + +def _run_child(command, env, timeout=86400, pass_fds=()): + """Bound both output streams while the engine is running.""" + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, pass_fds=pass_fds) + output = {process.stdout: bytearray(), process.stderr: bytearray()} + selector = selectors.DefaultSelector() + for stream in output: + selector.register(stream, selectors.EVENT_READ) + deadline = time.monotonic() + timeout + try: + while selector.get_map(): + if time.monotonic() >= deadline: + raise TimeoutError("Experiment operation timed out.") + for key, _ in selector.select(timeout=min(1, max(0, deadline - time.monotonic()))): + chunk = os.read(key.fileobj.fileno(), 65536) + if not chunk: + selector.unregister(key.fileobj) + continue + output[key.fileobj].extend(chunk) + if sum(len(value) for value in output.values()) > MAX_OUTPUT_BYTES: + raise OverflowError("Experiment output exceeded the supported size.") + return subprocess.CompletedProcess(command, process.wait(timeout=max(0.1, deadline - time.monotonic())), + output[process.stdout].decode("utf-8", errors="replace"), + output[process.stderr].decode("utf-8", errors="replace")) + finally: + selector.close() + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=3) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + for stream in output: + stream.close() + + +def run(argv, legacy=False): + arguments = list(argv) + if not arguments: + arguments = ["--help"] + if "--database" not in arguments: + arguments = ["--database", str(state_root() / "experiments.sqlite3")] + arguments + if "--artifacts" not in arguments: + arguments = ["--artifacts", str(state_root() / "artifacts")] + arguments + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join((str(ENGINE_ROOT), str(Path(__file__).resolve().parents[1]))) + env["PYTHONDONTWRITEBYTECODE"] = "1" + try: + module = { + "workspace": "mlx_agent.lab_workspace", + "queue": "mlx_agent.queue_cli", + "training-data": "video_lab.training_data_cli", + }.get(_operation(arguments), "video_lab.cli") + from .compute import ComputeLease, ComputeBusy + operation = _operation(arguments) + heavy = (operation == "worker" and any(item in arguments for item in ("run-once", "recover"))) or (operation == "adapter" and "probe" in arguments) or (operation == "adapter" and "feasibility" in arguments and "resume" in arguments) + with ComputeLease("lab-" + str(operation)) if heavy else nullcontext() as lease: + result = _run_child([sys.executable, "-m", module, *arguments], env, + pass_fds=(lease.descriptor,) if lease is not None else ()) + except ComputeBusy as error: + return _error("compute_busy", str(error), "Stop the owning model operation before retrying.") + except OverflowError as error: + return _error("lab_output_limit", str(error)) + except (TimeoutError, subprocess.TimeoutExpired): + return _error("lab_timeout", "Experiment operation timed out.") + except OSError: + return _error("lab_unavailable", "The experiment runtime could not start.") + if legacy or "--json" not in arguments: + sys.stdout.write(result.stdout) + sys.stderr.write(result.stderr) + return result.returncode + if len(result.stdout.encode()) > MAX_OUTPUT_BYTES: + return _error("lab_output_limit", "Experiment output exceeded the supported size.") + try: + payload = json.loads(result.stdout) + except ValueError: + return _error("lab_unavailable", "The experiment runtime returned no structured result.") + if not isinstance(payload, dict): + return _error("lab_invalid_output", "The experiment runtime returned an invalid result.") + success = payload.get("ok") is True or payload.get("status") == "ok" + data = payload.get("data") + if isinstance(data, list): + data = {"experiments": data} if "experiment" in arguments else {"items": data} + envelope = {"schema": "lab/1", "engine_schema": payload.get("schema_version"), + "status": "ok" if success and result.returncode == 0 else "error", + "data": data} + if envelope["status"] == "error": + envelope["error"] = payload.get("error") or {"code": "lab_failed", "message": "Experiment operation failed."} + print(json.dumps(envelope)) + return result.returncode if envelope["status"] == "ok" else (result.returncode or 1) + + +def worker_main(argv=None): + from .compute import ComputeLease, ComputeBusy + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join((str(ENGINE_ROOT), str(Path(__file__).resolve().parents[1]))) + env["PYTHONDONTWRITEBYTECODE"] = "1" + arguments = list(sys.argv[1:] if argv is None else argv) + try: + with ComputeLease("lab-worker") if "--help" not in arguments else nullcontext() as lease: + result = _run_child([sys.executable, "-m", "video_lab.worker", *arguments], env, + pass_fds=(lease.descriptor,) if lease is not None else ()) + except ComputeBusy as error: + return _error("compute_busy", str(error), "Stop the owning model operation before retrying.") + sys.stdout.write(result.stdout) + sys.stderr.write(result.stderr) + return result.returncode diff --git a/packages/codex/mlx-agent/src/mlx_agent/lab_adoption.py b/packages/codex/mlx-agent/src/mlx_agent/lab_adoption.py new file mode 100644 index 0000000..2f5f675 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/lab_adoption.py @@ -0,0 +1,193 @@ +"""Explicit, reversible adoption of a quiescent legacy experiment store.""" +from __future__ import annotations + +from contextlib import ExitStack, contextmanager +import ctypes +import hashlib +import json +import os +from pathlib import Path +import shutil +import stat +import sys +import uuid + +from video_lab.database import Database, SCHEMA_VERSION +from video_lab.local_authority import RetainedDirectoryAuthority, RetainedSqliteAuthority + + +def _paths(source_database, source_artifacts, destination): + paths = tuple(Path(value) for value in (source_database, source_artifacts, destination)) + if any(not path.is_absolute() for path in paths): + raise ValueError("Adoption requires absolute paths.") + database, artifacts, target = paths + if not database.is_file() or not artifacts.is_dir() or not target.parent.is_dir(): + raise ValueError("The source store and destination parent must exist.") + if target.exists() or target.is_symlink(): + raise ValueError("Adoption requires a new destination; existing stores are never replaced.") + if target == artifacts or artifacts in target.parents: + raise ValueError("The destination must be outside the source artifact store.") + return paths + + +def _preflight(database_path, artifact_root, destination, stack): + source = RetainedSqliteAuthority.capture(database_path) + stack.callback(source.close) + artifact_authority = RetainedDirectoryAuthority.capture(artifact_root) + stack.callback(artifact_authority.close) + with source.connect_readonly() as connection: + if connection.execute("SELECT max(version) FROM schema_migrations").fetchone()[0] > SCHEMA_VERSION: + raise ValueError("The source schema is newer than this engine.") + active = connection.execute( + "SELECT count(*) FROM jobs WHERE state NOT IN ('queued','completed','failed','cancelled')" + ).fetchone()[0] + active += connection.execute( + "SELECT count(*) FROM run_attempts WHERE status='active'" + ).fetchone()[0] + tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")} + if "training_runs" in tables: + active += connection.execute("SELECT count(*) FROM training_runs WHERE state='running'").fetchone()[0] + if active: + raise ValueError("Stop or recover active generation and training jobs before adoption.") + records = tuple(dict(row) for row in connection.execute("SELECT * FROM artifacts ORDER BY id")) + for record in records: + with _artifact(record, artifact_root): + pass + source.reverify() + artifact_authority.reverify() + material = {"schema": "lab-adoption/1", "source_database": str(database_path), + "source_database_digest": source.digest, "source_artifacts": str(artifact_root), + "destination": str(destination), "artifact_count": len(records), + "artifact_bytes": sum(record["byte_size"] for record in records), + "artifacts": [{key: record[key] for key in ("id","relative_path","sha256","byte_size","media_type")} + for record in records]} + encoded = json.dumps(material, sort_keys=True, separators=(",", ":")).encode() + plan = {key: value for key, value in material.items() if key != "artifacts"} + plan["preview_hash"] = hashlib.sha256(encoded).hexdigest() + return plan, source, artifact_authority, records + + + +def _identity(info): + return info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns + + +@contextmanager +def _artifact(record, root): + relative = Path(record["relative_path"]) + if relative.is_absolute() or not relative.parts or any(part in ("", ".", "..") for part in relative.parts): + raise ValueError("Artifact paths must remain inside their store.") + authority = RetainedDirectoryAuthority.capture(root / relative.parent) + try: + descriptor = authority.open_file(relative.name, os.O_RDONLY) + with os.fdopen(descriptor, "rb") as stream: + identity = _identity(os.fstat(stream.fileno())) + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode) or identity[2] != record["byte_size"]: + raise ValueError("Artifact size or file type changed.") + def verify(): + digest = hashlib.sha256() + stream.seek(0) + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + observed = os.stat(relative.name, dir_fd=authority.descriptor, follow_symlinks=False) + if digest.hexdigest() != record["sha256"] or _identity(observed) != identity or _identity(os.fstat(stream.fileno())) != identity: + raise ValueError("Artifact identity changed.") + authority.reverify() + verify() + stream.seek(0) + yield stream + verify() + finally: + authority.close() + + +def preview(source_database, source_artifacts, destination): + paths = _paths(source_database, source_artifacts, destination) + with ExitStack() as stack: + plan, *_ = _preflight(*paths, stack) + return plan + + +def _publish(parent, staged_name, name): + parent.reverify() + libc = ctypes.CDLL(None, use_errno=True) + if sys.platform == "darwin": + rename = libc.renameatx_np + flag = 4 # RENAME_EXCL: never replace a destination. + elif hasattr(libc, "renameat2"): + rename = libc.renameat2 + flag = 1 # RENAME_NOREPLACE + else: + raise ValueError("Atomic exclusive directory publication is unavailable on this host.") + rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] + rename.restype = ctypes.c_int + if rename(parent.descriptor, os.fsencode(staged_name), parent.descriptor, os.fsencode(name), flag): + number = ctypes.get_errno() + raise OSError(number, os.strerror(number)) + os.fsync(parent.descriptor) + parent.reverify() + + +def adopt(source_database, source_artifacts, destination, preview_hash): + paths = _paths(source_database, source_artifacts, destination) + source_path, source_root, target = paths + with ExitStack() as stack: + plan, source, artifacts, records = _preflight(*paths, stack) + if preview_hash != plan["preview_hash"]: + raise ValueError("Adoption preview changed; inspect a fresh preview.") + parent = RetainedDirectoryAuthority.capture(target.parent) + stack.callback(parent.close) + staged_name = ".lab-adoption-" + uuid.uuid4().hex + staging = target.parent / staged_name + os.mkdir(staged_name, 0o700, dir_fd=parent.descriptor) + retained = RetainedDirectoryAuthority.capture(staging) + identity = staging.stat().st_dev, staging.stat().st_ino + published = False + try: + backup = source.clone_to(retained, "before-adoption.sqlite3") + backup.close() + current = source.clone_to(retained, "experiments.sqlite3") + current.close() + (staging / "artifacts").mkdir(mode=0o700) + for record in records: + with _artifact(record, source_root) as handle: + output = staging / "artifacts" / record["relative_path"] + output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + authority = RetainedDirectoryAuthority.capture(output.parent) + try: + digest = hashlib.sha256() + with os.fdopen(authority.open_exclusive(output.name), "wb") as stream: + while chunk := handle.read(1024 * 1024): + stream.write(chunk) + digest.update(chunk) + stream.flush() + os.fsync(stream.fileno()) + authority.reverify() + if digest.hexdigest() != record["sha256"]: + raise ValueError("An artifact changed during adoption.") + finally: + authority.close() + Database(staging / "experiments.sqlite3").migrate() + with Database(staging / "experiments.sqlite3", read_only=True).connect() as connection: + if connection.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + raise ValueError("The adopted database failed its integrity check.") + source.reverify() + artifacts.reverify() + receipt = {**plan, "backup": str(target / "before-adoption.sqlite3"), + "store_path": str(target), "status": "adopted"} + with os.fdopen(retained.open_exclusive("adoption.json"), "w") as stream: + json.dump(receipt, stream, sort_keys=True) + stream.flush() + os.fsync(stream.fileno()) + retained.reverify() + retained.close() + _publish(parent, staged_name, target.name) + published = True + return receipt + finally: + retained.close() + if not published and staging.exists(): + parent.reverify() + info = staging.lstat() + if not staging.is_symlink() and (info.st_dev, info.st_ino) == identity: + shutil.rmtree(staging) diff --git a/packages/codex/mlx-agent/src/mlx_agent/lab_studies.py b/packages/codex/mlx-agent/src/mlx_agent/lab_studies.py new file mode 100644 index 0000000..33ac180 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/lab_studies.py @@ -0,0 +1,52 @@ +"""Reviewer-scoped native operations over the existing adaptive study service.""" +import hashlib +from video_lab.adaptive_study_models import AdaptiveStudyDefinition, adaptive_study_input_fingerprint +from video_lab.adaptive_study_repository import AdaptiveStudyRepository +from video_lab.adaptive_study_service import AdaptiveStudyService +from video_lab.domain import Manifest, DisplayPreference, utc_now +from video_lab.preference_optimizer import OptimizerConfig +from mlx_agent.compute import ComputeLease + +def execute(service, reviews, registry, reviewer, request, worker_factory): + repository = AdaptiveStudyRepository(service.experiments.database) + def bound(definition): + return AdaptiveStudyService(repository, service, reviews, + registry.capabilities(definition.base_manifest.adapter_id), reviewer_id=reviewer) + operation = request["operation"] + if operation == "study-list": + items = [] + for definition in repository.list(): + if repository.owner(definition.id) == reviewer: + items.append(bound(definition).show(definition.id).model_dump(mode="json")) + return {"studies": items} + if operation == "study-create": + manifest = Manifest.model_validate(request["manifest"]) + config = OptimizerConfig() + fingerprint = adaptive_study_input_fingerprint(manifest, config) + digest = hashlib.sha256(b"video-lab.adaptive-cli-study-id.v1\0" + reviewer.encode() + b"\0" + fingerprint.encode()).hexdigest() + definition = AdaptiveStudyDefinition(id="study_"+digest[:24], prompt=manifest.prompt, + negative_prompt=manifest.negative_prompt, seed=manifest.seed, base_manifest=manifest, + optimizer_config=config, input_fingerprint=fingerprint, created_at=utc_now()) + return {"study": bound(definition).create_study(definition).model_dump(mode="json")} + definition = repository.get(request["study_id"]) + study = bound(definition) + view = study.show(definition.id) # Verifies reviewer binding before any action. + if operation == "study-show": + return {"study": view.model_dump(mode="json")} + if operation == "study-resume": + return {"study": study.resume(definition.id).model_dump(mode="json")} + if operation == "study-submit": + return {"study": study.record_preference(definition.id, request["session_id"], + DisplayPreference(request["preference"])).model_dump(mode="json")} + if operation == "study-run": + if view.next_action != "complete_generation": + raise ValueError("Study has no generation pair awaiting execution.") + events = repository.list_events(definition.id) + experiment_id = events[-1].payload["experiment_id"] + with ComputeLease("adaptive-study-generation"): + worker = worker_factory() + for job in service.jobs.list_for_experiment(experiment_id): + if job.state.value == "queued": + worker.run_once(job.id) + return {"study": study.resume(definition.id).model_dump(mode="json")} + raise ValueError("Unknown adaptive operation.") diff --git a/packages/codex/mlx-agent/src/mlx_agent/lab_workspace.py b/packages/codex/mlx-agent/src/mlx_agent/lab_workspace.py new file mode 100644 index 0000000..5da56d3 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/lab_workspace.py @@ -0,0 +1,246 @@ +"""Native and agent projections over the same experiment services.""" +from __future__ import annotations + +import argparse +from dataclasses import asdict +import hashlib +import json +import os +from pathlib import Path +import re +import sqlite3 +import stat +import uuid +import sys + +from video_lab.adapter_registry import AdapterRegistry +from video_lab.artifacts import ArtifactStore, ArtifactError +from video_lab.database import Database +from video_lab.domain import DisplayPreference, ExperimentDraft, RubricScore +from video_lab.evidence_service import EvidenceService +from video_lab.feasibility_repository import FeasibilityRepository +from video_lab.repositories import ExperimentRepository, JobRepository +from video_lab.review_service import ReviewService +from video_lab.reviews import ReviewRepository +from video_lab.services import ExperimentService +from video_lab.worker import Worker +from mlx_agent.compute import ComputeLease +from video_lab.local_authority import RetainedDirectoryAuthority + +_FIELDS = { + "recipe-catalog": set(), "recipe-prepare": {"settings"}, + "adopt-preview": {"source_database", "source_artifacts", "destination"}, + "adopt": {"source_database", "source_artifacts", "destination", "preview_hash"}, + "study-list": set(), "study-create": {"manifest"}, + "study-show": {"study_id"}, "study-resume": {"study_id"}, "study-run": {"study_id"}, + "study-submit": {"study_id", "session_id", "preference"}, + "training-preview": {"receipt", "prompt", "max_tokens", "prompt_suite"}, + "training-evaluate": {"receipt", "prompt", "max_tokens", "prompt_suite", "preview_hash"}, + "training-review": {"evaluation_id", "artifact_sha256", "prompt_id", "preference", "notes"}, + "training-history": set(), "training-inspect": {"evaluation_id"}, + "snapshot": set(), "preview": {"draft"}, "create": {"draft"}, + "inspect": {"experiment_id"}, "run": {"job_id", "worker_id"}, + "cancel": {"job_id"}, "recover": {"worker_id"}, + "review-start": {"experiment_id"}, "review-media": {"session_id"}, + "review-submit": {"session_id", "preference", "scores", "notes"}, + "reveal": {"session_id"}, +} +_MAX_REQUEST = 1024 * 1024 + +def validate_request(request): + if not isinstance(request, dict) or request.get("operation") not in _FIELDS: + raise ValueError("Unknown workspace operation.") + operation = request["operation"] + if set(request) - (_FIELDS[operation] | {"operation"}): + raise ValueError("Unexpected workspace fields.") + for field in ("experiment_id", "job_id", "worker_id", "session_id", "study_id", "evaluation_id", "prompt_id"): + if field in request and (not isinstance(request[field], str) or not request[field].strip() + or not re.fullmatch(r"[A-Za-z0-9_-]{1,256}", request[field])): + raise ValueError("Invalid " + field) + required = { + "recipe-prepare": {"settings"}, + "adopt-preview": {"source_database", "source_artifacts", "destination"}, + "adopt": {"source_database", "source_artifacts", "destination", "preview_hash"}, + "study-create": {"manifest"}, + "study-show": {"study_id"}, "study-resume": {"study_id"}, "study-run": {"study_id"}, + "study-submit": {"study_id", "session_id", "preference"}, + "training-inspect": {"evaluation_id"}, + "training-preview": {"receipt"}, + "training-evaluate": {"receipt", "preview_hash"}, + "training-review": {"evaluation_id", "artifact_sha256", "prompt_id", "preference"}, + "inspect": {"experiment_id"}, "cancel": {"job_id"}, "run": {"job_id"}, + "review-start": {"experiment_id"}, "review-media": {"session_id"}, + "review-submit": {"session_id", "preference", "scores"}, "reveal": {"session_id"}, + }.get(operation, set()) + if not required <= set(request): + raise ValueError("Missing workspace fields.") + if operation in ("training-preview", "training-evaluate"): + from mlx_agent.training_evaluation import validate_suite, _prompt + if "prompt_suite" in request: + if "prompt" in request or "max_tokens" in request: + raise ValueError("Choose either a prompt suite or a single prompt.") + validate_suite(request["prompt_suite"]) + else: + _prompt(request.get("prompt"), request.get("max_tokens", 128)) + for field in ("source_database", "source_artifacts", "destination", "preview_hash"): + if field in request and (not isinstance(request[field], str) or not request[field] + or len(request[field]) > 4096 or any(ord(c) < 32 for c in request[field])): + raise ValueError("Invalid " + field) + if operation in ("preview", "create"): + ExperimentDraft.model_validate(request.get("draft")) + if operation == "recipe-prepare": + from mlx_agent.study_recipes import validate_settings + validate_settings(request["settings"]) + if operation == "review-submit": + DisplayPreference(request.get("preference")) + for item in request.get("scores", []): + RubricScore.model_validate(item) + return request + +def execute(database_path, artifact_root, reviewer, request): + request = validate_request(request) + if not reviewer.strip(): + raise ValueError("A reviewer identity is required.") + operation = request["operation"] + if operation.startswith("recipe-"): + from mlx_agent import study_recipes + if operation == "recipe-catalog": + return {"recipes": study_recipes.catalog()} + return {"manifest": study_recipes.prepare(request["settings"])} + if operation in ("adopt-preview", "adopt"): + from mlx_agent.lab_adoption import preview, adopt + arguments = [request["source_database"], request["source_artifacts"], request["destination"]] + return preview(*arguments) if operation == "adopt-preview" else adopt(*arguments, request["preview_hash"]) + service = ExperimentService.from_paths(database_path, artifact_root) + database = service.experiments.database + reviews = ReviewService(service.experiments, service.jobs, service.artifacts, ReviewRepository(database)) + registry = AdapterRegistry.with_defaults(feasibility_repository=FeasibilityRepository(database)) + evidence = EvidenceService(database, service.artifacts) + if operation.startswith("study-"): + from mlx_agent.lab_studies import execute as study_execute + return study_execute(service, reviews, registry, reviewer, request, lambda: Worker( + repository=service.jobs, artifact_store=service.artifacts, adapter_registry=registry, + scratch_root=artifact_root / ".scratch", worker_id="native-study-" + reviewer)) + if operation.startswith("training-"): + from mlx_agent import training_evaluation + if operation == "training-preview": + return {"plan": training_evaluation.preview(request["receipt"], request.get("prompt"), request.get("max_tokens", 128), request.get("prompt_suite"))} + if operation == "training-review": + return training_evaluation.review(database, service.artifacts, reviewer, request) + if operation == "training-history": + return {"evaluations": training_evaluation.history(database)} + if operation == "training-inspect": + return training_evaluation.inspect(database, service.artifacts, request["evaluation_id"]) + return training_evaluation.run(database, service.artifacts, request) + if operation == "snapshot": + history = evidence.list(reviewer_id=reviewer).model_dump(mode="json") + jobs = [] + for item in history["items"]: + for job in service.get(item["id"]).jobs: + attempts = service.jobs.attempts_for_job(job.id) + jobs.append({"id": job.id, "experiment_id": item["id"], + "state": job.state.value, "error": ("Generation failed; inspect operational evidence after review." if item["sealed"] else attempts[-1].error_message) if attempts and attempts[-1].error_message else None}) + return {"history": history, "queue": jobs} + if operation in ("preview", "create"): + draft = ExperimentDraft.model_validate(request["draft"]) + capabilities = registry.capabilities(draft.baseline.adapter_id) + pair = service.preview(draft, capabilities) + if operation == "preview": + return {"changed_paths": list(pair.changed_paths)} + view = service.create(draft, capabilities) + return {"experiment_id": view.experiment.id, "job_ids": [job.id for job in view.jobs]} + if operation == "inspect": + return evidence.detail(request["experiment_id"], reviewer).model_dump(mode="json") + if operation == "cancel": + job = service.jobs.request_cancellation(request["job_id"]) + return {"job_id": job.id, "state": job.state.value} + if operation in ("run", "recover"): + worker = Worker(repository=service.jobs, artifact_store=service.artifacts, + adapter_registry=registry, scratch_root=artifact_root / ".scratch", + worker_id=request.get("worker_id", "workbench")) + if operation == "recover": + with ComputeLease("lab-recovery"): + return {"jobs": [job.model_dump(mode="json") for job in worker.recover()]} + with ComputeLease("lab-generation"): + return asdict(worker.run_once(request.get("job_id"))) + if operation == "review-start": + view = reviews.start(request["experiment_id"], reviewer) + return {"session_id": view.session_id, "prompt": view.prompt} + if operation == "review-media": + outputs = {} + for position in ("left", "right"): + artifact = reviews.artifact_for_display(request["session_id"], position, reviewer) + parent = RetainedDirectoryAuthority.capture(artifact_root.absolute() / ".native-reviews") + parent.close() + folder = artifact_root.absolute() / ".native-reviews" / request["session_id"] + authority = RetainedDirectoryAuthority.capture(folder) + temporary = position + "-" + uuid.uuid4().hex + ".tmp" + target_name = position + ".mp4" + target = folder / target_name + try: + try: + target_stat = os.stat(target_name, dir_fd=authority.descriptor, follow_symlinks=False) + if not stat.S_ISREG(target_stat.st_mode): + raise ValueError("Review media must be a regular file.") + except FileNotFoundError: + pass + with database.connect() as connection: + with service.artifacts.open_verified_with_connection( + connection, artifact.id, sha256=artifact.sha256, + byte_size=artifact.byte_size, media_type=artifact.media_type, + ) as source: + digest = hashlib.sha256() + with os.fdopen(authority.open_exclusive(temporary), "wb") as stream: + while chunk := source.read(1024 * 1024): + stream.write(chunk) + digest.update(chunk) + stream.flush() + os.fsync(stream.fileno()) + service.artifacts.reverify_open_handle(connection, source) + if digest.hexdigest() != artifact.sha256: + raise ValueError("Review artifact changed.") + authority.reverify() + os.replace(temporary, target_name, src_dir_fd=authority.descriptor, + dst_dir_fd=authority.descriptor) + os.fsync(authority.descriptor) + authority.reverify() + finally: + try: + authority.unlink(temporary, missing_ok=True) + finally: + authority.close() + outputs[position] = str(target) + return {"session_id": request["session_id"], **outputs} + if operation == "review-submit": + result = reviews.submit(request["session_id"], reviewer, + DisplayPreference(request["preference"]), + tuple(RubricScore.model_validate(item) for item in request["scores"]), + request.get("notes")) + return result.model_dump(mode="json") + if operation == "reveal": + return reviews.reveal(request["session_id"], reviewer).model_dump(mode="json") + raise ValueError("Unsupported operation.") + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("--database", type=Path, required=True) + parser.add_argument("--artifacts", type=Path, required=True) + parser.add_argument("--reviewer-id", default="local-operator") + parser.add_argument("workspace", choices=["workspace"]) + parser.add_argument("--request", type=Path, required=True) + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + try: + if args.request.is_symlink() or not args.request.is_file() or args.request.stat().st_size > _MAX_REQUEST: + raise ValueError("Request must be a bounded regular file.") + request = validate_request(json.loads(args.request.read_text())) + data = execute(args.database, args.artifacts, args.reviewer_id, request) + print(json.dumps({"schema_version": "workspace.v1", "ok": True, "data": data})) + return 0 + except (ValueError, TypeError, KeyError, LookupError, PermissionError, OSError, sqlite3.Error, ArtifactError) as error: + print(json.dumps({"schema_version": "workspace.v1", "ok": False, + "error": {"code": "workspace_rejected", "message": str(error)}})) + return 2 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/src/mlx_agent/lora.py b/packages/codex/mlx-agent/src/mlx_agent/lora.py new file mode 100644 index 0000000..9abf3d8 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/lora.py @@ -0,0 +1,568 @@ +"""Confirmation-gated LoRA training jobs on the convert machinery.""" + +from __future__ import annotations + +import hashlib +import json +import os +import signal +import stat +import struct +import sys +import uuid +from datetime import datetime, timezone +from pathlib import Path + +from .convert import ( + _read_receipt, + _write_receipt, + receipts_root, +) +from .serve import _argv_matches, _pid_alive, _pid_command +from .wiring import _MODEL + + +LORA_RECEIPT_SCHEMA_VERSION = "1.0" +LORA_RECEIPT_KIND = "lora" +EXECUTABLE = "mlx_lm.lora" +ITERS_MIN, ITERS_MAX, ITERS_DEFAULT = 1, 100000, 1000 +BATCH_MIN, BATCH_MAX, BATCH_DEFAULT = 1, 64, 4 +LR_MIN, LR_MAX, LR_DEFAULT = 1e-6, 1e-2, 1e-5 +LAYERS_MIN, LAYERS_MAX, LAYERS_DEFAULT = -1, 128, 16 +MAX_DATASET_LINES = 10000 +MAX_DATASET_BYTES = 64 * 1024 * 1024 +MAX_RECORD_BYTES = 1024 * 1024 + + +class LoraError(RuntimeError): + """Classified lora failure safe to surface in a result envelope.""" + + def __init__(self, code, message, remediation): + super().__init__(message) + self.code = code + self.remediation = remediation + + +def _utc_now(): + return datetime.now(timezone.utc).isoformat() + + +def inspect_lora(repo=None, data=None, batch_size=BATCH_DEFAULT, hf_cache=None): + """Read independent prerequisites without a plan, receipt, download or model load.""" + from .convert import cached_snapshot + from .serve import _default_which + + _validate_hyperparameters(ITERS_DEFAULT, batch_size, LR_DEFAULT, LAYERS_DEFAULT) + checks = [] + + def check(identity, inspect): + item = {"id": identity} + try: + item.update(inspect()) + item.setdefault("status", "ready") + except LoraError as error: + item.update(status="blocked", code=error.code, message=str(error), remediation=error.remediation) + except (OSError, UnicodeError, ValueError) as error: + item.update(status="blocked", code="input_unreadable", message=str(error), + remediation="Choose readable local inputs, then check setup again.") + checks.append(item) + + def runtime(): + executable = _default_which(EXECUTABLE) + if executable is None: + raise LoraError("runtime_not_installed", "The mlx_lm.lora executable is not installed.", + "Install the training runtime with make install, then check setup again.") + return {"message": "Training executable found.", "path": str(executable)} + + def base(): + if not repo: + raise LoraError("repo_required", "Choose a base model.", "Select a cached language-model base or enter its repo id.") + if not isinstance(repo, str) or not _MODEL.fullmatch(repo): + raise LoraError("invalid_repo", "The base needs a publisher/model repo id.", "Enter the exact cached repo id.") + snapshot = cached_snapshot(repo, hf_cache) + if snapshot is None: + raise LoraError("model_not_local", "The base model is not in the local Hugging Face cache.", + "Cache the base model before training; checking setup never downloads it.") + identity = _training_base_identity(snapshot) + return {"message": "Cached architecture is supported by MLX-LM.", "path": str(snapshot), "identity": identity} + + def split(filename): + if not data: + raise LoraError("dataset_required", "Choose a dataset directory.", "Choose a directory with train.jsonl and valid.jsonl.") + path = Path(data) / filename + if not path.exists(): + code = "validation_required" if filename == "valid.jsonl" else "dataset_invalid" + raise LoraError(code, filename + " is missing.", "Provide separate training and validation examples.") + digest = _dataset_digest(path) + count = _validate_jsonl(path) + if _dataset_digest(path) != digest: + raise LoraError("dataset_changed", filename + " changed during inspection.", "Finish editing the dataset, then check setup again.") + result = {"examples": count, "sha256": digest, "path": str(path)} + if count < batch_size: + result.update(status="blocked", code="dataset_too_small", + message="{0} has {1} examples; batch size is {2}.".format(filename, count, batch_size), + remediation="Reduce batch size or add examples to this split, then check setup again.") + else: + result["message"] = "{0} examples; enough for batch size {1}.".format(count, batch_size) + return result + + check("runtime", runtime) + check("base", base) + for name, filename in (("train", "train.jsonl"), ("valid", "valid.jsonl")): + check(name, lambda filename=filename: split(filename)) + return {"schema_version": 1, "ready": all(item["status"] == "ready" for item in checks), + "repo": repo, "data": data, "batch_size": batch_size, "checks": checks} + + +def validate_dataset(data_dir): + """Validate an mlx_lm training dataset directory; bounded and read-only.""" + root = Path(data_dir) + train = root / "train.jsonl" + if not root.is_dir() or not train.is_file(): + raise LoraError( + "dataset_invalid", + "The dataset directory must contain train.jsonl: {0}".format(data_dir), + "Provide a directory with train.jsonl (text or messages per line).", + ) + summary = {"train_lines": _validate_jsonl(train), "valid_lines": 0} + valid = root / "valid.jsonl" + if valid.is_file(): + summary["valid_lines"] = _validate_jsonl(valid) + summary["sha256"] = {} + for name in ("train.jsonl", "valid.jsonl"): + path = root / name + if path.exists(): + summary["sha256"][name] = _dataset_digest(path) + return summary + + +def _dataset_digest(path): + if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_DATASET_BYTES: + raise LoraError("dataset_invalid", "Dataset files must be bounded regular files.", "Use a local dataset without links.") + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _validate_jsonl(path): + count = 0 + _dataset_digest(path) + try: + with open(path, "r", encoding="utf-8") as handle: + for line in iter(lambda: handle.readline(MAX_RECORD_BYTES + 1), ""): + count += 1 + if count > MAX_DATASET_LINES: + raise LoraError( + "dataset_invalid", + "{0} exceeds {1} lines.".format(path.name, MAX_DATASET_LINES), + "Split the dataset; lora validates bounded datasets only.", + ) + if len(line.encode("utf-8")) > MAX_RECORD_BYTES: + raise LoraError("dataset_invalid", "{0} line {1} exceeds the record size limit.".format(path.name, count), "Split oversized examples.") + stripped = line.strip() + if not stripped: + raise LoraError("dataset_invalid", "{0} line {1} is an empty record.".format(path.name, count), "Remove blank lines.") + try: + record = json.loads(stripped) + except json.JSONDecodeError: + raise LoraError( + "dataset_invalid", + "{0} line {1} is not valid JSON.".format(path.name, count), + "Fix the dataset line; every line must be one JSON object.", + ) + _validate_record(record, path.name, count) + except (OSError, UnicodeError) as error: + raise LoraError( + "dataset_invalid", + "The dataset could not be read: {0}".format(error), + "Pass a readable dataset directory.", + ) + if count == 0: + raise LoraError( + "dataset_invalid", + "{0} is empty.".format(path.name), + "Provide at least one training example.", + ) + return count + + +def _validate_record(record, name, line): + if not isinstance(record, dict): + raise LoraError( + "dataset_invalid", + "{0} line {1} must be a JSON object.".format(name, line), + "Use {\"text\": ...} or {\"messages\": [...]} per line.", + ) + text = record.get("text") + if isinstance(text, str) and text.strip(): + return + messages = record.get("messages") + if isinstance(messages, list) and messages: + for message in messages: + if ( + not isinstance(message, dict) + or not isinstance(message.get("role"), str) + or not isinstance(message.get("content"), str) + ): + raise LoraError( + "dataset_invalid", + "{0} line {1} has a malformed message.".format(name, line), + "Each message needs string role and content fields.", + ) + return + raise LoraError( + "dataset_invalid", + "{0} line {1} needs a non-empty text field or a messages array.".format(name, line), + "Use {\"text\": ...} or {\"messages\": [{\"role\": ..., \"content\": ...}]}.", + ) + + +def _validate_hyperparameters(iters, batch_size, learning_rate, num_layers): + if not isinstance(iters, int) or isinstance(iters, bool) or not ITERS_MIN <= iters <= ITERS_MAX: + raise LoraError( + "invalid_arguments", + "iters must be between {0} and {1}.".format(ITERS_MIN, ITERS_MAX), + "Pass a bounded --iters value.", + ) + if not isinstance(batch_size, int) or isinstance(batch_size, bool) or not BATCH_MIN <= batch_size <= BATCH_MAX: + raise LoraError( + "invalid_arguments", + "batch-size must be between {0} and {1}.".format(BATCH_MIN, BATCH_MAX), + "Pass a bounded --batch-size value.", + ) + if ( + not isinstance(learning_rate, (int, float)) + or isinstance(learning_rate, bool) + or not LR_MIN <= float(learning_rate) <= LR_MAX + ): + raise LoraError( + "invalid_arguments", + "learning-rate must be between {0} and {1}.".format(LR_MIN, LR_MAX), + "Pass a bounded --learning-rate value.", + ) + if not isinstance(num_layers, int) or isinstance(num_layers, bool) or not LAYERS_MIN <= num_layers <= LAYERS_MAX: + raise LoraError( + "invalid_arguments", + "num-layers must be between {0} and {1}.".format(LAYERS_MIN, LAYERS_MAX), + "Pass a bounded --num-layers value (-1 trains all layers).", + ) + + +def _training_base_identity(snapshot): + from .backends import BackendError, load_manifests, load_registries, lookup + from .jit_serve import model_fingerprint + from .taxonomy import classify, read_config_type + + model_type, keys = read_config_type(snapshot) + if not model_type: + raise LoraError("model_config_unavailable", "Training needs a readable local config.json with model_type.", + "Choose a complete cached base model, then preview again.") + if classify("", model_type=model_type, config_keys=keys, local=True)["type"] == "speculative_draft": + raise LoraError("unsupported_training_model", "A speculative drafter is not a standalone training base.", + "Choose the complete target language model instead.") + try: + manifests = {"mlx-lm": load_manifests()["mlx-lm"]} + registries = load_registries(manifests) + hits = lookup(model_type, manifests, registries) + except (BackendError, OSError, ValueError, KeyError) as error: + raise LoraError("training_backend_unavailable", "The MLX-LM model registry could not be read.", + "Restore the declared MLX-LM backend before previewing training.") from error + if not hits: + raise LoraError("unsupported_training_model", "MLX-LM does not declare the {0} model architecture.".format(model_type), + "Choose a cached language-model base recognized by MLX-LM; other backends are not used for text LoRA.") + return {"path": str(snapshot), "revision": snapshot.name, "fingerprint": model_fingerprint(snapshot), + "model_type": model_type, "loader": hits[0]["module"], "registry_source": registries["mlx-lm"]["source"]} + + +def plan_lora(repo, data, iters=ITERS_DEFAULT, batch_size=BATCH_DEFAULT, + learning_rate=LR_DEFAULT, num_layers=LAYERS_DEFAULT, out=None, hf_cache=None): + """Validate inputs and render the exact training plan; side-effect free.""" + if not isinstance(repo, str) or not _MODEL.fullmatch(repo): + raise LoraError( + "invalid_repo", + "lora requires a safe publisher/model base identifier.", + "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", + ) + _validate_hyperparameters(iters, batch_size, learning_rate, num_layers) + dataset = validate_dataset(data) + if not dataset["valid_lines"]: + raise LoraError( + "validation_required", "Training requires a nonempty valid.jsonl validation set.", + "Provide separate training and validation examples; the MLX-LM trainer requires both splits.", + ) + for filename, key in (("train.jsonl", "train_lines"), ("valid.jsonl", "valid_lines")): + if dataset[key] < batch_size: + raise LoraError( + "dataset_too_small", + "{0} has {1} examples, fewer than batch-size {2}.".format(filename, dataset[key], batch_size), + "Reduce --batch-size or add examples to this split, then preview again.", + ) + from .convert import cached_snapshot + snapshot = cached_snapshot(repo, hf_cache) + base = _training_base_identity(snapshot) if snapshot else None + if out is None: + out = "{0}-lora".format(repo.split("/", 1)[1]) + out = str(Path(out).expanduser().absolute()) + plan = { + "repo": repo, + "base_identity": base, + "data": str(data), + "iters": iters, + "batch_size": batch_size, + "learning_rate": float(learning_rate), + "num_layers": num_layers, + "out": str(out), + "dataset": dataset, + "argv": [ + EXECUTABLE, + "--model", repo, + "--train", + "--data", str(data), + "--iters", str(iters), + "--batch-size", str(batch_size), + "--learning-rate", repr(float(learning_rate)), + "--num-layers", str(num_layers), + "--adapter-path", str(out), + ], + } + canonical = json.dumps( + {key: plan[key] for key in sorted(plan) if key != "preview_hash"}, + sort_keys=True, separators=(",", ":"), + ) + plan["preview_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return plan + + +def _receipt_filename(plan): + return "{0}-lora.json".format(plan["repo"].split("/", 1)[1]) + + +def start_lora(plan, receipts_dir=None, confirm=False, preview_hash=None, + which=None, model_present=None, spawn=None, now=_utc_now, + pid_alive=None): + """Execute a reviewed training plan; the only mutating entry point.""" + from .serve import _default_spawn, _default_which + + which = which or _default_which + native_spawn = spawn is None + spawn = spawn or _default_spawn + pid_alive = pid_alive or _pid_alive + root = receipts_root(receipts_dir, LORA_RECEIPT_KIND) + + if not confirm: + return {"status": "preview", "plan": plan, "requires_confirmation": True} + if not preview_hash: + raise LoraError( + "preview_hash_required", + "--confirm requires the hash from a reviewed lora preview.", + "Run lora start without --confirm, inspect the plan, then pass --preview-hash.", + ) + if preview_hash != plan["preview_hash"]: + raise LoraError( + "preview_stale", + "The supplied preview hash does not match this lora plan.", + "Re-run lora start without --confirm and review the fresh plan.", + ) + if which(EXECUTABLE) is None: + raise LoraError( + "runtime_not_installed", + "The {0} executable is not installed.".format(EXECUTABLE), + "Install mlx-lm yourself (pip install mlx-lm); lora never installs runtimes.", + ) + if model_present is not None and not model_present(plan["repo"]): + raise LoraError( + "model_not_local", + "The base model is not present in the local Hugging Face cache.", + "Download it with the runtime's own pull command first; lora never downloads.", + ) + if Path(plan["out"]).exists(): + raise LoraError( + "output_exists", + "The adapter output path already exists: {0}".format(plan["out"]), + "Pick a fresh --out path; lora never overwrites.", + ) + for receipt_path in sorted(root.glob("*.json")) if root.is_dir() else []: + existing = _read_receipt(receipt_path, LORA_RECEIPT_KIND) + if existing is None or existing.get("completed_at"): + continue + if pid_alive(existing["pid"]): + raise LoraError( + "job_in_progress", + "A lora job is still running (pid {0}).".format(existing["pid"]), + "Wait for it to finish (lora status) before starting another.", + ) + + root.mkdir(parents=True, exist_ok=True) + run_id = uuid.uuid4().hex + name = "{0}-{1}-lora".format(plan["repo"].split("/", 1)[1], run_id) + log_path = root / (name + ".log") + result_path = root / (name + ".result") + if validate_dataset(plan["data"]) != plan["dataset"]: + raise LoraError("preview_stale", "Dataset changed after preview.", "Review a fresh training plan.") + if plan.get("base_identity"): + from .jit_serve import model_fingerprint + if model_fingerprint(plan["base_identity"]["path"]) != plan["base_identity"]["fingerprint"]: + raise LoraError("preview_stale", "Base model changed after preview.", "Review a fresh training plan.") + dataset_root = root / (name + ".dataset") + dataset_root.mkdir(mode=0o700) + for filename, digest in plan["dataset"]["sha256"].items(): + source = Path(plan["data"]) / filename + payload = source.read_bytes() + if hashlib.sha256(payload).hexdigest() != digest: + raise LoraError("preview_stale", "Dataset changed during snapshot.", "Review a fresh training plan.") + target = dataset_root / filename + target.write_bytes(payload) + target.chmod(0o600) + argv = list(plan["argv"]) + argv[argv.index("--data") + 1] = str(dataset_root) + if plan.get("base_identity"): + argv[argv.index("--model") + 1] = plan["base_identity"]["path"] + training_argv = list(argv) + if native_spawn: + argv = [sys.executable, str(Path(__file__).with_name("training_runner.py")), "--result", str(result_path), "--", *argv] + pid = spawn(argv, str(log_path)) + receipt = { + "schema_version": LORA_RECEIPT_SCHEMA_VERSION, + "kind": LORA_RECEIPT_KIND, + "run_id": run_id, + "dataset": plan["dataset"], + "dataset_snapshot": str(dataset_root), + "base_identity": plan.get("base_identity"), + "result_path": str(result_path), + "repo": plan["repo"], + "data": plan["data"], + "iters": plan["iters"], + "out": plan["out"], + "argv": training_argv, + "pid": pid, + "log_path": str(log_path), + "started_at": now(), + "preview_hash": plan["preview_hash"], + "completed_at": None, + "exit_status": None, + } + _write_receipt(root, receipt, name + ".json") + return {"status": "started", "receipt": receipt} + + +def status_lora(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, + now=_utc_now): + """Cross-check lora receipts against live processes; marks exits once.""" + root = receipts_root(receipts_dir, LORA_RECEIPT_KIND) + entries = [] + if not root.is_dir(): + return entries + for path in sorted(root.glob("*.json")): + receipt = _read_receipt(path, LORA_RECEIPT_KIND) + if receipt is None: + continue + entry = { + "receipt": str(path), + "repo": receipt["repo"], + "data": receipt.get("data"), + "out": receipt["out"], + "pid": receipt["pid"], + "log_path": receipt.get("log_path"), + "started_at": receipt.get("started_at"), + "completed_at": receipt.get("completed_at"), + } + if receipt.get("completed_at"): + entry["adapter_identity"] = receipt.get("adapter_identity") + entry["state"] = receipt.get("exit_status") or "done" + entries.append(entry) + continue + if pid_alive(receipt["pid"]): + command = pid_command(receipt["pid"]) + entry["state"] = "running" if _argv_matches( + {"argv": receipt["argv"], "port": None, "repo": None, "path": receipt.get("base_identity", {}).get("path") if receipt.get("base_identity") else receipt["repo"]}, + command, + require_port=False, + ) else "unknown" + entries.append(entry) + continue + result = None + if receipt.get("result_path"): + try: + candidate = Path(receipt["result_path"]) + if not candidate.is_symlink() and candidate.stat().st_size <= 4096: + result = json.loads(candidate.read_text()) + except (OSError, ValueError): + pass + if not isinstance(result, dict) or type(result.get("exit_code")) is not int: + entry["state"] = "unknown" + entries.append(entry) + continue + adapter = adapter_identity(receipt["out"]) if result["exit_code"] == 0 else None + exit_status = "done" if result["exit_code"] == 0 and adapter else "failed" + receipt["adapter_identity"] = adapter + receipt["process_exit_code"] = result["exit_code"] + entry["adapter_identity"] = adapter + receipt["completed_at"] = now() + receipt["exit_status"] = exit_status + _write_receipt(root, receipt, path.name) + entry["state"] = exit_status + entry["completed_at"] = receipt["completed_at"] + entries.append(entry) + return entries + + +def adapter_identity(output): + """Verify adapter tensor structure and return immutable content identities.""" + root = Path(output) + tensor = root / "adapters.safetensors" + config = root / "adapter_config.json" + try: + if root.is_symlink() or tensor.is_symlink() or config.is_symlink(): + return None + if not tensor.is_file() or not config.is_file() or config.stat().st_size > 1024 * 1024: + return None + json.loads(config.read_text()) + size = tensor.stat().st_size + with tensor.open("rb") as stream: + prefix = stream.read(8) + if len(prefix) != 8: + return None + length = struct.unpack(" 16 * 1024 * 1024 or length + 8 >= size: + return None + header = json.loads(stream.read(length)) + entries = [value for key, value in header.items() if key != "__metadata__"] + if not entries: + return None + for value in entries: + offsets = value["data_offsets"] + if len(offsets) != 2 or any(type(x) is not int for x in offsets): + return None + if not 0 <= offsets[0] < offsets[1] <= size - 8 - length: + return None + identities = {} + for path in (tensor, config): + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + identities[path.name] = {"sha256": digest.hexdigest(), "bytes": path.stat().st_size} + return identities + except (OSError, ValueError, TypeError, KeyError, AttributeError): + return None + + +def stop_lora(pid, receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, terminate=os.killpg): + root = receipts_root(receipts_dir, LORA_RECEIPT_KIND) + for path in sorted(root.glob("*.json")) if root.is_dir() else []: + receipt = _read_receipt(path, LORA_RECEIPT_KIND) + if receipt is None or receipt["pid"] != pid or receipt.get("completed_at"): + continue + if not pid_alive(pid): + raise LoraError("job_not_running", "Training is no longer running.", "Refresh training status.") + if not _argv_matches({"argv": receipt["argv"], "port": None, "repo": None, "path": receipt.get("base_identity", {}).get("path") if receipt.get("base_identity") else receipt["repo"]}, + pid_command(pid), require_port=False) or os.getpgid(pid) != pid: + raise LoraError("process_mismatch", "The training process identity changed.", "Inspect the retained receipt before stopping it.") + terminate(pid, signal.SIGTERM) + receipt["completed_at"] = _utc_now() + receipt["exit_status"] = "cancelled" + _write_receipt(root, receipt, path.name) + return {"status": "cancelled", "receipt": str(path)} + raise LoraError("receipt_not_found", "No active training receipt matches this PID.", "Refresh training status.") diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/modality.py b/packages/codex/mlx-agent/src/mlx_agent/modality.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/modality.py rename to packages/codex/mlx-agent/src/mlx_agent/modality.py diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/model_doctor.py b/packages/codex/mlx-agent/src/mlx_agent/model_doctor.py similarity index 96% rename from packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/model_doctor.py rename to packages/codex/mlx-agent/src/mlx_agent/model_doctor.py index b10511a..19cc504 100644 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/model_doctor.py +++ b/packages/codex/mlx-agent/src/mlx_agent/model_doctor.py @@ -17,6 +17,7 @@ import urllib.request from pathlib import Path +from .transactions import replay_step_journal from .verification import _safe_error, installed_model_ids from .wiring import ( _OLLAMA, @@ -42,7 +43,14 @@ def default_hf_cache(): - return Path.home() / ".cache" / "huggingface" / "hub" + configured = os.environ.get("HF_HUB_CACHE") or os.environ.get("HUGGINGFACE_HUB_CACHE") + if configured: + return Path(configured).expanduser() + home = os.environ.get("HF_HOME") + if home: + return Path(home).expanduser() / "hub" + cache = Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache").expanduser() + return cache / "huggingface" / "hub" def inspect_hf_cache(cache_dir): @@ -251,6 +259,10 @@ def scan_receipt_after_hashes(roots): value = json.loads(receipt.read_text(encoding="utf-8")) except (OSError, ValueError, json.JSONDecodeError): continue + try: + value = replay_step_journal(value, receipt.parent) + except (OSError, ValueError): + pass after_hashes = value.get("after_hashes") if not isinstance(after_hashes, dict): continue diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/models.py b/packages/codex/mlx-agent/src/mlx_agent/models.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/models.py rename to packages/codex/mlx-agent/src/mlx_agent/models.py diff --git a/packages/codex/mlx-agent/src/mlx_agent/music.py b/packages/codex/mlx-agent/src/mlx_agent/music.py new file mode 100644 index 0000000..b06dafb --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/music.py @@ -0,0 +1,83 @@ +"""Generate music from existing local weights in the isolated audio backend.""" + +import json +import math +import subprocess +from pathlib import Path + +from .backends import backend_python, is_installed, load_manifests, load_registries, lookup +from .inference_runner import InferenceError, RunnerContract, invoke +from .music_runner import tokenizer_directory + +MUSIC_RUNNER = Path(__file__).with_name("music_runner.py") + + +class MusicError(InferenceError): + """Classified music-generation failure safe to surface in a result envelope.""" + + +def validate_parameters(caption, lyrics, duration=15.0, steps=30, seed=42): + """Validate generation settings without accessing weights or a backend.""" + for name, text, limit in (("caption", caption, 2000), ("lyrics", lyrics, 10000)): + if not isinstance(text, str) or not text.strip() or len(text) > limit or any(ord(c) < 32 and c not in "\n\t" for c in text): + raise MusicError("invalid_arguments", "Invalid {0}.".format(name), "Provide a caption and lyrics; use [instrumental] for instrumental music.") + if isinstance(duration, bool) or not isinstance(duration, (float, int)) or not math.isfinite(duration) or not 0 < duration <= 360: + raise MusicError("invalid_arguments", "Duration must be greater than 0 and at most 360 seconds.", "Use --duration 15.") + if type(steps) is not int or not 1 <= steps <= 30 or type(seed) is not int or not 0 <= seed <= 2**32 - 1: + raise MusicError("invalid_arguments", "Steps or seed is out of range.", "Use 1..30 steps and a 32-bit unsigned seed.") + return {"caption": caption, "lyrics": lyrics, "duration": duration, "steps": steps, "seed": seed} + + +def plan_music(model_path, caption, lyrics, out, duration=15.0, steps=30, seed=42, + manifests=None, registries=None, root=None): + model = Path(model_path).expanduser() + try: + config = json.loads((model / "config.json").read_text(encoding="utf-8")) + if not model.is_absolute() or not isinstance(config, dict): + raise ValueError("An absolute local model directory is required") + except (OSError, ValueError) as error: + raise MusicError("model_unreadable", str(error), "Pass an existing converted music model directory.") from error + validate_parameters(caption, lyrics, duration, steps, seed) + target = Path(out).expanduser() + if not target.is_absolute() or target.suffix.lower() != ".wav" or target.exists() or target.is_symlink() or not target.parent.is_dir(): + raise MusicError("invalid_arguments", "Output must be a new absolute WAV path.", "Choose a new .wav file in an existing directory.") + manifests = load_manifests() if manifests is None else manifests + registries = load_registries(manifests, root) if registries is None else registries + hits = [hit for hit in lookup(config.get("model_type", ""), manifests, registries) if hit["category"] == "music_generation"] + if not hits: + raise MusicError("not_music_model", "No music backend implements this model type.", "Use a supported converted music generation model.") + if config.get("model_type") == "minimax_music3": + try: + tokenizer_directory(model) + except ValueError as error: + raise MusicError("model_unreadable", str(error), "Restore the checkpoint's tokenizer files before generating music.") from error + manifest = manifests[sorted(hits, key=lambda hit: hit["backend"])[0]["backend"]] + if not is_installed(manifest, root): + raise MusicError("backend_not_installed", "The audio backend is not installed.", "Install mlx-audio first.") + return {"backend": manifest["id"], "model": str(model), "out": str(target), + "duration": duration, "steps": steps, "seed": seed, + "argv": [str(backend_python(manifest, root)), str(MUSIC_RUNNER), "--model", str(model), + "--caption=" + caption, "--lyrics=" + lyrics, "--out", str(target), + "--duration", str(duration), "--steps", str(steps), "--seed", str(seed)]} + + +def _wrote_output(value, plan): + target = Path(plan["out"]) + return value.get("path") == plan["out"] and target.is_file() and not target.is_symlink() + + +CONTRACT = RunnerContract( + verb="music", error=MusicError, accept=_wrote_output, action="generate the music", + remediation="Check local model completeness and the audio backend.", + activity="Music generation", timeout_remediation="Use a shorter duration or a larger timeout.", + # Generation never resolves or downloads missing weights or tokenizer files. + environment={"HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1"}, +) + + +def run_music(plan, manifests=None, root=None, timeout=3600, runner=subprocess.run): + result = invoke(plan, CONTRACT, manifests, root, timeout, runner) + keys = ("path", "sample_rate", "audio_seconds", "seconds", "load_seconds", "real_time_factor", "peak_memory_gb") + return dict({"schema": "music/1", "backend": plan["backend"], "model": plan["model"], + "duration": plan["duration"], "steps": plan["steps"], "seed": plan["seed"]}, + **{key: result.get(key) for key in keys}) diff --git a/packages/codex/mlx-agent/src/mlx_agent/music_requantize_runner.py b/packages/codex/mlx-agent/src/mlx_agent/music_requantize_runner.py new file mode 100644 index 0000000..1d1928f --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/music_requantize_runner.py @@ -0,0 +1,71 @@ +"""Requantize an exported local MLX music checkpoint without fetching weights. + +The upstream raw MiniMax converter expects component checkpoints. Exported +MLX weights instead go through the music loader and its quantization policy. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import shutil + + +def copy_supporting_files(source, destination): + """Keep tokenizer/scheduler assets; never duplicate checkpoint weights/code.""" + extensions = {".json", ".txt", ".jinja", ".model", ".tiktoken", ".yaml", ".yml"} + for item in source.rglob("*"): + relative = item.relative_to(source) + if len(relative.parts) > 1 and relative.parts[0] not in {"tokenizer", "scheduler"}: + continue + if (item.suffix not in extensions or item.name in {"config.json", "model.safetensors.index.json"} + and len(relative.parts) == 1): + continue + if item.is_file(): + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(item, target) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--hf-path", required=True, type=Path) + parser.add_argument("--mlx-path", required=True, type=Path) + parser.add_argument("--quantize", required=True, action="store_true") + parser.add_argument("--q-bits", required=True, type=int, choices=(4, 8)) + args = parser.parse_args(argv) + source = args.hf_path.expanduser().resolve(strict=True) + destination = args.mlx_path.expanduser().absolute() + if destination.exists() or destination.is_symlink() or destination.resolve().is_relative_to(source): + raise ValueError("Choose a new output directory outside the source checkpoint.") + config = json.loads((source / "config.json").read_text(encoding="utf-8")) + if config.get("model_type") != "minimax_music3" or not isinstance(config.get("quantization"), dict): + raise ValueError("Expected a quantized local MiniMax Music 3 MLX checkpoint.") + + os.environ["HF_HUB_OFFLINE"] = "1" + os.environ["TRANSFORMERS_OFFLINE"] = "1" + import mlx.core as mx + from mlx_audio.music import load + from mlx_audio.convert import build_quant_predicate + from mlx_audio.lm.convert import dequantize_model, quantize_model, save_config, save_model + + model = dequantize_model(load(str(source))) + mx.eval(model.parameters()) + mx.clear_cache() + # Remove the source policy after dequantization. Keeping it makes the + # backend record per-layer overrides under the old global bit width. + config.pop("quantization", None) + config.pop("quantization_config", None) + model, config = quantize_model(model, config, 64, args.q_bits, + quant_predicate=build_quant_predicate(model)) + mx.eval(model.parameters()) + destination.mkdir(parents=True, exist_ok=False) + save_model(destination, model, donate_model=True) + copy_supporting_files(source, destination) + save_config(config, destination / "config.json") + + +if __name__ == "__main__": + main() diff --git a/packages/codex/mlx-agent/src/mlx_agent/music_runner.py b/packages/codex/mlx-agent/src/mlx_agent/music_runner.py new file mode 100644 index 0000000..8f3d6c8 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/music_runner.py @@ -0,0 +1,89 @@ +"""Run inside the isolated music backend; write a new WAV and measured metrics.""" + +import argparse +import json +from pathlib import Path +import time +from types import MethodType +import wave + + +def tokenizer_directory(model_path): + """Resolve the real tokenizer in raw and exported MiniMax checkpoint layouts.""" + root = Path(model_path) + for directory in (root / "tokenizer", root): + if (directory / "tokenizer.json").is_file(): + return directory + raise ValueError("MiniMax Music 3 requires its real local tokenizer.json; synthetic tokens are refused") + + +def configure_tokenizer(model, model_path): + """Use the pinned backend's official encoder, including root-level exports. + + mlx-audio 0.5.7 only checks a nested tokenizer directory, then silently + uses tiny-model test tokens. Bind the real encoder to this loaded instance + without changing the installed backend or the source checkpoint. + """ + if model.model_type != "minimax_music3": + return + directory = tokenizer_directory(model_path) + from mlx_audio.music.models.minimax_music3.minimax_music3 import _encode_official_text_pair + from mlx_audio.music.models.minimax_music3.prompt import assemble_prompt + + def text_ids(instance, caption, lyrics): + return _encode_official_text_pair(assemble_prompt(caption, lyrics), instance.config, directory) + + model._text_ids = MethodType(text_ids, model) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("model", "caption", "lyrics", "out"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--duration", type=float, required=True) + parser.add_argument("--steps", type=int, required=True) + parser.add_argument("--seed", type=int, required=True) + args = parser.parse_args(argv) + import mlx.core as mx + import numpy as np + from mlx_audio.music import load + + started = time.perf_counter() + model = load(args.model) + configure_tokenizer(model, args.model) + load_seconds = time.perf_counter() - started + mx.reset_peak_memory() + started = time.perf_counter() + chunks, sample_rate = [], None + for result in model.generate(text=args.caption, lyrics=args.lyrics, duration=args.duration, steps=args.steps, seed=args.seed): + if sample_rate is not None and sample_rate != result.sample_rate: + raise RuntimeError("Inconsistent sample rates") + sample_rate = result.sample_rate + audio = np.asarray(result.audio, dtype=np.float32) + if audio.ndim == 1: + audio = audio[:, None] + if audio.ndim != 2 or audio.shape[1] not in (1, 2) or not np.isfinite(audio).all(): + raise RuntimeError("Invalid audio output") + chunks.append(audio) + seconds = time.perf_counter() - started + if not chunks or not sample_rate: + raise RuntimeError("No audio output") + audio = np.concatenate(chunks, axis=0) + if audio.shape[0] == 0: + raise RuntimeError("Empty audio output") + pcm = (np.clip(audio, -1, 1) * 32767).astype(" 2: + return ".".join(parts[:2]) + return parts[0] + + +def _prefix_role(prefix, roles): + lowered = prefix.lower() + if "audio" in lowered and "audio" in roles: + return "audio" + if ("vision" in lowered or "visual" in lowered) and "vision" in roles: + return "vision" + if any(marker in lowered for marker in _TEXT_MARKERS) or lowered in ("model.layers", "model.norm", "model.embed_tokens"): + return "text" if "text" in roles else "model" + return None + + +def weight_prefixes(index, roles): + weight_map = index.get("weight_map") if isinstance(index, dict) else None + if not isinstance(weight_map, dict): + return [] + counts = {} + for key in weight_map: + if isinstance(key, str): + prefix = _prefix(key) + counts[prefix] = counts.get(prefix, 0) + 1 + return [ + {"prefix": prefix, "tensors": counts[prefix], "component": _prefix_role(prefix, roles)} + for prefix in sorted(counts) + ] + + +def _classes(text): + try: + tree = ast.parse(text) + except (SyntaxError, ValueError): + return {"parsed": False, "classes": []} + classes = [] + for node in tree.body: + if isinstance(node, ast.ClassDef): + classes.append({ + "name": node.name, + "bases": [ast.unparse(base) for base in node.bases], + "methods": [item.name for item in node.body if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef))], + "lineno": node.lineno, + }) + return {"parsed": True, "classes": classes} + + +def _code(client, repo, revision, names, warnings): + ordered = sorted(names, key=lambda name: (not name.startswith("modeling_"), name)) + files, sources, used, truncated = [], {}, 0, False + for name in ordered: + try: + text = client.fetch_raw_text(repo, revision, name) + except TRANSPORT_ERRORS as error: + warnings.append("{0} unreadable: {1}".format(name, error)) + continue + size = len(text.encode("utf-8")) + if used + size > MAX_CODE_BYTES: + truncated = True + warnings.append("{0} skipped: code budget of {1} bytes reached".format(name, MAX_CODE_BYTES)) + continue + used += size + sources[name] = text + entry = {"name": name, "bytes": size} + entry.update(_classes(text)) + files.append(entry) + return {"files": files, "truncated": truncated}, sources + + +def _fetch_json(client, repo, revision, name, warnings): + try: + value = json.loads(client.fetch_raw_text(repo, revision, name)) + except TRANSPORT_ERRORS as error: + warnings.append("{0} unreadable: {1}".format(name, error)) + return None + return value if isinstance(value, dict) else None + + +def analyze_with_sources(text, revision=None, client=None, manifests=None, registries=None): + source = parse_hf_source(text) + if source.get("subfolder"): + raise PortAnalysisError( + "invalid_source", "Port analysis reads the repository root, not {0}/.".format(source["subfolder"]), + "Pass the repository without the folder.", + ) + if revision: + source["revision"] = validate_revision(revision) + client = client or HuggingFaceClient() + manifests = load_manifests() if manifests is None else manifests + registries = load_registries(manifests) if registries is None else registries + repo, rev = source["repo"], source["revision"] + try: + info = client.fetch_model_info(repo, revision=rev) + except HuggingFaceHTTPError as error: + if error.status in (401, 403, 404, 410): + raise PortAnalysisError("not_found_or_private", str(error), "Check the repository id and that it is public.") from error + raise PortAnalysisError("hub_unreachable", str(error), "Check your network and retry.") from error + except TRANSPORT_ERRORS as error: + raise PortAnalysisError("hub_unreachable", str(error), "Check your network and retry.") from error + if not isinstance(info, dict): + raise PortAnalysisError("hub_unreachable", "The model API returned a non-object document.", "Retry later.") + if info.get("gated"): + raise PortAnalysisError("gated", "{0} is gated.".format(repo), "Gated repositories need a token, which intake does not handle.") + files = summarize_files(info) + if "config.json" not in files["names"]: + raise PortAnalysisError("no_config", "{0} has no config.json.".format(repo), "Port analysis needs a transformers-style config.json.") + warnings = [] + config = _fetch_json(client, repo, rev, "config.json", warnings) + if config is None: + raise PortAnalysisError("no_config", "config.json could not be read.", "Retry; if it persists the repository config is malformed.") + model_type = config.get("model_type") if isinstance(config.get("model_type"), str) else None + comps = components(model_type, config, manifests, registries) + for component in comps: + component["status"] = "exists" if component["matches"] else "missing" + roles = {component["role"] for component in comps} + index = None + if "model.safetensors.index.json" in files["names"]: + index = _fetch_json(client, repo, rev, "model.safetensors.index.json", warnings) + prefixes = weight_prefixes(index, roles) + indexed_files = set(index.get("weight_map", {}).values()) if isinstance(index, dict) else set() + extra_files = sorted( + name for name in files["names"] + if name.endswith(".safetensors") and "/" not in name and indexed_files and name not in indexed_files + ) + code, sources = _code(client, repo, rev, files["summary"]["python"], warnings) + missing = ( + [{"kind": "component", "name": c["role"]} for c in comps if c["status"] == "missing"] + + [{"kind": "weights", "name": p["prefix"]} for p in prefixes if p["component"] is None] + + [{"kind": "file", "name": name} for name in extra_files] + ) + architectures = [item for item in config.get("architectures") or [] if isinstance(item, str)] + payload = { + "schema": PORT_ANALYSIS_SCHEMA, + "source": {"input": text.strip(), "repo": repo, "revision": rev, "file": source["file"], + "url": "https://huggingface.co/{0}".format(repo)}, + "model_type": model_type, + "architectures": architectures, + "processor_class": config.get("processor_class") if isinstance(config.get("processor_class"), str) else None, + "transformers_version": config.get("transformers_version") if isinstance(config.get("transformers_version"), str) else None, + "components": [ + {"role": c["role"], "config_key": c["config_key"], "model_type": c["model_type"], + "status": c["status"], "matches": c["matches"]} + for c in comps + ], + "weights": {"index_available": index is not None, "prefixes": prefixes, "extra_files": extra_files}, + "code": code, + "missing": missing, + "warnings": warnings, + } + return payload, sources + + +def analyze(text, revision=None, client=None, manifests=None, registries=None): + return analyze_with_sources(text, revision, client, manifests, registries)[0] diff --git a/packages/codex/mlx-agent/src/mlx_agent/port_plan.py b/packages/codex/mlx-agent/src/mlx_agent/port_plan.py new file mode 100644 index 0000000..4913549 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/port_plan.py @@ -0,0 +1,113 @@ +"""Draft a porting plan from a port analysis using a local loopback model.""" + +from __future__ import annotations + +import hashlib +import json +import os +from datetime import datetime, timezone +from pathlib import Path + +from .intake import TRANSPORT_ERRORS +from .port_analysis import PortAnalysisError, analyze_with_sources +from .transactions import _atomic_in_directory +from .verification import _http_json_request, _validated_loopback_url + +PORT_PLAN_SCHEMA = "port-plan/1" +DRAFT_TIMEOUT_SECONDS = 900.0 +DRAFT_MAX_RESPONSE_BYTES = 4 * 1024 * 1024 +SYSTEM_PROMPT = "You write precise engineering plans. You only state facts present in the provided material." +INSTRUCTIONS = ( + "Write an implementation plan for porting the Hugging Face model below to MLX (Apple's array " + "framework), reusing the existing MLX modules listed. Sections: 1. Summary. 2. Reused modules " + "(exact module paths). 3. Modules to write, each naming the PyTorch class it mirrors and the " + "weight prefixes it loads. 4. Weight conversion (prefix renames, dtypes, sharding). " + "5. Pre- and post-processing (feature extractor, tokenizer). 6. Validation against the " + "transformers reference (same inputs, compare outputs). 7. Risks and unknowns. Only claim facts " + "present in the analysis or the source; label anything else as an assumption." +) + + +def port_plans_root(env=None): + env = os.environ if env is None else env + base = env.get("XDG_STATE_HOME") + root = Path(base).expanduser() if base else Path.home() / ".local" / "state" + return root / "mlx-workbench" / "port-plans" + + +def build_prompt(payload, sources, budget_chars): + analysis_json = json.dumps(payload, indent=1, sort_keys=True) + lines = [INSTRUCTIONS, "", "## Deterministic analysis (JSON)", analysis_json, "", "## Existing MLX modules to reuse"] + for component in payload["components"]: + for match in component["matches"]: + lines.append("- {0} `{1}` -> {2} ({3}, {4})".format( + component["role"], component["model_type"], match["module"], match["backend"], match["match"] + )) + lines += ["", "## Source of the custom code"] + prompt = "\n".join(lines) + truncated = False + for name in sorted(sources, key=lambda item: (not item.startswith("modeling_"), item)): + block = "\n\n### {0}\n```python\n{1}\n```".format(name, sources[name]) + room = budget_chars - len(prompt) + if room <= 0: + truncated = True + break + if len(block) > room: + prompt += block[:room] + "\n[truncated]" + truncated = True + break + prompt += block + return prompt, truncated + + +def draft_port_plan(text, endpoint, model, revision=None, context_tokens=32768, max_tokens=4096, + out_dir=None, client=None, manifests=None, registries=None, post=None, now=None): + try: + parsed = _validated_loopback_url(endpoint) + except ValueError as error: + raise PortAnalysisError("endpoint_not_loopback", str(error), + "Point --endpoint at a model served on 127.0.0.1 or localhost.") from error + origin = "{0}://{1}".format(parsed.scheme, parsed.netloc) + payload, sources = analyze_with_sources(text, revision, client, manifests, registries) + analysis_sha = hashlib.sha256(json.dumps(payload, sort_keys=True).encode("utf-8")).hexdigest() + budget = max(4000, (int(context_tokens) - int(max_tokens) - 512) * 3) + prompt, truncated = build_prompt(payload, sources, budget) + post = post or (lambda url, body, timeout: _http_json_request( + url, "POST", payload=body, timeout=timeout, max_response_bytes=DRAFT_MAX_RESPONSE_BYTES)) + try: + response = post( + "{0}/v1/chat/completions".format(origin), + {"model": model, "temperature": 0, "max_tokens": int(max_tokens), + "messages": [{"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": prompt}]}, + DRAFT_TIMEOUT_SECONDS, + ) + except TRANSPORT_ERRORS as error: + raise PortAnalysisError("draft_failed", "The local model request failed: {0}".format(error), + "Check that {0} is serving {1}, then retry.".format(origin, model)) from error + try: + content = response["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError): + content = None + if not isinstance(content, str) or not content.strip(): + raise PortAnalysisError("draft_empty", "The local model returned no plan text.", + "Retry with a larger --max-tokens or a stronger local model.") + stamp = (now or (lambda: datetime.now(timezone.utc)))() + repo = payload["source"]["repo"] + document = ( + "# Port plan draft: {0}\n\n" + "> Draft by {1} via {2}; analysis sha256 {3}; generated {4}. " + "The analysis is the evidence; this prose is an unverified model draft.\n\n{5}\n" + ).format(repo, model, origin, analysis_sha, stamp.isoformat(), content.strip()) + directory = Path(out_dir) if out_dir is not None else port_plans_root() + if directory.is_symlink(): + raise PortAnalysisError("symlink_refused", "{0} is a symbolic link.".format(directory), + "Use a real directory for port plans.") + directory.mkdir(parents=True, exist_ok=True) + filename = "{0}-{1}.md".format(repo.replace("/", "--"), stamp.strftime("%Y%m%dT%H%M%SZ")) + encoded = document.encode("utf-8") + _atomic_in_directory(directory, filename, encoded, 0o600) + return { + "schema": PORT_PLAN_SCHEMA, "path": str(directory / filename), "model": model, + "endpoint": origin, "analysis_sha256": analysis_sha, "prompt_chars": len(prompt), + "truncated": truncated, "bytes": len(encoded), + } diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/probe_assets.py b/packages/codex/mlx-agent/src/mlx_agent/probe_assets.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/probe_assets.py rename to packages/codex/mlx-agent/src/mlx_agent/probe_assets.py diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/project_blueprint.py b/packages/codex/mlx-agent/src/mlx_agent/project_blueprint.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/project_blueprint.py rename to packages/codex/mlx-agent/src/mlx_agent/project_blueprint.py diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/providers.py b/packages/codex/mlx-agent/src/mlx_agent/providers.py similarity index 60% rename from packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/providers.py rename to packages/codex/mlx-agent/src/mlx_agent/providers.py index 8e5fad1..4f18145 100644 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/providers.py +++ b/packages/codex/mlx-agent/src/mlx_agent/providers.py @@ -9,11 +9,33 @@ import shutil import subprocess import time +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from pathlib import Path -_GEMINI_EXTENSION_SUFFIX = Path(".gemini") / "extensions" / "mlx-agent" +_AGY_USER_PLUGIN_SUFFIX = Path(".gemini") / "config" / "plugins" / "mlx-agent" +_AGY_PROJECT_PLUGIN_SUFFIX = Path(".agents") / "plugins" / "mlx-agent" + + +def _tree_files(location): + """Regular files under a resolved directory, relative and in path order; any symbolic link is refused.""" + found, pending = [], [()] + while pending: + parts = pending.pop() + with os.scandir(location.joinpath(*parts)) as entries: + for entry in entries: + child = parts + (entry.name,) + if entry.is_symlink(): + raise ValueError( + "provider artifact source must not contain symlinks: {0}".format(location.joinpath(*child)) + ) + if entry.is_dir(follow_symlinks=False): + pending.append(child) + elif ("__pycache__" not in child and os.path.splitext(entry.name)[1] != ".pyc" + and entry.is_file(follow_symlinks=False)): + found.append(child) + return [Path(*parts) for parts in sorted(found)] @dataclass(frozen=True) @@ -72,6 +94,7 @@ class ProviderDetection: last_tested_version: str = None error: str = None install_mode: str = "direct" + warning: str = None def to_dict(self): return { @@ -85,9 +108,52 @@ def to_dict(self): "last_tested_version": self.last_tested_version, "error": self.error, "install_mode": self.install_mode, + "warning": self.warning, } +def _logical(path): + """Absolute, `..`-free spelling of a path without resolving symlinks. + + Containment is judged on the spelling a user configured, not on where the + filesystem happens to point. A provider directory that is a symlink to + another volume — a dotfile checkout, config on external storage — is still + that provider's directory, and resolving it would misread it as an escape. + Traversal is rejected outright, so a rendered template can never climb out + of an approved root. + """ + value = Path(path).expanduser() + if ".." in value.parts: + raise ValueError("path traversal is not allowed: {0}".format(path)) + return Path(os.path.normpath(os.path.abspath(str(value)))) + + +def _physical(path): + """Resolve a validated root to the spelling the filesystem will accept. + + Artifacts are written through a descriptor walk that opens every component + with ``O_NOFOLLOW``, so a symlinked directory anywhere above them cannot be + traversed — and a provider config directory that is a symlink is a normal + arrangement (dotfiles checkout, storage on another volume). Resolving the + existing prefix once, here, means the walk only ever sees real directories + while receipts, previews, and ownership checks all use one stable spelling. + + Containment is checked on the logical path before this runs. Where a user's + own symlink points is that user's arrangement, not something a manifest can + influence. + """ + value = Path(path) + missing = [] + cursor = value + while cursor != cursor.parent and not cursor.exists(): + missing.append(cursor.name) + cursor = cursor.parent + resolved = Path(os.path.realpath(str(cursor))) + for name in reversed(missing): + resolved = resolved / name + return resolved + + def _safe_relative(value, field, allow_current=False): path = Path(value) if allow_current and str(value) == ".": @@ -100,16 +166,38 @@ def _safe_relative(value, field, allow_current=False): class ProviderRegistry: """Load immutable installation data from the canonical plugin manifest.""" - def __init__(self, manifest_path, home=None, config_root=None, xdg_config_home=None): + def __init__( + self, manifest_path, home=None, config_root=None, xdg_config_home=None, + openclaw_state_dir=None, hermes_home=None, + ): self.manifest_path = Path(manifest_path).resolve() self.home = Path(home).expanduser().resolve() if home else Path.home().resolve() + # Receipts are state, not configuration, and belong in the XDG state + # directory. The previous default put them in a dotless ``~/mlx-agent``. self.config_root = ( - Path(config_root).expanduser().resolve() if config_root else self.home + Path(config_root).expanduser().resolve() if config_root + else (self.home / ".local" / "state").resolve() ) self.xdg_config_home = ( Path(xdg_config_home).expanduser().resolve() if xdg_config_home else (self.home / ".config").resolve() ) + self.openclaw_state_dir = ( + Path(openclaw_state_dir).expanduser().resolve() + if openclaw_state_dir else (self.home / ".openclaw").resolve() + ) + self.hermes_home = ( + Path(hermes_home).expanduser().resolve() + if hermes_home else (self.home / ".hermes").resolve() + ) + + def _template_roots(self): + return { + "home": str(self.home), "config_root": str(self.config_root), + "xdg_config_home": str(self.xdg_config_home), + "openclaw_state_dir": str(self.openclaw_state_dir), + "hermes_home": str(self.hermes_home), + } def definitions(self): try: @@ -212,25 +300,20 @@ def _definition(self, provider_id, value): if location.is_file(): artifacts.append(ProviderArtifact(location, destination, project_destination, scopes)) else: - for child in sorted(location.rglob("*")): - if child.is_symlink(): - raise ValueError( - "provider artifact source must not contain symlinks: {0}".format(child) - ) - resolved = child.resolve() - relative = child.relative_to(location) - if "__pycache__" in relative.parts or child.suffix == ".pyc": - continue - if child.is_file() and location in resolved.parents: - project_child = project_destination / child.relative_to(location) if project_destination else None - artifacts.append(ProviderArtifact(resolved, destination / child.relative_to(location), project_child, scopes)) + for relative in _tree_files(location): + project_child = project_destination / relative if project_destination else None + artifacts.append(ProviderArtifact(location / relative, destination / relative, project_child, scopes)) config_paths = value["config_paths"] if not isinstance(config_paths, list) or not all(isinstance(item, str) for item in config_paths): raise ValueError("provider {0}.config_paths is invalid".format(provider_id)) - if provider_id == "gemini": - self._validate_gemini_extension_layout(user_root, project_root, artifacts) + if provider_id == "agy": + self._validate_agy_plugin_layout(user_root, project_root, artifacts) if provider_id == "opencode": self._validate_opencode_layout(user_root, project_root, artifacts) + if provider_id == "openclaw": + self._validate_openclaw_layout(user_root, project_root, artifacts) + if provider_id == "hermes": + self._validate_hermes_layout(user_root, project_root, artifacts) return ProviderDefinition( id=provider_id, detect_commands=tuple(commands), @@ -247,21 +330,20 @@ def _definition(self, provider_id, value): config_paths=tuple(config_paths), ) - def _validate_gemini_extension_layout(self, user_root, project_root, artifacts): - """Keep Gemini's install roots aligned with its extension discovery layout.""" - if user_root != (self.home / _GEMINI_EXTENSION_SUFFIX).resolve(): - raise ValueError("provider gemini.user_root must target home/.gemini/extensions/mlx-agent") - if project_root != _GEMINI_EXTENSION_SUFFIX: - raise ValueError("provider gemini.project_root must target .gemini/extensions/mlx-agent") - if not any(item.destination == Path("gemini-extension.json") for item in artifacts): - raise ValueError("provider gemini must install gemini-extension.json") - for item in artifacts: - if item.project_destination is not None and item.project_destination.parts[:1] != (".gemini",): - raise ValueError("provider gemini project artifacts must stay under .gemini") + def _validate_agy_plugin_layout(self, user_root, project_root, artifacts): + """Keep Agy's convention-discovered plugin roots exact.""" + if user_root != _physical(_logical(self.home / _AGY_USER_PLUGIN_SUFFIX)): + raise ValueError("provider agy.user_root must target home/.gemini/config/plugins/mlx-agent") + if project_root != _AGY_PROJECT_PLUGIN_SUFFIX: + raise ValueError("provider agy.project_root must target .agents/plugins/mlx-agent") + if not any(item.destination == Path("plugin.json") for item in artifacts): + raise ValueError("provider agy must install its plugin.json marker") + if not any(item.destination == Path("skills/mlx-scout/SKILL.md") for item in artifacts): + raise ValueError("provider agy must install native skills") def _validate_opencode_layout(self, user_root, project_root, artifacts): """Keep OpenCode's documented global and project discovery locations exact.""" - if user_root != (self.xdg_config_home / "opencode").resolve(): + if user_root != _physical(_logical(self.xdg_config_home / "opencode")): raise ValueError("provider opencode.user_root must target the OpenCode XDG config directory") if project_root != Path(".opencode"): raise ValueError("provider opencode.project_root must target .opencode") @@ -278,30 +360,49 @@ def _validate_opencode_layout(self, user_root, project_root, artifacts): if not any(item.destination == Path("src/mlx_agent/command_executor.py") for item in artifacts): raise ValueError("provider opencode must install its custom-tool runtime") + def _validate_openclaw_layout(self, user_root, project_root, artifacts): + """Keep OpenClaw's managed and workspace skill roots exact.""" + if user_root != _physical(_logical(self.openclaw_state_dir)): + raise ValueError("provider openclaw.user_root must target the OpenClaw state directory") + if project_root != Path("."): + raise ValueError("provider openclaw.project_root must target the workspace root") + if not any(item.destination == Path("skills/mlx-scout/SKILL.md") for item in artifacts): + raise ValueError("provider openclaw must install native skills") + if not any(item.destination == Path("skills/mlx-scout/scripts/mlx-agent") for item in artifacts): + raise ValueError("provider openclaw must install each skill's bundled launcher") + + def _validate_hermes_layout(self, user_root, project_root, artifacts): + """Keep Hermes's profile and project skill roots exact.""" + if user_root != _physical(_logical(self.hermes_home)): + raise ValueError("provider hermes.user_root must target the Hermes home directory") + if project_root != Path(".hermes"): + raise ValueError("provider hermes.project_root must target .hermes") + if not any(item.destination == Path("skills/mlx-scout/SKILL.md") for item in artifacts): + raise ValueError("provider hermes must install native skills") + if not any(item.destination == Path("skills/mlx-scout/scripts/mlx-agent") for item in artifacts): + raise ValueError("provider hermes must install each skill's bundled launcher") + def _user_root(self, template, provider_id): if not isinstance(template, str) or "{project}" in template: raise ValueError("provider {0}.user_root is invalid".format(provider_id)) try: - path = Path(template.format( - home=str(self.home), config_root=str(self.config_root), - xdg_config_home=str(self.xdg_config_home), - )).resolve() + path = _logical(template.format(**self._template_roots())) except (KeyError, ValueError) as error: raise ValueError("provider {0}.user_root is invalid: {1}".format(provider_id, error)) - allowed_roots = (self.home, self.config_root, self.xdg_config_home) + allowed_roots = ( + self.home, self.config_root, self.xdg_config_home, + self.openclaw_state_dir, self.hermes_home, + ) if not any(root == path or root in path.parents for root in allowed_roots): raise ValueError("provider {0}.user_root must be under an approved user configuration root".format(provider_id)) - return path + return _physical(path) def _project_root(self, template, provider_id): if not isinstance(template, str) or "{project}" not in template: raise ValueError("provider {0}.project_root is invalid".format(provider_id)) marker = Path("/__mlx_agent_project_marker__") try: - rendered = Path(template.format( - project=str(marker), home=str(self.home), config_root=str(self.config_root), - xdg_config_home=str(self.xdg_config_home), - )).resolve() + rendered = Path(template.format(project=str(marker), **self._template_roots())).resolve() return rendered.relative_to(marker) except (KeyError, ValueError): raise ValueError("provider {0}.project_root is invalid".format(provider_id)) @@ -363,7 +464,7 @@ def detect_providers(definitions, env=None, path=None, executable_lookup=None, p environment = dict(os.environ if env is None else env) search_path = path if path is not None else environment.get("PATH", "") lookup = executable_lookup or shutil.which - detections = [] + detections, probes = [], [] for definition in definitions: if definition.install_mode == "portable": detections.append(ProviderDetection( @@ -386,51 +487,65 @@ def detect_providers(definitions, env=None, path=None, executable_lookup=None, p install_mode=definition.install_mode, )) continue - try: - argv = [command_path] + list(definition.version_probe_args) - if probe_runner is None: - completed = _run_bounded_probe( - argv, - timeout=definition.version_probe_timeout, - env=environment, - ) - else: - completed = probe_runner( - argv, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=definition.version_probe_timeout, - check=False, - shell=False, - env=environment, - ) - raw = completed.stdout or b"" - if isinstance(raw, bytes): - output = raw[:_MAX_VERSION_OUTPUT].decode("utf-8", errors="replace") - else: - output = str(raw)[:_MAX_VERSION_OUTPUT] - match = _SEMVER.search(output) - if completed.returncode != 0 or match is None: - raise ValueError("version probe did not return a SemVer core") - version = match.group(1) - supported = definition.last_tested_version is not None and ( - _version_tuple(definition.minimum_version) - <= _version_tuple(version) - <= _version_tuple(definition.last_tested_version) - ) - state = "native-visible" if supported else "unsupported" - detections.append(ProviderDetection( - definition.id, supported, command, command_path, state=state, - version=version, minimum_version=definition.minimum_version, - last_tested_version=definition.last_tested_version, - error=None if supported else "provider version is outside the validated range", - install_mode=definition.install_mode, - )) - except (OSError, subprocess.SubprocessError, ValueError) as error: - detections.append(ProviderDetection( - definition.id, False, command, command_path, state="unsupported", - minimum_version=definition.minimum_version, - last_tested_version=definition.last_tested_version, - error=str(error), install_mode=definition.install_mode, - )) + detections.append(None) + probes.append((len(detections) - 1, definition, command, command_path)) + if probes: + # Version probes are independent processes; run them together so detection waits for the slowest one only. + with ThreadPoolExecutor(max_workers=len(probes)) as pool: + pending = [(index, pool.submit(_probe_detection, definition, command, command_path, environment, probe_runner)) + for index, definition, command, command_path in probes] + for index, future in pending: + detections[index] = future.result() return detections + + +def _probe_detection(definition, command, command_path, environment, probe_runner): + try: + argv = [command_path] + list(definition.version_probe_args) + if probe_runner is None: + completed = _run_bounded_probe( + argv, + timeout=definition.version_probe_timeout, + env=environment, + ) + else: + completed = probe_runner( + argv, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=definition.version_probe_timeout, + check=False, + shell=False, + env=environment, + ) + raw = completed.stdout or b"" + if isinstance(raw, bytes): + output = raw[:_MAX_VERSION_OUTPUT].decode("utf-8", errors="replace") + else: + output = str(raw)[:_MAX_VERSION_OUTPUT] + match = _SEMVER.search(output) + if completed.returncode != 0 or match is None: + raise ValueError("version probe did not return a SemVer core") + version = match.group(1) + # A release newer than the last tested one is used with a warning; one below the minimum is refused. + supported = definition.last_tested_version is not None and ( + _version_tuple(definition.minimum_version) <= _version_tuple(version) + ) + untested = supported and _version_tuple(version) > _version_tuple(definition.last_tested_version) + state = "native-visible" if supported else "unsupported" + return ProviderDetection( + definition.id, supported, command, command_path, state=state, + version=version, minimum_version=definition.minimum_version, + last_tested_version=definition.last_tested_version, + error=None if supported else "provider version is outside the validated range", + install_mode=definition.install_mode, + warning="provider version {0} is newer than the last tested {1}".format( + version, definition.last_tested_version) if untested else None, + ) + except (OSError, subprocess.SubprocessError, ValueError) as error: + return ProviderDetection( + definition.id, False, command, command_path, state="unsupported", + minimum_version=definition.minimum_version, + last_tested_version=definition.last_tested_version, + error=str(error), install_mode=definition.install_mode, + ) diff --git a/packages/codex/mlx-agent/src/mlx_agent/queue_bridge.py b/packages/codex/mlx-agent/src/mlx_agent/queue_bridge.py new file mode 100644 index 0000000..1aaec50 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/queue_bridge.py @@ -0,0 +1,139 @@ +"""Conversion queue calls the canonical agent CLI without a second scheduler.""" +from contextlib import redirect_stdout +import io +import json +from pathlib import Path +import sys + +CLI_RELATIVE = Path("scripts/mlx-agent") +DEFAULT_TIMEOUT = 300 +MAX_LOG_BYTES = 64 * 1024 +_receipts_directory = None + +def run(agent_path, argv, timeout=DEFAULT_TIMEOUT, runner=None): + arguments = list(argv) + ["--json"] + if runner is not None: + result = runner([sys.executable, str(Path(agent_path) / "scripts/mlx-agent"), *arguments], timeout=timeout) + try: + return json.loads(result["stdout"]) + except (KeyError, ValueError, TypeError) as error: + raise BridgeError("queue_execution_invalid", str(error), "Inspect the agent operation.") from error + if _receipts_directory is not None: + arguments += ["--receipts-dir", str(_receipts_directory)] + from mlx_agent.cli import main + output = io.StringIO() + with redirect_stdout(output): + main(arguments) + return json.loads(output.getvalue()) + +class BridgeError(RuntimeError): + """A classified failure to reach or run the agent CLI.""" + + def __init__(self, code, message, remediation): + super().__init__(message) + self.code = code + self.remediation = remediation + + def to_dict(self): + return { + "code": self.code, + "message": str(self), + "remediation": self.remediation, + } + +def unwrap(envelope): + """Return the envelope's data, or raise its classified error.""" + if envelope.get("status") == "ok": + return envelope.get("data") or {} + error = envelope.get("error") or {} + raise BridgeError( + error.get("code", "skill_failed"), + error.get("message", "The agent reported an error."), + error.get("remediation", "Inspect the agent output and retry."), + ) + +def start(agent_path, gguf_path, preview_hash, q_bits=4, out=None, + timeout=DEFAULT_TIMEOUT, runner=None): + """Start a reviewed GGUF conversion. The hash must come from a preview.""" + argv = [ + "convert", "start", "--gguf", gguf_path, "--q-bits", str(q_bits), + "--confirm", "--preview-hash", preview_hash, + ] + if out: + argv.extend(["--out", out]) + return unwrap(run(agent_path, argv, timeout=timeout, runner=runner)) + +def start_repo(agent_path, repo, preview_hash, q_bits=4, out=None, hf_cache=None, + timeout=DEFAULT_TIMEOUT, runner=None): + """Start a reviewed HF-cache conversion. The hash must come from a preview.""" + argv = [ + "convert", "start", "--repo", repo, "--q-bits", str(q_bits), + "--confirm", "--preview-hash", preview_hash, + ] + if out: + argv.extend(["--out", out]) + if hf_cache: + argv.extend(["--hf-cache", hf_cache]) + return unwrap(run(agent_path, argv, timeout=timeout, runner=runner)) + +def jobs(agent_path, timeout=DEFAULT_TIMEOUT, runner=None): + """Cross-check conversion receipts against live processes.""" + return unwrap(run(agent_path, ["convert", "status"], timeout=timeout, runner=runner)) + +def read_convert_receipts(entries, max_bytes=MAX_LOG_BYTES): + """Read bounded receipt JSON from paths advertised by convert status.""" + receipts = [] + for entry in entries or []: + if not isinstance(entry, dict) or not isinstance(entry.get("receipt"), str): + continue + advertised = Path(entry["receipt"]).expanduser() + try: + location = advertised.resolve(strict=True) + except FileNotFoundError as error: + raise BridgeError( + "receipt_missing", + "A conversion receipt no longer exists: {0}".format(advertised), + "Run convert status again or inspect the mlx-agent receipt directory.", + ) from error + except OSError as error: + raise BridgeError( + "receipt_unreadable", + "A conversion receipt could not be resolved: {0}".format(error), + "Check the receipt path and file permissions.", + ) from error + if not location.is_file(): + raise BridgeError( + "receipt_missing", + "A conversion receipt is not a regular file: {0}".format(location), + "Run convert status again or inspect the mlx-agent receipt directory.", + ) + try: + size = location.stat().st_size + except OSError as error: + raise BridgeError( + "receipt_unreadable", + "A conversion receipt could not be inspected: {0}".format(error), + "Check the receipt file permissions.", + ) from error + if size > max_bytes: + raise BridgeError( + "receipt_too_large", + "A conversion receipt exceeds the {0}-byte read limit.".format(max_bytes), + "Inspect or remove the malformed receipt in the mlx-agent state directory.", + ) + try: + receipt = json.loads(location.read_text(encoding="utf-8")) + except (OSError, UnicodeError, ValueError) as error: + raise BridgeError( + "receipt_unreadable", + "A conversion receipt is not readable JSON: {0}".format(error), + "Inspect or remove the malformed receipt in the mlx-agent state directory.", + ) from error + if not isinstance(receipt, dict): + raise BridgeError( + "receipt_unreadable", + "A conversion receipt is not a JSON object.", + "Inspect or remove the malformed receipt in the mlx-agent state directory.", + ) + receipts.append(receipt) + return receipts diff --git a/packages/codex/mlx-agent/src/mlx_agent/queue_cli.py b/packages/codex/mlx-agent/src/mlx_agent/queue_cli.py new file mode 100644 index 0000000..236ad74 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/queue_cli.py @@ -0,0 +1,190 @@ +"""Versioned, process-serialized conversion queue operations. Stdlib only.""" +from __future__ import annotations + +import argparse +from contextlib import contextmanager +import fcntl +import hashlib +import json +import os +from pathlib import Path +import stat +import sys + +from mlx_agent.compute import compute_root +from mlx_agent.convert import load_receipts, status_convert +from mlx_agent.convert_queue import ConvertQueue, QueueStore, QueueOperationError, QueuePersistenceError, _validated_items +from mlx_agent import queue_bridge +from mlx_agent.transactions import _assert_safe_directory, _assert_safe_target, _open_directory + +MAXIMUM_BYTES = 8 * 1024 * 1024 +_FIELDS = { + "snapshot": set(), "run": set(), "clear": set(), + "cancel": {"id"}, "retry": {"id"}, "move": {"id", "direction"}, + "enqueue": {"kind", "preview_hash", "q_bits", "out", "path", "repo", "hf_cache", "label"}, + "adopt-preview": {"source"}, "adopt": {"source", "preview_hash"}, +} + + +def default_path(): + return compute_root().parent / "queues/convert-queue.json" + + +def _read(path): + location = _assert_safe_target(path) + descriptor = os.open(location, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_size > MAXIMUM_BYTES: + raise ValueError("Queue input must be a bounded regular file.") + chunks, size = [], 0 + while True: + chunk = os.read(descriptor, 65536) + if not chunk: + break + size += len(chunk) + if size > MAXIMUM_BYTES: + raise ValueError("Queue input exceeded its limit.") + chunks.append(chunk) + if (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns) != ( + os.fstat(descriptor).st_dev, os.fstat(descriptor).st_ino, + os.fstat(descriptor).st_size, os.fstat(descriptor).st_mtime_ns): + raise ValueError("Queue input changed while reading.") + return b"".join(chunks) + finally: + os.close(descriptor) + + +@contextmanager +def _guard(path): + _assert_safe_directory(path.parent, create=True) + directory = _open_directory(path.parent) + descriptor = None + try: + descriptor = os.open(path.name + ".lock", + os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600, dir_fd=directory) + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1: + raise ValueError("Queue lock must be an owned regular file.") + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + _assert_safe_target(path) + yield + finally: + if descriptor is not None: + os.close(descriptor) + os.close(directory) + + +def _adoption(source, target, receipts_dir): + if target.exists() or target.is_symlink(): + raise ValueError("Queue adoption never replaces an existing queue.") + source = Path(source).expanduser() + payload = _read(source) + items = _validated_items(json.loads(payload)) + if any(item["state"] == "starting" for item in items): + raise ValueError("Reconcile starting conversions before transferring queue ownership.") + if any(item["state"] in ("running", "unknown") for item in status_convert(receipts_dir)): + raise ValueError("Stop active conversions before transferring queue ownership.") + digest = hashlib.sha256(payload).hexdigest() + plan = {"source": str(source), "queue_path": str(target), "source_digest": digest, + "item_count": len(items), "preview_hash": hashlib.sha256( + (str(source) + "\\0" + str(target) + "\\0" + digest).encode()).hexdigest()} + return plan, payload, items + + +def execute(request, path=None, receipts_dir=None): + if not isinstance(request, dict) or request.get("operation") not in _FIELDS: + raise ValueError("Unknown conversion queue operation.") + operation = request["operation"] + if set(request) - (_FIELDS[operation] | {"operation"}): + raise ValueError("Unexpected conversion queue fields.") + required = {"enqueue": {"kind", "preview_hash", "q_bits"}, "cancel": {"id"}, + "retry": {"id"}, "move": {"id", "direction"}, + "adopt-preview": {"source"}, "adopt": {"source", "preview_hash"}}.get(operation, set()) + if not required <= set(request): + raise ValueError("Missing conversion queue fields.") + location = Path(path).expanduser() if path is not None else default_path() + with _guard(location): + if operation in ("adopt-preview", "adopt"): + plan, original, items = _adoption(request["source"], location, receipts_dir) + if operation == "adopt-preview": + return plan + if request["preview_hash"] != plan["preview_hash"]: + raise ValueError("Queue adoption preview changed.") + backup = location.with_name(location.name + "." + plan["source_digest"] + ".backup") + if backup.exists(): + if _read(backup) != original: + raise ValueError("Queue backup does not match the source.") + else: + _write_backup(backup, original) + try: + QueueStore(location).save(items) + if _read(Path(request["source"])) != original: + location.unlink() + raise ValueError("Source queue changed during adoption; retry from a fresh preview.") + except BaseException: + if location.exists(): + location.unlink() + raise + return {**plan, "status": "adopted", "backup": str(backup), + "conversion_queue": {"schema_version": "1.1", "items": items}} + if location.exists() and location.stat().st_size > MAXIMUM_BYTES: + raise ValueError("Queue state exceeds the supported size.") + queue = ConvertQueue(location) + if queue.load_error: + raise QueuePersistenceError(queue.load_error["code"], queue.load_error["message"], + queue.load_error["remediation"], location, + queue.load_error.get("preserved_path")) + result = {} + if operation == "enqueue": + result["item"] = queue.enqueue(**{key: value for key, value in request.items() if key != "operation"}) + elif operation == "cancel": + result["changed"] = queue.cancel(request["id"]) + elif operation == "retry": + result["item"] = queue.retry(request["id"]) + elif operation == "move": + result["changed"] = queue.move(request["id"], request["direction"]) + elif operation == "clear": + result["removed_count"] = queue.clear() + elif operation == "run": + queue_bridge._receipts_directory = receipts_dir + receipts = load_receipts(receipts_dir) + for item in queue.snapshot(): + if item["state"] == "queued" and any(queue._receipt_matches(item, receipt) for receipt in receipts): + queue.cancel(item["id"]) + result = queue.try_start_next(str(Path(__file__).resolve().parents[2])) or {"status": "waiting"} + return {**result, "queue_path": str(location), + "conversion_queue": {"schema_version": "1.1", "items": queue.snapshot()}} + + +def _write_backup(backup, original): + descriptor = os.open(backup, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(original) + stream.flush() + os.fsync(stream.fileno()) + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("--database") + parser.add_argument("--artifacts") + parser.add_argument("--path", type=Path) + parser.add_argument("--receipts-dir") + parser.add_argument("queue", choices=["queue"]) + parser.add_argument("--request", type=Path, required=True) + parser.add_argument("--json", action="store_true") + arguments = parser.parse_args(argv) + try: + data = execute(json.loads(_read(arguments.request)), arguments.path, arguments.receipts_dir) + print(json.dumps({"schema_version": "queue.v1", "ok": True, "data": data})) + return 0 + except (ValueError, TypeError, KeyError, OSError, QueueOperationError, QueuePersistenceError, queue_bridge.BridgeError) as error: + detail = error.to_dict() if hasattr(error, "to_dict") else { + "code": "queue_rejected", "message": str(error), "remediation": "Inspect the queue request and retained state."} + print(json.dumps({"schema_version": "queue.v1", "ok": False, "error": detail})) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/research.py b/packages/codex/mlx-agent/src/mlx_agent/research.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/research.py rename to packages/codex/mlx-agent/src/mlx_agent/research.py diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/resources/adapter-runtime.json b/packages/codex/mlx-agent/src/mlx_agent/resources/adapter-runtime.json similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/resources/adapter-runtime.json rename to packages/codex/mlx-agent/src/mlx_agent/resources/adapter-runtime.json diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mflux.json b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mflux.json new file mode 100644 index 0000000..c944c16 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mflux.json @@ -0,0 +1,39 @@ +{ + "schema": "backend/1", + "id": "mflux", + "version": "0.20.0", + "builtin": false, + "package": "mflux", + "convert": "mflux.models.common.cli.save", + "lock": "mflux.lock", + "ports": { + "image_generation": [ + "qwen_image_21" + ] + }, + "port_convert": { + "qwen_image_21": "convert" + }, + "port_pipelines": { + "QwenImage21Pipeline": "qwen_image_21" + }, + "port_recipes": { + "abenzerps/Qwen-Image-2.1-Uncensored-GGUF": { + "port": "qwen_image_21", + "base": "Qwen/Qwen-Image-2.1", + "base_revision": "d26bb61231c349cf6b7896fa83353113880e1ba3", + "lora": "qwen-image-2.1-uncensored-lora.safetensors", + "lora_scale": 1.0 + } + }, + "categories": { + "image_generation": "mlx_agent_ports" + }, + "remap_files": {}, + "registry": { + "image_generation": { + "model_types": [], + "remapping": {} + } + } +} diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mflux.lock b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mflux.lock new file mode 100644 index 0000000..024805a --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mflux.lock @@ -0,0 +1,1447 @@ +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb + # via typer +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ + --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 + # via httpx +certifi==2026.10.19 \ + --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b \ + --hash=sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae + # via + # httpcore + # httpx + # requests +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f + # via requests +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via huggingface-hub +contourpy==1.4.0 \ + --hash=sha256:018227e134b73090b06f911e773a526c77c9af426be29065500ce096a5accd4c \ + --hash=sha256:072a702e2e178f4fcf96f04775d0c059e4c917925abf0d3208ebb6865df3c23d \ + --hash=sha256:0c7a4c2716a4e98342221954416a836cca77c996c14ddbf22b5f01d5d93ca09c \ + --hash=sha256:107ec46f7aa1664266d69b1181aadc953d58d39f7d8d648fa5b795d4a060b0de \ + --hash=sha256:181bea01bc742734ae672fa00c717d855dd35e1f029536406538c52e7b0cd73d \ + --hash=sha256:186ba929df36d61b6127da2e89cd1357e4cb79aca647381ab1a6feb0b152877b \ + --hash=sha256:196759a3e4db60e1546167e361090ebb6d199aa3aafc9e34ea17a5ac3b23e814 \ + --hash=sha256:20156f5a1ac4f8ce02656e39a61e82164a3d359796dc8026f75b062783d500e1 \ + --hash=sha256:20da2f86bfaed60dbba729fe738c8241b1dd22b6cf7e8cbbf30df290bd04ba28 \ + --hash=sha256:2deb580178ca19437a84bd77e4bc2cd91a8ccad212413a83c273900868b5978c \ + --hash=sha256:2e741a39dfc96babe1722561e81351aeec104526c57dbe91c167c1db606c2d55 \ + --hash=sha256:3863ef2e2b13fe93f8c0ebb08ee400cb07153b8b0e91c5acb26e4537f283634f \ + --hash=sha256:3e0a2392533e5e2abdf277c951047516acc8d2f3c42b2dd9dc34c907c9ebabf2 \ + --hash=sha256:404dbcd9233513dfd1323b66ea593fef90e899f658f6b8a9ed8e93bd0ca669db \ + --hash=sha256:417be048f7e122cefbe2a34c51a1f2b0410f8eb35796f794d45475ddb7872d9d \ + --hash=sha256:423bd5f4b3f11d54a8c597234e513382e3454bb106b8ec7ae32ba4ed63f8d99b \ + --hash=sha256:432e89f835cf01f8d2123a130a3126b17ee6e0348e4d62c0973872b30aa873bf \ + --hash=sha256:43c3ccbb32c6294b183dcc8e8c46dacf5ecef809497e3d48a5be298eef185ad0 \ + --hash=sha256:43d80072a32299bf945de0a6dd4e034ea0162e832261de2e07a274a5adbba9c9 \ + --hash=sha256:4bffcb2e8cc5c0e837631acab59dd04265fdac376550e7c3e310523877c68cb7 \ + --hash=sha256:510f7d93d94cf6ebf4e2cd0640bea16ab8affac7547230175b6745b596ce0599 \ + --hash=sha256:5450f091ac1be0be3ad3a2a3b3f23b5e443e78c670ced4fd347d626f92a28fd2 \ + --hash=sha256:5545ff38229c15d11d822fc6f7415b932da9f4975f53e305b7b96ebfb01f889c \ + --hash=sha256:5a4c89c7f38a0d7a94356d74e3391073c4325c4d47ad2fd065c3fe7dbae16c0c \ + --hash=sha256:61624f6722480aa7e168fd746164e7bfdf48f8ccb6542f1613741200dc37e2b1 \ + --hash=sha256:618137ec5778fb76d494c9ec854b104a377fddba0d128bef7136cce2c3bf0df8 \ + --hash=sha256:630d32f06cedf37b7f1dd2b6a12e4492826bd477a4c9c7b2e2fee6ce3a2cea76 \ + --hash=sha256:64039341e2d8804f1a13bda8c69083eab935167bbd7e856cf096241f17d5bd45 \ + --hash=sha256:6507a016976a75a15aee47809a34605de6244ccc056e6164bf9fb14b27eecad4 \ + --hash=sha256:6ced1670fafee703b8277ab1645072a226e74f167166eae146fb743916119b67 \ + --hash=sha256:6e697d94e69f499ff6bebb899cae97a58d5d14f0e1fe9568b43a0248d2f9af8c \ + --hash=sha256:6f963ea9f0d68d2b6a02d861e3c0aae09ae25ba21f243fcf281e9fed468b078d \ + --hash=sha256:738c44fa71735a617f36da58e32810512407d283d5d1bb5b1cecb00d7eeba7cf \ + --hash=sha256:758e7496cec3195fc28a28945eab3ceef348c47fb95c7afc2eca2f3901209c8c \ + --hash=sha256:78ed5c5f962b3e156109f0531b174a65ba80d13cc681a70227b12524c162e184 \ + --hash=sha256:79b06c60d5e569ce12c5da8f0c51217cdc484386e2349fa717c30fafb56d2871 \ + --hash=sha256:7f861af508fca2384eef392bc1d8377cfa349b2b854cb11b45d89c75780e2561 \ + --hash=sha256:80c7fc8e7ac217777ce7ba2db3cf8478652b5a4b335283111012ff9eaac9d842 \ + --hash=sha256:86d05cec773c9507a3950122e0e40ce77c23c75ceeb2fc189514e71893cbb34b \ + --hash=sha256:875f42444c9cf48d56f724f2637e60d0f73b3b12c9041e1484580a233edf9591 \ + --hash=sha256:90feb8da006803a95ba573fddc9536357d0c3faddc3a4c9e22816ca49af52878 \ + --hash=sha256:912c6afaa106e2f74ba22b30416b77ef7eb94e3bdc5a4df51ab147845dba9b6d \ + --hash=sha256:96019f059bcb3774acc0104be3360a57b36d33eaaa2e1d7f77c800ca8e07618b \ + --hash=sha256:9c0e07c691f3b3321913ed9b8161c50ea5f77fadd006f52f5e755359b0dcbfc5 \ + --hash=sha256:a03b32c2e7eda8b17757c022162c13c86f5c3d6f937dddf9ba3c3ff7b513953b \ + --hash=sha256:a1c8a74744fa746eeaa12f0f9ed7f8b4add9d8f1b14d95e3b5e133675eac888f \ + --hash=sha256:a3e67bc1a6a4618a1dac7c3053a9ffece5ddfa2046b2670b342cf430bb0b87d1 \ + --hash=sha256:a58b5f130afec61a093d743cdda435cc4047748b6791627e34a68f41ec3a9b07 \ + --hash=sha256:a5e2bb871b90cd7a52bdee63bf80ef7acb46398d27efc66d7679240016efc5aa \ + --hash=sha256:abd0f7e51ecc52bf8c95713bd80b1c17c516a71fa391c54475308a4377903c2b \ + --hash=sha256:b7794ea07cab575633daad8d6e963b662053391022294d3aacc1d9ba9ef54114 \ + --hash=sha256:b9dc493e7924e5aa32d89d0b8e50986280f3d4b284968a792787bd7bcffd3cad \ + --hash=sha256:be85d160e7c795113c2a93254dde2fc7e86b375322570eee78b420969426973c \ + --hash=sha256:bee6ff96653e0807cd0f8eaef00174a031e3a2effa5c20f49a5a8d574b05af23 \ + --hash=sha256:c76f3a5318164db7d9401132fc1b5364b784c613c93fa506e3b5e6d1bf353ec8 \ + --hash=sha256:c8be3392b84cf43373a488874dc1ebf46f60784a70977f92586e18019d71c7ed \ + --hash=sha256:c927ec747633e68c9920bcaad48b0bcefe648812b822f18c34e2d0e2270a3d2a \ + --hash=sha256:cc87f2ffa84a49a77d76cb792f28a48ff3b05a222aa0b2bddbd839ba78e1d5ca \ + --hash=sha256:dd59df9e2fb0aff8bd7fd1adb9f349b7c835245a9d0f18c2f2deeb537190f2b1 \ + --hash=sha256:ddf5a2596d716fd3793434844caf31abc7a40b1e8718431420c89858268fb909 \ + --hash=sha256:de503609fdb71f597634ca64fb4bd2f13d27c97f140e912b1e8ed93544840df0 \ + --hash=sha256:e21d1d4a9db5b3a793649c7ab0ddb4697f2818929eec871a7b9c45fce7b2a1ea \ + --hash=sha256:e439ab450c93455feb0218532ded3e946ec7a9b5f459069f122cfa44b89229f6 \ + --hash=sha256:ea09dc704029930cbd097f75cb0cc1db9852adbcf1a151fac7e9559f7bcbfd19 \ + --hash=sha256:ef47fef9a912c77e3d84b014f701e5a9340f25703e9ed3bcebe37f91fb69dc49 \ + --hash=sha256:ef9440f6f8506246269a82734f5ff9e2e4c5e775b3996fc883cc491c5257eca6 \ + --hash=sha256:f1219a8898523cba821085f2da8a1b962cc696325763f09d7f93538ba43b2d70 \ + --hash=sha256:f863c6100bf926cf47d13f3cd75f9bb8ebd98aaab230eeb4468b91f98f39a6b3 \ + --hash=sha256:fa1b787362a3856e63dd2b89449f6c384d55ee1beb8f94f4bcc871b40f02462c \ + --hash=sha256:fc9feef8f1f001c5b87decadc67c4a5d1eebb62ca39c4763d1237ff62cf2b707 + # via matplotlib +cycler==0.12.1 \ + --hash=sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30 \ + --hash=sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c + # via matplotlib +filelock==4.0.9 \ + --hash=sha256:635e7d67fa92654eed444e75e9ca18426d34e77ad9c469bf4373f75a932f7b22 \ + --hash=sha256:9287fd61b99a806e5202be29a83034c0808a1b9830e820537c2f5773981f7eeb + # via + # huggingface-hub + # mflux + # torch +fonttools==4.66.1 \ + --hash=sha256:058cd823b80bac59e64dfad9e3b6fcd677852f9a3804971bbf6b48cc611e785c \ + --hash=sha256:05aeb146451f37289f782c3c861f3d0f4b86c2dd2e4620b46683544c7406640e \ + --hash=sha256:05c0fff6b4a5d872ed89cab2c4f81060b86ace263903eb4e8d0edcac47a60dfa \ + --hash=sha256:08d8956e3ec990c75230d92f1630b215e8f3738c83a003421c22b31ebfd0ce15 \ + --hash=sha256:09ae73bd219e1245debd8376077a0fa6e03175e255c4f51bae5f6a271bfe384a \ + --hash=sha256:0dc6fd99cb8c30941036308b148da9432640442a6f26f36d71dad9be24cbd0e9 \ + --hash=sha256:1801fdad5600118327171e0e8aa79f7cc48831dd55ab36998c9de03bd5ffe6cd \ + --hash=sha256:261d8dc95845e751f975fe8d6075600593ee253470d46d1b84801688051b09f6 \ + --hash=sha256:2aeb745f2664eb811026997c95628071137a777ea2ad296deec9cb393f0b23cf \ + --hash=sha256:2c7340497cf53490293e0c2b61011e0191633022ede0a0a964a68157a98b0fb4 \ + --hash=sha256:2ce4c93160535761f22c80b2afbc96cabc09855363a5d1a5554265b8a4c85901 \ + --hash=sha256:2d320483928c7831f0139ecb361954a26b2e2a8995681200155835dd8cd4a7d5 \ + --hash=sha256:2d637468dac23aac0e223bd52e66f8faa3b0dfcef57435460fa2107e830226cd \ + --hash=sha256:3087a430722aba8de429c2539fd2a58a9cf05238cdfefd8626460001052ca878 \ + --hash=sha256:34378db9a398b59de18cc79d942f0a907c6fc6301945e065ec888202f607aa3f \ + --hash=sha256:36bb24d4b98faacaff04af1d5e0a4285feba6ed1da6728cd34b6b6deb6bbb934 \ + --hash=sha256:38ce8f5fbd5c17dd2153d47d7c8d4108f3deda3f2b4a79b60ddc470a58faded3 \ + --hash=sha256:43d1284c1964666ee833f2badd3017dc138f53d4889043ffca66c5ce4188f188 \ + --hash=sha256:53e5854ea8003efec34adc0863c18ce91da923018354d27366f7fee7db928d7a \ + --hash=sha256:56d41d650cb8fc6cfe1d85ed7c62a0a56cbeed07bc65ca795475b914d401312a \ + --hash=sha256:5de5d80fbc0e50ff794c244e8fb7afd3eadfe0fa232ba8b162b8c551df22fcb4 \ + --hash=sha256:60f5ea17aed4262630afa43f26997ceabd6417fa05dcedf54c665f5a29193e18 \ + --hash=sha256:64967c6ddb0d4c610dfd8cb1485981b2d27972ddfb7d4bbbd9e199d2a089c450 \ + --hash=sha256:668f092bc0de8902167df6a0d5c5aedc3b4f9e43cf88eea92e9b46a2bd3968f5 \ + --hash=sha256:66fad3b7874062c2a2692f0ae6dea56d24f01b778c7f191950ca3ff997e25a88 \ + --hash=sha256:6946fe7bfb28590a1fd4061a17609c9a843952deb65dcf30d1fe725070c3e7a4 \ + --hash=sha256:71c7ca1b5f46f5dd549f56b47d47c0b709217675c23d3a7bc6aa1a69b6d9bbae \ + --hash=sha256:72299346b96b9244dabcc051b24e4653da4edfda6105544cfb10ce856a1afaac \ + --hash=sha256:7234ae9e28db64273fbbfa72caebd0a97e3bdba6b05064114741b9539ef339d0 \ + --hash=sha256:7b8ff9e0edbcee2fbf7dff0c41b9041c1901c26acf64e23adb67495012df11de \ + --hash=sha256:7cf4f996f9b1cb549bff9ea4c50813988a26ec922c95cfa85c7e4f1270447e06 \ + --hash=sha256:7f49f2834f5d006fe0f3bb10fec73b261806c50941f0cfbc08294074ffc32210 \ + --hash=sha256:83572afe48733bad7a4a9c11721d3a726c2e976d82b063fc9bdd049d76955abd \ + --hash=sha256:8526b2b7ec4db6b81efb83438be52b1264eda9a4994d867163cfe8c65581ce8d \ + --hash=sha256:8aed2bbcd6216253ef1b015763593365ee8084f621dfa53bb957c19d5e05f7cd \ + --hash=sha256:90de3477394c73481d27d2b86091c1c736053ee13ff52c42f0e151948e8578c6 \ + --hash=sha256:9261ef507f2dd74203443a472b65b5a26429eb378f975016dec7dc7305b24898 \ + --hash=sha256:9ea6c93091cbf83161a544388746a0911550bd98cb911faca3591cf5ead166ac \ + --hash=sha256:b13c8c541ce0b794add3211b3641cc0e113d707f73e06235e6fe9731bd7c45a9 \ + --hash=sha256:b18803cbdef248e7ee1be59cb277fbbe1da1faaa6f726fa5d3557904e6a3d967 \ + --hash=sha256:b878c78b2af11b879bd4f26bb0d8bda2a4c64543fdd3f28efe2c80f97f043885 \ + --hash=sha256:b8b71db96d605784e2c5ebf0788a406018ea8fdd80338491f4c83613d5cd1fec \ + --hash=sha256:b913b8e9f7ca9bec44d1eb919f591c596c61041aa357c96be55ff93169859e91 \ + --hash=sha256:c258eba62260beb33c110b03a6912cefa3635239c4ab5615b7225fb6f7b85238 \ + --hash=sha256:c47299bca4b5acaaeb32100f77b944feea151de9ef1773365a410dc3d49b945b \ + --hash=sha256:c666fefdd5613a0e99aa4516e6ff4ef87aa86cf1c7ba12a73550f4770e46b750 \ + --hash=sha256:c724e56213494c6695335577822b2d1628d102e71614de8b7eb8e30886d6a314 \ + --hash=sha256:d3b5403e82d0c7659ff1d9f956e29a3a68d094f043e9f5bc0442796fc3a4fb58 \ + --hash=sha256:d4f76868aea9cc4ce47fdbeaa904c02ee7d85dd0ad095071ae77f0bda6e62cf5 \ + --hash=sha256:d84ac0bf776b68396185bd919dd29e633d94300660335efc40b55b294b886903 \ + --hash=sha256:d8f0a8f16c4f3a5a87ca971de2631792d8cb4d570951f2000acf712f157d40db \ + --hash=sha256:dbb7b950f8c02deaffb6968994691e8589d671b7ef8396bc9d5b5c0dfbb7292f \ + --hash=sha256:dfba62cc93199ba62c376f90f2a9147d92730d301e44f88e013e50ff5edf6193 \ + --hash=sha256:e1cde50b3ec84ca6fe63ca815de183dbecb88e8adf8ada82d8ea130ef12b2b43 \ + --hash=sha256:e7ea7a08547a453fa000db96ed5714a3dc7e2b4255b9243f897921f8c10c169a \ + --hash=sha256:eef76d5796e604f9d6753fa6d323c4eb9f4e0e43f1dcca553f3e6914f1667b64 \ + --hash=sha256:f08ab7f8461c37ecfdd29ad97fb0c0780b50501bd664bb0f46b6e83ed2b9d2a7 \ + --hash=sha256:fdf4afd75c643e60ef4a96fe64fc8a9def27d2a542112332371a9e5066885f9a + # via + # matplotlib + # mflux +fsspec==2026.9.0 \ + --hash=sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe \ + --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f + # via + # huggingface-hub + # torch +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore +hf-transfer==0.1.9 \ + --hash=sha256:035572865dab29d17e783fbf1e84cf1cb24f3fcf8f1b17db1cfc7fdf139f02bf \ + --hash=sha256:0d991376f0eac70a60f0cbc95602aa708a6f7c8617f28b4945c1431d67b8e3c8 \ + --hash=sha256:16f208fc678911c37e11aa7b586bc66a37d02e636208f18b6bc53d29b5df40ad \ + --hash=sha256:1a6bd16c667ebe89a069ca163060127a794fa3a3525292c900b8c8cc47985b0d \ + --hash=sha256:2c7fc1b85f4d0f76e452765d7648c9f4bfd0aedb9ced2ae1ebfece2d8cfaf8e2 \ + --hash=sha256:3a736dfbb2c84f5a2c975478ad200c0c8bfcb58a25a35db402678fb87ce17fa4 \ + --hash=sha256:3ebc4ab9023414880c8b1d3c38174d1c9989eb5022d37e814fa91a3060123eb0 \ + --hash=sha256:435cc3cdc8524ce57b074032b8fd76eed70a4224d2091232fa6a8cef8fd6803e \ + --hash=sha256:504b8427fd785dd8546d53b9fafe6e436bd7a3adf76b9dce556507650a7b4567 \ + --hash=sha256:57fd9880da1ee0f47250f735f791fab788f0aa1ee36afc49f761349869c8b4d9 \ + --hash=sha256:5828057e313de59300dd1abb489444bc452efe3f479d3c55b31a8f680936ba42 \ + --hash=sha256:5d561f0520f493c66b016d99ceabe69c23289aa90be38dd802d2aef279f15751 \ + --hash=sha256:6e94e8822da79573c9b6ae4d6b2f847c59a7a06c5327d7db20751b68538dc4f6 \ + --hash=sha256:8669dbcc7a3e2e8d61d42cd24da9c50d57770bd74b445c65123291ca842a7e7a \ + --hash=sha256:8674026f21ed369aa2a0a4b46000aca850fc44cd2b54af33a172ce5325b4fc82 \ + --hash=sha256:89a23f58b7b7effbc047b8ca286f131b17728c99a9f972723323003ffd1bb916 \ + --hash=sha256:8fd0167c4407a3bc4cdd0307e65ada2294ec04f1813d8a69a5243e379b22e9d8 \ + --hash=sha256:a5b366d34cd449fe9b20ef25941e6eef0460a2f74e7389f02e673e1f88ebd538 \ + --hash=sha256:cdca9bfb89e6f8f281890cc61a8aff2d3cecaff7e1a4d275574d96ca70098557 \ + --hash=sha256:d2fde99d502093ade3ab1b53f80da18480e9902aa960dab7f74fb1b9e5bc5746 \ + --hash=sha256:dc7fff1345980d6c0ebb92c811d24afa4b98b3e07ed070c8e38cc91fd80478c5 \ + --hash=sha256:e66acf91df4a8b72f60223059df3003062a5ae111757187ed1a06750a30e911b \ + --hash=sha256:e6ac4eddcd99575ed3735ed911ddf9d1697e2bd13aa3f0ad7e3904dd4863842e \ + --hash=sha256:ee8b10afedcb75f71091bcc197c526a6ebf5c58bbbadb34fdeee6160f55f619f \ + --hash=sha256:fc6bd19e1cc177c66bdef15ef8636ad3bde79d5a4f608c158021153b4573509d + # via mflux +hf-xet==1.6.0 \ + --hash=sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7 \ + --hash=sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675 \ + --hash=sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef \ + --hash=sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9 \ + --hash=sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3 \ + --hash=sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb \ + --hash=sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e \ + --hash=sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338 \ + --hash=sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d \ + --hash=sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d \ + --hash=sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765 \ + --hash=sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c \ + --hash=sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522 \ + --hash=sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f \ + --hash=sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a \ + --hash=sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b \ + --hash=sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b + # via huggingface-hub +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via huggingface-hub +huggingface-hub==1.33.0 \ + --hash=sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899 \ + --hash=sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e + # via + # mflux + # tokenizers + # transformers +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # anyio + # httpx + # requests +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via torch +kiwisolver==1.5.1 \ + --hash=sha256:007a5553dfc4f4e8d184f588a0200e2cd4b63a59cc8796df3c39909e679dc7a0 \ + --hash=sha256:0324cd2567259b7a095f6cf18a52b0ffc6f3de9e69528ff1bc0e7a37bd43ff1a \ + --hash=sha256:0627b9bceb9c3cdcf12b8a18655eedfed2692b038df27423383c120d0b7dc2d6 \ + --hash=sha256:06a6917674de9e0fe3f66f5430787f59a9f2ddb64af9b714eaec547e29ef5c19 \ + --hash=sha256:072bdb15a3c19a5b5dbc8f8fb1f4e1884bf4f3507eeb4cc6334401274d37a5c0 \ + --hash=sha256:0a4faea5c6db201c6a21391d2ac926ea97acf7dacdbc3c417189e1adb1a00837 \ + --hash=sha256:0ba9527afc80ae3d7814ed98b6572d02bf85eaf48065678342c5f0c6dab7a8c7 \ + --hash=sha256:0d8924877ce22e17326a99a418c3c82037da078df3c6a260b13eca677444e6e7 \ + --hash=sha256:0ebdef3eae5336568147c39a55be6a2036ffde53faa9ca2d978989ae7c2da12c \ + --hash=sha256:1209042a623ddfda5497e4066c7b77651dde8e1d3a9dd97599dc7e97f3b9b78c \ + --hash=sha256:16895f553ee6620a827d2da56b871f835fb70b9216cca5d188e885caf6e3bd23 \ + --hash=sha256:17851e5dad4484be0cbccbde3b15331deae036de9aebd45eed964487802b172f \ + --hash=sha256:1798e83840c3f627246104c4d8a9639c60fa068adf9ce92b61791781fa8a68c1 \ + --hash=sha256:18170a77ddfecf40ec60d0928268dc95880c881864e015a8f34094ed18b9b9ad \ + --hash=sha256:186884a58486651e3c217b6acea0a53eaa9498fdd472057c46f2f0fb5c25aad5 \ + --hash=sha256:18a0cfb124546a4c2e6087c5f3029c7f44b37c85b142e0ced71f73a7599ac208 \ + --hash=sha256:1983f0974a750a6f6556f368ba11105d1d8369c735b944747c9f12ae5aea7aae \ + --hash=sha256:1a7587dc335f2c0f5bd577fd0540bd16c66006bdb60f759a1059f025e6c4f071 \ + --hash=sha256:1acc7e5b7ef05e9da8bb70cd6c7c4513090213d2e1ad9720f599f0bf6c52aec5 \ + --hash=sha256:1d852545c4d0e35a72728d072cbaa59e2fa7dd84bdf01e068d670dd0ceb58eb6 \ + --hash=sha256:1ed0f5e49d0ceff8b72190824d9e59c062fbbc02c231b853112c78474b3f5ec2 \ + --hash=sha256:1fff05e239575b1481b6ed1a782f6fad616efbf1f0b1f44e6e85c4dfe426e483 \ + --hash=sha256:21e46b23a2da695c364124817bc01d970effd5483147f8d66a6a7167e3f6b851 \ + --hash=sha256:22d5e5aaad6be121f2515765e3b1c444352cb8eb4c86510801db8f2e50757316 \ + --hash=sha256:2551cf9917af48ee7c4b29cc82320489508cf96fd26a51f6fc124de661cd44c7 \ + --hash=sha256:255605693a483db7bd5c79f60437f7bf658f7f520d61aa42722e32257c941951 \ + --hash=sha256:26e8268480be5061d509e29669d59103c067a26377a56491630ece11762e3858 \ + --hash=sha256:27add358abe374ebaa3b8763ef380bc99051b5a4b18d94878366a9e4f59efef0 \ + --hash=sha256:2ae70bc59790d2af72a3f76f24b272403e135070340281108b447cb77ea70819 \ + --hash=sha256:2e10ae1bba1899188b33557c10d73affcc12033edd18adddb57d209039976a4c \ + --hash=sha256:3221f78211074f561c44ca42eac0619828171bec15a2c4cf6f7747d07df76e8e \ + --hash=sha256:34633ecf50d16187ab8e5528b7a2530f2feb4e23f300db4672538b51cfc5cd38 \ + --hash=sha256:34ec467940442c9943016fb2d4c81d1ba84351eeca2f1a78f8bc87f1ba0d414c \ + --hash=sha256:37f801b5d7cc0e5a548921308e059fd2b057bb42972b591cfa3049f95423c4ed \ + --hash=sha256:38f6e0deb4d0a4615efe0c4efc5990b06ae450ab50a0b321c0b078b6d238c083 \ + --hash=sha256:3c24cd69455e1b00ddf770c13b6e2c33e07d6dc3f2d34add0bf9277c5c6bbd46 \ + --hash=sha256:3cc210010fd2f438a3ed430b45f1b501fd13a8618bf984dc2c5ce5b69b78752e \ + --hash=sha256:3fa5855898f6d3d01b72ccd48a2d65cbdee301251603fefe34e2025bddba219c \ + --hash=sha256:416ba7ff9f233b7036689bb5a3783537e838ad483f63558d2a800f75afe738b1 \ + --hash=sha256:431dc224a1a92a5c8f582d96e505196a3b5997a7271076678da2dfde67b77e9a \ + --hash=sha256:43844c1a7ad6d723d5b5b4c4fc7f5bd399c40e288120d16257c7c9e8765c6e85 \ + --hash=sha256:44b8faef94f1857e77fa0238f3390ff1ac51d2ea20a487e2e452a59fd2b5f5ca \ + --hash=sha256:470d420f98d368d6f010633a20659b544c5fdfa5329e6b70219f2ef08fd4a7ef \ + --hash=sha256:482676e5bd48d70ac99d9fc78863469845421e01184fa83f1f9366dc49f7e974 \ + --hash=sha256:4d4ca09bf13cff792b1884f64b98ee6c2467930d632233be25c56b442d99f10e \ + --hash=sha256:5025e36fb4fb275cef0a4e30dbb11cb4ae61d1c83deb90189cb5d7e4cafd6b55 \ + --hash=sha256:509735237ae0d849e8a843551d423d2500d2e0a9ac1611a145658b29c0fb9f85 \ + --hash=sha256:534f02c1abb31ed6dbd3515545285c330b2f12d00fdb1fdb71658b9ca5a13a6a \ + --hash=sha256:5978c3340f16a35c30f8ab2fa7bcf559973c55f1a5ef6970e1f621acf3c4db13 \ + --hash=sha256:5b973887ff782cfd6b67c9904ad8ca542e0bc5e4961503408b423b5a688b4d38 \ + --hash=sha256:5c490db2168a508088f59140dd392556a54b8bd1048fc6383c8baff13c359673 \ + --hash=sha256:5d142e352eb13facc7dd047489aebdff6ba78576c239f1ea04931979caaf0567 \ + --hash=sha256:5daa1f19e097050b9c4d9a78fcc9263cb96c9dfae08037ddc1b7c4ad1889f2a2 \ + --hash=sha256:61e9a64c7635095a6bfe483e2ff055d437c59bd45f3617a228b37277f0185d62 \ + --hash=sha256:63fb7294b768f444eb4b068965f2662f28c2fd4161e23bd60fcf3ff27b74c046 \ + --hash=sha256:685929988b208a911f1285e2f8ed54210b0d681a3dc0f03e00d599d291986e7e \ + --hash=sha256:6a797a1cefc8b9c93170db580337e1fe3d011ad18b1299943231279406342048 \ + --hash=sha256:6b92f60017dda7d877fdc546438b5e28f31c523264f49cf5a48c1d0ce1a0dfbc \ + --hash=sha256:70ed9a45c7484d2b30cdacf60d220f494a1763b9fec1ad03285c6553fa0889f2 \ + --hash=sha256:719a35fa1156db3640555f95ebb94f60a444e64d1c69626b0edef5df78eba225 \ + --hash=sha256:74ad5c3dad54a4641b4c28cd15ded70899d04459c6c7aeacafea716be97cce6d \ + --hash=sha256:74ea337e0ec3f6f342a36a4f1b5cd94dd9affddcd28ba9aae2905af932ee8c6b \ + --hash=sha256:75d9b1cf8258462dbdc1eeda718c96ea7f079324c09067f6daabfcf37712b7fe \ + --hash=sha256:77a4c8187a5948d7f8795adb765a3c7b553d07d86d88e43038fc32fc1fb9a3f3 \ + --hash=sha256:7824b5e8bdbf0bccb4ccd37bbb115849a1dc45437fb4de8351385ed07c437ee0 \ + --hash=sha256:7d38b0c279c3032e8c9cc013b405c6df8e1668dbf15465779aa7f15f61201812 \ + --hash=sha256:7e9c01d3dd7ceba4d1d436cc021d40d592466e40b9bc7f5d83dc4e98a5c9cd8c \ + --hash=sha256:7fd82debf43c6acd0a94359d232f6bb516ee13f269a7993736a9ac9f988bb5d9 \ + --hash=sha256:824c3d763a05ea9e9003610145186b0e9848c7584a5575c79bac5a8e7cd80bad \ + --hash=sha256:828f75af2b0080c8a972e75f649ab46af008e92c6104a57a759157200b835b75 \ + --hash=sha256:83f78128fa28705fa85d01c59771c72fe81c11bd0e6155edbb9f818983a7d761 \ + --hash=sha256:876bbfd276473d3daffe30e8c975df4ed9429967b41a6cb362dbb5155b6f13ad \ + --hash=sha256:886fc26012f0e8b5f69d1cfe6d711f6b11f194621539bf8e6bb1c25c5dc82724 \ + --hash=sha256:8a34616dc2521cc8dc1d7d081734da63539f021ac0450ce950908340c6e7aa2f \ + --hash=sha256:8a708a47ade1fe19e8371d5da076bac0dd4b0a5a7985ad6c637f7f7e361b6baa \ + --hash=sha256:8af9b142ad719ae3a911ebf616bc4b78b32bbab84d6a40d3ad2f129670509957 \ + --hash=sha256:8bf4df63592c2a66b4f8edc5df2544998c288aa02f96ce0acd880cd1de8c8127 \ + --hash=sha256:8de6f2a4ce7e7bd27d23dd94abf0ccafe0e0e5cc9c764b0577191f2c25f08f26 \ + --hash=sha256:8f8fddb8e323bd6eee4e54e69a39243beab22689070f4c66b472c4cc88bb89d8 \ + --hash=sha256:8fca690b00c4c48f6c2a547b0160ed511357093a4e4c9b47e0fadf3128066d89 \ + --hash=sha256:9506e892bcc3b409831d363c6f53e5985e1c8d1f6f6b0256d00358684ff85378 \ + --hash=sha256:958254518717542d02d0688d0d20cbf771da5e415e6f49543f92481c850a4540 \ + --hash=sha256:95a02752aa032eef4aed01cda6d9b687c669bd0396bf4519eef8bba22a286720 \ + --hash=sha256:96c30002424670b5e1e46495c2b8cbffef39cf77c1d79e76462029d50339785b \ + --hash=sha256:98b208a7cc42c803445ef551d6753cc42a5ea13e9cab1ee66cd8b9cb70195330 \ + --hash=sha256:9b3092d8992a1d69b7a59c3e39f35e1b9be327a17f68a7c35fc17329e337d6f2 \ + --hash=sha256:9e51c119992ea8820706871c30a4642ec76de20ae82f9b50b9a45517d8e9f810 \ + --hash=sha256:a5716a33bfabb2c6ce27b6cf03253467b3804f83e215f4d202685cf93c6c9874 \ + --hash=sha256:a5a00665d1a0e26763a7338d7e911d4598fbc1d50dd0d6b7919b7dc6c5d6569f \ + --hash=sha256:a5ca5aebae78a0bc13c1943af4af615d4966c5b650b05d5aa83b50e427196fee \ + --hash=sha256:a7b85b2cc6ea45e5f7e8c9a30bc9fabd47cda09106cbb4b967335c3e6c43b69d \ + --hash=sha256:a83ee7107df13abe42a54a6654670eef9bb39425cf2e27f65e0007465e1286ab \ + --hash=sha256:aa7d00b1700966d2917e54d278aba86897890ca9276dd8b76cf6446b6c181b92 \ + --hash=sha256:ab620eb663952455271ac37f9aaad86b73c969c02f11f53cea405b38e96a4300 \ + --hash=sha256:ad8b9671348d7c8716715652ae11f85ed0eb99e265a2df2ca490577d69860b2c \ + --hash=sha256:aefe930d113798330e9462f7874542977869c0613cba3262e2de3a8d5dee8f3a \ + --hash=sha256:b03af77d77e50edba2030fd5f7c352ff209314b09030a3cba7c14edf9a09a444 \ + --hash=sha256:b390aec180a7c054919c04898835e1c77bced23ea8383eb2c570213bf25d1a86 \ + --hash=sha256:b3d78f7bb2b9d9a30345be1474b9aaa8685430b54afb51ba3639b5c6c11e9ed6 \ + --hash=sha256:b5664603a253efd3a75716d793d1d3a6a82723b61dc6db767b2460bbbeec4c0f \ + --hash=sha256:b69602970994a2ed8bbfa78c2f0394a7435226c6040489702d9f0a0ad0c07052 \ + --hash=sha256:b6ae6a0328f0bc035741820fdeecdcd67bf4694eee03972e843663107122f450 \ + --hash=sha256:bad20d4c69c851c982a1e3606f4c293edfd5a87885786c50082412240c4b1ffd \ + --hash=sha256:bb7c99f0673c03017a3ee01e54a5c2617a05468b11eabe513b0080e063ed95b1 \ + --hash=sha256:bebb89489b279b2f5661bbbb2abcc87bcd4a46607bb4a5c966f04f1db6b8df9a \ + --hash=sha256:bfd1de989b3330420e29de39352f5c049905c9e3ee67233a50d550e3d652c148 \ + --hash=sha256:c2306e8bb53601979fcb3fa09cc65e031876d9ae01eff2fcbcd7a84ef94d5bc1 \ + --hash=sha256:c3a4e41e3096bf1f0f1b76e2ffd6d828d6547f574f702d59bdbef7acfa59db9c \ + --hash=sha256:c6834b92dd2428e2dd85ef3d85f723d3c12f20aaf43a2ddd4f944ca25d833408 \ + --hash=sha256:c90d3022d8a94778939cda8638c6c8da8fa757b8958dad7ec868ce29c87681b8 \ + --hash=sha256:ca307d6c259e5c98d3cb9ade55342b47a6839762caf2536f3d7b46ee660cc82e \ + --hash=sha256:ca7f6fe0f37ca978a1e5eb7a3a68e6413f417e78e838324947ffd420202b198b \ + --hash=sha256:cb6fae641357ed2f6e533c0d3c6504a4a5703621a50c89459e46051d56b61140 \ + --hash=sha256:cdaeeb6c350106df6bf9d873395973e5f066a9713200b72cd64f55d0a3eafab6 \ + --hash=sha256:cea20da04494e662b83c872683bf4ff2345206043d036315ed0e924b652e7294 \ + --hash=sha256:cea90547bfd93807e0013a004dc76552be44fad3bc1cc2b38610a9e889ed098f \ + --hash=sha256:d09037ca068d784ebc4aec290ef952ca27ac15dd9c0b5801a88c6e1096b83e6b \ + --hash=sha256:d27c2123977cb9269c30a49ba45f03a4323017ef693e19db4ec9dbe1299a3002 \ + --hash=sha256:d50de98e8d807dc31822fff96f50293163a62418eb65487a21b42713d72ed0b7 \ + --hash=sha256:d66a64dd5dec136040ec2ae94aa026a912ee60fdd45bc28d3db30037fd809e88 \ + --hash=sha256:d79308fa689fac89cbcfbd4dbfc80b5f95c54c5a7fd4d194be221f9d33d026e6 \ + --hash=sha256:da3275833be0edbaf4830fae08bae3dc7219f40ce0c37eaa6c25825957e06612 \ + --hash=sha256:dc1a26b8e53395a01c2c611e58602fa47461f136fba7cd5542e6db6d64be1839 \ + --hash=sha256:dc23390afe9f4ef9ac3bcc72a03a56eebbde03f4c571a32cb38f859cff9a6524 \ + --hash=sha256:e05c2f7925f1d88778e53cb44f14e0223204a3bdd09a41664750363acfb1f2ef \ + --hash=sha256:e12dfea7f5fc2a34a9080efbf79c4c44eb380ec5b9c6fea09407e08f0d1e941d \ + --hash=sha256:e4e4523d6f336708d732516e6cfca7796cf3d96c9474eb5aecf6165f2f1fefc3 \ + --hash=sha256:e4e49f7e1a4e7191bdf9dc67a974db714501b1fc52c24324103d06a86abd5c08 \ + --hash=sha256:e68e151428b5384f766cd25739bf77c7e4a3dc93b5ded7a12118d9fbfdf78ab6 \ + --hash=sha256:e8e4d953faaded9ec7ede36824e9814082d22d4c7b1eafbfa079ecba8cd0d076 \ + --hash=sha256:ee9df1f0d77b9c6e94f4ac0fec533fbddd5ea3a327807f18d7b069ae019ded80 \ + --hash=sha256:f0a887b6565bbfe80efde2b7f6e8890d7d9bbdb11bdb17028a3690c32fe0621f \ + --hash=sha256:f0f4a42db92d6ec7677ab9d12830a2a8ec145a9c6d15db2b593466bc875c78d7 \ + --hash=sha256:f1303ef2eec81262a4b708c3e858afe58d7c75ad91c1c05266eda7673369859a \ + --hash=sha256:f1d56ec54d257d05e0b50f5780d967540cd07beeaf9e5f645b26d50cce79f4d8 \ + --hash=sha256:f4167e87b397f273dc2356fcf1eaf50a6bac51e6105f45103ef7129c8efb0255 \ + --hash=sha256:f76fc85bd054c806960f917ec0f329e24e436f1712267d90588e4c39890caa63 \ + --hash=sha256:f942903fde7363d1d879057ec5de01310efda2597161784d752fa9953a01a71a \ + --hash=sha256:f9b1c4900736e489a812c529100de4b8fb617d4db075e931e213c57424b83d9b \ + --hash=sha256:fc271a6f0a2126958f4090e5507b9da5848927dae331f8f763bd4aa642b3d2cd \ + --hash=sha256:febcce10f2bcdbb80b4ea919238a6a4ac13dbc4c7cadbe8d5d75c3682f8b5404 + # via matplotlib +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +markupsafe==3.0.4 \ + --hash=sha256:007e1ffd9bf65bb6ee96df7b258fc632a4868dd5566037986c64781f35a36e98 \ + --hash=sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002 \ + --hash=sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b \ + --hash=sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653 \ + --hash=sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c \ + --hash=sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e \ + --hash=sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc \ + --hash=sha256:0764a13d34cae40db7bbf3a09b7e9b491bf4603e20b263a7a9d6b8e324975d0a \ + --hash=sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92 \ + --hash=sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f \ + --hash=sha256:0cee7cb0f9a1b6892ea482237d9403b3d1b4603aee057d0ff01f0fac2d019a97 \ + --hash=sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4 \ + --hash=sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7 \ + --hash=sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691 \ + --hash=sha256:14bd2d845d62ab678eaf81da89d7b621b51756c72346745c1a594c09d49207a2 \ + --hash=sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc \ + --hash=sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde \ + --hash=sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99 \ + --hash=sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9 \ + --hash=sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df \ + --hash=sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5 \ + --hash=sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17 \ + --hash=sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8 \ + --hash=sha256:2a6ef68ae94aed8721934072b27a3b654ea2100b97e4ab864cf1489c90926fbc \ + --hash=sha256:2b2b1e18af909b448bb3cf9e3433366f7a8726271fc214e8b10e0f62a78c724b \ + --hash=sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea \ + --hash=sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248 \ + --hash=sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741 \ + --hash=sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5 \ + --hash=sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6 \ + --hash=sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7 \ + --hash=sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1 \ + --hash=sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67 \ + --hash=sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f \ + --hash=sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9 \ + --hash=sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c \ + --hash=sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc \ + --hash=sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba \ + --hash=sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17 \ + --hash=sha256:3d23795802fc8bd72534836d64489bbf0f67c088959091bdb22e10735a5107bf \ + --hash=sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6 \ + --hash=sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2 \ + --hash=sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163 \ + --hash=sha256:4a540e2d3192792fc84eced57bef37851ccb2b41f73291bb17408eea77bcd278 \ + --hash=sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d \ + --hash=sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b \ + --hash=sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634 \ + --hash=sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38 \ + --hash=sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed \ + --hash=sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c \ + --hash=sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148 \ + --hash=sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a \ + --hash=sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7 \ + --hash=sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f \ + --hash=sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811 \ + --hash=sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e \ + --hash=sha256:57f9947a7e57a081c1e3e0a2dd0d2dcf290a4531450e6f611e30084c222a7295 \ + --hash=sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2 \ + --hash=sha256:5c22873ad1f0532ba40fa1727f3c0fc1bbbaab6d373d4cbe3f0dc74b2e2521c7 \ + --hash=sha256:5e8b3d0b18fd623afa12ecb2ce8d8becef69f9b5440c6330c7972200e0bb84b0 \ + --hash=sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6 \ + --hash=sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed \ + --hash=sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378 \ + --hash=sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0 \ + --hash=sha256:6768d67d1bce64270e0fdc2e69309d68b9b18ae56ddf6c711d168e9d051c2cac \ + --hash=sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b \ + --hash=sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96 \ + --hash=sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59 \ + --hash=sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808 \ + --hash=sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2 \ + --hash=sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb \ + --hash=sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65 \ + --hash=sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72 \ + --hash=sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8 \ + --hash=sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e \ + --hash=sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91 \ + --hash=sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a \ + --hash=sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2 \ + --hash=sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e \ + --hash=sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707 \ + --hash=sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21 \ + --hash=sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef \ + --hash=sha256:8698d70a8081ee8c090dbb394768b5789a1da8b131b5499f89d071dd3cfaf6be \ + --hash=sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453 \ + --hash=sha256:88d59b473bfb03259722600839af9bbd7fa13a2eb514beefeedb95997882f69a \ + --hash=sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6 \ + --hash=sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977 \ + --hash=sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 \ + --hash=sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692 \ + --hash=sha256:9240187afb63d2f9ddc3e032c670356fe941f6e20662ea168a5dc3f1f317e1b3 \ + --hash=sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369 \ + --hash=sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a \ + --hash=sha256:9388003072b95f2f1e3fd908604194d653ba21330d811961a78b7da1a77e9e36 \ + --hash=sha256:9438a2648b2195980cb2dd8e53ed7b8df91319e2d0b70ae61a9e1d1bc8d3bec9 \ + --hash=sha256:94e4c421742086aeee4c32a506eec8859d7634aad943f7e6aacf70f813478768 \ + --hash=sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916 \ + --hash=sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b \ + --hash=sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f \ + --hash=sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346 \ + --hash=sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c \ + --hash=sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464 \ + --hash=sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9 \ + --hash=sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee \ + --hash=sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300 \ + --hash=sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6 \ + --hash=sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d \ + --hash=sha256:ac0c7c9f1609b0c4c114feb1d7a3409564c7fb77e360bed9e97e5d25dfeaf868 \ + --hash=sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46 \ + --hash=sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97 \ + --hash=sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733 \ + --hash=sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe \ + --hash=sha256:b61687d0828e72bf5cda24a2690188f37170bd31c9359ac97e4e66569f120a16 \ + --hash=sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429 \ + --hash=sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39 \ + --hash=sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894 \ + --hash=sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c \ + --hash=sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c \ + --hash=sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169 \ + --hash=sha256:bf053da3c97a4bc5ecfbb218cdd2983febd91c617be8367d139882aa11e490aa \ + --hash=sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77 \ + --hash=sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe \ + --hash=sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad \ + --hash=sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85 \ + --hash=sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e \ + --hash=sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34 \ + --hash=sha256:cf63c214fe879a65e69a386f915e36104fc84254ab141240f8854602d8e0be2a \ + --hash=sha256:d1aca03ede943eb80ab3d63bb082c84b7aab85ea83bd0fd0c200260945fb49d9 \ + --hash=sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c \ + --hash=sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749 \ + --hash=sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214 \ + --hash=sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932 \ + --hash=sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494 \ + --hash=sha256:dd8ea6ebee7aedbf7c749fa80521d9ccf1ba473e0d1e14805caafbaad281c889 \ + --hash=sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1 \ + --hash=sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0 \ + --hash=sha256:dff05cb7016dff1e9fd68f4122c127b65dfc59de5306cfb7ad92f956f230bee2 \ + --hash=sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786 \ + --hash=sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78 \ + --hash=sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e \ + --hash=sha256:e841068dc0be4cb6dfb5c890eb88cbdcff2f4a332393c7ec94e8e618bd32c1a8 \ + --hash=sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289 \ + --hash=sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c \ + --hash=sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe \ + --hash=sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237 \ + --hash=sha256:f291bcf42ae98eb5107edb162c3c998b4a89648fd8e99ed4cbd12705292788cd \ + --hash=sha256:f61efe1d2fe0de16158a5fe1d1cf3c14bdb6aecd54d8938fd26512c525c1f624 \ + --hash=sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19 \ + --hash=sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977 \ + --hash=sha256:fd9f8797427910198f95bced71ddfed61130d7e349213bfb8466c9c99e2c46a8 \ + --hash=sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3 + # via jinja2 +matplotlib==3.11.2 \ + --hash=sha256:01dc8eaaab5a9fce9ff615eca82345728f289e4715b186ee10c6d85272fc26bb \ + --hash=sha256:02329432ae5c6af87cf208ee575b701d698bdf0b1a3bb28cc6d53c36e967e575 \ + --hash=sha256:07d9b9fa60cd4c393692f50d0bb03123242ddf61c99bb0e95e75feb354e7c1a8 \ + --hash=sha256:116cdb0eb0eb5644fc98eb2975d4b4dd4ad35e5c8e6b851c22e6976f371f7ab5 \ + --hash=sha256:1944895967f87c84c9b4bad29a707b31f5b36df4a3a2339ea1f8ea3ce5105539 \ + --hash=sha256:1a3040b209f3968b4e84161df7b174f07a9fad33b0f2d7e48ea3bbd3075e2863 \ + --hash=sha256:1b9a7ad579856284135e401ecc918c5f8a017ee30539298862a109f51b971710 \ + --hash=sha256:254d4ddb2fa8df3b4c689c0c306063aee10521df82cfb438185e499c75fe37c1 \ + --hash=sha256:2a8285cea8ef4d92aa041d1c33788bcae82248503300f93f1ca2136b9049452f \ + --hash=sha256:30ec15d7eefee71de16b7c689b42ba49715a4644650b76a6c7c70d79daf24e91 \ + --hash=sha256:399fef672f7046ef7d6a57572b2a6f9845f3f5afcff04e7b2df7a363a9f42190 \ + --hash=sha256:3aa4b8516fd26659e4363abbf317c703d9116496c5db2e9d0609a2866dd39dd2 \ + --hash=sha256:3da3bc0cbf7245e7db72cc6d29d12c5abef72cb73059c73b945f14e3545f3eb2 \ + --hash=sha256:3e8576f7c47e02fd4f21f44171302d2d1d58d4471d46da3d71fe8899d19539d9 \ + --hash=sha256:57b9ea60a835937c2012861923cbb91f47db8565775d326d1c42fc926aa10351 \ + --hash=sha256:5e1e923a3fc3326b99ec0a6ff1ab1338ac6c6cc62ad9d8a9c944197c7f8c6221 \ + --hash=sha256:643ff850d8e0f5b8319337f87ed3cb59506afb3df3cc48de777d85871233be7b \ + --hash=sha256:75b6d88402770e181b5d06a67dda4129c31da7d05004d63a21c310ec78d1b83c \ + --hash=sha256:79a258f58253dfa025af80a9e9bb228d75fced007f0e93ae7423fefbde81a74d \ + --hash=sha256:7d43ff8cebb50840648cb6429b2228621dbd709010f3117b3740abb20abf21c0 \ + --hash=sha256:7e5a90f8a707ebb6004a713b1cff091a40cc5df4c7c1cb165e5a505ebc11c292 \ + --hash=sha256:7ef7a53b66780e5d942923724f08577fbc5be1f7322da0f0f3f9dcaa45dd803d \ + --hash=sha256:854df8d7dfe9fdffcbaa6f39e44a6c24b409cb4d7561fbc09213b157d833f6a6 \ + --hash=sha256:894a9cbbecbe30ae6787d464df2e8fc7a8d475cfc68f87c3029f7c11152899b3 \ + --hash=sha256:8c8255de28f986d935a64c9ca71c0ec2d2f41d355691f5ea684725dc91413f71 \ + --hash=sha256:930efb28f59fda124e39265d177bab302625297bb147d2910de725a2fc2aef54 \ + --hash=sha256:a24d5fd36e4f0e742c3851dcd20810e56a95633a342e4bf6cb591c678e8fe61f \ + --hash=sha256:a6939df7567114b6bac7f4c5e06c84a67f197c1b2f2e4b234d4eecb3bfec9482 \ + --hash=sha256:a8756cc73d9af9a7fe0deb54ea2e75ef73b01d9e575877e72acad5458e660943 \ + --hash=sha256:af2661f6ac6bbd1d081996f54fdd9385715c625ace8cec0869055c0cfbf38981 \ + --hash=sha256:bbf1062991d826ed27e2144f3afa4461afbb8ff56e8f703043e191a8163b1ee9 \ + --hash=sha256:bc067c462a86f0e57bf52fc6e058d90171a5420007dac00c46e90153050c69a7 \ + --hash=sha256:bf3fe71fbfb8ec0e310e0bc8537c3405a01f38f25f9394ed2135e6202fed542b \ + --hash=sha256:c0b83f044ce10a98027b105b3931548719a6e8c7ef986b4362651e0b5367c8dc \ + --hash=sha256:c27e577ece613ea12a0790e00b4eb80d901c59a3e16cad474f31d8b1529690b9 \ + --hash=sha256:c5c1c68ee401fc98271263410f0e5ce88285abacf7627132914e8adf3d70ff43 \ + --hash=sha256:c9721f81275499da1feeb36a2cf8192ea086283b3bd16b7dc4c9d7aedb7396d6 \ + --hash=sha256:cc82dde2a0d3e3ad472edce04897ad7146b8d8bfd1df8a32992eebb81af18fdc \ + --hash=sha256:cec596316640f2b394b8f0daa0ea61a8eae82d017b620b9f202befb972a59ea4 \ + --hash=sha256:cf41ecd1b0c0b6f7177ed965a54c2afbe888715c7cf6054dc12d53bc1494002c \ + --hash=sha256:d3304eb5a59442a8867f6920d484591c0fa09ffc29e9260be2feec3351e25869 \ + --hash=sha256:d480038c83691532ed52ff3147db51fa902fc78cb2d8349993a1cdb684435bff \ + --hash=sha256:df4f7784aca81a94f254c0a2767d592ee25f407e488f5fa7203e51093fb6ca27 \ + --hash=sha256:e43b188f0a5b75447bcc197728258166aa64365770ed1caa36595a5e1ca4bbba \ + --hash=sha256:e60cf3047a51edecdc4535a9196bbd6a732936b8e9f8184aabf4d16165884aaf \ + --hash=sha256:eac4b07d4e3743b172451e122ea964f72f152879d4f9adcf3f3d33e518f12ead \ + --hash=sha256:eb3712dc9b464793de0a4e42a7313d50293c751f94bddaf7332a1bc71bccdda9 \ + --hash=sha256:ecea603dd2fbf8242fd31a305a8b12a4ece2de28096870c65fdd0d1e35b8d9a6 \ + --hash=sha256:ef31985c4dedb5f1424e1aec6849a47dd37689cb7fa3c20b1b82187f26806261 \ + --hash=sha256:ef752769cd962f39ea0b6ffc82d1ea43a0012c5a6157c7a075212fa509cfcff2 \ + --hash=sha256:f25446b2981717dca9786bac841cb3fd7efb568e3e3c755dd980481c5cb9228d \ + --hash=sha256:f2ac30cf5eb5dff1b584627ae0b0e1186551a4f69ae3c75073911da497a29170 \ + --hash=sha256:fea03cf56568cc1cba08b470be6a0559e71c3a5b688d54b7179bb35ba23d0821 + # via mflux +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +mflux==0.20.0 \ + --hash=sha256:5d60a278d67289be0da2e4ba0e8dcf938dd3c676a01c9064f64e3fb10e30767d \ + --hash=sha256:8fec3407bbc821cfba9cebd8ae55e35450b9429ec9b5445c9557aaeb78087de0 + # via -r req.in +mlx==0.32.3 \ + --hash=sha256:0512049c81fded99aa6af2d039ae1fccf113ff1a5f2df937a596578aabd46869 \ + --hash=sha256:097856504e6cf5944b9aaa7c2e1d5be0693428abd9175429cae3893e7ec2f681 \ + --hash=sha256:15e8578c9d10b4fd4728243cd6c926bc46e1da267c6a849cdbceca21d51caa51 \ + --hash=sha256:177bbb598bcb37963568f69b84e7d972f7f9499cdd39d1c6d6dab35a718aeb92 \ + --hash=sha256:19da79adad1aa9653c70819040ac58d65d7a9610a375a30d03c7cce708d4d2bb \ + --hash=sha256:231a278b4fd2804ea16d297960ab169d1749ee1adbf820173e704fbed97bf1b6 \ + --hash=sha256:2dc1085f5c2eba18cdac095d7fb0b68ddfadd51bf695b18a1102eda15e77e76d \ + --hash=sha256:32fa23e57297fdca23e016570229b459f20f645eeb5965527cd890f74143f1d0 \ + --hash=sha256:448fdbc99c5fb0c26f11337e5a182cd6516f8a1a65848184be063d00ad3107dc \ + --hash=sha256:452c621862684e8769be93c1517420c8ba2fd6e8b01a1c970c8ba8069022f0ac \ + --hash=sha256:46df0a5489edcc071cec3d9dd1f7894d492888c9d425effdc8831f4e7021108d \ + --hash=sha256:4e1a8fa78461c6cf1f9473e3bc8a3c4e362f1aaa613b435787f2844fad522bc7 \ + --hash=sha256:50939a8bd54250e6092ef3427a724c7a72c0911dcee09d70f6543431e7f28d3c \ + --hash=sha256:5d12de01f23d2fae739ecfecd9c1d4c07c1f9b778a3729f7c0884ea1168ec14d \ + --hash=sha256:65122a7eeb718df56641d7ac6f7fdb0ada573fd841d1fb0a61a6a443e33f11b5 \ + --hash=sha256:656df5e36b9bfa862fd633d55dee4da072d292def9f27f22134c594e03ff28a7 \ + --hash=sha256:712ca29119d9e2aae04af8f8e416bfb0097538d9ff73e6fab8b028053fb45ff8 \ + --hash=sha256:72711cb23cc6dccc5a460f63f39c11931a00b2c6dcd6fba253921090072e21bc \ + --hash=sha256:75333afef55819afd2a31e87ffbed0bf59f379bca21771e6b13a95a675e4c85b \ + --hash=sha256:78372731c18fadb6d6960c9188cb752d5ea4d87de9be430a311a385e01d7fb42 \ + --hash=sha256:7aecc3d4a904279492dae58e28378a3c4bec4533e7066e64ae5c78c7a5511f22 \ + --hash=sha256:8439baa207db769df1df82a722b0ee0c2362f75c23c0740b7472c3d279253bae \ + --hash=sha256:8564a59943346fac01ab498870a9350d0f79b1cdea95c46c895934ee027d6197 \ + --hash=sha256:8edf996749b331659ab9bb8ecdc895c292f9e7020d02e534147e07cae22bec7c \ + --hash=sha256:8ef541373a94a78fd59ac59368501e976cef5fa07738438362e942315be22ea4 \ + --hash=sha256:936cc82b0c050686c43230c85f3a0bd9f179f650377ce9590f7af3ba659e2fce \ + --hash=sha256:942f0da5328986cdae29ec0eba57b2a08eb9dc6125cfe196853de7498f2c5359 \ + --hash=sha256:951ba2ab75148f61e9a5c222a75fdb0ab481be0b196c21a0d087e9b886d204f0 \ + --hash=sha256:a14387cd679018a32e802fdaa58b1e9845a278cacd309955eddca95757988ce0 \ + --hash=sha256:a5e165179e3d86672cc0d594587bba2b9c3be511ccb99ffdc63e4cc7119eec55 \ + --hash=sha256:a6241ea8f56a08a6c3a06bbaad7d7fc2fc1c05f23757c61fe0a8797325ac6d46 \ + --hash=sha256:ac4472834f38e61d294aa1bf2fdc9f687f3b3162b9729dc9dc15e8c149f8c216 \ + --hash=sha256:b14744a1b3a9dd48e06a62c448710a798ceda9451d7d37a4654a58b5adfa97f8 \ + --hash=sha256:b2acd057e4a27eb1543bff0e9030c34fdc64e18815fe4bc23c2dbcc47b60f0e3 \ + --hash=sha256:b57996fd2b72207a796656bc39508f20089692815297120039284edb296b5fff \ + --hash=sha256:b91012fdd5ec5cbedcec2cae6adfaad9a8f547023e570fc5781cb7eb4bfe2567 \ + --hash=sha256:be5cb6554873d3c86f083fb0b35aedf4a1f12657adee4ac0fc90665ffd7e6722 \ + --hash=sha256:c0764ce1a7a7f19768d6b3473387dcac03cac8b0dc74b7e1c8d05b451ff4786c \ + --hash=sha256:d17f0c33789009a3b3e609a52af34dc687ed39816836e20ab466aab669c14e6c \ + --hash=sha256:d3b9494b45400b803eeed7d05e6e566aa135f1dd2d121c29016ee9492e7ead67 \ + --hash=sha256:d61942263d4603814622cd72bb5991b8a3cb59970bde8a7b40b897a7f088fb84 \ + --hash=sha256:d9ff05b868748b846a02f59913e9aa9d7b576827db92cc358ec1cf660f7f90a5 \ + --hash=sha256:e13f5ce391293dab946edf18e7ae7bb55768beef1140deb509f67a2b14edcf7c \ + --hash=sha256:e32d595df067f19739f1e214e723035ed2a9c216fb1ed3cd118d08f19b9cfabd \ + --hash=sha256:e75fa55161d9477e694f852cad0d383c6b4a5c3b505a5d08339f0e2260c2556c \ + --hash=sha256:f2909f683ffc5a2720f6c8099398a64fd1be53f1b6538e4e728ee55816ce699b \ + --hash=sha256:f348705a72fa236f310c2cbb9c393d3b620a942c9b34ac641fd0f098bdc5be44 \ + --hash=sha256:f5716b51fea3d06352d4d8c851fb6c5141b1a7de5a5164d44de5a6d368d84a7e + # via mflux +mlx-metal==0.32.3 \ + --hash=sha256:34ae9b83ad2f0ccdd3e5d48ec35176e7119f57069eef187122916dc941a4ae1f \ + --hash=sha256:fdba88ed9131448cf163be2c7ad6374746d8818205f28657aa91b6c69050f691 \ + --hash=sha256:feea99fe4d01f9e9f33ca73d869f988219a1725de1c18870baa7d5cd5aef6cb3 + # via mlx +mpmath==1.3.0 \ + --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ + --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c + # via sympy +networkx==3.7 \ + --hash=sha256:e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0 \ + --hash=sha256:fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a + # via torch +numpy==2.5.3 \ + --hash=sha256:012e66aca395d795496446e52aeeb5866312a5d4d3f27da270e5a0b43f70dc5c \ + --hash=sha256:09d5a423c71ad5feb5625844ad58050e35df43871004b52ac9c0ad44a56775be \ + --hash=sha256:09ffa5d903faeaa5c4dd05009cf81c8bab9f2cb37c548b8d39b65b4cfa7c97f7 \ + --hash=sha256:0a59a421a32580a009e8a1751345bf829631b990dc1794b80514ab722b435def \ + --hash=sha256:116f96cadd935c6122e9228d676fe7ede19e741f5c8bb1c3cddbe0c51ccebea2 \ + --hash=sha256:1302b90c0e52281681b2975adfe8a860cb7b12216a27b4b0b4207c44bf7bccf0 \ + --hash=sha256:15aa985ac73a8db02db7663381aa109510449d3819d37206caed27b33a65a8a6 \ + --hash=sha256:1aad64d99730d013cfc6debafed22783b4fc5a7f4b8bc744d2d8cf7dcc880551 \ + --hash=sha256:1c80eabb4035ecf4ca9cd49cde8a9fdd69a729e63e6474887d1523ade7aa277f \ + --hash=sha256:1f3ed25271581281f2fccb1adcedfcde4c07362eec69189b50baf6f90e3ae159 \ + --hash=sha256:1fb6f8fb9ff0b3a69f52c66ce397b0246583e9f28616231b0e32ca49259a5fa6 \ + --hash=sha256:214045a5bf00113a146ab9ee9730c44501af6723cdf1f6830932f7b5ef2e7af0 \ + --hash=sha256:26e15e4aecd8617dfbaecb37d223e365d7b39411fba20454be2670a96aa74cb5 \ + --hash=sha256:2c25dfa72943e4336ddb6b0ee4277b47a0c85bede0807530ec68103bf58e2c10 \ + --hash=sha256:2d8240cb4c16fd831074aa2b2cf9fc54664d826341d61c372245b96a74a49a9a \ + --hash=sha256:350ba9783ce969cf9f7ce6e6a9a58e1a6e2a19ca025b7ee448c4db727706212a \ + --hash=sha256:4c8a6d2ebce6305fd82fbefca827775437147052a976ee7c94b36a0c1b52ac6c \ + --hash=sha256:4f8929ee6c96bfbd7b4ed2032e0c03af86fe1826740ab61ddabf9072d06e57ff \ + --hash=sha256:536f963710a4e63934d80ac0dc4f478804a83e9a84b6828018f25d09953ada33 \ + --hash=sha256:54a115e5a73b8fc44f0cebef486365a1894b5c9760685d4558b72b7c3eb846e0 \ + --hash=sha256:595d020938c84e320bcf40ad71089e108eac0d377cd018e14a8c094f39e98d85 \ + --hash=sha256:66a78fe4556c60aceda5916f9eacd638b18e9e681016ec302dcb4682d6d4d034 \ + --hash=sha256:6b05c171afb3aa07adbd20abc00aea86fe375beb0fdb9ef780ec5b7f63bab1c0 \ + --hash=sha256:6cef4bb1706dfec49243c05d921eefb4e190d41e2528b30d8035ea1f36b4c24a \ + --hash=sha256:6f24021b9f22bc6301c37b196974a92c1c18dccedb6fef3dd252e95f2d6adbe4 \ + --hash=sha256:71b39d9f935b6ec0f8753e3e2afb51e3efba6f2e05b68b32a40754d24bcd4a3c \ + --hash=sha256:71cad2b2a7451ab79d8f5e71b453485b6775963d5cf794179144a7463fe6e8ec \ + --hash=sha256:76c2c1e6bfa5c84adc6434dfbf013aa92096a7985221762c8f11fedfd20fff58 \ + --hash=sha256:8617bbfae4486cf99c9f899966699428d19da931d06ca94ad3da986c76e15997 \ + --hash=sha256:86bff898a431c0fb71f7610b75726e75a54d47b37edc9d537f48de63bb3c0b90 \ + --hash=sha256:8e4dd766076855b5ff7ea52fa5f07ce26286726e0f8bff446b7739d02e6ea204 \ + --hash=sha256:92f30e89b8ee0ecf363033576c422b2f58fed6a80bed0aa48dff6d14c654663e \ + --hash=sha256:93e1f5447e2b1e479d7bd74701e84746b86450cff1fc368b132d195e2b8f8211 \ + --hash=sha256:9a37475425b431b4d060f23b4f52cd2f3aef6bc7c654bd760adf0040eec9d435 \ + --hash=sha256:9deb49575e5b0b94ed72c8a64ec4d033381adc27e9060ae842971f697ba96104 \ + --hash=sha256:a5fa86b80fd24bcd1aff83ad23be44ea323de3f787be8f8b15d4a65621e25321 \ + --hash=sha256:a6391fafaba97500887132cd582abc6e19452b1ac775a47caa7b24490e152058 \ + --hash=sha256:a72f874bc9e10e4b8f80426fb49716d5141f64442a0c8418065093ec8017fbb0 \ + --hash=sha256:ac7bb1c52d445bd4f8f7f97fefe6abc3a084dc4d63df50d79b17fa2b78e89297 \ + --hash=sha256:adc1ada2662f8a5f960b8a10d9986897e7499ef07e06d4cfe7197f8cce923c07 \ + --hash=sha256:b00eefbcf0f292945c4b4dec2ae845389ef5bcdcd596e6e4328051db5b5ba694 \ + --hash=sha256:b0521d0f4aebb6e06189451025fa17a913287b13c03d5fe05c017333b654ea5b \ + --hash=sha256:b5d93cf48f687479941d12b69c873ad2cc76bbd487f0091c2200636497f34034 \ + --hash=sha256:b7e18c623bb5c95acb3b3328861272816ba199fb531921c5d6d0b675f1fde9e3 \ + --hash=sha256:bd4cb9ad3c7889b9b3fe0a9a9fb5d2ed26f9879bff2608d9f01aed147a20d231 \ + --hash=sha256:be5a8381859b6da607c84f4f7d6847725f1cf1853ef8a2c9e115b7d58bef47dc \ + --hash=sha256:befa1ae5bd6030b3f512b43ff3fa5290bbed6b84411a44244b14adf835f5b89d \ + --hash=sha256:bf63afbe037eb5d2fe87fbcc7778e61da53ebaf21d938a4515aa73b62532a5d4 \ + --hash=sha256:c00abe94c1a69d75d827dcf1c025b25c8a45d230b3bcd77a9020883a1b047653 \ + --hash=sha256:c2381f82999704f818e2c987a865050e285ec3621262c66d40f5a96c8f899f8e \ + --hash=sha256:c76d5dde9f445058f83d0c02af00557a4db91de9a9a57c0df87d1535001d654b \ + --hash=sha256:cb189f09db39283b26bfd061ec16189e14f71c6755207f72a0f7540867afe5b9 \ + --hash=sha256:ccb32e0525d29e8b0572eb84c9a57af0e7a4e615726927506f55063c62414034 \ + --hash=sha256:ccbc4665079665c3cf3bab4db9f6b095370cd6437d66be549b6c2a1fd19e1958 \ + --hash=sha256:d1c89973648c85069c5046ad460f7b8a00218b29a2e42359ac8cc63e9ab94832 \ + --hash=sha256:df2d5874ff183595a4ba404edd04f6bd9b5505c1d7708573f6a6c17489a67563 \ + --hash=sha256:e01c918ac3d48e18a927cf7b14a26a3e29ff2bdf2eacb976da0aecd6a43ed034 \ + --hash=sha256:e6ab667ba76450084eb64013762c438ea76d9d29cc676dcd6c2e9892ba37f841 \ + --hash=sha256:e931e4f499e0dc7ef29d269a8e5b35dd722e5d14be07df6240166ea7c6532fae \ + --hash=sha256:f54660b0eb6b0b9f36e7fe1cdfdff472028dd0d14acd9b9b65098efbad059469 \ + --hash=sha256:f59a878c33d6b88122d80d239bb3b845d58708750b0cb06a09aebb9b18ec696c \ + --hash=sha256:f7fabeb6cea87d65f3b926de33d03fb016cfdc29314c90974383b5582ae72891 \ + --hash=sha256:f9579f383d1bf9df80081e72760e84960a7fd4f88cf0c9e535a8597c9bb646f5 \ + --hash=sha256:f9a2353b37a1a9e78fd82b27ad7e2a32a2d036604d18f02b05e3136c62ca3b09 \ + --hash=sha256:fc36dc566135b5eceec4cf89758fcb719266a019ef07dae1754ae7c9f617ef3e \ + --hash=sha256:ffdc76bfcae6b255dff75202c5e7feaf95b40246bc0a17944facc1fecf9f79ab + # via + # contourpy + # matplotlib + # mflux + # opencv-python + # transformers +opencv-python==4.14.0.94 \ + --hash=sha256:04456937c8f12ca64ff94df8bfd813ca98f5c9767da792948969a759c716ebe2 \ + --hash=sha256:3198fcbcb8f98fd68063bf2c75d129b0f50d46e555a4e47578a6eb086eb38e1f \ + --hash=sha256:4ae0b5a2acbcfb9813035f1644cfba751edd02797fac994e32180813eeef8420 \ + --hash=sha256:549c2401f8654708c702bdf0e26d45816b9c411369a0efa41907ec2b4ec0e707 \ + --hash=sha256:6a49842cc0ba9f43e35a289e61c0e15a1f8be8df60145ee5c7d42593b0d4f4b8 \ + --hash=sha256:726a2b40c1120a0bbd0116394c6a5b2aa0a4dfa4e609a81ee10d12c6a34cd372 \ + --hash=sha256:ace53616cdffc9643e17e075397b493e607c427d388c7508798ef7ad2ed577cc \ + --hash=sha256:add600736d0d65a39e797a9ceeb494f3cdc5736d4bd96b4f89d65a8c92451528 \ + --hash=sha256:afadc76a38a07713a4c2b4702da917cf0ed489be57e2a40a34e670189731de7d + # via mflux +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # huggingface-hub + # matplotlib + # transformers +piexif==1.1.3 \ + --hash=sha256:3bc435d171720150b81b15d27e05e54b8abbde7b4242cddd81ef160d283108b6 \ + --hash=sha256:83cb35c606bf3a1ea1a8f0a25cb42cf17e24353fd82e87ae3884e74a302a5f1b + # via mflux +pillow==13.0.0 \ + --hash=sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27 \ + --hash=sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782 \ + --hash=sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22 \ + --hash=sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15 \ + --hash=sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502 \ + --hash=sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692 \ + --hash=sha256:10b5019883c98f1dd819ca48ffc6f33a91cee04948fe637c605f4345c7cafd6f \ + --hash=sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8 \ + --hash=sha256:12fbb851cf323cc19aac3eea1df5ad17b57e91e9fe3cfd06100dba7bb8208c13 \ + --hash=sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3 \ + --hash=sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369 \ + --hash=sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88 \ + --hash=sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d \ + --hash=sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb \ + --hash=sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f \ + --hash=sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0 \ + --hash=sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40 \ + --hash=sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599 \ + --hash=sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055 \ + --hash=sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033 \ + --hash=sha256:37290fbef0c73e2d51d594eb78fded7dc7170d1c3ad0a890269e6fd8c8c4e061 \ + --hash=sha256:405be6058e206a86fedce2f715ecc2343f94072d7925251632dffa2c05f70dca \ + --hash=sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f \ + --hash=sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca \ + --hash=sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c \ + --hash=sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4 \ + --hash=sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8 \ + --hash=sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620 \ + --hash=sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071 \ + --hash=sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc \ + --hash=sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a \ + --hash=sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6 \ + --hash=sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e \ + --hash=sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582 \ + --hash=sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f \ + --hash=sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79 \ + --hash=sha256:6a0920cb55df5a4d3f6862964908078c0bd59332bdd1bcd20d14e8c70ef27cc5 \ + --hash=sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683 \ + --hash=sha256:6f38f3f885a09cafc93fd667b1173a92f379e43785d635e7983c893f0da319fb \ + --hash=sha256:73ee25cf8cf8b75d5a5aeed7cb6588f5e5dbfbb0fde51707b630bbb23897645c \ + --hash=sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa \ + --hash=sha256:7c88c10dd432f3b39ccdafc9ed81d0c21b8fd634502574da76d1bfa4fe2c76d0 \ + --hash=sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521 \ + --hash=sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e \ + --hash=sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c \ + --hash=sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058 \ + --hash=sha256:83b37abd87b590acd8f47ca8a61f70893322b11834d36e07c2298dacad4d14d2 \ + --hash=sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463 \ + --hash=sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d \ + --hash=sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81 \ + --hash=sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf \ + --hash=sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63 \ + --hash=sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0 \ + --hash=sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42 \ + --hash=sha256:a192360d512305fe92d3053e4f7627bbad1f7708f1f4ad38cccd9336acabe9ad \ + --hash=sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81 \ + --hash=sha256:a625df9a2a5597fa5e5266e01aefc82604bb85d291e3547981e863560d75da6c \ + --hash=sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4 \ + --hash=sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981 \ + --hash=sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43 \ + --hash=sha256:b84be41387fc00db28c13c02e30d42b02d8d8f6a4a741a1aee6012ca7f1f6039 \ + --hash=sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76 \ + --hash=sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51 \ + --hash=sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2 \ + --hash=sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630 \ + --hash=sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac \ + --hash=sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510 \ + --hash=sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6 \ + --hash=sha256:d7710eb676e9bb68965cb045be0d333b4c0d7f9dfbaa7fbf121385b1588683d3 \ + --hash=sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d \ + --hash=sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61 \ + --hash=sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932 \ + --hash=sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1 \ + --hash=sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d \ + --hash=sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00 \ + --hash=sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe \ + --hash=sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313 \ + --hash=sha256:f391dc92a015c478745192fe4f2e120c2af2aba18d9fe0616341cbf537599806 \ + --hash=sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a \ + --hash=sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0 \ + --hash=sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52 \ + --hash=sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b \ + --hash=sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe + # via + # matplotlib + # mflux +platformdirs==4.12.2 \ + --hash=sha256:29dbf06d96c500bc6bdbce75fb0a14d63279c93b1842f97e72a135b33e856983 \ + --hash=sha256:eab5f70271a490ef74618bb314fbb86e3c7e82fa3b9c922c2ea0e0a1a155d329 + # via mflux +protobuf==7.36.2 \ + --hash=sha256:497d0463ff3316681da6c0b9e8d06cb465d61abce00b613ab42226175644d1bb \ + --hash=sha256:89f23aa53c24553a2416fd4fd1ec06f74fa42b14b546d8883128813f775bbfd2 \ + --hash=sha256:912c1221170e16c08d1f086762f563dd61ff83c18b5fa6652952dfaded66f728 \ + --hash=sha256:a300819d441e078a5608c0d3c709796bb548136058fda017ae51d425b44fd353 \ + --hash=sha256:bdb3a345d48db958e6ce1f18e508beb0cc981d64f24088427549c866cd039f1e \ + --hash=sha256:cbc70b17ee27e28894c7fee8bb04be1abead49e936bc70eb60052531eee2079e \ + --hash=sha256:e11e1f0180583a2af89db6a2ecd9e8dc40aa6d2988ca175bfd0e6d12ea72d74e \ + --hash=sha256:f4fee11ec330d238b34a05c9b675f693c20415d1c5bd7d5320cc2f8a798eb9cf + # via mflux +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via rich +pyparsing==3.3.3 \ + --hash=sha256:928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36 \ + --hash=sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4 + # via matplotlib +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via matplotlib +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # huggingface-hub + # mflux + # transformers +regex==2026.9.29 \ + --hash=sha256:01000ddf0e3ffef97f2413ceb514f6313040106b6d18a03ee00a4fe35c1eb1db \ + --hash=sha256:0166844493626c5015c6088ee15c9ca2fd060ca15b7641d1657da6a58432ae33 \ + --hash=sha256:044265d77d94f5e3cb2fd72c76723807c429cb8c533e9d4672d0334a6f14f588 \ + --hash=sha256:0476e5bcbe6e1ba3d1c4cc7bbb1c3ba78e3b979b5c8a88d0a6a8cdd4992b8c84 \ + --hash=sha256:066d0e3dbfdd739bce2bf8c2a41dd16f73e3d8adc2eb06dd803a36a307f56075 \ + --hash=sha256:0b65c72739f981377c9c22e0c5c3cd7f42da7bd8a3c9209330fac772c7d893ed \ + --hash=sha256:0c992c19cd45058a4b92f68f139c93db168b48fb1f322c9a7cd620806afb6b51 \ + --hash=sha256:0cc63b5e47c12a48d90c7e9d7de6a035dd14f62868aaedbb4e0ff8ba2b8bfe7b \ + --hash=sha256:0dd8af32e9f7b56b7f95cc1fd79b23054c3bdc172392ae560acc24d57b7ffe71 \ + --hash=sha256:0def9fb6abac55492d6d51cddb7225d07d6f279e774e0adc08569a54a5fc8d46 \ + --hash=sha256:0fd2c901cc307a745ad4bc87f20060d7a0825a3371d1e93488af22e7a387f78f \ + --hash=sha256:1043aedf5917caa861bcb25a9c11460049656bdf0017a90a309fa8f255467725 \ + --hash=sha256:121a76a0985db80ceae9e171c337f8c927868e37d01b54e3ce87bc87f9c6a208 \ + --hash=sha256:143533cc4b6fbc5b95aca0a5b8d541088d374831593def000ec89322c220221d \ + --hash=sha256:14e953ff3607c92d7675bf79c4d4509ef6782aa8c08509f179f9b3d6d0679e86 \ + --hash=sha256:18ae8eed4526e35bdb754d61562b90bf5c00a67fdcf3cc1380dd59597486631b \ + --hash=sha256:19959129885356df0e97556856f77eb2888380dac18bed075a7c05c5128c618d \ + --hash=sha256:1ba8c6a416569ce0d37e83e28a254a61dc99a419084dfb6476cea02d997f74fa \ + --hash=sha256:1c2a0026062abcc321a53db4a185ceba0b59a66b5d37b0808917a88b55a5257f \ + --hash=sha256:1d9fe8091b2e89d470df68a9331111ed008ae8aae6bf1e8e1fba4086a495c84e \ + --hash=sha256:2089fe39c406784d90101c726755ffa1497bb74638fd434300d2b88006186de8 \ + --hash=sha256:23ae6fdad9e63e54038f5ef78aba2933faca61e24d432786589e737bc5522ebb \ + --hash=sha256:26ec4ccce55aa533fbd603d08911b01101a8fcfec987845ac3ae2c7087b2bde3 \ + --hash=sha256:2f7f7aa47b229f2b39a2ae2596d2ad5625d77b5eb9856fac2dab3eb506cdd0a0 \ + --hash=sha256:31b003f9a070335e2a8233ee9b14a3ca8e6d792012ae011f741bf0aaf11744c5 \ + --hash=sha256:32ab11df9677ca80bcbb5fe4eb1da9109a5019239a054836efc6fa1c64e683cf \ + --hash=sha256:33026515aebc0e70d1c89978e53e8d695d35d9e472f8d5b34465ba3c74028650 \ + --hash=sha256:34b6925af9853bf461950e6508910f179fd6e9b1a7ec8548e069606b7e51a26b \ + --hash=sha256:352cf115a810b357caa35193ab656ecf5ef41056855e82f292c99e8514f8d954 \ + --hash=sha256:39ab5894d971f9ac68baa6eca5c50387db579cfcacf36ae8df3feceb1815e6d0 \ + --hash=sha256:3a21a9509d0ee88e7a70e1ad228cd2f0e0fd1e187458db132e8a8d18c97daf9d \ + --hash=sha256:3c5c2ef13797466aa64170cbb66ad98a32351dd4127694cea7199f80f213750d \ + --hash=sha256:3e778bfccd63075167709136afbc251c1f683758d5bf49c803c60ac3f894ce6b \ + --hash=sha256:3f1e6cb402a89457582cd696f982559217d13484a193202c394015297968c86d \ + --hash=sha256:42e82e578c904445d4c8a35b8f28052cf567593215fa5db06266fbc6f77aaa2e \ + --hash=sha256:4408b2b27a95ca8cc48b7411945753773353b5c93b307754781086c99d3a576f \ + --hash=sha256:446654b29bfaa30500d80947eda42cef1449dc8a87f4e3cf061cc8485d3a1f0b \ + --hash=sha256:45010bcfe66df41522d56c9b6114e87ecc597a08970ff6a2ced24415c141ae5f \ + --hash=sha256:49ee178ca31c94621294bf9b8b676a92a2e6bba8af0529591753719e57edb621 \ + --hash=sha256:4d7d93613b01b0199961330e49cfc52d479b3d5776c56c691db31130c0a07d91 \ + --hash=sha256:4fb41211d2333eb930a51e0546a65999761cf1f572a4da56ef9b8a62966c06f2 \ + --hash=sha256:4fe97894d1b306c919b4e50def1e6f6c522f4d03a7283811f4d108f1ce5d3ac2 \ + --hash=sha256:554bffadcbcb6d5f4e5fb10a61cc52084b9a63d1dab5f10bcd2c4343972e8e2c \ + --hash=sha256:59b49507f47479e299a9e1bc41b5cb83a7afda0540625f1dbae886615978acbf \ + --hash=sha256:5eeb8edc6110d9194a4d0d54610f64c37a31c605b5dbb7e407fc6ec7fa34a4a1 \ + --hash=sha256:612b709381c0355b70d89cdb51b7f670591ed5cbbc0e3b5337488019dc667b65 \ + --hash=sha256:61956f074ecd123f55adca68ee3eab46e6a07ad3f8e64e6db95dfacb444f55c4 \ + --hash=sha256:6398d5145689503412cc1748895242598d8846b8967b851133b20dc2ed1e21e8 \ + --hash=sha256:65b408d8fcb273e3499e7ef2ce796810da1becd208c7fb4373692a242d79d461 \ + --hash=sha256:686ac5350fceae63830bb98805fcb8039325bf4c06d9f6f048ff65229d5bffa5 \ + --hash=sha256:6a1a824fbed817e0a891103886b68f063b1e83cc51bc97192a90a60195a9291f \ + --hash=sha256:6abb75ab16bc3281714a5b99548a2225db70dba1f995f6d7f7419b76eb5a8fbe \ + --hash=sha256:6f7121a8914ed13fcfe2099f895341bfb789f004d4c5a0bdece8fa667da10849 \ + --hash=sha256:7020ed44df30b3aa492c00ee3b52d0548c1f30c2c6c5bb13ae897680900d3413 \ + --hash=sha256:720537c7ea6f80dc61913184edb0ce2497a306b39ef19f28505b322553d52bdb \ + --hash=sha256:724184b4aafed865e4f13ca313fdcb43024300c028ec67319cfa16847d84685e \ + --hash=sha256:7c03031610e3e6ed1768a2b7a8fc84637c1257b50c5eacaf094c6e17a84fc563 \ + --hash=sha256:80a5ea3b4fd9d6a5b9a44f7976a9acaaab35aa3c1f6b29e5bd857dfabaded223 \ + --hash=sha256:80c7cadd3fd2bfde5df8aa0787e315812cad0c313a753095d02f4c2b6c01677b \ + --hash=sha256:80ea96f5c1a30bf09007d48466521d9c294bebe197c708c3359096e3e3691632 \ + --hash=sha256:864e9b87ac33c3fb9fb4ad48166d4fdb579c351d5c77deb0d34bccb36a775cd9 \ + --hash=sha256:87fb80cbe3557e27e7b28b995c2b2eedf689b8886f941ab93e0e288f0976518a \ + --hash=sha256:8873c4a11c50b9989168881aeb3f08859f469d809941866aa1feefd8be5431f6 \ + --hash=sha256:888d60953908dcf761aa320c3e390ab8556efbdb551ace63921de90f6ae0848d \ + --hash=sha256:8b5fcc4771732191b2b7d1dd68d8f0353f47f8d90b6150f6dce58bf1112442cb \ + --hash=sha256:8f39588af4731c8923c26810eb3b33f76f17633985e40f59c3cd45a33805a895 \ + --hash=sha256:9173db3be74a35cb6731701094b98120f7ee4876a287882a59cdea1fa7da342f \ + --hash=sha256:92f05c9c42bde5785dc48770bc2194d9f7442544156f951e19cd31b096cec562 \ + --hash=sha256:951733b1bbdb71e377cec567b409f1a7881b47cfcad84121aa74cb575fa425ea \ + --hash=sha256:957bb708e8057ab1649ba566456429d691ec9b90d1c9ad1af1ba7ffbbeaf05f2 \ + --hash=sha256:9916fda742cd4eede63b286f58c06718324265d727ce0856eb1aac86d0d150d6 \ + --hash=sha256:9e1d3a4cb7993b708f0ada8d0c84590efd853f169e7147d2202c9da503180242 \ + --hash=sha256:9e4482589065c8ecd761cff522dcd85f2d39e62f551e37e025d1c7d54772def3 \ + --hash=sha256:a5300757f8a68f5b6cc33f57338d72a0e3589c5cc9ad5f8504ea06f028be582a \ + --hash=sha256:a540abfab208e1b7ef2df231c40ef3b6cbb30a0aad6204e9b6a81c10a6794628 \ + --hash=sha256:a5758353650079898dc1b2b0e95aa51fa23a30d020e06f62c430dd08ee56cdd8 \ + --hash=sha256:a64b85a4760337cfefdb27d42da6ed8b58e8cde3f2d57b6ef43e76ef6ea9ef47 \ + --hash=sha256:a655d34b2a6943af32401f3d94f72e9d731f6ad16285815550bf2b4ee69d420a \ + --hash=sha256:a714befaacbd10092ffe4cea0d3c5f008fb9efe9bc322c715bcdfdee414b9a3d \ + --hash=sha256:a760da040b47767b4b873adfb7c3b691e9ba2fc60f113f9d0b88f1a62f323e85 \ + --hash=sha256:addd736a0547d553283adaf4e05d7104e7f2c7b0b092e9b4d28756825f14531f \ + --hash=sha256:ae4613d7d9dda60fcba95f846cc6f808017f1843f392cf9daad14a6534493d71 \ + --hash=sha256:b11b589e00095ec69cf79841a76360f9b079e95b0368a25b5ebb951ab0c157ff \ + --hash=sha256:b3e445b66c80b4eb4234e855ce94d9adc183eedbd632816228d89930b91b2c5b \ + --hash=sha256:b7b893976e7fe42053da64f2aa27239c24252fd2ec6df471e1be197c0addc3b1 \ + --hash=sha256:b84f186a7f0536fe4ff9a9fa12d06d007b9b71d4b5352ddcc41f59ad6522a312 \ + --hash=sha256:b89efc38431793d28b7cd91227e2f952ad7c48df19132b17f43a5fec3c14143b \ + --hash=sha256:b97a38fb4c732b6832db6bf108963adbcd82ef1268ba2025dce390f45af75efa \ + --hash=sha256:b9d74e4eee9ddb64c2e92d5d61472c59c21684c059eb7b68767be9628e977859 \ + --hash=sha256:bb90e7177944b6684738c1fc36aabd2dd00d1de3be7dbe09f91e196f1bc0dc81 \ + --hash=sha256:bec37990e3d6121f29ecfb594bd8f1bf009e9f7926daba2e50e3b27d3892a783 \ + --hash=sha256:bf3c49863c23a1ad6da9c30351aed6cff8d5ddbeb63c5c8420ae54e98c7d0138 \ + --hash=sha256:bf48516e35cf848390ea68850aba53e7c333720d2945b4d2c25b69fc5171723f \ + --hash=sha256:bfc71e6d970419c1309b3640305298643e2a734cad3f7cfb6d2ddee4175ab53d \ + --hash=sha256:c0094897d7d01f184b2d7fe8c56c66d64efe01b31f4b7d34205b391387df1111 \ + --hash=sha256:c03c6eb6ece86dfdcbb34799efaa339b093132e1aceed491ba5e08fe06cdf699 \ + --hash=sha256:c1a9a6651197fbed6f0212591418b9def774fc3f8324f78d1bf0e6a63e5f8aa1 \ + --hash=sha256:c3589f40749acce747510bf5d589d54e376cb0930ea58b35effac97e5312b0c1 \ + --hash=sha256:c4e38dd8f39c43a91d2410ad2b85610701b0979342c3df1d69eaf8e838c757d8 \ + --hash=sha256:c6c8fabf1dafc1f1ddcbb67896d3f93efb092e8c4b6322d7389b944e76a484e5 \ + --hash=sha256:c90fcf7804ea0a54b896ce0f2b9565350220b8d4890fd0db461a476a4c687963 \ + --hash=sha256:c9b602fae1e00b7c035d661ce85575365719192a7b46784bd71cf64c68053aa0 \ + --hash=sha256:ccb64d887a9db1cd76dbc0f92051a1a478a2a67e7f56c62d915cb881d7734704 \ + --hash=sha256:d06fcdecc10fc7954d7c8f27a03c96055fe525274dc84a7b0dbdc3d6b9e03dab \ + --hash=sha256:d0c3082bf79bcd6a614d55916590ad4b8f93200e10b97f463ea5d9d07c9b5f23 \ + --hash=sha256:d49c18f1ea294cf4adde2e5ac256e98c82ea9d708462ce4bf799dffa7cfe8a2c \ + --hash=sha256:d60030baaa7bfbb02d650c126cdcddcb6e33dbff14d819434c8fa2fdcaeeeba5 \ + --hash=sha256:d7cab119d0df0b9413f106b4d7fc34f2872d3574ed3806fb48959c830b1537da \ + --hash=sha256:d9b77b25b4f395f92de6099ab08e8ae2bc7e51dfe157f22900902243a5cc90c7 \ + --hash=sha256:dabee8f4935e731fb46b2a3091bdda0d3d94b3bbfb907d2b4f12eefce4009619 \ + --hash=sha256:db5e82ba15c142425b8406690032df89e39cca4a2e8afbbb9a3d84edc2373ac3 \ + --hash=sha256:dc79d36d0618752265f0d575915bdc5c5130ecb9c9f6b3bcefeae32e4bdfafcf \ + --hash=sha256:ddfa987262763c3c22a8367d2a49c244b018a74c3a8e3ab1a864119ad45c5633 \ + --hash=sha256:e1172147d28d8fbcf8cb8d26c41506169f5ad8fe9ec969cb116835a19d4d8eca \ + --hash=sha256:e11edba5bc344a32b029a7af9d4b3173982dd79eeafa0b9dbd787364414b0509 \ + --hash=sha256:e2c89e9b762c57f59d5e99ee8b20202adb892e35f8d3485741340999ca55058e \ + --hash=sha256:e31f72490b7c12f7790e1e25c3afffd20503ee1bfb43461d7838b871ff244b19 \ + --hash=sha256:e8c65ef3862a8ad6e86492b6ed9327805dd66904c012bd3649dc67d822ed6c34 \ + --hash=sha256:ebb8912f565b8cdbbf27debfe00df04202c20e2f651b9e32767930c5eace3621 \ + --hash=sha256:ed511a0708e2297e1d6431e7fb217e3402791e491e02da800658ace4973df1bb \ + --hash=sha256:edf06545875f3efa31560d94121e95c7fd70d98b1dfedc0157097d79b13b52ea \ + --hash=sha256:f0fe9834e5aeccaf19a0d8feb296d66a24be1a7c9922002f842a682cd5abb787 \ + --hash=sha256:f1a0d5117230dd46b399a30a38afa44f79c99f3168988fdc4f425c3f928b39df \ + --hash=sha256:f37964e4a5e993d2fd45147741e9dff7f34a2d8c00ab94c4ea0514a4677f959e \ + --hash=sha256:f57dc6b8fef170f105d2cf5cdce254f47b137d7755086cf7050f47e16582abba \ + --hash=sha256:f93bc1c3486ef3747e07c9d7c1d0a147b8fbaab975f80e348aed6f71309dfaca \ + --hash=sha256:fb00027a09a8f9f08028b40dce4c933cf73e4833240ed356583fdc9cfa721566 \ + --hash=sha256:fb99cc9d45f48895d9d67f6a0b8a57f08d39c174d9f25ad97a313e0470267b1c \ + --hash=sha256:fdd88ed5e20b1bcdd234421e454962c971aa44b653bdb7f1ea9ef683e90fb649 \ + --hash=sha256:fe3fa1dd453ed5c7f5ea23a26218329790ed7197a99b90e94330e313959a7f52 + # via + # mflux + # transformers +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via mflux +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via typer +safetensors==0.8.0 \ + --hash=sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358 \ + --hash=sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f \ + --hash=sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d \ + --hash=sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d \ + --hash=sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0 \ + --hash=sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc \ + --hash=sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235 \ + --hash=sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98 \ + --hash=sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4 \ + --hash=sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846 \ + --hash=sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca \ + --hash=sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0 \ + --hash=sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25 \ + --hash=sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452 \ + --hash=sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d \ + --hash=sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78 \ + --hash=sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774 + # via + # mflux + # transformers +sentencepiece==0.2.2 \ + --hash=sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497 \ + --hash=sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e \ + --hash=sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0 \ + --hash=sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719 \ + --hash=sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107 \ + --hash=sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3 \ + --hash=sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b \ + --hash=sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a \ + --hash=sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c \ + --hash=sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838 \ + --hash=sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5 \ + --hash=sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6 \ + --hash=sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d \ + --hash=sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf \ + --hash=sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25 \ + --hash=sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d \ + --hash=sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e \ + --hash=sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b \ + --hash=sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd \ + --hash=sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936 \ + --hash=sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85 \ + --hash=sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b \ + --hash=sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708 \ + --hash=sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb \ + --hash=sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9 \ + --hash=sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0 \ + --hash=sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790 \ + --hash=sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e \ + --hash=sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78 \ + --hash=sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99 \ + --hash=sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9 \ + --hash=sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d \ + --hash=sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8 \ + --hash=sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906 \ + --hash=sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383 \ + --hash=sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b \ + --hash=sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53 \ + --hash=sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b \ + --hash=sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab \ + --hash=sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0 \ + --hash=sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914 \ + --hash=sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91 \ + --hash=sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e \ + --hash=sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a \ + --hash=sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032 \ + --hash=sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711 \ + --hash=sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da \ + --hash=sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c \ + --hash=sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a \ + --hash=sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d \ + --hash=sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151 \ + --hash=sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563 \ + --hash=sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811 \ + --hash=sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b \ + --hash=sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820 \ + --hash=sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090 \ + --hash=sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c + # via mflux +setuptools==84.0.0 \ + --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \ + --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73 + # via torch +shellingham==1.5.4 \ + --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ + --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de + # via typer +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +sympy==1.14.0 \ + --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ + --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 + # via torch +tokenizers==0.23.2 \ + --hash=sha256:12f0835dc2ee694746a76adf7b1567d4346a4a502ebe93fb1f5f80ea49799b78 \ + --hash=sha256:2e96f5699d5249c9c64aa8412e044f727aae3a4098cf830f9901ec1afc361cde \ + --hash=sha256:325fee2e0418a9dc6c9ecf736a5f5f0db7875183ace9549ae339da76f7a1fbb7 \ + --hash=sha256:41c2f84d172449b4dadb9cdc508e3e364076613c35b16e76ecfe47a60d1e3305 \ + --hash=sha256:43e4f2071e3cc8d5d86421c874aebc82659bb51a68bcdef5a0da75ee89511ccb \ + --hash=sha256:5c56bda1511921587789163e524d196ed8284174ac23abd7685d5ea8da6c4718 \ + --hash=sha256:7b7e37ba198f24150f523e1242e83c4970de4a525480586be5dcc24d9add32c5 \ + --hash=sha256:7f0f085686b9de0d0079e6f874ae053600db64c5d13049e0bbc0119926d25aac \ + --hash=sha256:85a9a357a3764aecc904ee76bdaf8cf1ad8e5a67a1b929a487c4a39b49ed0e90 \ + --hash=sha256:950d7c9426fa72406a0ffeacdbc0bb9985f5db20eb8b263f29c79aaf83105703 \ + --hash=sha256:986670e43691469dcee610ea0f846f91a8f84e91fc6f7a48d4c064414c0ec2bf \ + --hash=sha256:a37039b5dfc4af84eb3ef0a92f4307e28936c8f9adccba2629d36f652e9bf7a2 \ + --hash=sha256:bef235815a067b2648caf6dcc7a71091b0b0fff9ee8057f6451eb9335fae52ef \ + --hash=sha256:debf978920d93ba9c219bd67cc4bbfaf912c9039e41e7a28b91ec15e3728c95a \ + --hash=sha256:e49c394456dd9985787fec76132438ba3fb8911f857b1bf3d40119f9292d41aa \ + --hash=sha256:eb2f9c8a24da020ea8c11a01a19c1c2547912d92121ae4a01cfbca46125dee40 \ + --hash=sha256:f486f402f6f9abee5bb032553736813af0c710a86b2e0ca592634c55cea1f835 + # via transformers +toml==0.10.2 \ + --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ + --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f + # via mflux +torch==2.14.1 \ + --hash=sha256:06c3ec25f3b497f9a73dc0c36f293f295cfe8d5584fd446238d9a501b30a66d5 \ + --hash=sha256:0b13acde294b401509830d85a8906cbf2fc97425a12a66cce1d9fa13c5106a2a \ + --hash=sha256:0bd2fb0c4098856bc5af59fe5988bf38bdbcdb6550b44273c12ae425cb8ec17e \ + --hash=sha256:1d4df07be7338bbcc4d54085adee21363c91314702b9bd4d78ef72ffac9465ba \ + --hash=sha256:1fee6b20869a90ccec05e35559a706338ac8aab40a9c4fb58f16667e93af36ae \ + --hash=sha256:23011fe29a99b591eabb3a31a25080c62e2c2f0690a3c798744e5489dae50451 \ + --hash=sha256:305a61f61f35f128579f299c5bd33d475f6a01c6307336139632e30856c4854d \ + --hash=sha256:38bee9f2a2ccfc6898172a5075e4ba52522fbe143eeb099674fc67ab390e858d \ + --hash=sha256:420dbf314c180ee4b86e9bc00aee5746a7d6e5bacdd7df925af671ed393f0b2e \ + --hash=sha256:6d530bd11479fb574567af3a1f384af9a77bf5bb9550cde8080b356ca7220d5a \ + --hash=sha256:711713391d26a1ce5e9fbc6c996d954a8c12e8825374cc77b809a6af29539b8c \ + --hash=sha256:9113f94f70429f9f302bf55b090b5411269083e8a72a5d015ffcc5a7f83f2c69 \ + --hash=sha256:957df236525bcd940ccd9ca59074e9821182171253417c0d98e7b82647254799 \ + --hash=sha256:9ee7d50ec80ae5b4258cafc0ea108711988d961f21d0a68cf685650da2812e5f \ + --hash=sha256:b6074b130fd26bc50f5d171f07dfed70db22dd0c354ab34767729249f9e0f042 \ + --hash=sha256:c8f71aabc67bcbfc9373dc131537a5968d04edce73e88add21354a7cd0a76985 \ + --hash=sha256:cee091caf2a6229e248daf41d18ceebf590ba02d9179205610062c64ee5fef03 \ + --hash=sha256:d02a4c48a2ca5fb7654e36e71f710f74494d83f1f10aaff8e059dd554adca956 \ + --hash=sha256:dbe359d705f4d67236743794c296c6dee93a922fd8117eff8c3e880d7d0fb2b9 \ + --hash=sha256:e07306caa1de2a4ac1467e11ecfc92fc44f523dd6a521145039aef46d913963c \ + --hash=sha256:e24b9815001d18e73e76eae1ea25f8ef908c6103e857378ad20f8f6cf5c954f5 \ + --hash=sha256:e65d5fe136e533b23c2d134377f7f126721c7c78dd75b3dbb735af082a8aeb85 \ + --hash=sha256:ef1276fd07adb8d463ac402a24ea7482e45c9a42df8ebdac211aa757779200ba \ + --hash=sha256:f68f5476e2bc0e8f60b74f7ca21acda885c4477af6392f8977e4f0d1ea1aa162 + # via mflux +tqdm==4.70.1 \ + --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 \ + --hash=sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4 + # via + # huggingface-hub + # mflux + # transformers +transformers==5.18.0 \ + --hash=sha256:d79e5a515a572ee3deb33eb0d578be47d91e70c75d54cf21bf6eede6edef14a9 \ + --hash=sha256:d89c206e42e841af7cd314b3d2b21c03e977d819cdb4aef65ada048769a644df + # via mflux +typer==0.27.2 \ + --hash=sha256:269b7eb9d3c202ca84b4bc9618cb04ebb43d3d4d1e567e4c768607232c05f945 \ + --hash=sha256:b3a5fc4342d5fc8fda8fc3010b1cf117e9249aab7fae800c2eff62fd3842d97d + # via transformers +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # huggingface-hub + # torch +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via + # mflux + # requests diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-audio-build.lock b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-audio-build.lock new file mode 100644 index 0000000..49357a9 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-audio-build.lock @@ -0,0 +1,9 @@ +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +setuptools==84.2.1 \ + --hash=sha256:3ff8b3b5492a0aa66960f4419365a86b84e013070d4313e4d4a6894f5d64d6bc \ + --hash=sha256:ca140b45d5325a9478f880e351484f62fef15f336462e449ca0c03c873d589c2 +wheel==0.48.0 \ + --hash=sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab \ + --hash=sha256:94800765601e9171bf5d58d066e640662842bcedcbab982b2c90787a2c987322 diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-audio.json b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-audio.json new file mode 100644 index 0000000..48a3d51 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-audio.json @@ -0,0 +1,176 @@ +{ + "schema": "backend/1", + "id": "mlx-audio", + "version": "0.5.7", + "builtin": false, + "package": "mlx_audio", + "convert": "mlx_audio.convert", + "lock": "mlx-audio.lock", + "ports": { + "speech_to_text": [ + "audio8_asr_infinite" + ] + }, + "port_quantize": { + "audio8_asr_infinite": { + "include": [ + "language_model." + ], + "exclude": [ + "ada_rms_norm" + ] + } + }, + "categories": { + "speech_to_text": "stt/models", + "text_to_speech": "tts/models", + "music_generation": "music/models" + }, + "remap_files": { + "music_generation": ["music/utils.py"], + "speech_to_text": [ + "stt/utils.py" + ], + "text_to_speech": [ + "tts/utils.py" + ] + }, + "registry": { + "music_generation": { + "model_types": ["minimax_music3"], + "remapping": {"minimax_music3": "minimax_music3"} + }, + "speech_to_text": { + "model_types": [ + "canary", + "cohere_asr", + "fireredasr2", + "fun_asr_nano", + "glmasr", + "granite_speech", + "granite_speech5_ctc", + "granite_speech_nar", + "higgs_audio_3", + "lasr_ctc", + "mega_asr", + "mms", + "moonshine", + "moss_music", + "moss_transcribe_diarize", + "nemotron_asr", + "parakeet", + "phonon", + "qwen2_audio", + "qwen3_asr", + "qwen3_forced_aligner", + "sensevoice", + "vibevoice_asr", + "voxtral", + "voxtral_realtime", + "wav2vec", + "whisper" + ], + "remapping": { + "canary": "canary", + "cohere_asr": "cohere_asr", + "fireredasr2": "fireredasr2", + "fun_asr_nano": "fun_asr_nano", + "glm": "glmasr", + "granite_speech": "granite_speech", + "granite_speech5_ctc": "granite_speech5_ctc", + "granite_speech_nar": "granite_speech_nar", + "higgs_audio_3": "higgs_audio_3", + "mega_asr": "mega_asr", + "mms": "mms", + "moonshine": "moonshine", + "moss_music": "moss_music", + "moss_transcribe_diarize": "moss_transcribe_diarize", + "parakeet": "parakeet", + "parakeet_tdt": "parakeet", + "phonon": "phonon", + "qwen2_audio": "qwen2_audio", + "qwen3_asr": "qwen3_asr", + "sensevoice": "sensevoice", + "vibevoice": "vibevoice_asr", + "voxtral": "voxtral", + "voxtral_realtime": "voxtral_realtime" + } + }, + "text_to_speech": { + "model_types": [ + "arktts", + "bailingmm", + "bark", + "breeze_tts", + "chatterbox", + "chatterbox_turbo", + "confucius4", + "dense", + "dia", + "dramabox", + "echo_tts", + "fish_qwen3_omni", + "higgs_audio", + "higgs_audio_v3", + "indextts", + "irodori_tts", + "kitten_tts", + "kokoro", + "kugelaudio", + "llama", + "longcat_audiodit", + "melotts", + "moss_tts", + "moss_tts_delay", + "moss_tts_local", + "moss_tts_nano", + "omnivoice", + "outetts", + "pocket_tts", + "qwen3", + "qwen3_tts", + "rumik_oss", + "sesame", + "soprano", + "spark", + "tada", + "vibevoice", + "voxcpm", + "voxcpm2", + "voxtral_tts", + "zonos2" + ], + "remapping": { + "audiodit": "longcat_audiodit", + "bailingmm": "bailingmm", + "breeze": "breeze_tts", + "breeze_tts": "breeze_tts", + "chatterbox_turbo": "chatterbox_turbo", + "csm": "sesame", + "dramabox_tts": "dramabox", + "echo_tts": "echo_tts", + "fish_qwen3_omni": "fish_qwen3_omni", + "higgs_multimodal_qwen3": "higgs_audio_v3", + "irodori_tts": "irodori_tts", + "kitten": "kitten_tts", + "kugelaudio": "kugelaudio", + "longcat": "longcat_audiodit", + "marvis": "sesame", + "melotts": "melotts", + "moss_tts_delay": "moss_tts", + "moss_tts_local": "moss_tts", + "moss_tts_nano": "moss_tts_nano", + "omnivoice": "omnivoice", + "outetts": "outetts", + "qwen3_tts": "qwen3_tts", + "soprano": "soprano", + "spark": "spark", + "vibevoice_streaming": "vibevoice", + "voxcpm": "voxcpm", + "voxcpm1.5": "voxcpm", + "voxcpm2": "voxcpm2", + "voxtral_tts": "voxtral_tts" + } + } + } +} diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-audio.lock b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-audio.lock new file mode 100644 index 0000000..49e942f --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-audio.lock @@ -0,0 +1,2262 @@ +addict==2.4.0 \ + --hash=sha256:249bb56bbfd3cdc2a004ea0ff4c2b6ddc84d53bc2194761636eb314d5cfa5dfc \ + --hash=sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494 +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb + # via typer +annotated-types==0.8.0 \ + --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 + # via pydantic +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ + --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 + # via httpx +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ + --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 + # via + # jsonschema + # referencing +babel==2.18.0 \ + --hash=sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d \ + --hash=sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35 +blis==1.3.3 \ + --hash=sha256:034d4560ff3cc43e8aa37e188451b0440e3261d989bb8a42ceee865607715ecd \ + --hash=sha256:1e647341f958421a86b028a2efe16ce19c67dba2a05f79e8f7e80b1ff45328aa \ + --hash=sha256:1ef6d6e2b599a3a2788eb6d9b443533961265aa4ec49d574ed4bb846e548dcdb \ + --hash=sha256:27f82b8633030f8d095d2b412dffa7eb6dbc8ee43813139909a20012e54422ea \ + --hash=sha256:2a1c74e100665f8e918ebdbae2794576adf1f691680b5cdb8b29578432f623ef \ + --hash=sha256:30b8a5b90cb6cb81d1ada9ae05aa55fb8e70d9a0ae9db40d2401bb9c1c8f14c4 \ + --hash=sha256:3f6c595185176ce021316263e1a1d636a3425b6c48366c1fd712d08d0b71849a \ + --hash=sha256:3f966ca74f89f8a33e568b9a1d71992fc9a0d29a423e047f0a212643e21b5458 \ + --hash=sha256:45866a9027d43b93e8b59980a23c5d7358b6536fc04606286e39fdcfce1101c2 \ + --hash=sha256:6297e7616c158b305c9a8a4e47ca5fc9b0785194dd96c903b1a1591a7ca21ddf \ + --hash=sha256:62fb8c731347b0f98f5f81d19d339049e61489798738467d156c66cc329b0754 \ + --hash=sha256:631836d4f335e62c30aa50a1aa0170773265c73654d296361f95180006e88c04 \ + --hash=sha256:650f1d2b28e3c875927c63deebda463a6f9d237dff30e445bfe2127718c1a344 \ + --hash=sha256:66e6249564f1db22e8af1e0513ff64134041fa7e03c8dd73df74db3f4d8415a7 \ + --hash=sha256:6f165930e8d3a85c606d2003211497e28d528c7416fbfeafb6b15600963f7c9b \ + --hash=sha256:7260da065958b4e5475f62f44895ef9d673b0f47dcf61b672b22b7dae1a18505 \ + --hash=sha256:7a0fc4b237a3a453bdc3c7ab48d91439fcd2d013b665c46948d9eaf9c3e45a97 \ + --hash=sha256:7e88181e9dd8430029ebaf22d41bf79e756e8c95363e9471717102c66beb4a6d \ + --hash=sha256:8177879fd3590b5eecdd377f9deafb5dc8af6d684f065bd01553302fb3fcf9a7 \ + --hash=sha256:878d4d96d8f2c7a2459024f013f2e4e5f46d708b23437dae970d998e7bff14a0 \ + --hash=sha256:8c888438ae99c500422d50698e3028b65caa8ebb44e24204d87fda2df64058f7 \ + --hash=sha256:9b0d42420ddd543eec51ccb99d38364a0c0833b6895eced37127822de6ecacff \ + --hash=sha256:9de26fbd72bac900c273b76d46f0b45b77a28eace2e01f6ac6c2239531a413bb \ + --hash=sha256:9e5fdf4211b1972400f8ff6dafe87cb689c5d84f046b4a76b207c0bd2270faaf \ + --hash=sha256:c3e33cfbf22a418373766816343fcfcd0556012aa3ffdf562c29cddec448a415 \ + --hash=sha256:c4ae70629cf302035d268858a10ca4eb6242a01b2dc8d64422f8e6dcb8a8ee74 \ + --hash=sha256:d0114cf2d8f19e0ed210f9ae92594cd0a12efa1bbbce444028b0fc365bbbb8af \ + --hash=sha256:d563160f874abb78a57e346f07312c5323f7ad67b6370052b6b17087ef234a8e \ + --hash=sha256:d734b19fba0be7944f272dfa7b443b37c61f9476d9ab054a9ac53555ceadd2e0 \ + --hash=sha256:e10c8d3e892b1dbdff365b9d00e08291876fc336915bf1a5e9f188ed087e1a91 \ + --hash=sha256:e5a662c48cd4aad5dae1a950345df23957524f071315837a4c6feb7d3b288990 \ + --hash=sha256:e9327a6ca67de8ae76fe071e8584cc7f3b2e8bfadece4961d40f2826e1cda2df \ + --hash=sha256:e9f5c53b277f6ac5b3ca30bc12ebab7ea16c8f8c36b14428abb56924213dc127 \ + --hash=sha256:f0628a030d44aa71cac5973e40c9e95ec767abaaf2fd366a094b9398885f82f2 \ + --hash=sha256:f20f7ad69aaffd1ce14fe77de557b6df9b61e0c9e582f75a843715d836b5c8af \ + --hash=sha256:f36c0ca84a05ee5d3dbaa38056c4423c1fc29948b17a7923dd2fed8967375d74 +catalogue==2.0.10 \ + --hash=sha256:4f56daa940913d3f09d589c191c74e5a6d51762b3a9e37dd53b7437afd6cda15 \ + --hash=sha256:58c2de0020aa90f4a2da7dfad161bf7b3b054c86a5f09fcedc0b2b740c109a9f +certifi==2026.10.19 \ + --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b \ + --hash=sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae + # via + # httpcore + # httpx + # requests +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via + # miniaudio + # sounddevice + # soundfile +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f + # via requests +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via huggingface-hub +cloudpathlib==0.26.0 \ + --hash=sha256:6ea02248f2db7cb10729caf1edbdec92587dfdbbd3df8c71a32aedd4ae55715a \ + --hash=sha256:abcd7f54bcbb2fbb79c0e96b79241f08866760aee6276586882b3558672d571b +cloudpickle==3.1.2 \ + --hash=sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414 \ + --hash=sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a +confection==1.3.3 \ + --hash=sha256:b9fef9ee84b237ef4611ec3eb5797b70e13063e6310ad9f15536373f5e313c82 \ + --hash=sha256:f0f6810d567ff73993fe74d218ca5e1ffb6a44fb03f391257fc5d033546cbfaa +csvw==4.1.1 \ + --hash=sha256:5714d43d744e4069e2b6d0e58a67829ca1bf8d0578aad92cfc74e87d69e6cb31 \ + --hash=sha256:fce2c461899290771eaec28c770e2fd0f6bc1324edb81d688c89e09cf222fcdb +cymem==2.0.13 \ + --hash=sha256:03cb7bdb55718d5eb6ef0340b1d2430ba1386db30d33e9134d01ba9d6d34d705 \ + --hash=sha256:042e8611ef862c34a97b13241f5d0da86d58aca3cecc45c533496678e75c5a1f \ + --hash=sha256:0d78a27c88b26c89bd1ece247d1d5939dba05a1dae6305aad8fd8056b17ddb51 \ + --hash=sha256:0dca715e708e545fd1d97693542378a00394b20a37779c1ae2c8bdbb43acef79 \ + --hash=sha256:1366c7437a209230f4b797fae10227a8206d4021d37c9f9c0d31fd97ea4feb35 \ + --hash=sha256:1710390e7fb2510a8091a1991024d8ae838fd06b02cdfdcd35f006192e3c6b0e \ + --hash=sha256:1775d3fd34cf099929b79c3e48469283642463f977af6801231f3c0e5d9c9369 \ + --hash=sha256:18ad5b116a82fa3674bc8838bd3792891b428971e2123ae8c0fd3ca472157c5e \ + --hash=sha256:1c91a92ae8c7104275ac26bd4d29b08ccd3e7faff5893d3858cb6fadf1bc1588 \ + --hash=sha256:2fae4a29a36267772010a05ca85220e53f4fb3e1083d4f686342d8a3cc7620db \ + --hash=sha256:2ff1c41fd59b789579fdace78aa587c5fc091991fa59458c382b116fc36e30dc \ + --hash=sha256:30c4e75a3a1d809e89106b0b21803eb78e839881aa1f5b9bd27b454bc73afde3 \ + --hash=sha256:38aefeb269597c1a0c2ddf1567dd8605489b661fa0369c6406c1acd433b4c7ba \ + --hash=sha256:45dcaba0f48bef9cc3d8b0b92058640244a95a9f12542210b51318da97c2cf28 \ + --hash=sha256:52b76216c59e0078cebd988754426c41ee60e7c972f4c5185ac32fbd53b6d035 \ + --hash=sha256:666ce6146bc61b9318aa70d91ce33f126b6344a25cf0b925621baed0c161e9cc \ + --hash=sha256:673183466b0ff2e060d97ec5116711d44200b8f7be524323e080d215ee2d44a5 \ + --hash=sha256:68489bf0035c4c280614067ab6a82815b01dc9fcd486742a5306fe9f68deb7ef \ + --hash=sha256:6bbd701338df7bf408648191dff52472a9b334f71bcd31a21a41d83821050f67 \ + --hash=sha256:6d36710760f817194dacb09d9fc45cb6a5062ed75e85f0ef7ad7aeeb13d80cc3 \ + --hash=sha256:717270dcfd8c8096b479c42708b151002ff98e434a7b6f1f916387a6c791e2ad \ + --hash=sha256:742fc19764467a49ed22e56a4d2134c262d73a6c635409584ae3bf9afa092c33 \ + --hash=sha256:7700b116524b087e0169f10f267539223b48240ef2734c3a727a9e6b4db9a671 \ + --hash=sha256:7a8d404a48af953084fcec4239487ab05e14b829da251628827d51c87880aeb8 \ + --hash=sha256:7e1a863a7f144ffb345397813701509cfc74fc9ed360a4d92799805b4b865dd1 \ + --hash=sha256:8431347901026e9b554daeb982f5f1ecf3780ce46435c7a8e0cb82490e58f13c \ + --hash=sha256:84c1168c563d9d1e04546cb65e3e54fde2bf814f7c7faf11fc06436598e386d1 \ + --hash=sha256:84e2976e38cd663f758e40b5497fa5cd183d7c5fb0d04ce81a4b42a1ba124ff0 \ + --hash=sha256:891fd9030293a8b652dc7fb9fdc79a910a6c76fc679cd775e6741b819ffea476 \ + --hash=sha256:89c4889bd16513ce1644ccfe1e7c473ba7ca150f0621e66feac3a571bde09e7e \ + --hash=sha256:8efc4f308169237aade0e82877a65a563833dec32eb7ab2326120253e0e9e918 \ + --hash=sha256:90c2d0c04bcda12cd5cebe9be93ce3af6742ad8da96e1b1907e3f8e00291def1 \ + --hash=sha256:92a2ce50afa5625fb5ce7c9302cee61e23a57ccac52cd0410b4858e572f8614b \ + --hash=sha256:9d441d0e45798ec1fd330373bf7ffa6b795f229275f64016b6a193e6e2a51522 \ + --hash=sha256:a84ba3178d9128b9ffb52ce81ebab456e9fe959125b51109f5b73ebdfc6b60d6 \ + --hash=sha256:a868c1e3fe4f6f51232883b7bee7319a35e36644604577f4c6032cf957b9056b \ + --hash=sha256:a9150be2896fbe0096911bc9138a54e6fcc3814f197bc0c3d64d7c005a5e579d \ + --hash=sha256:ac699c8ec72a3a9de8109bd78821ab22f60b14cf2abccd970b5ff310e14158ed \ + --hash=sha256:bc116a70cc3a5dc3d1684db5268eff9399a0be8603980005e5b889564f1ea42f \ + --hash=sha256:bee2791b3f6fc034ce41268851462bf662ff87e8947e35fb6dd0115b4644a61f \ + --hash=sha256:c0046a619ecc845ccb4528b37b63426a0cbcb4f14d7940add3391f59f13701e6 \ + --hash=sha256:c16cb80efc017b054f78998c6b4b013cef509c7b3d802707ce1f85a1d68361bf \ + --hash=sha256:c3ff57aec4f264451739b18b7eab9d161d51af42b570ca0392fc051dcca28cb1 \ + --hash=sha256:c8dbfddfe5c604974e17c6f373cedd4d25cd67f84812ede7dea12128fa0c2015 \ + --hash=sha256:c8f30971cadd5dcf73bcfbbc5849b1f1e1f40db8cd846c4aa7d3b5e035c7b583 \ + --hash=sha256:c90a6ecba994a15b17a3f45d7ec74d34081df2f73bd1b090e2adc0317e4e01b6 \ + --hash=sha256:c9251d889348fe79a75e9b3e4d1b5fa651fca8a64500820685d73a3acc21b6a8 \ + --hash=sha256:ce821e6ba59148ed17c4567113b8683a6a0be9c9ac86f14e969919121efb61a5 \ + --hash=sha256:d1c950eebb9f0f15e3ef3591313482a5a611d16fc12d545e2018cd607f40f472 \ + --hash=sha256:d2a4bf67db76c7b6afc33de44fb1c318207c3224a30da02c70901936b5aafdf1 \ + --hash=sha256:d670329ee8dbbbf241b7c08069fe3f1d3a1a3e2d69c7d05ea008a7010d826298 \ + --hash=sha256:d8d06ea59006b1251ad5794bcc00121e148434826090ead0073c7b7fedebe431 \ + --hash=sha256:e02d3e2c3bfeb21185d5a4a70790d9df40629a87d8d7617dc22b4e864f665fa3 \ + --hash=sha256:e03bb575a96c59bc210d7d59862747f0012696b0dac3427ce8af33c7afb3d4a2 \ + --hash=sha256:e8afbc5162a0fe14b6463e1c4e45248a1b2fe2cbcecc8a5b9e511117080da0eb \ + --hash=sha256:e9027764dc5f1999fb4b4cabee1d0322c59e330c0a6485b436a68275f614277f \ + --hash=sha256:e96848faaafccc0abd631f1c5fb194eac0caee4f5a8777fdbb3e349d3a21741c \ + --hash=sha256:ec99efa03cf8ec11c8906aa4d4cc0c47df393bc9095c9dd64b89b9b43e220b04 \ + --hash=sha256:ed9de1b9b042f76fe5c312e4359eab58bf52ac7dfdf6887368a760410d809440 \ + --hash=sha256:f190a92fe46197ee64d32560eb121c2809bb843341733227f51538ce77b3410d \ + --hash=sha256:f3aee3adf16272bca81c5826eed55ba3c938add6d8c9e273f01c6b829ecfde22 \ + --hash=sha256:fb8291691ba7ff4e6e000224cc97a744a8d9588418535c9454fd8436911df612 \ + --hash=sha256:fe5424b38f61709b046df2ba7a6d7b54272ee1e2a2c56772d9cd309e4c1ea5ef \ + --hash=sha256:fece5229fd5ecdcd7a0738affb8c59890e13073ae5626544e13825f26c019d3c \ + --hash=sha256:ff036bbc1464993552fd1251b0a83fe102af334b301e3896d7aa05a4999ad042 +dlinfo==2.0.0 \ + --hash=sha256:88a2bc04f51d01bc604cdc9eb1c3cc0bde89057532ca6a3e71a41f6235433e17 \ + --hash=sha256:b32cc18e3ea67c0ca9ca409e5b41eed863bd1363dbc9dd3de90fedf11b61e7bc +docopt==0.6.2 \ + --hash=sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491 +en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl#sha256=1932429db727d4bff3deed6b34cfc05df17794f4a52eeb26cf8928f7c1a0fb85 \ + --hash=sha256:1932429db727d4bff3deed6b34cfc05df17794f4a52eeb26cf8928f7c1a0fb85 +espeakng-loader==0.2.4 \ + --hash=sha256:08721baf27d13d461f6be6eed9a65277e70d68234ff484fd8b9897b222cdcb6d \ + --hash=sha256:41f1e08ac9deda2efd1ea9de0b81dab9f5ae3c4b24284f76533d0a7b1dd7abd7 \ + --hash=sha256:b77477ae2ddf62a748e04e49714eabb2f3a24f344166200b00539083bd669904 \ + --hash=sha256:d1e798141b46a050cdb75fcf3c17db969bb2c40394f3f4a48910655d547508b9 \ + --hash=sha256:d27cdca31112226e7299d8562e889d3e38a1e48055c9ee381b45d669072ee59f \ + --hash=sha256:d7a2928843eaeb2df82f99a370f44e8a630f59b02f9b0d1f168a03c4eeb76b89 +filelock==4.0.8 \ + --hash=sha256:325ff22f358c18443b1fcdfa0a7aa3faec4b500c2807c554719da4567b533d31 \ + --hash=sha256:733d9b6b153fc63672f86104324186818b6bbe9dd7db84e9bb9887b6a04a2775 + # via huggingface-hub +fsspec==2026.9.0 \ + --hash=sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe \ + --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f + # via huggingface-hub +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore +hf-xet==1.6.0 \ + --hash=sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7 \ + --hash=sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675 \ + --hash=sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef \ + --hash=sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9 \ + --hash=sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3 \ + --hash=sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb \ + --hash=sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e \ + --hash=sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338 \ + --hash=sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d \ + --hash=sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d \ + --hash=sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765 \ + --hash=sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c \ + --hash=sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522 \ + --hash=sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f \ + --hash=sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a \ + --hash=sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b \ + --hash=sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b + # via huggingface-hub +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via huggingface-hub +huggingface-hub==1.33.0 \ + --hash=sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899 \ + --hash=sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e + # via + # mlx-audio + # tokenizers + # transformers +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # anyio + # httpx + # requests +isodate==0.7.2 \ + --hash=sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15 \ + --hash=sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6 +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 +joblib==1.6.0 \ + --hash=sha256:2ccc96785b12046c08fd6d55839c12857831b54a3c1673ffadd2f04bfc4eda03 \ + --hash=sha256:3dbbf9f6e4b592a2357b854608e980fe6390d131d7a82f011a377ef2ebef7aba +jsonschema==4.26.0 \ + --hash=sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326 \ + --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce + # via mistral-common +jsonschema-specifications==2025.9.1 \ + --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \ + --hash=sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d + # via jsonschema +language-tags==1.3.1 \ + --hash=sha256:b15f05505dad3ad296a1782d5a6083fd141309186094d1ab08095348f4203f37 \ + --hash=sha256:f7db7ef8879523019603e09644a86d95ba60595ce5e5ea82d46e8971b0f68f7b +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +markupsafe==3.0.4 \ + --hash=sha256:007e1ffd9bf65bb6ee96df7b258fc632a4868dd5566037986c64781f35a36e98 \ + --hash=sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002 \ + --hash=sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b \ + --hash=sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653 \ + --hash=sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c \ + --hash=sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e \ + --hash=sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc \ + --hash=sha256:0764a13d34cae40db7bbf3a09b7e9b491bf4603e20b263a7a9d6b8e324975d0a \ + --hash=sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92 \ + --hash=sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f \ + --hash=sha256:0cee7cb0f9a1b6892ea482237d9403b3d1b4603aee057d0ff01f0fac2d019a97 \ + --hash=sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4 \ + --hash=sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7 \ + --hash=sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691 \ + --hash=sha256:14bd2d845d62ab678eaf81da89d7b621b51756c72346745c1a594c09d49207a2 \ + --hash=sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc \ + --hash=sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde \ + --hash=sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99 \ + --hash=sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9 \ + --hash=sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df \ + --hash=sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5 \ + --hash=sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17 \ + --hash=sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8 \ + --hash=sha256:2a6ef68ae94aed8721934072b27a3b654ea2100b97e4ab864cf1489c90926fbc \ + --hash=sha256:2b2b1e18af909b448bb3cf9e3433366f7a8726271fc214e8b10e0f62a78c724b \ + --hash=sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea \ + --hash=sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248 \ + --hash=sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741 \ + --hash=sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5 \ + --hash=sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6 \ + --hash=sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7 \ + --hash=sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1 \ + --hash=sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67 \ + --hash=sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f \ + --hash=sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9 \ + --hash=sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c \ + --hash=sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc \ + --hash=sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba \ + --hash=sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17 \ + --hash=sha256:3d23795802fc8bd72534836d64489bbf0f67c088959091bdb22e10735a5107bf \ + --hash=sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6 \ + --hash=sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2 \ + --hash=sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163 \ + --hash=sha256:4a540e2d3192792fc84eced57bef37851ccb2b41f73291bb17408eea77bcd278 \ + --hash=sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d \ + --hash=sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b \ + --hash=sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634 \ + --hash=sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38 \ + --hash=sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed \ + --hash=sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c \ + --hash=sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148 \ + --hash=sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a \ + --hash=sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7 \ + --hash=sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f \ + --hash=sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811 \ + --hash=sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e \ + --hash=sha256:57f9947a7e57a081c1e3e0a2dd0d2dcf290a4531450e6f611e30084c222a7295 \ + --hash=sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2 \ + --hash=sha256:5c22873ad1f0532ba40fa1727f3c0fc1bbbaab6d373d4cbe3f0dc74b2e2521c7 \ + --hash=sha256:5e8b3d0b18fd623afa12ecb2ce8d8becef69f9b5440c6330c7972200e0bb84b0 \ + --hash=sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6 \ + --hash=sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed \ + --hash=sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378 \ + --hash=sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0 \ + --hash=sha256:6768d67d1bce64270e0fdc2e69309d68b9b18ae56ddf6c711d168e9d051c2cac \ + --hash=sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b \ + --hash=sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96 \ + --hash=sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59 \ + --hash=sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808 \ + --hash=sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2 \ + --hash=sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb \ + --hash=sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65 \ + --hash=sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72 \ + --hash=sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8 \ + --hash=sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e \ + --hash=sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91 \ + --hash=sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a \ + --hash=sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2 \ + --hash=sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e \ + --hash=sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707 \ + --hash=sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21 \ + --hash=sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef \ + --hash=sha256:8698d70a8081ee8c090dbb394768b5789a1da8b131b5499f89d071dd3cfaf6be \ + --hash=sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453 \ + --hash=sha256:88d59b473bfb03259722600839af9bbd7fa13a2eb514beefeedb95997882f69a \ + --hash=sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6 \ + --hash=sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977 \ + --hash=sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 \ + --hash=sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692 \ + --hash=sha256:9240187afb63d2f9ddc3e032c670356fe941f6e20662ea168a5dc3f1f317e1b3 \ + --hash=sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369 \ + --hash=sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a \ + --hash=sha256:9388003072b95f2f1e3fd908604194d653ba21330d811961a78b7da1a77e9e36 \ + --hash=sha256:9438a2648b2195980cb2dd8e53ed7b8df91319e2d0b70ae61a9e1d1bc8d3bec9 \ + --hash=sha256:94e4c421742086aeee4c32a506eec8859d7634aad943f7e6aacf70f813478768 \ + --hash=sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916 \ + --hash=sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b \ + --hash=sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f \ + --hash=sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346 \ + --hash=sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c \ + --hash=sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464 \ + --hash=sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9 \ + --hash=sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee \ + --hash=sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300 \ + --hash=sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6 \ + --hash=sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d \ + --hash=sha256:ac0c7c9f1609b0c4c114feb1d7a3409564c7fb77e360bed9e97e5d25dfeaf868 \ + --hash=sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46 \ + --hash=sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97 \ + --hash=sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733 \ + --hash=sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe \ + --hash=sha256:b61687d0828e72bf5cda24a2690188f37170bd31c9359ac97e4e66569f120a16 \ + --hash=sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429 \ + --hash=sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39 \ + --hash=sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894 \ + --hash=sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c \ + --hash=sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c \ + --hash=sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169 \ + --hash=sha256:bf053da3c97a4bc5ecfbb218cdd2983febd91c617be8367d139882aa11e490aa \ + --hash=sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77 \ + --hash=sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe \ + --hash=sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad \ + --hash=sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85 \ + --hash=sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e \ + --hash=sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34 \ + --hash=sha256:cf63c214fe879a65e69a386f915e36104fc84254ab141240f8854602d8e0be2a \ + --hash=sha256:d1aca03ede943eb80ab3d63bb082c84b7aab85ea83bd0fd0c200260945fb49d9 \ + --hash=sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c \ + --hash=sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749 \ + --hash=sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214 \ + --hash=sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932 \ + --hash=sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494 \ + --hash=sha256:dd8ea6ebee7aedbf7c749fa80521d9ccf1ba473e0d1e14805caafbaad281c889 \ + --hash=sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1 \ + --hash=sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0 \ + --hash=sha256:dff05cb7016dff1e9fd68f4122c127b65dfc59de5306cfb7ad92f956f230bee2 \ + --hash=sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786 \ + --hash=sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78 \ + --hash=sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e \ + --hash=sha256:e841068dc0be4cb6dfb5c890eb88cbdcff2f4a332393c7ec94e8e618bd32c1a8 \ + --hash=sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289 \ + --hash=sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c \ + --hash=sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe \ + --hash=sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237 \ + --hash=sha256:f291bcf42ae98eb5107edb162c3c998b4a89648fd8e99ed4cbd12705292788cd \ + --hash=sha256:f61efe1d2fe0de16158a5fe1d1cf3c14bdb6aecd54d8938fd26512c525c1f624 \ + --hash=sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19 \ + --hash=sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977 \ + --hash=sha256:fd9f8797427910198f95bced71ddfed61130d7e349213bfb8466c9c99e2c46a8 \ + --hash=sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +miniaudio==1.71 \ + --hash=sha256:06222d80b057ca4beccb6f97a134c2c2bf646ef7890e1759cfc09db7eecec44d \ + --hash=sha256:12bc33e7e61072b4b541c14e10ef76119d5643e6bbb98e2dec0c0738889438fb \ + --hash=sha256:14892ad9b884e637029a22a781dea569b292a1be13682380fd14cefcf80ea4ed \ + --hash=sha256:154b085dd914a0e79e3d93160e1a07aacb27d66c65f9ef6a0d87c1a194f32c04 \ + --hash=sha256:166516449e2bb5f628d89cedbbb8720dceb96a0562c7e08a0e8e3cb10f58647c \ + --hash=sha256:19be6f0a1e601c2237433e579734cfaf6469191b224c20c9e5f73c32ef9ee2b9 \ + --hash=sha256:19dc58e4c50ffc48db2ce988019c28f05ca0eaa7c10b45b5c99b70107e610c8a \ + --hash=sha256:1bf93aeede652926f27f430f0fd69ef0cf8a949c07b537d6a2f295602c747037 \ + --hash=sha256:33986d5d725ebcbc253551e7358689bc81b19b6950b33cec8e8c1142ca4fc0a9 \ + --hash=sha256:3bbeb1e068fe42475e017e8150e9e345182b583d0dd4d9e77ffa20c39935d9ec \ + --hash=sha256:3ef441d139264f8a5dcb9aa6fcd0b1e1e69f58715baae416ff33f045ffba6ad5 \ + --hash=sha256:4c849ccb1349f7b3553a77a66fe7e972315185f5c4c44a0bbda7ebcdd224db37 \ + --hash=sha256:5009b4e29cd43de3631d2d5ab09cc074192c085b4c8dd8a121b856ce1af6bab7 \ + --hash=sha256:50d66729e1dd7a4cf13edc25115ac54f776dd9f67803ba1a7cd1128ebf2e8cfe \ + --hash=sha256:61b86f26d653040db32d9d15b05446321dd10e45beba25b44f841e26935213d5 \ + --hash=sha256:62db602651bc20a2698f36a0d356d7217ed6f4f917550c7ffb3705c8e8be90cf \ + --hash=sha256:70fa2ea5353e6919aca59b8c5768144af009d18c3bca251749d66fb497424563 \ + --hash=sha256:84139a10ef172acd762ccf120142877b037a1aaf71def99d2c75f66329f89d8b \ + --hash=sha256:8a28ff4ad23e55bbde8808ce525d3bb7d249d7612f77646b30e06fc6b7a778ac \ + --hash=sha256:8fc1a4f084cc1b4b25c567d22f54d1e46bfa505c17ed777c8b198e5c53d0f785 \ + --hash=sha256:978cc4d58d8beef1a705e1141dc177a8a357c10ba3a16f7d71482ee722023bbd \ + --hash=sha256:9f379d4995f1fac6dcae65810f6a31cba264339b3e591a14b233f85a6d03d81e \ + --hash=sha256:ab100e5240b104b5326e4ec1be07b6ae461f7d3d4d7a694857fd2f0493d210f9 \ + --hash=sha256:ac4a37ebbbfbfbeca50f4390e50f9952807ca61ac62f0c3bcbbc7dd698531dd3 \ + --hash=sha256:aee8e4eec8d7bde4ee78066561329235a04231a221c9b247f1ffaf850551087d \ + --hash=sha256:d9dc15eff711bcfc62a9d05e0c78e4bc34821a455595e049629f2fea7491a523 \ + --hash=sha256:e6287f15caa808a88aad0700a182bec1ff6d98769717425adf9ebf41259d1936 \ + --hash=sha256:ea86ae04ddbbf2beed20b9970af4a0baca8e6ed0e9625e1ed957be5540c943fd \ + --hash=sha256:f4a44b70b66628b0c307e40ae0ae857695978cae18462179b806d8edc807d416 \ + --hash=sha256:f7042af3a4db5b90e5efaea257b6dfcff9389239ea643e6b0faa80169528e2e6 \ + --hash=sha256:ff51e2887bb673e2e757752b586b3dc924d59aa5fbcae9bbc45f4a111bd3262b + # via mlx-audio +misaki==0.9.4 \ + --hash=sha256:3960fa3e6de179a90ee8e628446a4a4f6b8c730b6e3410999cf396189f4d9c40 \ + --hash=sha256:90e2eeb169786c014c429e5058d2ea6bcd02d651f2a24450ba6c9ffc0f8da15a +mistral-common==1.9.1 \ + --hash=sha256:550583d70a395c3586cfb748ffab53bd1d7c3409507f0efc0118bff30ffb26e9 \ + --hash=sha256:9e2b2520b6f67bac2e2bb06fcf985b7a1277b01938da2b7cda8cf0fdbfa92e91 + # via mlx-audio +mlx==0.32.3 \ + --hash=sha256:0512049c81fded99aa6af2d039ae1fccf113ff1a5f2df937a596578aabd46869 \ + --hash=sha256:097856504e6cf5944b9aaa7c2e1d5be0693428abd9175429cae3893e7ec2f681 \ + --hash=sha256:15e8578c9d10b4fd4728243cd6c926bc46e1da267c6a849cdbceca21d51caa51 \ + --hash=sha256:177bbb598bcb37963568f69b84e7d972f7f9499cdd39d1c6d6dab35a718aeb92 \ + --hash=sha256:19da79adad1aa9653c70819040ac58d65d7a9610a375a30d03c7cce708d4d2bb \ + --hash=sha256:231a278b4fd2804ea16d297960ab169d1749ee1adbf820173e704fbed97bf1b6 \ + --hash=sha256:2dc1085f5c2eba18cdac095d7fb0b68ddfadd51bf695b18a1102eda15e77e76d \ + --hash=sha256:32fa23e57297fdca23e016570229b459f20f645eeb5965527cd890f74143f1d0 \ + --hash=sha256:448fdbc99c5fb0c26f11337e5a182cd6516f8a1a65848184be063d00ad3107dc \ + --hash=sha256:452c621862684e8769be93c1517420c8ba2fd6e8b01a1c970c8ba8069022f0ac \ + --hash=sha256:46df0a5489edcc071cec3d9dd1f7894d492888c9d425effdc8831f4e7021108d \ + --hash=sha256:4e1a8fa78461c6cf1f9473e3bc8a3c4e362f1aaa613b435787f2844fad522bc7 \ + --hash=sha256:50939a8bd54250e6092ef3427a724c7a72c0911dcee09d70f6543431e7f28d3c \ + --hash=sha256:5d12de01f23d2fae739ecfecd9c1d4c07c1f9b778a3729f7c0884ea1168ec14d \ + --hash=sha256:65122a7eeb718df56641d7ac6f7fdb0ada573fd841d1fb0a61a6a443e33f11b5 \ + --hash=sha256:656df5e36b9bfa862fd633d55dee4da072d292def9f27f22134c594e03ff28a7 \ + --hash=sha256:712ca29119d9e2aae04af8f8e416bfb0097538d9ff73e6fab8b028053fb45ff8 \ + --hash=sha256:72711cb23cc6dccc5a460f63f39c11931a00b2c6dcd6fba253921090072e21bc \ + --hash=sha256:75333afef55819afd2a31e87ffbed0bf59f379bca21771e6b13a95a675e4c85b \ + --hash=sha256:78372731c18fadb6d6960c9188cb752d5ea4d87de9be430a311a385e01d7fb42 \ + --hash=sha256:7aecc3d4a904279492dae58e28378a3c4bec4533e7066e64ae5c78c7a5511f22 \ + --hash=sha256:8439baa207db769df1df82a722b0ee0c2362f75c23c0740b7472c3d279253bae \ + --hash=sha256:8564a59943346fac01ab498870a9350d0f79b1cdea95c46c895934ee027d6197 \ + --hash=sha256:8edf996749b331659ab9bb8ecdc895c292f9e7020d02e534147e07cae22bec7c \ + --hash=sha256:8ef541373a94a78fd59ac59368501e976cef5fa07738438362e942315be22ea4 \ + --hash=sha256:936cc82b0c050686c43230c85f3a0bd9f179f650377ce9590f7af3ba659e2fce \ + --hash=sha256:942f0da5328986cdae29ec0eba57b2a08eb9dc6125cfe196853de7498f2c5359 \ + --hash=sha256:951ba2ab75148f61e9a5c222a75fdb0ab481be0b196c21a0d087e9b886d204f0 \ + --hash=sha256:a14387cd679018a32e802fdaa58b1e9845a278cacd309955eddca95757988ce0 \ + --hash=sha256:a5e165179e3d86672cc0d594587bba2b9c3be511ccb99ffdc63e4cc7119eec55 \ + --hash=sha256:a6241ea8f56a08a6c3a06bbaad7d7fc2fc1c05f23757c61fe0a8797325ac6d46 \ + --hash=sha256:ac4472834f38e61d294aa1bf2fdc9f687f3b3162b9729dc9dc15e8c149f8c216 \ + --hash=sha256:b14744a1b3a9dd48e06a62c448710a798ceda9451d7d37a4654a58b5adfa97f8 \ + --hash=sha256:b2acd057e4a27eb1543bff0e9030c34fdc64e18815fe4bc23c2dbcc47b60f0e3 \ + --hash=sha256:b57996fd2b72207a796656bc39508f20089692815297120039284edb296b5fff \ + --hash=sha256:b91012fdd5ec5cbedcec2cae6adfaad9a8f547023e570fc5781cb7eb4bfe2567 \ + --hash=sha256:be5cb6554873d3c86f083fb0b35aedf4a1f12657adee4ac0fc90665ffd7e6722 \ + --hash=sha256:c0764ce1a7a7f19768d6b3473387dcac03cac8b0dc74b7e1c8d05b451ff4786c \ + --hash=sha256:d17f0c33789009a3b3e609a52af34dc687ed39816836e20ab466aab669c14e6c \ + --hash=sha256:d3b9494b45400b803eeed7d05e6e566aa135f1dd2d121c29016ee9492e7ead67 \ + --hash=sha256:d61942263d4603814622cd72bb5991b8a3cb59970bde8a7b40b897a7f088fb84 \ + --hash=sha256:d9ff05b868748b846a02f59913e9aa9d7b576827db92cc358ec1cf660f7f90a5 \ + --hash=sha256:e13f5ce391293dab946edf18e7ae7bb55768beef1140deb509f67a2b14edcf7c \ + --hash=sha256:e32d595df067f19739f1e214e723035ed2a9c216fb1ed3cd118d08f19b9cfabd \ + --hash=sha256:e75fa55161d9477e694f852cad0d383c6b4a5c3b505a5d08339f0e2260c2556c \ + --hash=sha256:f2909f683ffc5a2720f6c8099398a64fd1be53f1b6538e4e728ee55816ce699b \ + --hash=sha256:f348705a72fa236f310c2cbb9c393d3b620a942c9b34ac641fd0f098bdc5be44 \ + --hash=sha256:f5716b51fea3d06352d4d8c851fb6c5141b1a7de5a5164d44de5a6d368d84a7e + # via mlx-audio +mlx-audio==0.5.7 \ + --hash=sha256:2f2b3c666241ff4a45df7e92057d176b3c1079541a5e9062a0d1e8088db5c123 \ + --hash=sha256:4446704bd8305b3957a6e422234118142e32b87a55d7d3e33b0fb8dbca516543 + # via -r .tmp/hf-intake/locks/mlx-audio.in +mlx-metal==0.32.3 \ + --hash=sha256:34ae9b83ad2f0ccdd3e5d48ec35176e7119f57069eef187122916dc941a4ae1f \ + --hash=sha256:fdba88ed9131448cf163be2c7ad6374746d8818205f28657aa91b6c69050f691 \ + --hash=sha256:feea99fe4d01f9e9f33ca73d869f988219a1725de1c18870baa7d5cd5aef6cb3 + # via mlx +murmurhash==1.0.15 \ + --hash=sha256:0861cb11039409eaf46878456b7d985ef17b6b484103a6fc367b2ecec846891d \ + --hash=sha256:1349a7c23f6092e7998ddc5bd28546cc31a595afc61e9fdb3afc423feec3d7ad \ + --hash=sha256:189a8de4d657b5da9efd66601b0636330b08262b3a55431f2379097c986995d0 \ + --hash=sha256:213d710fb6f4ef3bc11abbfad0fa94a75ffb675b7dc158c123471e5de869f9af \ + --hash=sha256:2224f30f7729717644745a6f513ea7662517dfe7b1867cf1588177f64c61df3c \ + --hash=sha256:22aa3ceaedd2e57078b491ed08852d512b84ff4ff9bb2ff3f9bf0eec7f214c9e \ + --hash=sha256:231dc7982e1aeae8bbab21f8f21953e3b9fb0a34e3ffe2d2342ff32e4f07af9d \ + --hash=sha256:263807eca40d08c7b702413e45cca75ecb5883aa337237dc5addb660f1483378 \ + --hash=sha256:2680851af6901dbe66cc4aa7ef8e263de47e6e1b425ae324caa571bdf18f8d58 \ + --hash=sha256:26fd7c7855ac4850ad8737991d7b0e3e501df93ebaf0cf45aa5954303085fdba \ + --hash=sha256:32c6fde7bd7e9407003370a07b5f4addacabe1556ad3dc2cac246b7a2bba3400 \ + --hash=sha256:342277d8d7f712d136507fb3ccdba26c076a34ca0f8d1b96f65f0daa556da2e9 \ + --hash=sha256:34e5a91139c40b10f98d0b297907f5d5267b4b1b2e5dd2eb74a021824f751b98 \ + --hash=sha256:3c69b4d3bcd6233782a78907fe10b9b7a796bdc5d28060cf097d067bec280a5d \ + --hash=sha256:43bf4541892ecd95963fcd307bf1c575fc0fee1682f41c93007adee71ca2bb40 \ + --hash=sha256:43cc6ac3b91ca0f7a5ae9c063ba4d6c26972c97fd7c25280ecc666413e4c5535 \ + --hash=sha256:44d211bcc3ec203c47dac06f48ee871093fcbdffa6652a6cc5ea7180306680a8 \ + --hash=sha256:4a70ca4ae19e600d9be3da64d00710e79dde388a4d162f22078d64844d0ebdda \ + --hash=sha256:4fd8189ee293a09f30f4931408f40c28ccd42d9de4f66595f8814879339378bc \ + --hash=sha256:539d8405885d1d19c005f3a2313b47e8e54b0ee89915eb8dfbb430b194328e6c \ + --hash=sha256:55e1a7f65095f0af141c8a460765e026e35d9ec526f0daa2f88fed29a292b2ff \ + --hash=sha256:5678a3ea4fbf0cbaaca2bed9b445f556f294d5f799c67185d05ffcb221a77faf \ + --hash=sha256:58e2b27b7847f9e2a6edf10b47a8c8dd70a4705f45dccb7bf76aeadacf56ba01 \ + --hash=sha256:5a301decfaccfec70fe55cb01dde2a012c3014a874542eaa7cc73477bb749616 \ + --hash=sha256:5d8b43a7011540dc3c7ce66f2134df9732e2bc3bbb4a35f6458bc755e48bde26 \ + --hash=sha256:66395b1388f7daa5103db92debe06842ae3be4c0749ef6db68b444518666cdcc \ + --hash=sha256:671979f15b24817968ff6fab5e3a468e2b05555bac5e92f11155610a37165ffc \ + --hash=sha256:694fd42a74b7ce257169d14c24aa616aa6cd4ccf8abe50eca0557e08da99d055 \ + --hash=sha256:6cb4e962ec4f928b30c271b2d84e6707eff6d942552765b663743cfa618b294b \ + --hash=sha256:7c4280136b738e85ff76b4bdc4341d0b867ee753e73fd8b6994288080c040d0b \ + --hash=sha256:8155d106c63c5a509ec58c19b0c30804f3a0931bc65cca9826efc53373332536 \ + --hash=sha256:847d712136cb462f0e4bd6229ee2d9eb996d8854eb8312dff3d20c8f5181fda5 \ + --hash=sha256:88dc1dd53b7b37c0df1b8b6bce190c12763014492f0269ff7620dc6027f470f4 \ + --hash=sha256:898a629bf111f1aeba4437e533b5b836c0a9d2dd12d6880a9c75f6ca13e30e22 \ + --hash=sha256:899068ba3d7c371e7edd093852c634cce802fefd9aaddfcc0d2fda1d7433c7f9 \ + --hash=sha256:8a181494b5f03ba831f9a13f2de3aab9ef591e508e57239043d65c5c592f5837 \ + --hash=sha256:95d7c52598dce7a8543e5a5f61a893cbb762a62a907c6c9cd025d5f618bb8522 \ + --hash=sha256:9aba94c5d841e1904cd110e94ceb7f49cfb60a874bbfb27e0373622998fb7c7c \ + --hash=sha256:a2ea4546ba426390beff3cd10db8f0152fdc9072c4f2583ec7d8aa9f3e4ac070 \ + --hash=sha256:a32054edb567417ac81f7172b7dd7731846f25c63084edeb20545fa7abc849d7 \ + --hash=sha256:aadac5fd5f3f465094a5e4ecd00d739a2b870651cad06c8d1005153c2e815fb3 \ + --hash=sha256:b3ba6d05de2613535b5a9227d4ad8ef40a540465f64660d4a8800634ae10e04f \ + --hash=sha256:b65a5c4e7f5d71f7ccac2d2b60bdf7092d7976270878cfec59d5a66a533db823 \ + --hash=sha256:bba0e0262c0d08682b028cb963ac477bd9839029486fa1333fc5c01fb6072749 \ + --hash=sha256:bc54facccb32fe1e97d6231edd4f3e2937467c35658b26aa35bbd6a87ebb7cb0 \ + --hash=sha256:c22e56c6a0b70598a66e456de5272f76088bc623688da84ef403148a6d41851d \ + --hash=sha256:c4cd739a00f5a4602201b74568ddabae46ec304719d9be752fd8f534a9464b5e \ + --hash=sha256:cb8ebafae60d5f892acff533cc599a359954d8c016a829514cb3f6e9ee10f322 \ + --hash=sha256:cc93769619b6b42740cab8eebb587709daaa3d8f813372cc60358a571de20d2d \ + --hash=sha256:d37e3ae44746bca80b1a917c2ea625cf216913564ed43f69d2888e5df97db0cb \ + --hash=sha256:d4d681f474830489e2ec1d912095cfff027fbaf2baa5414c7e9d25b89f0fab68 \ + --hash=sha256:d7e47c5746785db6a43b65fac47b9e63dd71dfbd89a8c92693425b9715e68c6e \ + --hash=sha256:dc35606868a5961cf42e79314ca0bddf5a400ce377b14d83192057928d6252ec \ + --hash=sha256:e43a69496342ce530bdd670264cb7c8f45490b296e4764c837ce577e3c7ebd53 \ + --hash=sha256:e525bbd8e26e6b9ab1b56758a59b16c2fffd73bad2f7b8bf361c16f70ff1d980 \ + --hash=sha256:e8e674f02a99828c8a671ba99cd03299381b2f0744e6f25c29cadfc6151dc724 \ + --hash=sha256:ef19f38c6b858eef83caf710773db98c8f7eb2193b4c324650c74f3d8ba299e0 \ + --hash=sha256:f32307fb9347680bb4fe1cbef6362fb39bd994f1b59abd8c09ca174e44199081 \ + --hash=sha256:f3e99a6ee36ef5372df5f138e3d9c801420776d3641a34a49e5c2555f44edba7 \ + --hash=sha256:f4989c16053a9a83b02c520dd00a31f0877d5fd2ab8a9b6b75ed9eba0e25c489 \ + --hash=sha256:f4ac15a2089dc42e6eb0966622d42d2521590a12c92480aafecf34c085302cca \ + --hash=sha256:f9bf47101354fb1dc4b2e313192566f04ba295c28a37e2f71c692759acc1ba3c \ + --hash=sha256:fa1b70b3cc2801ab44179c65827bbd12009c68b34e9d9ce7125b6a0bd35af63c \ + --hash=sha256:fe50dc70e52786759358fd1471e309b94dddfffb9320d9dfea233c7684c894ba \ + --hash=sha256:fe883982114de576c793fd1cf55945c8ee6453ad4c4785ac1a48f84e74fdc650 +num2words==0.5.14 \ + --hash=sha256:1c8e5b00142fc2966fd8d685001e36c4a9911e070d1b120e1beb721fa1edb33d \ + --hash=sha256:b066ec18e56b6616a3b38086b5747daafbaa8868b226a36127e0451c0cf379c6 +numpy==2.5.3 \ + --hash=sha256:012e66aca395d795496446e52aeeb5866312a5d4d3f27da270e5a0b43f70dc5c \ + --hash=sha256:09d5a423c71ad5feb5625844ad58050e35df43871004b52ac9c0ad44a56775be \ + --hash=sha256:09ffa5d903faeaa5c4dd05009cf81c8bab9f2cb37c548b8d39b65b4cfa7c97f7 \ + --hash=sha256:0a59a421a32580a009e8a1751345bf829631b990dc1794b80514ab722b435def \ + --hash=sha256:116f96cadd935c6122e9228d676fe7ede19e741f5c8bb1c3cddbe0c51ccebea2 \ + --hash=sha256:1302b90c0e52281681b2975adfe8a860cb7b12216a27b4b0b4207c44bf7bccf0 \ + --hash=sha256:15aa985ac73a8db02db7663381aa109510449d3819d37206caed27b33a65a8a6 \ + --hash=sha256:1aad64d99730d013cfc6debafed22783b4fc5a7f4b8bc744d2d8cf7dcc880551 \ + --hash=sha256:1c80eabb4035ecf4ca9cd49cde8a9fdd69a729e63e6474887d1523ade7aa277f \ + --hash=sha256:1f3ed25271581281f2fccb1adcedfcde4c07362eec69189b50baf6f90e3ae159 \ + --hash=sha256:1fb6f8fb9ff0b3a69f52c66ce397b0246583e9f28616231b0e32ca49259a5fa6 \ + --hash=sha256:214045a5bf00113a146ab9ee9730c44501af6723cdf1f6830932f7b5ef2e7af0 \ + --hash=sha256:26e15e4aecd8617dfbaecb37d223e365d7b39411fba20454be2670a96aa74cb5 \ + --hash=sha256:2c25dfa72943e4336ddb6b0ee4277b47a0c85bede0807530ec68103bf58e2c10 \ + --hash=sha256:2d8240cb4c16fd831074aa2b2cf9fc54664d826341d61c372245b96a74a49a9a \ + --hash=sha256:350ba9783ce969cf9f7ce6e6a9a58e1a6e2a19ca025b7ee448c4db727706212a \ + --hash=sha256:4c8a6d2ebce6305fd82fbefca827775437147052a976ee7c94b36a0c1b52ac6c \ + --hash=sha256:4f8929ee6c96bfbd7b4ed2032e0c03af86fe1826740ab61ddabf9072d06e57ff \ + --hash=sha256:536f963710a4e63934d80ac0dc4f478804a83e9a84b6828018f25d09953ada33 \ + --hash=sha256:54a115e5a73b8fc44f0cebef486365a1894b5c9760685d4558b72b7c3eb846e0 \ + --hash=sha256:595d020938c84e320bcf40ad71089e108eac0d377cd018e14a8c094f39e98d85 \ + --hash=sha256:66a78fe4556c60aceda5916f9eacd638b18e9e681016ec302dcb4682d6d4d034 \ + --hash=sha256:6b05c171afb3aa07adbd20abc00aea86fe375beb0fdb9ef780ec5b7f63bab1c0 \ + --hash=sha256:6cef4bb1706dfec49243c05d921eefb4e190d41e2528b30d8035ea1f36b4c24a \ + --hash=sha256:6f24021b9f22bc6301c37b196974a92c1c18dccedb6fef3dd252e95f2d6adbe4 \ + --hash=sha256:71b39d9f935b6ec0f8753e3e2afb51e3efba6f2e05b68b32a40754d24bcd4a3c \ + --hash=sha256:71cad2b2a7451ab79d8f5e71b453485b6775963d5cf794179144a7463fe6e8ec \ + --hash=sha256:76c2c1e6bfa5c84adc6434dfbf013aa92096a7985221762c8f11fedfd20fff58 \ + --hash=sha256:8617bbfae4486cf99c9f899966699428d19da931d06ca94ad3da986c76e15997 \ + --hash=sha256:86bff898a431c0fb71f7610b75726e75a54d47b37edc9d537f48de63bb3c0b90 \ + --hash=sha256:8e4dd766076855b5ff7ea52fa5f07ce26286726e0f8bff446b7739d02e6ea204 \ + --hash=sha256:92f30e89b8ee0ecf363033576c422b2f58fed6a80bed0aa48dff6d14c654663e \ + --hash=sha256:93e1f5447e2b1e479d7bd74701e84746b86450cff1fc368b132d195e2b8f8211 \ + --hash=sha256:9a37475425b431b4d060f23b4f52cd2f3aef6bc7c654bd760adf0040eec9d435 \ + --hash=sha256:9deb49575e5b0b94ed72c8a64ec4d033381adc27e9060ae842971f697ba96104 \ + --hash=sha256:a5fa86b80fd24bcd1aff83ad23be44ea323de3f787be8f8b15d4a65621e25321 \ + --hash=sha256:a6391fafaba97500887132cd582abc6e19452b1ac775a47caa7b24490e152058 \ + --hash=sha256:a72f874bc9e10e4b8f80426fb49716d5141f64442a0c8418065093ec8017fbb0 \ + --hash=sha256:ac7bb1c52d445bd4f8f7f97fefe6abc3a084dc4d63df50d79b17fa2b78e89297 \ + --hash=sha256:adc1ada2662f8a5f960b8a10d9986897e7499ef07e06d4cfe7197f8cce923c07 \ + --hash=sha256:b00eefbcf0f292945c4b4dec2ae845389ef5bcdcd596e6e4328051db5b5ba694 \ + --hash=sha256:b0521d0f4aebb6e06189451025fa17a913287b13c03d5fe05c017333b654ea5b \ + --hash=sha256:b5d93cf48f687479941d12b69c873ad2cc76bbd487f0091c2200636497f34034 \ + --hash=sha256:b7e18c623bb5c95acb3b3328861272816ba199fb531921c5d6d0b675f1fde9e3 \ + --hash=sha256:bd4cb9ad3c7889b9b3fe0a9a9fb5d2ed26f9879bff2608d9f01aed147a20d231 \ + --hash=sha256:be5a8381859b6da607c84f4f7d6847725f1cf1853ef8a2c9e115b7d58bef47dc \ + --hash=sha256:befa1ae5bd6030b3f512b43ff3fa5290bbed6b84411a44244b14adf835f5b89d \ + --hash=sha256:bf63afbe037eb5d2fe87fbcc7778e61da53ebaf21d938a4515aa73b62532a5d4 \ + --hash=sha256:c00abe94c1a69d75d827dcf1c025b25c8a45d230b3bcd77a9020883a1b047653 \ + --hash=sha256:c2381f82999704f818e2c987a865050e285ec3621262c66d40f5a96c8f899f8e \ + --hash=sha256:c76d5dde9f445058f83d0c02af00557a4db91de9a9a57c0df87d1535001d654b \ + --hash=sha256:cb189f09db39283b26bfd061ec16189e14f71c6755207f72a0f7540867afe5b9 \ + --hash=sha256:ccb32e0525d29e8b0572eb84c9a57af0e7a4e615726927506f55063c62414034 \ + --hash=sha256:ccbc4665079665c3cf3bab4db9f6b095370cd6437d66be549b6c2a1fd19e1958 \ + --hash=sha256:d1c89973648c85069c5046ad460f7b8a00218b29a2e42359ac8cc63e9ab94832 \ + --hash=sha256:df2d5874ff183595a4ba404edd04f6bd9b5505c1d7708573f6a6c17489a67563 \ + --hash=sha256:e01c918ac3d48e18a927cf7b14a26a3e29ff2bdf2eacb976da0aecd6a43ed034 \ + --hash=sha256:e6ab667ba76450084eb64013762c438ea76d9d29cc676dcd6c2e9892ba37f841 \ + --hash=sha256:e931e4f499e0dc7ef29d269a8e5b35dd722e5d14be07df6240166ea7c6532fae \ + --hash=sha256:f54660b0eb6b0b9f36e7fe1cdfdff472028dd0d14acd9b9b65098efbad059469 \ + --hash=sha256:f59a878c33d6b88122d80d239bb3b845d58708750b0cb06a09aebb9b18ec696c \ + --hash=sha256:f7fabeb6cea87d65f3b926de33d03fb016cfdc29314c90974383b5582ae72891 \ + --hash=sha256:f9579f383d1bf9df80081e72760e84960a7fd4f88cf0c9e535a8597c9bb646f5 \ + --hash=sha256:f9a2353b37a1a9e78fd82b27ad7e2a32a2d036604d18f02b05e3136c62ca3b09 \ + --hash=sha256:fc36dc566135b5eceec4cf89758fcb719266a019ef07dae1754ae7c9f617ef3e \ + --hash=sha256:ffdc76bfcae6b255dff75202c5e7feaf95b40246bc0a17944facc1fecf9f79ab + # via + # mistral-common + # mlx-audio + # scipy + # soundfile + # soxr + # transformers +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # huggingface-hub + # transformers +phonemizer-fork==3.3.2 \ + --hash=sha256:10e16e827d0443b087062e21b55e805c00989cf1343b2e81e734cae5f6c0cf69 \ + --hash=sha256:97305c76f4183b3825dae8f4c032265fe78c9946ce58c47d4b62161349264b74 +pillow==13.0.0 \ + --hash=sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27 \ + --hash=sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782 \ + --hash=sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22 \ + --hash=sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15 \ + --hash=sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502 \ + --hash=sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692 \ + --hash=sha256:10b5019883c98f1dd819ca48ffc6f33a91cee04948fe637c605f4345c7cafd6f \ + --hash=sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8 \ + --hash=sha256:12fbb851cf323cc19aac3eea1df5ad17b57e91e9fe3cfd06100dba7bb8208c13 \ + --hash=sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3 \ + --hash=sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369 \ + --hash=sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88 \ + --hash=sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d \ + --hash=sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb \ + --hash=sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f \ + --hash=sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0 \ + --hash=sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40 \ + --hash=sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599 \ + --hash=sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055 \ + --hash=sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033 \ + --hash=sha256:37290fbef0c73e2d51d594eb78fded7dc7170d1c3ad0a890269e6fd8c8c4e061 \ + --hash=sha256:405be6058e206a86fedce2f715ecc2343f94072d7925251632dffa2c05f70dca \ + --hash=sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f \ + --hash=sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca \ + --hash=sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c \ + --hash=sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4 \ + --hash=sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8 \ + --hash=sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620 \ + --hash=sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071 \ + --hash=sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc \ + --hash=sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a \ + --hash=sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6 \ + --hash=sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e \ + --hash=sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582 \ + --hash=sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f \ + --hash=sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79 \ + --hash=sha256:6a0920cb55df5a4d3f6862964908078c0bd59332bdd1bcd20d14e8c70ef27cc5 \ + --hash=sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683 \ + --hash=sha256:6f38f3f885a09cafc93fd667b1173a92f379e43785d635e7983c893f0da319fb \ + --hash=sha256:73ee25cf8cf8b75d5a5aeed7cb6588f5e5dbfbb0fde51707b630bbb23897645c \ + --hash=sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa \ + --hash=sha256:7c88c10dd432f3b39ccdafc9ed81d0c21b8fd634502574da76d1bfa4fe2c76d0 \ + --hash=sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521 \ + --hash=sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e \ + --hash=sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c \ + --hash=sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058 \ + --hash=sha256:83b37abd87b590acd8f47ca8a61f70893322b11834d36e07c2298dacad4d14d2 \ + --hash=sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463 \ + --hash=sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d \ + --hash=sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81 \ + --hash=sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf \ + --hash=sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63 \ + --hash=sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0 \ + --hash=sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42 \ + --hash=sha256:a192360d512305fe92d3053e4f7627bbad1f7708f1f4ad38cccd9336acabe9ad \ + --hash=sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81 \ + --hash=sha256:a625df9a2a5597fa5e5266e01aefc82604bb85d291e3547981e863560d75da6c \ + --hash=sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4 \ + --hash=sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981 \ + --hash=sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43 \ + --hash=sha256:b84be41387fc00db28c13c02e30d42b02d8d8f6a4a741a1aee6012ca7f1f6039 \ + --hash=sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76 \ + --hash=sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51 \ + --hash=sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2 \ + --hash=sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630 \ + --hash=sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac \ + --hash=sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510 \ + --hash=sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6 \ + --hash=sha256:d7710eb676e9bb68965cb045be0d333b4c0d7f9dfbaa7fbf121385b1588683d3 \ + --hash=sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d \ + --hash=sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61 \ + --hash=sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932 \ + --hash=sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1 \ + --hash=sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d \ + --hash=sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00 \ + --hash=sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe \ + --hash=sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313 \ + --hash=sha256:f391dc92a015c478745192fe4f2e120c2af2aba18d9fe0616341cbf537599806 \ + --hash=sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a \ + --hash=sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0 \ + --hash=sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52 \ + --hash=sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b \ + --hash=sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe + # via mistral-common +preshed==3.0.13 \ + --hash=sha256:04d8f13f2986e5d11af5ac51f55ce3106c70c41b483d20ea392e6180bdd0f870 \ + --hash=sha256:09397592d333a77f88454e72b7f1f941b2afaf040b392b9e74898dbc4648cdf5 \ + --hash=sha256:09f96b477c987755b3c945df214ea1c1c80bfb350e9f34e78da89585535b77e8 \ + --hash=sha256:0e5b2865aecbd2e1e10e5d19bb8bfad765863c1307c6c3e51f2a08bd64122409 \ + --hash=sha256:183b339956a9e1d7a4a00038a3b9587a734db9e8bd915939a49791bd1b372156 \ + --hash=sha256:19318dc1cd8cac6663c6c830bf7e0002d2de853769fb03e056774e97c21bedfd \ + --hash=sha256:208dcebbe294bf1881ce33fb015d56ab2a7587aece85a09147727174207892e4 \ + --hash=sha256:2b704e46cb7b88f656ef16a3e5347b36525a1c53721d327a4ba1457404101f85 \ + --hash=sha256:2e77bed56aded7cbe5d28d6bd2178bc5b13eda0e0e464dab205fb578fa915000 \ + --hash=sha256:35d6c5acb3ee3b12b87a551913063f0cec784055c2af16e028c19fe875f079d0 \ + --hash=sha256:3e3528f6628329349e281b607aad746ae3c06c15ba59fd5b6599c7c2fd77911b \ + --hash=sha256:40e9445911051bc67cf84ba12745e3be4d010aaf4f8ade3ba3def82fdb45d18e \ + --hash=sha256:42c58b07e8b431e33d0ad9922e896632453821cad8b09171b619b8c61101916f \ + --hash=sha256:461327f8dd36520dcf1fd55a671e0c3c2c97a2d95e22fc85faa31173f4785dda \ + --hash=sha256:4a7bc48220de579be6bdb0a8715482cf36e2a625a6fd5ad26c9f43485a4a23b5 \ + --hash=sha256:4e9ae86c982e49f58620d45eeb51e073e50feb88f52ac10a95fe117c1d222a84 \ + --hash=sha256:4f8856ca3d88e9b250630d70abb4f260d8933151ddfb413024784b25b009868e \ + --hash=sha256:502f93f49a22788203f02d3067d4ea077a0cca3864de6a792eae12e7ce589e14 \ + --hash=sha256:5268c0e6fa96f50cdf87f516c2d4b32563c12706ee768e75c00e8d0098acd545 \ + --hash=sha256:5d14eea14bd01291388928991d7df7d60b9fd19ae970e55006eb4d29b0c1e8eb \ + --hash=sha256:5e2753779832e411e93eb727f3d409c0a6b7408e5ce4dd868076d8ece48c7693 \ + --hash=sha256:62cf7f3113132891d6bba70ff547ad81c6fe50a31930bbbb8499f1d47cd122b7 \ + --hash=sha256:670db59a52e1823b5f088c764df474e65b686592d4093adbeef14581c95ee2cb \ + --hash=sha256:70d502e081348df207d90f347f21770ed596822bb04eb3c3b32b7281579e90c6 \ + --hash=sha256:7557963d0125a3a7bcdb2eb6948f3e45da31b5a7f066b55320de3dea22d7557f \ + --hash=sha256:7770987c2e57497cd26124a9be5f652b5b3ccd0def89859ab0da8bca6144a3de \ + --hash=sha256:7c333f18e9a81c8a6de0603fd8781e17115324b117c445ca91abdf7bfb1abe49 \ + --hash=sha256:7da9d931e7660dcdd757e5870269f0c159126d682ed73ed313971d199eb0f334 \ + --hash=sha256:867aa73abbf4ee3b4d7662148091c33a8c039271269e3a7f1e0ca995f91995c8 \ + --hash=sha256:8b82d7a7bb63d248a6cbbfcabb4a570c993d54d964e39dc5d85c14018ba2079e \ + --hash=sha256:8b8de3f58043070a354477995acdd98626ce43e4193c708ebd0f694e467f5155 \ + --hash=sha256:8d6acc1f5031a535a55a6f7148e2f274554a8343a16309c700cebea0fe7aee8c \ + --hash=sha256:985cb9b097beda76cd13c01a0499707103e8915f888fa30f8aa8324ef2cc6b08 \ + --hash=sha256:9ca43ecbc3783eda4d6ab3416ae2ecd9ef23dca5f53995843f69f7457bcd0677 \ + --hash=sha256:a06e27f4e5b9d7943840087828c6a0dae4a3475576d12c2e95b71abbb325a80b \ + --hash=sha256:a3ac301b065e67e9541f8e3ab3f67533e53deb57c2d258395c5bb98f9723f99b \ + --hash=sha256:a8682988e47739adba369bf43789fc870b554a21e2d1f30a3d17ed336d05f451 \ + --hash=sha256:acd4d89abeca3678c5d8c89b3cd351314465bc67c7fa053d2644f8513e543386 \ + --hash=sha256:b03e21b0bf95eb56e23973f32cabb930e94f352228652f81c0955dbd6967d904 \ + --hash=sha256:b980f3ea9bb74b7f94464bc3d6eb3c9162b6b79b531febd14c6465c24344d2cc \ + --hash=sha256:bef84b225d226af43adfee78ce5ddede72a6155ce5292c1a41dcd1f0b9c87c30 \ + --hash=sha256:c046736239cc8d72670749b79b526e4111839a2fc461a58545d212797649129c \ + --hash=sha256:c0d0c14187dc0078d8a63bf190ec045a4d13e7748b6caeb557a7d575e411410b \ + --hash=sha256:c4bc60dc994864095d784b7e4d77dba3e64188d169ac88722b699d175561fddb \ + --hash=sha256:c8596e41a258ff213553a441e0bb3eb388fd8158e84a7bf3aae6d8ede2c166d3 \ + --hash=sha256:cf8e1a7a1823b2a7765121446c630140ac6e8650c07a6efbf375e168d1fef4f7 \ + --hash=sha256:d0e114300e5577e806c17fb1cc9f07bc6584188d84545401f66e690c8315feef \ + --hash=sha256:d2f1efae396cadab5f3890a2fd43d2ee65373ef9096ccbb805e51e8d8bcc563b \ + --hash=sha256:d4ae5cfe075bb7a07982e382bca44f41ddf041f4d24cbd358e8cccfc049259b8 \ + --hash=sha256:d75f718bbfd97e992f7827e0fa7faf6a91bdd9c922d5baa4b50d62731396cb89 \ + --hash=sha256:dbd7c735a613857ae39ac23bf4690b0d92adc30add977828529b50ba09e33fbc \ + --hash=sha256:de87fbabb0f37c3c92d4dd9b94fc82ab73cdab4247cdfbd57ab3926caa983919 \ + --hash=sha256:df642547a1a94079978a0ea8f4593ab4b8d3bd43f767bef0ef64d9a214f8c4c9 \ + --hash=sha256:e1ab099b2f5843b19e875502b64001b0705e375fb5bd1ca6240aa14e4ffc31e4 \ + --hash=sha256:e5c8462472f790c16708306aef3a102a762bd19dfe3d2f8ee08bd5e12f51b835 \ + --hash=sha256:f05b08ce92399c0655b5e0eb5a1cc1f9e295703ed3aabdfaf6538dfa8ae23d57 \ + --hash=sha256:f8e6fe0620ed0f96a246d46447055c447e071cd8222731a045c235e8a758c918 +pycountry==26.2.16 \ + --hash=sha256:115c4baf7cceaa30f59a4694d79483c9167dbce7a9de4d3d571c5f3ea77c305a \ + --hash=sha256:5b6027d453fcd6060112b951dd010f01f168b51b4bf8a1f1fc8c95c8d94a0801 + # via pydantic-extra-types +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pydantic==2.13.5 \ + --hash=sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73 \ + --hash=sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08 + # via + # mistral-common + # pydantic-extra-types +pydantic-core==2.46.5 \ + --hash=sha256:013d6f3483d81e02e7c328831808f336c8596ee33b4bd4026b9ffb1e960b8942 \ + --hash=sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821 \ + --hash=sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5 \ + --hash=sha256:0bddb4020d8f04175865ccd17eff3040874fc11fb593f424edb452653b4b947c \ + --hash=sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184 \ + --hash=sha256:0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2 \ + --hash=sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc \ + --hash=sha256:15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5 \ + --hash=sha256:18a09e1e1011b462f2e32774f25859ef1223d5c2b0546a633cf56654710721e0 \ + --hash=sha256:193375f3548919d3f0b60936ca113ada3e38f264f91b9b8e0508efaad57be931 \ + --hash=sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e \ + --hash=sha256:1e449def1945a462c464331254e5a44fca7c3b4f9aedf59ec2f50f8066dd8e25 \ + --hash=sha256:1e5aad1220a1192c42341c8fd4a8686657e73ab2a920c970bdc4de334fe3193d \ + --hash=sha256:200aa3dc9f8d54f0754f43247c0bad0999fdcfbfd2488384dd44f37279271fe6 \ + --hash=sha256:2471fd51c61c610e1dcf7de44d7299283661654d11264ab4802b303368d69c47 \ + --hash=sha256:24922243639cbdac66c75fcb6fd6495a9cb52b213d62f9a0d16f0310b1ff8038 \ + --hash=sha256:28a6a556cd3b6066bea827857f9d9cce027c96f776e512f544a581f9e42161f8 \ + --hash=sha256:2bc9419666990c06d7397831f2126a1ecc3594aaa3ff7de5bf2d066802f4e07b \ + --hash=sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a \ + --hash=sha256:2d330aaba8621b1edcec8ae2c4050f63b84ccf6d98723a8f212e9684713abf0e \ + --hash=sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074 \ + --hash=sha256:337639ba62a11acde6ef3aeb08c8ea755f8ef1fe5e513356c0f36a2b0d7568b0 \ + --hash=sha256:347ec774390c87326a2e4929d58d3f7e8763a104d5d35f4cd595a4c952366433 \ + --hash=sha256:356c8368cbc321050b169595683a2e1d63413b1e0e2868b330af9fc14c616d3f \ + --hash=sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034 \ + --hash=sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21 \ + --hash=sha256:3a3e26b6a8274211bddee2d0e4d0d42778f17a34510f49d2ec44b58abfc41736 \ + --hash=sha256:3aa166e99c4f2985407fb8714aebede877ecb5455cf321b606adca926d30d5a0 \ + --hash=sha256:3d2652072b2d774947ba5cf78a9e59644ac62ee572daf6dd2e1dfe905e15b2b7 \ + --hash=sha256:40375c2d05acec10323e45dfe2077ac44bc74659008614af5069034e2cfc781c \ + --hash=sha256:413a717a410d0c817ef5b786a059415550b3794e1d0c2abffd9efb93a3d9f7b4 \ + --hash=sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c \ + --hash=sha256:49776eab08766a08dfff7012f8b422dcd7e25e43b316eedf0477c24fcfa84b7c \ + --hash=sha256:4d44cf99ddebf875f9b68cc267aa684c99b7b44fe63ee1cac4ec163807290069 \ + --hash=sha256:4dedce55295becb61921e386b99d4f2706045306e7fa52249a33004c837379fb \ + --hash=sha256:4f8507560a9284e1370bb048ed4282012fbef4e8d109875b95e884d228552061 \ + --hash=sha256:4fdc8b93a41521988916eeaa271173fcca7fa0803d62f87675aac8dcec1c8e29 \ + --hash=sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3 \ + --hash=sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa \ + --hash=sha256:53feb344243bb9510a9dec7bf3cf1b64d88a98af5dc7872a5160465f8b198c8e \ + --hash=sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38 \ + --hash=sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f \ + --hash=sha256:5cb482e9e84c851f4e623fe4acc1ced89168cf1fe18f7089db4548c8f5bbb65b \ + --hash=sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8 \ + --hash=sha256:5ee239d575f80b08eca11f6e20f90c4c695de7825c67eefe6091fbf20dda648e \ + --hash=sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c \ + --hash=sha256:5f93c5fe914d75fbec9a49209b00da5f08e9e467d69da2b1510c81940cfd10be \ + --hash=sha256:657b40d6240c0a7b6a64b30f22d1e3aa631c7e846c621b0c0f6d1d75e2e15ea6 \ + --hash=sha256:6d30e1a4f138b8951063e9a394752a9179b51da288ffa507b1e659222f4c1793 \ + --hash=sha256:6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1 \ + --hash=sha256:701b2e04b560eeb4bddf7a25ab8ca476176e34fdbd9a0e18196f0d12d4685f0b \ + --hash=sha256:771cf63ae0b1b50dd22e5f3e3549fab5f3f4ff1635d352a9e1a97fe01c7b2e64 \ + --hash=sha256:79bdfa52f843137045b2d081cc05c120ba6665d29b7559c2c47690906f39279f \ + --hash=sha256:7ac031912d54f3d83ef3b3eb98dfabc1608802e2202263d25957eeed40b94761 \ + --hash=sha256:7b0fc826b16c55e561e5d2a0c5c77b051ba1d92808118c4e4b5390f5e0cf191d \ + --hash=sha256:7c6be839a5a8312626b32029a415644a0846b420bc8b52b95b28cd92da162168 \ + --hash=sha256:816ff0a6550ffc06c098ccd2e0698600f9aa7da192a79eaa6f9af504a35db869 \ + --hash=sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111 \ + --hash=sha256:837b396ca3d7b74091ca623f6cbd8351bd42d670a79c2683e79fb089f06a2de5 \ + --hash=sha256:850a08d167dde16db8702c274f320c7be9d7da6f6dff2b58b18f9e815bd94f5b \ + --hash=sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7 \ + --hash=sha256:892a881d5f68c2b9ea304b7a6c2c60d9343df578a311b0f86b94bc8f1ffe8129 \ + --hash=sha256:895395f8918627b04efb1ad2a4cf605387143300ba03304cd1dfa6d03f5e095e \ + --hash=sha256:8b10e3e8fd7ddc2bd915848a2768e44c15b22936f1cc54c462ad1164deb02655 \ + --hash=sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c \ + --hash=sha256:8feeac04b5794e513e710af2f9c87d49f31a6dc47967bb264a1fed61a8989bec \ + --hash=sha256:9432f3598db432cb51c5b37fdbf29a60fcccc79e30d37a05022776a6bc4ab689 \ + --hash=sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f \ + --hash=sha256:978e7b97d4824b5be09c69fb70507cbde3b0323fc147332ca40a94d9a6a0ebbf \ + --hash=sha256:97bf8de4d541598c94a59344eeb988a94c08ff76b5723c41f6567ec18c7892ea \ + --hash=sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9 \ + --hash=sha256:9b68938dd5b0c783d88ff8e2dcc69451b5eb936fe212d516b21b9d5567f6d464 \ + --hash=sha256:9c4b71f10dd532fb7a5cbc8f58707779e64f03a258c2bf8bfbaecfcd9970b519 \ + --hash=sha256:9f47b8a949e60f027f0aa0a6f6c7b7e9c55cbf4380d10b344e282fa4e7ab1e1b \ + --hash=sha256:a1dee1b804ff4d11c663636cf15d2ea47e9f79cd56c033fb1cbf08924842a48f \ + --hash=sha256:a2468d93d181667a7abd66e1b64bb9f76f361b0fef8faddf687456453576f5ee \ + --hash=sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a \ + --hash=sha256:a39ac25a9a2fa4072efdb429833c4a4c8009a51ff9eea3eeae131713cd27991e \ + --hash=sha256:a445486499897b88a7d6c310c88ed64dd37b1b59bfd7ae9107490bbb362f47d6 \ + --hash=sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2 \ + --hash=sha256:ab4b66edffb32d9e951efb3814bd104b8367a7501b81b955cacb5726d897389f \ + --hash=sha256:aca6c767f552b21b10f774aeac128e828eafb796adfa1b666a18bf6321453c3a \ + --hash=sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f \ + --hash=sha256:b10ec717381bdbfafef34607824db4c91de69ff085e4fca3b2af91b4fa17e68a \ + --hash=sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47 \ + --hash=sha256:b6acfb46a814762367fb7ba0828b0a17d441b92ce249a0e007474c9072662dda \ + --hash=sha256:b7ca9034437b6022f941f4857459562ee00a560b97e7cce8a0ec5a74fc6766e0 \ + --hash=sha256:b98134087d9de723658d17a42c7d0da8d6e2ef08015dee7dc93889047315f5e4 \ + --hash=sha256:b9fe6fb92520e3fd61f2e49000b6911b188824f089b75973ea06d6267f0b476d \ + --hash=sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3 \ + --hash=sha256:c14ad3bdc85ee7f318742c457ca3968a92126d144b15721c759033bfb06296c2 \ + --hash=sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355 \ + --hash=sha256:c3471e5c4a949c26ec00a77f01df59096aa9495877de76fd60a980f8ee6be461 \ + --hash=sha256:c583b927a8838dab890706a6fa7573fbb8b70e24000ef9f7238e2d6f6435a5ed \ + --hash=sha256:c76fe65e607be28c7fd4d56fc3c42b1583aa058ce3408b7ad0fd540171d31f9f \ + --hash=sha256:c7ea57fc63aa7da93a1bd2d644e6577befae10c52c4e36377635eea1056a74f5 \ + --hash=sha256:cd5214352ae68f3b5e9af7768bdc5253695ee069675db3480518420b3be881f2 \ + --hash=sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829 \ + --hash=sha256:cdc8b74ecc48c0cb1e9607a05ec4e9e88db60a19ffcc9a1d5f9088ede40c8dc0 \ + --hash=sha256:d0a24b40877af2de4950252be9d21eaf7fb07660f3c2cae1f56c6b599ada5266 \ + --hash=sha256:d22a945598fb91236b4dd793a6e42e4f3dd7740bb5aace5ebd7d4c08d13bb575 \ + --hash=sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290 \ + --hash=sha256:d625a186a65201c23a9e3b8ed9c47e90a026e03256608cc91851c6709096844f \ + --hash=sha256:d925f3d9afd05a8c0fb3a1031463a8d59ebe5e2afad297e29c78be19e13b4e62 \ + --hash=sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f \ + --hash=sha256:e652ab17569c94bff5475520f907b7148b8c24036a8ebbe5cf7cf7493d28579a \ + --hash=sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7 \ + --hash=sha256:e80675d75ae2cd14372cb65cad5400d9347a3d3f6c13000183f22dfd027283ed \ + --hash=sha256:e9c134bb666dd54b778b9fc0d2b50cbb7f979b9e3716f26a88c9ab3b6fc1dd0f \ + --hash=sha256:eb7d8d0e5886a89a55d2eef490e272fa965a9d57c6b29a5b5088a7997ec2cad1 \ + --hash=sha256:ecb42011e12ee19cafbc312887cbf3546959fe02fbad44f272d4be5baa997615 \ + --hash=sha256:ef3fbbf161dc9351a2fe0422e51b129f9e97e42385bd0320b309c15f7d287dd8 \ + --hash=sha256:efd62a42486f1bda5d24cb4f63d15a3c7768375fe83d36f9417b4ad7a2fb20b3 \ + --hash=sha256:f077d0b97ab11fa7dcc633fca53515f290bca8a8a633e966d5b6d1879d9ed01a \ + --hash=sha256:f332f0e72a5a0400141f830744e141bf9f97917878dbe968669e8a7fefea78ff \ + --hash=sha256:f7b0ec93a2893de856652154d73b7ba622f26fa97726487dcac373de5f4c6084 \ + --hash=sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0 \ + --hash=sha256:fc5d783bd4a2387e97b8a2d5ec781cfb92b3d893bf82370548e99db5915935d3 \ + --hash=sha256:fc8515076c11f3cfdf4fb142dcca0fe384b1230a3b5415458ac84f3e0903ec13 \ + --hash=sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9 + # via pydantic +pydantic-extra-types==2.11.1 \ + --hash=sha256:1722ea2bddae5628ace25f2aa685b69978ef533123e5638cfbddb999e0100ec1 \ + --hash=sha256:46792d2307383859e923d8fcefa82108b1a141f8a9c0198982b3832ab5ef1049 + # via mistral-common +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via rich +pyparsing==3.3.3 \ + --hash=sha256:928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36 \ + --hash=sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4 +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # huggingface-hub + # transformers +rdflib==7.6.0 \ + --hash=sha256:30c0a3ebf4c0e09215f066be7246794b6492e054e782d7ac2a34c9f70a15e0dd \ + --hash=sha256:6c831288d5e4a5a7ece85d0ccde9877d512a3d0f02d7c06455d00d6d0ea379df +referencing==0.37.0 \ + --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 \ + --hash=sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8 + # via + # jsonschema + # jsonschema-specifications +regex==2026.9.29 \ + --hash=sha256:01000ddf0e3ffef97f2413ceb514f6313040106b6d18a03ee00a4fe35c1eb1db \ + --hash=sha256:0166844493626c5015c6088ee15c9ca2fd060ca15b7641d1657da6a58432ae33 \ + --hash=sha256:044265d77d94f5e3cb2fd72c76723807c429cb8c533e9d4672d0334a6f14f588 \ + --hash=sha256:0476e5bcbe6e1ba3d1c4cc7bbb1c3ba78e3b979b5c8a88d0a6a8cdd4992b8c84 \ + --hash=sha256:066d0e3dbfdd739bce2bf8c2a41dd16f73e3d8adc2eb06dd803a36a307f56075 \ + --hash=sha256:0b65c72739f981377c9c22e0c5c3cd7f42da7bd8a3c9209330fac772c7d893ed \ + --hash=sha256:0c992c19cd45058a4b92f68f139c93db168b48fb1f322c9a7cd620806afb6b51 \ + --hash=sha256:0cc63b5e47c12a48d90c7e9d7de6a035dd14f62868aaedbb4e0ff8ba2b8bfe7b \ + --hash=sha256:0dd8af32e9f7b56b7f95cc1fd79b23054c3bdc172392ae560acc24d57b7ffe71 \ + --hash=sha256:0def9fb6abac55492d6d51cddb7225d07d6f279e774e0adc08569a54a5fc8d46 \ + --hash=sha256:0fd2c901cc307a745ad4bc87f20060d7a0825a3371d1e93488af22e7a387f78f \ + --hash=sha256:1043aedf5917caa861bcb25a9c11460049656bdf0017a90a309fa8f255467725 \ + --hash=sha256:121a76a0985db80ceae9e171c337f8c927868e37d01b54e3ce87bc87f9c6a208 \ + --hash=sha256:143533cc4b6fbc5b95aca0a5b8d541088d374831593def000ec89322c220221d \ + --hash=sha256:14e953ff3607c92d7675bf79c4d4509ef6782aa8c08509f179f9b3d6d0679e86 \ + --hash=sha256:18ae8eed4526e35bdb754d61562b90bf5c00a67fdcf3cc1380dd59597486631b \ + --hash=sha256:19959129885356df0e97556856f77eb2888380dac18bed075a7c05c5128c618d \ + --hash=sha256:1ba8c6a416569ce0d37e83e28a254a61dc99a419084dfb6476cea02d997f74fa \ + --hash=sha256:1c2a0026062abcc321a53db4a185ceba0b59a66b5d37b0808917a88b55a5257f \ + --hash=sha256:1d9fe8091b2e89d470df68a9331111ed008ae8aae6bf1e8e1fba4086a495c84e \ + --hash=sha256:2089fe39c406784d90101c726755ffa1497bb74638fd434300d2b88006186de8 \ + --hash=sha256:23ae6fdad9e63e54038f5ef78aba2933faca61e24d432786589e737bc5522ebb \ + --hash=sha256:26ec4ccce55aa533fbd603d08911b01101a8fcfec987845ac3ae2c7087b2bde3 \ + --hash=sha256:2f7f7aa47b229f2b39a2ae2596d2ad5625d77b5eb9856fac2dab3eb506cdd0a0 \ + --hash=sha256:31b003f9a070335e2a8233ee9b14a3ca8e6d792012ae011f741bf0aaf11744c5 \ + --hash=sha256:32ab11df9677ca80bcbb5fe4eb1da9109a5019239a054836efc6fa1c64e683cf \ + --hash=sha256:33026515aebc0e70d1c89978e53e8d695d35d9e472f8d5b34465ba3c74028650 \ + --hash=sha256:34b6925af9853bf461950e6508910f179fd6e9b1a7ec8548e069606b7e51a26b \ + --hash=sha256:352cf115a810b357caa35193ab656ecf5ef41056855e82f292c99e8514f8d954 \ + --hash=sha256:39ab5894d971f9ac68baa6eca5c50387db579cfcacf36ae8df3feceb1815e6d0 \ + --hash=sha256:3a21a9509d0ee88e7a70e1ad228cd2f0e0fd1e187458db132e8a8d18c97daf9d \ + --hash=sha256:3c5c2ef13797466aa64170cbb66ad98a32351dd4127694cea7199f80f213750d \ + --hash=sha256:3e778bfccd63075167709136afbc251c1f683758d5bf49c803c60ac3f894ce6b \ + --hash=sha256:3f1e6cb402a89457582cd696f982559217d13484a193202c394015297968c86d \ + --hash=sha256:42e82e578c904445d4c8a35b8f28052cf567593215fa5db06266fbc6f77aaa2e \ + --hash=sha256:4408b2b27a95ca8cc48b7411945753773353b5c93b307754781086c99d3a576f \ + --hash=sha256:446654b29bfaa30500d80947eda42cef1449dc8a87f4e3cf061cc8485d3a1f0b \ + --hash=sha256:45010bcfe66df41522d56c9b6114e87ecc597a08970ff6a2ced24415c141ae5f \ + --hash=sha256:49ee178ca31c94621294bf9b8b676a92a2e6bba8af0529591753719e57edb621 \ + --hash=sha256:4d7d93613b01b0199961330e49cfc52d479b3d5776c56c691db31130c0a07d91 \ + --hash=sha256:4fb41211d2333eb930a51e0546a65999761cf1f572a4da56ef9b8a62966c06f2 \ + --hash=sha256:4fe97894d1b306c919b4e50def1e6f6c522f4d03a7283811f4d108f1ce5d3ac2 \ + --hash=sha256:554bffadcbcb6d5f4e5fb10a61cc52084b9a63d1dab5f10bcd2c4343972e8e2c \ + --hash=sha256:59b49507f47479e299a9e1bc41b5cb83a7afda0540625f1dbae886615978acbf \ + --hash=sha256:5eeb8edc6110d9194a4d0d54610f64c37a31c605b5dbb7e407fc6ec7fa34a4a1 \ + --hash=sha256:612b709381c0355b70d89cdb51b7f670591ed5cbbc0e3b5337488019dc667b65 \ + --hash=sha256:61956f074ecd123f55adca68ee3eab46e6a07ad3f8e64e6db95dfacb444f55c4 \ + --hash=sha256:6398d5145689503412cc1748895242598d8846b8967b851133b20dc2ed1e21e8 \ + --hash=sha256:65b408d8fcb273e3499e7ef2ce796810da1becd208c7fb4373692a242d79d461 \ + --hash=sha256:686ac5350fceae63830bb98805fcb8039325bf4c06d9f6f048ff65229d5bffa5 \ + --hash=sha256:6a1a824fbed817e0a891103886b68f063b1e83cc51bc97192a90a60195a9291f \ + --hash=sha256:6abb75ab16bc3281714a5b99548a2225db70dba1f995f6d7f7419b76eb5a8fbe \ + --hash=sha256:6f7121a8914ed13fcfe2099f895341bfb789f004d4c5a0bdece8fa667da10849 \ + --hash=sha256:7020ed44df30b3aa492c00ee3b52d0548c1f30c2c6c5bb13ae897680900d3413 \ + --hash=sha256:720537c7ea6f80dc61913184edb0ce2497a306b39ef19f28505b322553d52bdb \ + --hash=sha256:724184b4aafed865e4f13ca313fdcb43024300c028ec67319cfa16847d84685e \ + --hash=sha256:7c03031610e3e6ed1768a2b7a8fc84637c1257b50c5eacaf094c6e17a84fc563 \ + --hash=sha256:80a5ea3b4fd9d6a5b9a44f7976a9acaaab35aa3c1f6b29e5bd857dfabaded223 \ + --hash=sha256:80c7cadd3fd2bfde5df8aa0787e315812cad0c313a753095d02f4c2b6c01677b \ + --hash=sha256:80ea96f5c1a30bf09007d48466521d9c294bebe197c708c3359096e3e3691632 \ + --hash=sha256:864e9b87ac33c3fb9fb4ad48166d4fdb579c351d5c77deb0d34bccb36a775cd9 \ + --hash=sha256:87fb80cbe3557e27e7b28b995c2b2eedf689b8886f941ab93e0e288f0976518a \ + --hash=sha256:8873c4a11c50b9989168881aeb3f08859f469d809941866aa1feefd8be5431f6 \ + --hash=sha256:888d60953908dcf761aa320c3e390ab8556efbdb551ace63921de90f6ae0848d \ + --hash=sha256:8b5fcc4771732191b2b7d1dd68d8f0353f47f8d90b6150f6dce58bf1112442cb \ + --hash=sha256:8f39588af4731c8923c26810eb3b33f76f17633985e40f59c3cd45a33805a895 \ + --hash=sha256:9173db3be74a35cb6731701094b98120f7ee4876a287882a59cdea1fa7da342f \ + --hash=sha256:92f05c9c42bde5785dc48770bc2194d9f7442544156f951e19cd31b096cec562 \ + --hash=sha256:951733b1bbdb71e377cec567b409f1a7881b47cfcad84121aa74cb575fa425ea \ + --hash=sha256:957bb708e8057ab1649ba566456429d691ec9b90d1c9ad1af1ba7ffbbeaf05f2 \ + --hash=sha256:9916fda742cd4eede63b286f58c06718324265d727ce0856eb1aac86d0d150d6 \ + --hash=sha256:9e1d3a4cb7993b708f0ada8d0c84590efd853f169e7147d2202c9da503180242 \ + --hash=sha256:9e4482589065c8ecd761cff522dcd85f2d39e62f551e37e025d1c7d54772def3 \ + --hash=sha256:a5300757f8a68f5b6cc33f57338d72a0e3589c5cc9ad5f8504ea06f028be582a \ + --hash=sha256:a540abfab208e1b7ef2df231c40ef3b6cbb30a0aad6204e9b6a81c10a6794628 \ + --hash=sha256:a5758353650079898dc1b2b0e95aa51fa23a30d020e06f62c430dd08ee56cdd8 \ + --hash=sha256:a64b85a4760337cfefdb27d42da6ed8b58e8cde3f2d57b6ef43e76ef6ea9ef47 \ + --hash=sha256:a655d34b2a6943af32401f3d94f72e9d731f6ad16285815550bf2b4ee69d420a \ + --hash=sha256:a714befaacbd10092ffe4cea0d3c5f008fb9efe9bc322c715bcdfdee414b9a3d \ + --hash=sha256:a760da040b47767b4b873adfb7c3b691e9ba2fc60f113f9d0b88f1a62f323e85 \ + --hash=sha256:addd736a0547d553283adaf4e05d7104e7f2c7b0b092e9b4d28756825f14531f \ + --hash=sha256:ae4613d7d9dda60fcba95f846cc6f808017f1843f392cf9daad14a6534493d71 \ + --hash=sha256:b11b589e00095ec69cf79841a76360f9b079e95b0368a25b5ebb951ab0c157ff \ + --hash=sha256:b3e445b66c80b4eb4234e855ce94d9adc183eedbd632816228d89930b91b2c5b \ + --hash=sha256:b7b893976e7fe42053da64f2aa27239c24252fd2ec6df471e1be197c0addc3b1 \ + --hash=sha256:b84f186a7f0536fe4ff9a9fa12d06d007b9b71d4b5352ddcc41f59ad6522a312 \ + --hash=sha256:b89efc38431793d28b7cd91227e2f952ad7c48df19132b17f43a5fec3c14143b \ + --hash=sha256:b97a38fb4c732b6832db6bf108963adbcd82ef1268ba2025dce390f45af75efa \ + --hash=sha256:b9d74e4eee9ddb64c2e92d5d61472c59c21684c059eb7b68767be9628e977859 \ + --hash=sha256:bb90e7177944b6684738c1fc36aabd2dd00d1de3be7dbe09f91e196f1bc0dc81 \ + --hash=sha256:bec37990e3d6121f29ecfb594bd8f1bf009e9f7926daba2e50e3b27d3892a783 \ + --hash=sha256:bf3c49863c23a1ad6da9c30351aed6cff8d5ddbeb63c5c8420ae54e98c7d0138 \ + --hash=sha256:bf48516e35cf848390ea68850aba53e7c333720d2945b4d2c25b69fc5171723f \ + --hash=sha256:bfc71e6d970419c1309b3640305298643e2a734cad3f7cfb6d2ddee4175ab53d \ + --hash=sha256:c0094897d7d01f184b2d7fe8c56c66d64efe01b31f4b7d34205b391387df1111 \ + --hash=sha256:c03c6eb6ece86dfdcbb34799efaa339b093132e1aceed491ba5e08fe06cdf699 \ + --hash=sha256:c1a9a6651197fbed6f0212591418b9def774fc3f8324f78d1bf0e6a63e5f8aa1 \ + --hash=sha256:c3589f40749acce747510bf5d589d54e376cb0930ea58b35effac97e5312b0c1 \ + --hash=sha256:c4e38dd8f39c43a91d2410ad2b85610701b0979342c3df1d69eaf8e838c757d8 \ + --hash=sha256:c6c8fabf1dafc1f1ddcbb67896d3f93efb092e8c4b6322d7389b944e76a484e5 \ + --hash=sha256:c90fcf7804ea0a54b896ce0f2b9565350220b8d4890fd0db461a476a4c687963 \ + --hash=sha256:c9b602fae1e00b7c035d661ce85575365719192a7b46784bd71cf64c68053aa0 \ + --hash=sha256:ccb64d887a9db1cd76dbc0f92051a1a478a2a67e7f56c62d915cb881d7734704 \ + --hash=sha256:d06fcdecc10fc7954d7c8f27a03c96055fe525274dc84a7b0dbdc3d6b9e03dab \ + --hash=sha256:d0c3082bf79bcd6a614d55916590ad4b8f93200e10b97f463ea5d9d07c9b5f23 \ + --hash=sha256:d49c18f1ea294cf4adde2e5ac256e98c82ea9d708462ce4bf799dffa7cfe8a2c \ + --hash=sha256:d60030baaa7bfbb02d650c126cdcddcb6e33dbff14d819434c8fa2fdcaeeeba5 \ + --hash=sha256:d7cab119d0df0b9413f106b4d7fc34f2872d3574ed3806fb48959c830b1537da \ + --hash=sha256:d9b77b25b4f395f92de6099ab08e8ae2bc7e51dfe157f22900902243a5cc90c7 \ + --hash=sha256:dabee8f4935e731fb46b2a3091bdda0d3d94b3bbfb907d2b4f12eefce4009619 \ + --hash=sha256:db5e82ba15c142425b8406690032df89e39cca4a2e8afbbb9a3d84edc2373ac3 \ + --hash=sha256:dc79d36d0618752265f0d575915bdc5c5130ecb9c9f6b3bcefeae32e4bdfafcf \ + --hash=sha256:ddfa987262763c3c22a8367d2a49c244b018a74c3a8e3ab1a864119ad45c5633 \ + --hash=sha256:e1172147d28d8fbcf8cb8d26c41506169f5ad8fe9ec969cb116835a19d4d8eca \ + --hash=sha256:e11edba5bc344a32b029a7af9d4b3173982dd79eeafa0b9dbd787364414b0509 \ + --hash=sha256:e2c89e9b762c57f59d5e99ee8b20202adb892e35f8d3485741340999ca55058e \ + --hash=sha256:e31f72490b7c12f7790e1e25c3afffd20503ee1bfb43461d7838b871ff244b19 \ + --hash=sha256:e8c65ef3862a8ad6e86492b6ed9327805dd66904c012bd3649dc67d822ed6c34 \ + --hash=sha256:ebb8912f565b8cdbbf27debfe00df04202c20e2f651b9e32767930c5eace3621 \ + --hash=sha256:ed511a0708e2297e1d6431e7fb217e3402791e491e02da800658ace4973df1bb \ + --hash=sha256:edf06545875f3efa31560d94121e95c7fd70d98b1dfedc0157097d79b13b52ea \ + --hash=sha256:f0fe9834e5aeccaf19a0d8feb296d66a24be1a7c9922002f842a682cd5abb787 \ + --hash=sha256:f1a0d5117230dd46b399a30a38afa44f79c99f3168988fdc4f425c3f928b39df \ + --hash=sha256:f37964e4a5e993d2fd45147741e9dff7f34a2d8c00ab94c4ea0514a4677f959e \ + --hash=sha256:f57dc6b8fef170f105d2cf5cdce254f47b137d7755086cf7050f47e16582abba \ + --hash=sha256:f93bc1c3486ef3747e07c9d7c1d0a147b8fbaab975f80e348aed6f71309dfaca \ + --hash=sha256:fb00027a09a8f9f08028b40dce4c933cf73e4833240ed356583fdc9cfa721566 \ + --hash=sha256:fb99cc9d45f48895d9d67f6a0b8a57f08d39c174d9f25ad97a313e0470267b1c \ + --hash=sha256:fdd88ed5e20b1bcdd234421e454962c971aa44b653bdb7f1ea9ef683e90fb649 \ + --hash=sha256:fe3fa1dd453ed5c7f5ea23a26218329790ed7197a99b90e94330e313959a7f52 + # via + # tiktoken + # transformers +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # mistral-common + # tiktoken +rfc3986==1.5.0 \ + --hash=sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835 \ + --hash=sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97 +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via typer +rpds-py==2026.6.3 \ + --hash=sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5 \ + --hash=sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680 \ + --hash=sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9 \ + --hash=sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538 \ + --hash=sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804 \ + --hash=sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf \ + --hash=sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4 \ + --hash=sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97 \ + --hash=sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6 \ + --hash=sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96 \ + --hash=sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a \ + --hash=sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187 \ + --hash=sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975 \ + --hash=sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f \ + --hash=sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703 \ + --hash=sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9 \ + --hash=sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127 \ + --hash=sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f \ + --hash=sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa \ + --hash=sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05 \ + --hash=sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171 \ + --hash=sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba \ + --hash=sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c \ + --hash=sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223 \ + --hash=sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4 \ + --hash=sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885 \ + --hash=sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698 \ + --hash=sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f \ + --hash=sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7 \ + --hash=sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed \ + --hash=sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f \ + --hash=sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf \ + --hash=sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e \ + --hash=sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f \ + --hash=sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24 \ + --hash=sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a \ + --hash=sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41 \ + --hash=sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc \ + --hash=sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d \ + --hash=sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146 \ + --hash=sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e \ + --hash=sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e \ + --hash=sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4 \ + --hash=sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12 \ + --hash=sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7 \ + --hash=sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261 \ + --hash=sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6 \ + --hash=sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5 \ + --hash=sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93 \ + --hash=sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7 \ + --hash=sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda \ + --hash=sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8 \ + --hash=sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342 \ + --hash=sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c \ + --hash=sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb \ + --hash=sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0 \ + --hash=sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77 \ + --hash=sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3 \ + --hash=sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885 \ + --hash=sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826 \ + --hash=sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617 \ + --hash=sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb \ + --hash=sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577 \ + --hash=sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80 \ + --hash=sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e \ + --hash=sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945 \ + --hash=sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90 \ + --hash=sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7 \ + --hash=sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0 \ + --hash=sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140 \ + --hash=sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822 \ + --hash=sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba \ + --hash=sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9 \ + --hash=sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4 \ + --hash=sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a \ + --hash=sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8 \ + --hash=sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf \ + --hash=sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4 \ + --hash=sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324 \ + --hash=sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53 \ + --hash=sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b \ + --hash=sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41 \ + --hash=sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9 \ + --hash=sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca \ + --hash=sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1 \ + --hash=sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d \ + --hash=sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690 \ + --hash=sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107 \ + --hash=sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2 \ + --hash=sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76 \ + --hash=sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d \ + --hash=sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af \ + --hash=sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6 \ + --hash=sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db \ + --hash=sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369 \ + --hash=sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd \ + --hash=sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911 \ + --hash=sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504 \ + --hash=sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a \ + --hash=sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9 \ + --hash=sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13 \ + --hash=sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc \ + --hash=sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278 \ + --hash=sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868 \ + --hash=sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2 \ + --hash=sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd \ + --hash=sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4 \ + --hash=sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6 \ + --hash=sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9 \ + --hash=sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00 \ + --hash=sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f \ + --hash=sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e \ + --hash=sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442 \ + --hash=sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da \ + --hash=sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90 \ + --hash=sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef + # via + # jsonschema + # referencing +safetensors==0.8.0 \ + --hash=sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358 \ + --hash=sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f \ + --hash=sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d \ + --hash=sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d \ + --hash=sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0 \ + --hash=sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc \ + --hash=sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235 \ + --hash=sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98 \ + --hash=sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4 \ + --hash=sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846 \ + --hash=sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca \ + --hash=sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0 \ + --hash=sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25 \ + --hash=sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452 \ + --hash=sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d \ + --hash=sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78 \ + --hash=sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774 + # via transformers +scipy==1.18.1 \ + --hash=sha256:011413b7426b75012840e35649e00fe0a2c3bae89fed433876e3a99251572efc \ + --hash=sha256:0ac49ea97594532dd44b7136094d35f5440fa06e6d9c6384a74c01764df388c5 \ + --hash=sha256:0e82073ecc7acc6436fac4b31674109c7e1d3e596789767eda01258a8c9e8123 \ + --hash=sha256:0fcb3c93519f27bb4f0c4b0f7802cdcaca7fcf93267b75edda2e9f4e8a55cbd7 \ + --hash=sha256:10ac20c69d880f77f375db44c22e3e6a644f9fefa291d4cd2fb9790a89fc99fd \ + --hash=sha256:11c423f1049c5755ad4409af52a9ada1cff96fe9b50795d4af3619f292901239 \ + --hash=sha256:179ce34a8d0fe273d8883ba59e17e052247d08973dfcb743ca52bb1cce2d60b0 \ + --hash=sha256:1bca3b943fc2567ea49cd02c99abde49da4d5178ec46f624bd8255cda8755beb \ + --hash=sha256:1d73131e358976663dd969e1fb4ed1404b815cd977eaaedc3b3a133ba2d81c35 \ + --hash=sha256:2a0b02f9fc46f8520330c23d45e6560db7e3a0d927232139427637f98943e11d \ + --hash=sha256:2d3ab0e8c69a17dd3559eab8cbb88f258e285c94d572c2719033f90f83290c89 \ + --hash=sha256:30f464bee641fa8e282577c7dce027308403213c6ca8270bba73285c91024bc5 \ + --hash=sha256:33a834464fdabc0f26a45508df31b3cc5d028e04dbf6c5ed398541418e0a12fe \ + --hash=sha256:3ab3523da44749156e1f68b464dc56af11ae4cbc5c739a49d05f32b982eca9f3 \ + --hash=sha256:3c085faa2cfa879c5141df483f836f4d691045a078224a670fa570fa01612d89 \ + --hash=sha256:457fd7a2a8edeb044ab6ffbc0aa03ff6cd18491356e5e0c834d76ce621b916d1 \ + --hash=sha256:49023963c193dacee096301452f223ee24d86ec5807f8df93c0f7221d119e305 \ + --hash=sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307 \ + --hash=sha256:559ed65f60c1af5a03f3912605a1b5114f522c7c32fb23c3376ae8f03219fe28 \ + --hash=sha256:5632e3ae3d09197c446310cd5187de63e28448ce22f0f67b2b93d97503c0c230 \ + --hash=sha256:5e4d44984abc0020154ea81b247adeddcc3ac5527b975ff798bd1ba0adc513c2 \ + --hash=sha256:75b00eb8fb802090aa903f4ea1c7f5a584779f967361e68b7e98e531cc2d7174 \ + --hash=sha256:78a0d7c918e74a232394117160e7e3db503377572a45bcef8826e4ab8a35feba \ + --hash=sha256:78c0665edead396b1abb4897c41a5c1d9bf090c8a637a4c20a61678e0a264e66 \ + --hash=sha256:7bbf207c4453ce1ad2e00b17313852b33310b83090c2311bdaf97f93c0380d12 \ + --hash=sha256:7f4b8bc363b6d65ee2152bec57568e3c52639bb34c46057b09857a307ed5e21d \ + --hash=sha256:82f201b4c878551d48558337aab270d3c6cca5507b8737c8d8a608d234cccde0 \ + --hash=sha256:83de5453a7799afc9048b4616bd085cef126e36412f0ea2f6370c36a2a3a51e7 \ + --hash=sha256:88f0e784020649f88ea48c9f5ddfa403bf9205820667c0914740b392035afb82 \ + --hash=sha256:8bcf3c1ba5d6456e2effd30fcbd3459b044d683fcdac79a2e6830f0bdf7de487 \ + --hash=sha256:911de823097db8b63f034299d12662db93344e6ffa0b881cbb57748974b70168 \ + --hash=sha256:92c14f5bdbfb6216315ce33e78080474082de8b3830122ba97809bfbe65f75c0 \ + --hash=sha256:95298364e251be3e60249facbeeca03631d3bb7584f85879516ec55ac717b81f \ + --hash=sha256:9554bcc6d715ee87a633a3cc8e7703c6628b100dd29cb8a2efc4c0533c7ff729 \ + --hash=sha256:9f2897bf7737392ad0d5213ea7b6add72a4edf5679b3153106aeb88b6507b3b9 \ + --hash=sha256:a1d33a7836f7ddc1993427966a0823468ec41bcbdb1a9f9942d1d7e57f803ba3 \ + --hash=sha256:ac0333bdf38309aa3dcbe7e3fa7ea29e7a2c37c6ea306a757b700ded8e4596ad \ + --hash=sha256:bff0b729edd992766136b34e39cc76bc2fad905aa58897ee72a9cd000a6d8443 \ + --hash=sha256:c24acac1e18912761c4700239bbc1fd32f615af690f1584d49b35859be51324d \ + --hash=sha256:c35d74ce0e193ff740c2f2be2ac913ddc232fe6c1ff40b26cfecb9c670c63314 \ + --hash=sha256:c825cef2f49e46753726a7181a8e199804a912b29519ada542c6ebc654951899 \ + --hash=sha256:c9d18a33309122074ea483dd92dd444189166b8b2ec429fe9ed5ac73c7a0aa23 \ + --hash=sha256:cbf38d043c1aa4ab306e1ada6ab6eddacc3322a20b7af1b30bc93254b366fe09 \ + --hash=sha256:cd479fc04dd9401e3b4f49e76518768ef99c4f517a98c284eb091fd725719adf \ + --hash=sha256:ceb30a00ce7c92d459819443d29ca486d882b83fb6738bdcbb2a1cce94ac5daa \ + --hash=sha256:cfbf154f2ba187f2ed6cce2639efff7d105f1140573642c0161615b6d91d6a87 \ + --hash=sha256:d2924a03db38dc2e848bca2fe9f077dafb891480b91a00a0963a8cf86dfc31c1 \ + --hash=sha256:d416b16cccfd70fbf62400e84d0bb2f4e6af519a45557f1692c749b37f14b315 \ + --hash=sha256:d65d448389b8436493abcf629cc94ad0cf32aecaf06e1acca1de53cc795f2f12 \ + --hash=sha256:d84a09d0dad90ba6525d8ac1c2334b33e64bf3ccfe9e841f02feb867a22681e4 \ + --hash=sha256:ddef79fb382df40104a19bb7151b3b23e57c1778fcf857c71ceecd9bd264513f \ + --hash=sha256:e3b417bf8c2c7c16e8f58ad91db17783ec911ac16e7b50eb6eab6e809b4f5b07 \ + --hash=sha256:e402cf31eb68f453dbb2d36fc6d722b33f24a55d68b2ae1d92fa6305ca71c298 \ + --hash=sha256:e6fb6a55cc0ba97b59a1f288fb86dc6fce8bdfc0fffcbfd015e3a954bf2a2d93 \ + --hash=sha256:e708533e8b2ae2497d65346538a7dcc92814410b25b81432eac66de0f2af8265 \ + --hash=sha256:ea324d9dd34c38bfb9bec8ca4d1b407db97dbb74029f566b8e322b1b6fe56fe6 \ + --hash=sha256:eb0dfcf4e28a99c12c999744a2ff67c9b06200e20401c7c88186e33552a46331 \ + --hash=sha256:eda632a7981f69730d6281f451db9c1c370993a2c0d7ddb43e2a809a2862b83a \ + --hash=sha256:f29633129f9fa7e88a3f0fca835de2d030bfc9643f7799e1a0c46cee24d38fc7 \ + --hash=sha256:f55fa87b6c612ecd6b058f167c53231b1d14e412efe361d3d6e38b3631c73218 \ + --hash=sha256:fdaf5ea890a6183d0565f51a61799d67081bd5b1cf03c5f4b3fd3732108625c9 + # via mlx-audio +segments==2.4.0 \ + --hash=sha256:4021dc67f201cc03c864c74c618bdb163b1af629da3040babbaa37d8813f3db0 \ + --hash=sha256:bba71f5520ddd54c8aa2f4d765a60618c6862162d6e7356a4a097f2223166f5b +sentencepiece==0.2.2 \ + --hash=sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497 \ + --hash=sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e \ + --hash=sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0 \ + --hash=sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719 \ + --hash=sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107 \ + --hash=sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3 \ + --hash=sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b \ + --hash=sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a \ + --hash=sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c \ + --hash=sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838 \ + --hash=sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5 \ + --hash=sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6 \ + --hash=sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d \ + --hash=sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf \ + --hash=sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25 \ + --hash=sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d \ + --hash=sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e \ + --hash=sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b \ + --hash=sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd \ + --hash=sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936 \ + --hash=sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85 \ + --hash=sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b \ + --hash=sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708 \ + --hash=sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb \ + --hash=sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9 \ + --hash=sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0 \ + --hash=sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790 \ + --hash=sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e \ + --hash=sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78 \ + --hash=sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99 \ + --hash=sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9 \ + --hash=sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d \ + --hash=sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8 \ + --hash=sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906 \ + --hash=sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383 \ + --hash=sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b \ + --hash=sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53 \ + --hash=sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b \ + --hash=sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab \ + --hash=sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0 \ + --hash=sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914 \ + --hash=sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91 \ + --hash=sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e \ + --hash=sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a \ + --hash=sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032 \ + --hash=sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711 \ + --hash=sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da \ + --hash=sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c \ + --hash=sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a \ + --hash=sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d \ + --hash=sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151 \ + --hash=sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563 \ + --hash=sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811 \ + --hash=sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b \ + --hash=sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820 \ + --hash=sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090 \ + --hash=sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c + # via mlx-audio +setuptools==84.2.1 \ + --hash=sha256:3ff8b3b5492a0aa66960f4419365a86b84e013070d4313e4d4a6894f5d64d6bc \ + --hash=sha256:ca140b45d5325a9478f880e351484f62fef15f336462e449ca0c03c873d589c2 +shellingham==1.5.4 \ + --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ + --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de + # via typer +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 +smart-open==8.0.4 \ + --hash=sha256:8a3a67679942fa40d1a93ed3aea37619b4b4ec5458f1d5a9228dfc301db11b4c \ + --hash=sha256:8cff8321f0b0ae4bd9cb0cd0c507c135ad6aa980171248fca520b645fd6ff717 +sounddevice==0.5.6 \ + --hash=sha256:7f4162f514f007b0bf25a3ccfed3f1705bc2ec311888a90232729eec4f57a4f4 \ + --hash=sha256:8ec9fbfde2e32f020b167e348f3ab3bac6625a5f15af524d790108ac7147a410 \ + --hash=sha256:b36b807eb02abd257198bf84b2af05e4fea199a9d2f0019014169c7136d45e9c \ + --hash=sha256:c8ae19173e5f27f8c12d4b5eee2dbfe542cee125d591e663e0fb4dfb75246d45 \ + --hash=sha256:de099612311ad81e55d31ccbd83f43ea6bf4d87b48f9b6ea55a1fbcde0eee4e0 \ + --hash=sha256:e3aef00ad8b1d1740eb66d9a7671eab88a4d2b8fa4ab33498d742e63b65c309c + # via mlx-audio +soundfile==0.14.0 \ + --hash=sha256:0a6ae43c50c71b4e020cc55382925cb89451c1ed1a0c3d0f5d802da269226849 \ + --hash=sha256:19be05428da76ed61a4cad29b8e4bcf43a3e5c100089d2ec81dc961eed1b0dd4 \ + --hash=sha256:1e38bac1853412871318e82a1ba69a8be677619b56025bbfcccdb41b6cafe82d \ + --hash=sha256:299491d3499460fb1b74bb4bd78b57ffc2d243a5fafa7b6ec1b264875c78453e \ + --hash=sha256:8ba81ae3a89fd5ab3bef8a8eb481fbbe794e806309675a89b4df48b8d31908a8 \ + --hash=sha256:ba1c1a2d618bca5c406647c83b89f07cc8810fa506a50622a6993ba130c1de11 \ + --hash=sha256:d828d35a059626da52f1415b5faee610aeab393319cb3fc4a9aef47b619fc14c \ + --hash=sha256:e090704718e124e7c844695236f1fce8d18a5e761eaf7c82dfcd124620805f98 \ + --hash=sha256:e85724a90bc99a6e8062c0b4ddf725f53b2a3b70afd4da875e9d2cfc4e92f377 + # via mistral-common +soxr==1.1.0 \ + --hash=sha256:1577865e993f98ffb261257c3060fa76ec3db44ed3f181b16464268000424464 \ + --hash=sha256:26925618945f1a44dfbd783cc572874f0685e9ecdf46b96f4000f6b8c9c8b825 \ + --hash=sha256:318925f7281df61dfa7f17fe343952eb10cefd3954f2423a733fabe3a517bab2 \ + --hash=sha256:33525740fb7dbed8b09970bf0cd4219b365538845053987b11cc235b20562e09 \ + --hash=sha256:34cc92208c3c412c046813e69da639c04a792c6a41fbfd7d909d359cd3e97a2d \ + --hash=sha256:3b033078e86f3c4a658e5697fac8995764fad9e799563616b630136b613167f1 \ + --hash=sha256:3da87e3ffa3e41823d873b051c7ecb2acebd8d1b6b46b752f5facf10a0d84ab9 \ + --hash=sha256:474aabb9283f177e899747510d60661730538052fca0ed93a943d4686d6655b1 \ + --hash=sha256:52c9ca84e3dc656d83acc424574770e20ea8e0704dc3842d4e27b0fe9d3ba449 \ + --hash=sha256:588c7de1abafe59e66face9a074514658ac0398c85a774cdbb8efac131192692 \ + --hash=sha256:6ae2a174bffea94e8ead857dad85999d3f49f091774dbad5b046c0417d7092f4 \ + --hash=sha256:868a24d864c25024f60ca964f851a759f2ada5352608fc194d927b7facc2e28b \ + --hash=sha256:8e11e26f1718b5c2e5b96f2f71b9f00e31d247b065289661e3a6996c758669d9 \ + --hash=sha256:9443e5eb82152d8952422b7285692192cc7dcffa5218bb511b096203018bc273 \ + --hash=sha256:9564d82f7fa6bf548e5f18bb86235dff20eea8bd30727b64d49783c95c34fb8d \ + --hash=sha256:9f228ae21c78fa9359ca98d8a5e8e91f30639e438e574133dace62c5b5309e44 \ + --hash=sha256:a941f5aaa0b8abced24318105c1ea22576afcc1138c19f625716ce4e2f76ad64 \ + --hash=sha256:ae30c48ac795378cf23ba3c7c640b8ff794af714ac388b9fd6b31a40b39e6e86 \ + --hash=sha256:b2e94c713b7d96fb92841947b785bcee6606124bc852273fab70454b51bfe270 \ + --hash=sha256:bd30f7201eac896ebf5db7b09156e6f1a1b82601900d29d9c8449bdad8365b11 \ + --hash=sha256:bf98c0d7b7d5ef5bf072fee8d3020e8b664f2d195933ea7bc5089267c2e22a06 \ + --hash=sha256:d6a7ad82b8d5f3fcc04b1d2ca055562b96af571e1d4fa7c6c61d0fb509ac43b4 \ + --hash=sha256:e0e09fa633ce2e67df08b298afced4d184f6e753fc330f241022250f1d0d61da \ + --hash=sha256:e17d4ef9b0185214b2c0935605ae63f827ea423bc74964be44763d68d2b6c21e \ + --hash=sha256:f4977323ef9c3aa3c2a26ff5fe0191c84b8fd759daf7afb1f25a91a55ad8b730 \ + --hash=sha256:feebcba99ac99adb8009d46c8f4c1956b8c167576b0ae8a6fb47502e9a6f78e7 + # via mistral-common +spacy==3.8.16 \ + --hash=sha256:024ce6408ea00c7f8c6387a6de65bb67aad40c51c9d63705303c5cb9a8ef51b0 \ + --hash=sha256:04fd0206c9f33a0542a40049211528742b5492ef5f971d06b151b9cc49b9237b \ + --hash=sha256:111d817b32755d869e5ed6cc258b55c50c6687f47b78f6ebb2c14b1ce5ee707c \ + --hash=sha256:15908539b375bd8e3c627a076dac793b8fd790c5945f3a696837dd70776a4fc0 \ + --hash=sha256:2810fd2ce41f6a8dde62642dad0d11fd9db7cb6a1cd40b1c8b70a01586e6ff9f \ + --hash=sha256:32fe82bfbe6711a4687e427c8eec44af7cd5e09322a252b35bc8166aa84b9025 \ + --hash=sha256:39304dd9800065c09fa440983ac75cf444469b159959171c0e0d761b3e854d62 \ + --hash=sha256:397a80c4d09a6237eebaaee00a2e5f8732ff4cc165f94679b899f30179d38ea8 \ + --hash=sha256:44a641085abbe3a09ea56a89f2e50b5f51aea6cf69213b70305fb48e341b883b \ + --hash=sha256:49fe8d6a6cf343777caf59a3d4ba76f81cee0dfc1abe78be571cdc2db0d77672 \ + --hash=sha256:4e91c4f9a320d964a27ffae1b62e37bb6e8c391aa837890a082d82bd9a89ed43 \ + --hash=sha256:5991c334e71c23b798c25e0d403295dde4d2d1fb58c2e075450b964db03c05ea \ + --hash=sha256:5e32a51b115674d3f42c6cde696c583dc1594f5bff6b430de4aba4c753d47f93 \ + --hash=sha256:6a1523ce0a1358936fa1abdc3a43f2ec558d4166ac5c2f246b33fd88e3f238aa \ + --hash=sha256:6c51eac85344784ca7b184f0c3f7da0fca47c354d63e05e733d90cae35a2ecc4 \ + --hash=sha256:770cc0581fc06c0723cb1488dc1d1da0570801168da786674626f342d903ed37 \ + --hash=sha256:81fe468596678c7bf717650b12352c8e4d174c72d050dcc01c8ee3c6c781ccad \ + --hash=sha256:86227a0a0d3dfee3f3dcc15f73c1387b586d77b075348afc250ffafea88ffcec \ + --hash=sha256:8681eb07f6b0e48fb3ba4a5f7f2582c0fc6df991d240b431b628131029c4ade3 \ + --hash=sha256:8a8a2bf3eb3486a0992176b77ac1d38ca9c669623941fdb8d3dddacb44dfd28e \ + --hash=sha256:8d0f63c3124d0a34a37e9b519d004cee1269744be07f91f843902e6c9f3e557a \ + --hash=sha256:97bb04bd81a3690c45dfd62d4bd19584aa544bb955c8b497fb488256d10ac54c \ + --hash=sha256:9e89bebe168ec8714b21f0225950f0525d0ef87109cfcb11e5d18ebb1f4e658a \ + --hash=sha256:a237491463e351755f0167546f6a821d42971c5275a219a62d468f19f654138b \ + --hash=sha256:a2c46c35467d963a62dc0407c99d3a18562d4c85ee887a57e4a07dde020f1a38 \ + --hash=sha256:a3d19da23637cc396b42d22fc33852680f675d8ddcb847d3e2a0d094712d1794 \ + --hash=sha256:b022ebde7465334c0631f74e0cd21dff257ee5f80ca68918c56fd64681b49463 \ + --hash=sha256:b741266d901222dde979a802d5e9f3cf3d9bf77a15be3137b387f62905a74d57 \ + --hash=sha256:bc4e59799dcb0eb4823e5946515d3ca0d503ff78ef2502172d4b59ee0fc567ec \ + --hash=sha256:cb07d4b8255be6b6ff3a34ada93173176d0937d2023b35302a34eed3364681c0 \ + --hash=sha256:cc5a850ba2ce371ac13893ef4153a7f3cf0d7fee8ca4ddea21fac2d2628d7ea0 \ + --hash=sha256:cc7e449aec9a313bc037ef5ea45fb0ac99135d92dace8421d68414d16be39543 \ + --hash=sha256:ccd74917536fa82896f31c66db5f230301ec2662b7ad3c0d8c3eb790b0fc6121 \ + --hash=sha256:ce120d4077050352f344b987354be3e3fddb207436b537cf91a891837657ce2c \ + --hash=sha256:dc17227717aa254b63c90161d8de4ec672ca5bd8e5c92effba2a8510498ee355 \ + --hash=sha256:e045765035e9760f38637101f41a7c89d3b69659dcc70e716bb4011a95969ac9 \ + --hash=sha256:e67052bebdeba53847d3d09058f814708d587c4fb2216c65239964a341da2280 \ + --hash=sha256:f08555e5204da7d9c3f8440d9d88dc3011edcfc1fd21689069091da1973c883b \ + --hash=sha256:f42f257404e749d9048d3b3b97004692210057d38e03c2f156817258bf6daf2b \ + --hash=sha256:f765cb6cbef82b5d98c46936a1385e87fb05919433fbc6b953e3c093ac30f8ef +spacy-legacy==3.0.12 \ + --hash=sha256:476e3bd0d05f8c339ed60f40986c07387c0a71479245d6d0f4298dbd52cda55f \ + --hash=sha256:b37d6e0c9b6e1d7ca1cf5bc7152ab64a4c4671f59c85adaf7a3fcb870357a774 +spacy-loggers==1.0.5 \ + --hash=sha256:196284c9c446cc0cdb944005384270d775fdeaf4f494d8e269466cfa497ef645 \ + --hash=sha256:d60b0bdbf915a60e516cc2e653baeff946f0cfc461b452d11a4d5458c6fe5f24 +srsly==2.5.4 \ + --hash=sha256:0d45068f0aa30ac3902205b0d8bfdde876d5203c4920640c9dc815363b30faa6 \ + --hash=sha256:0f1bec0f3a859b55997c92cbc19f3500c2ceea407ceb7690959193a13ec7c315 \ + --hash=sha256:168a1b9b5d275276cdf9895fd7527dadc29c1d81d0a1fde7adb464e089051cec \ + --hash=sha256:2071242637b30520b543cbcaa217d2438a1236b2e702aaa7502b322449decf31 \ + --hash=sha256:236463ba0e1d17087c3f42189846a9509d24f64b4f450e1fd77af8ae0488a873 \ + --hash=sha256:23fb905191d81066bd9b91639cf272e3f55170bc40124af6aae98f4cb2f14e54 \ + --hash=sha256:2fa56076c0214805e11933f83aca7bc982f3abc93d11ac04974efd8693eb5098 \ + --hash=sha256:3ac6a5d1c132408e813fa3adee66cc2a3c4d217714926999f66a1b3e60a968ce \ + --hash=sha256:461e8d0bfc448960455afa53767434af23bace5b640f17ff0631e690597309dd \ + --hash=sha256:494beb477c35ffab46d63e6aee7771c88f23adaea731e4b52c0212a88fbfbaca \ + --hash=sha256:4cfff5ef7c6397d3d583b16ce0a75438794501ea3c0fad04d210b4916989edec \ + --hash=sha256:550defe364028366725bc6bd3cebf87aee5dc4eebe6c5dccb22708599b657794 \ + --hash=sha256:5bd3bb57db07e59a6366ffcf1cfca75a82d058ed4e08021408f29a7d8a0aa781 \ + --hash=sha256:60fd24af0feacb375dbebbfbaf6bd3c56d744ce44d4aa0d2e0474ed800a34d00 \ + --hash=sha256:67a798b18ead89a95974618a8f001fd7ebeb2455c91894f170f6edf2aa49cf6a \ + --hash=sha256:686b2df0c024ac5b3de28fc4c37c89ae5f4c0500c279869d63f250ff51eb735d \ + --hash=sha256:69d89a1e37db57bf031163315ab8e08d37f869ac7bb7ee77e463d7d9c0a00cf2 \ + --hash=sha256:70257eba7b3970155910c740f85df4c64f01c9ace4837c1cd223684ab23fea8e \ + --hash=sha256:71f0cffce9a48ea285c943239270f4ba386eec15ff857bb2dcaa9c758f5d8f6b \ + --hash=sha256:73674813f3536b401471f2012ef104258ef9723eca1e4ac2de5e7a5014051a7e \ + --hash=sha256:7589a35dd6bf1d177803c4ed454098f2f690b3704b80cca337a50be95ba59b3a \ + --hash=sha256:75c3b08bdab87b12872bd58159c4ca36d48d584968edfb9da4345b2c3643b47e \ + --hash=sha256:78266b8d138f28878a10a2604c464580532865e4ee446655bb13ae62f3a28fcf \ + --hash=sha256:7829bd3597cac2692bff0f9f3e7aa50e562c5b83ad35c7f547f601aa832bb1df \ + --hash=sha256:7a101c03c1b1a9fa26c81114192cf44a8fd905a6094bebceeb3be471ffa195c8 \ + --hash=sha256:7a68ac110530b4150bf11e4310aae48e083de511501fe7824c97c99f4cffda10 \ + --hash=sha256:7d4a04ba664ef387c2b9efaa53014f2215a2da6210cb215211bde67fac33233a \ + --hash=sha256:7de0faa1189975f51a62740854cb0a80d721abf3f03fe0c80c53ffd79c227ca0 \ + --hash=sha256:8208098727517e6d72f0a60e184a8c1c1e76c6287a0735f5d6a8ccb5da76e556 \ + --hash=sha256:8291d7b7de129b204f9462deb23cf72286790e9fd866b138610be4417470f03a \ + --hash=sha256:872249c20013578f940f0cd96ea697210e8113b02c92832c42250a38fc12b4a5 \ + --hash=sha256:8a2b41f2995976cb27102c87cdd84d308a083f901c150ae729e00f6f7741e138 \ + --hash=sha256:8e7b657dec809a5cf658c1a54e1d859384e958eb1a9bf2fe80e9faee3fd966f5 \ + --hash=sha256:90e1eff3d87d40a703f9be5720b3c01055c8529c3e6ef345ad9085c1a60c0d81 \ + --hash=sha256:91924491ea5a8a0b67b38f806d3e5766f19ca75d5c6364be6fd5349a5ec00323 \ + --hash=sha256:9a54d7e913f6aadbd46ad25504d083f4e51a8685815fe144661d350a65486494 \ + --hash=sha256:9ed763bfc0d634437aec4d779f2708b5022786dfd6340059b8bcab7554d016cb \ + --hash=sha256:a287627f6d35f83b8e8df99b4c7f7068b3cf386d4c28fc4b257b226512568f91 \ + --hash=sha256:a6904132d2f40e478430636e6a5d9deb90342a63d95fe3d1c560aa92bd88d270 \ + --hash=sha256:abf01c861a724da1e08f0268cac718e26cfd8e251a5d210fb2da05cba9677cce \ + --hash=sha256:aef76c62564593404770760bef01ffb22b8b2781a7dc2a4cc8a70ebf36c438b6 \ + --hash=sha256:b2768afa6fef019688aec70bd1cbe33e0d93dcf417feb6989013952627b7a477 \ + --hash=sha256:bb463e076e5efe61784c439493dc012e27fd05b4ddcf78f3353345c5d7e462c3 \ + --hash=sha256:be07e1cc7628834605c1e16f6af41730e37830a84fd1dd01e011f07faa8aeea3 \ + --hash=sha256:c6849bdda42fdb9028328c63998f710da720a2c04c62d1394e9b408c7a3de34c \ + --hash=sha256:c8cb46d773244c2d2415068d5333882f9fc910c9abee2c6d251c05813379e23d \ + --hash=sha256:d42e867817004670c7c234b72dd315298682abd1e16095e96673f8505021120d \ + --hash=sha256:d8d5563030a2bb9823f0ba54b35b8192630137d79d8c9a466fb1fbf224d540bd \ + --hash=sha256:dac865eaf3a8097e2bb59fa90705f89d6a7b7b6a6de1f66ac4e9d742c8425cf1 \ + --hash=sha256:dc2a84d4e3d652c0c027c1e8791f7047d0a414825f868e0d644ea382200dc566 \ + --hash=sha256:e37ff356439457f4160c5bf4c37d13cd3ae24022928eacb097d54140b1c1f6a4 \ + --hash=sha256:ec7e25f11a9763880f06df5d9f62250935d1e576b011527c1e5eea5312fd6fbe \ + --hash=sha256:ecbb13d9d12c9028d529588c323c777af00ae5af79ca8c49caf3429cf084ab44 \ + --hash=sha256:f12d104ab2e75ff51056e0eac3d18ea67de13b2e52dc91d1ef58a4be7b4b357c \ + --hash=sha256:f3f59d68903fc23e34574eb978c88aba1a25ede9f5833582b94df9432bc137f4 \ + --hash=sha256:f565e0401e6b316bd45c7ad91d047397c4f72ed9c490eacac3cf16a5990b61c3 \ + --hash=sha256:f7b67b8500fec7b53f33fd34094ae45864a17d5b357c03a021b38ec4f4877556 +termcolor==3.3.0 \ + --hash=sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5 \ + --hash=sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5 +thinc==8.3.13 \ + --hash=sha256:0355c37e40d1a9fc2a1b8e9c2e294d8586f6baa97bcac6b9002f2dddb4b82ae9 \ + --hash=sha256:0a0fa13dcfe4b319c3a396432c1dbff30d3de37dbbdee559e76600ee2b9486df \ + --hash=sha256:11754fada9ad5ba2e02d5f3f234f940e24015b82333db58372f4a6aedad9b43f \ + --hash=sha256:303477eb51b9b39c94a7fc7967ee8a039eca1ca37d95dcce1234c83b95b4ee9f \ + --hash=sha256:3710d318b4e5460cf366a6f7b5ddbefb5d39dbd4cfa408222750fdc6c27c4411 \ + --hash=sha256:3dac18a0fb0a42f711c2ce9c02cbb090385aecae92089aa17b9dfd808a542013 \ + --hash=sha256:433e3826e018da489f1a8068e6de677f6eff3cc93991a599d90f12cd1bc26cdc \ + --hash=sha256:4565102638038a01a2193c7f5d41ccbd6233fbdcb1f1b184322a06add4f51f18 \ + --hash=sha256:4b5ec9ff313819e7d8667794a3559463fa89ff45aaa73e3fd8d6273b1e0d7a7f \ + --hash=sha256:5593e6300cb1ebe0c0e546e9c9fb49e7c2627a0aa688795cd4f995a8b820d2ec \ + --hash=sha256:565300b7e13de799e5abff00d445f537e9256cf7da4dcb0d0f005fc16748a29e \ + --hash=sha256:5a08c87143a6d20177652dca1ec0dc815d88216d8fc62594a57e8bc45bf5ed49 \ + --hash=sha256:5c9a48f2bc1e04f138240ed5f9b815a9141a5de26accd0f08fa0137fcefed258 \ + --hash=sha256:68e658549fc1eb3ff92aed5147fcbb9c15d6e9cc0e623b4d0998d16522ffb4f9 \ + --hash=sha256:723949cab11d1925c15447928513a718276316cec6e0de28337cca0a62be0521 \ + --hash=sha256:77a41f66285321d20aaedaea1e87d7cd48dca6d2427bed1867ec7cba7109fc8d \ + --hash=sha256:79a29a44d76bd02f5ac0624268c6e42b3576ae472c791a8ae9c2d813ae789b59 \ + --hash=sha256:7a99d0e242d1ccd23f9ae6bea7cd502f8626efa65c156b91d84581d0356696c3 \ + --hash=sha256:7badb0be4825535e6362c19e8a41872b65409e9da46d3453a391b843a0720865 \ + --hash=sha256:81337dfbee37f58f36c0c70f9a819dce1b32cdc13d959181e10de079621f6ac6 \ + --hash=sha256:84fb50fe572a1860165f2e7a640c7cb70d43d6962366e69f643fa9a27e4a2127 \ + --hash=sha256:859fbd9d9b16af5278da23589b4afbe2ab6b0dd615df4d3229b7c4e67cd3107e \ + --hash=sha256:8ad40307f20e83f77af28ff5c6be0b86af7a8b251d1231c545508d2763157d8f \ + --hash=sha256:a518d5c761a0f2341e530e867de133dc3ed814558365b2a68ec53b89c482a43f \ + --hash=sha256:a61a31fd0ce3c2771cf4901ba6df70e774ffe32febf1024c5b43d63575cd58fe \ + --hash=sha256:ba8119daf84a12259ae4d251d36426417bafa0b34108890b4b7e2b50966bd990 \ + --hash=sha256:c17cef1900a1aba7e1487493d16b8aa0a8633116f1b2a51c6649a4000697f17b \ + --hash=sha256:c2811dfd8d46d8b5d3b39051b23e64006b2994a5143b1978b436938018792af8 \ + --hash=sha256:c6a049703a6011c8fe26ee41af7e70272145594140d82f79bb23de619c6a6525 \ + --hash=sha256:cd8a2b714c061969eee65802965167a6ada1fe708d82fe176d98dcb95ebe182a \ + --hash=sha256:d7a9654f9ca362a4be7f5e590fdfee26e2e2084da9fd3306032ec037e99f2f8e \ + --hash=sha256:e08b1577a56e7315770af280aabd8fa5f2a1fb6afd1c50a4183c06e907faf558 \ + --hash=sha256:e1f8d13bf92ee10595c40692fd4cf8e7bbe73bd9f260107e975fd5dbee1af42b \ + --hash=sha256:e676edd21a747afbe3e6b9f3fca8b962e36d146ded03b070cb0c28e2dfbe9499 \ + --hash=sha256:e7f046d8914055cad51e83ff0da1a892acb73cd58556d7c1a5d4015a3766a899 \ + --hash=sha256:e9c7c5c104737b414c8c4ec578e67d78b6c859afe25cbc0684402e721415bd7f \ + --hash=sha256:ed1dc709ac4f2f03b710457889e4e02f05de51bc8456980c241d0b28798bc7cb \ + --hash=sha256:f4f26d1eec9b2a6a8f2e0298a5515d13eb06d70730d0d9e1040bb329e12bf3fb \ + --hash=sha256:f697174d3fb474966ce50b430bbafa101a6d2f7ffb559dac4b5c59389ef72d22 \ + --hash=sha256:fbc0ee16edd260c6a4a9e365ff36d0a682c9e7ca6d7b985682659ef2e3e73826 +tiktoken==0.14.0 \ + --hash=sha256:087538c080e5ff421abd3a0785ed63c5111d06af98e6cd0d374dbe5969147ca3 \ + --hash=sha256:10f31e63e40313f2e518d87f7086cfa44e45f64cc14d8ae14103b41220c30a14 \ + --hash=sha256:11d8211b290855d2721334ff17dd9b3a17bfb26872be01f25d73612ef7ece890 \ + --hash=sha256:144a3fc369f92b7d548995217c5d6e84038d3572157a0f6f34080d65291d0f78 \ + --hash=sha256:149d97453c4c98c04b081d64a85e635921269b532710d6faf81e9e82b790e7d3 \ + --hash=sha256:14b47e3674f2624803a8acc8fb367b7e24fc53055f9df3296482fe9a3a34a232 \ + --hash=sha256:151d37a150c8f3dfc5f4345597b10e101876bd1bd13494e0185af6b508758d2e \ + --hash=sha256:18a1b651c4b032004bf7b4f1713391a54b2a341a52c6e8a2b59acae9d16e13c7 \ + --hash=sha256:19d643d701fdaa70e5b9c7f8f96abcaffe77ca5e482a3a1a7dde46feb4284695 \ + --hash=sha256:1b6e4adcfd285c44502aed51df98aaaca4f0fea028165dbf8a9e857b9f98d8ea \ + --hash=sha256:1f83081065ee5833d35b49e9180f3d8d15622a603dd1c435da0da6cc12b3662f \ + --hash=sha256:2157f52e4b4d7ac5ecc7457b3716834706e7ef9a46f5144029bfeb7cf71f4e06 \ + --hash=sha256:231dec90efcdccf1b565a1416107736f1e09b1a08fe736ef9d6363e626d03874 \ + --hash=sha256:26cc4b4840fa0e9f4b72ed489883e12f57e00d1021ca794720e3c29a12f0edef \ + --hash=sha256:26e60f6a956ee171ab728b37b8439905d7ea1db435c30f9822f291e9861c861d \ + --hash=sha256:2cc19ac87b41c9493c9778ff5847f0c8bbcf5bd0ec6b87ce06c1c802adc8a771 \ + --hash=sha256:2ea70afba6b9eddbf22c165142e5f0a2ad7aa36a452873c48b57bb2aeb8492ae \ + --hash=sha256:2ec16eb585332c55d022d86354e209ddf27326b1ea3477585ab248e7776d3b1f \ + --hash=sha256:2fc834fbe3f6a0736905c36ab709537e6840dbd63b982dc9e0216ae7d305ba1a \ + --hash=sha256:380873f330b741c4435574f37edb20813d04603ace2d53e0a63560e1fec83010 \ + --hash=sha256:3b12e54f8bec91433e41aff65d8d1f209a4f678081163747079806e5361f6c91 \ + --hash=sha256:3c5349c9f916283bba32bec8af69b763e4faa304dc004d0eaaea66a3cf004c1f \ + --hash=sha256:3de75343041a1c57333b1e707ac8a9769738241d7d6a55d39e12cf84548337c6 \ + --hash=sha256:3fd7c14b1cb45b486c39fc9b3443bb341f3e2fc7e6f31247f3435a5836651632 \ + --hash=sha256:447ada49af4898b5e992f0b5799d2f3af385921102c211947ce3fe960dd919da \ + --hash=sha256:4d8d91d68353bd167fdf26467e5ff9e56aaa5f87d6410c0238608629e4dc0d33 \ + --hash=sha256:50a7e5646cbac2a8f7c3e8c0934ffda1a4357ee9c44b652434b23c3ed54d0900 \ + --hash=sha256:561e7580f84a79859af1ef6f676968e9030fcc3fe195700b15235bca64f009c9 \ + --hash=sha256:60c47ca69ddda0dea8256fffd12e1b86f4b59734a20e4a70c61f63cc5f021df4 \ + --hash=sha256:6eb94895c45f26bb8f5546e5fd8a069efcf6e3f108ea9d5cbe3bf6f7f3983438 \ + --hash=sha256:728303a072163130c5b477b1f20d6211895569c1d5302c24ffc93a3009160871 \ + --hash=sha256:78571efc311c30b73f31eb949a921d6dac39a5d9dc42d1cfa8f8db157b3447b1 \ + --hash=sha256:7896eea257fe497a2b7134474d909156c6744ce8da35bce88011a960e008aa0d \ + --hash=sha256:7aab286a020660a039097912a088236b985d18a3090d73f136c4413d29d37ca0 \ + --hash=sha256:7b7acbb7a4b8383707bce22ad3c162006478c27b56368acd3e1fcb1658a80425 \ + --hash=sha256:7db45b98e94adf4173a5cd7422b150999a7ee11ff847783a14f6e1b80cc38cb6 \ + --hash=sha256:86951a971c53979ec857bd8c4a32dc227ab0fd33f6c12a3bd62d3fbf5f0bfcaa \ + --hash=sha256:86f66c85e796f5d05d5c4a60ec1d40cbfebc47a32464053528c797163fa9ab89 \ + --hash=sha256:8e947aefe98ef74cce94923f90e48c98fe34eb1ec0a6bfdfadfc5a96359bfc36 \ + --hash=sha256:90a762670c7f968184723769a06ed51f5cf5ce5dcd1e30164f25c72d85c2d1f1 \ + --hash=sha256:94f77b60a8ab23580db19ae822744c9716c1720020d2179ca5605112d12326f1 \ + --hash=sha256:979c1524f753b662b0f3cd261b135afe6659cce33caaa7a5ea00dd1756b3055c \ + --hash=sha256:a140e83317fef02faeeb78d9a8efac623887f2feaf0055c55dcdb2b17f0226ad \ + --hash=sha256:aa428a559d5fd02ae619aacaace86c7474a1f2702d2c01fc828908dd60f20f7a \ + --hash=sha256:b950248272f1b303dc32986396e2dccfa10cf6d1e83ec8f0bba1776660305482 \ + --hash=sha256:c2edf09b381fafbc014ae8e018ed25087abb9a3dafa8465a0ea63c6558c47a79 \ + --hash=sha256:c3093001ddce822b4587e6e94bf6de36a5f97b3f31de1c9fc8d4fda144c59ff4 \ + --hash=sha256:c6cb9896a82b9ee44e15ba0b5c8044072f2e4d48acaa704c8d3feeef5ad9487c \ + --hash=sha256:c77d4a3e1deb2707819df92046b89aad1ac81d27e07616b797cbff3f62c037da \ + --hash=sha256:ca4db6ff5c5bf600f9b7761a0070ed44dfe5797a76bd432fb978bc480ef40c58 \ + --hash=sha256:cbe2cc3bba939bcdaf103e03df9d5039d33887080b315624be28ec69059e5f94 \ + --hash=sha256:cd8ca1305c1c902fe42c486165f2e4808d9997625c98ffb05b9e0366d99d3948 \ + --hash=sha256:d0781223705199b289faa59601bb9c2441712d4c600dd13c43d8fd6a33d22cd5 \ + --hash=sha256:d6cebe67765569df3dafac8474e4eccf5c19d24140492567a5e58a11445732a4 \ + --hash=sha256:e067f4cbcc5d036e8aff7fe7a6b530a8f4de2e4616ad9005a24a1879e24e6450 \ + --hash=sha256:e2eca764c53490f8930dbce329e0769f11108d87d908282a80c5c130e26e7037 \ + --hash=sha256:e3442bbb2f0c588cec876061e37ae67b455b9df9978b003c8fe30e45f2ef5b42 \ + --hash=sha256:e4ddf863b59347deaa92302dcd90e5eb003cdc9be06ec2b692c38d1bdd9efd49 \ + --hash=sha256:e9c5fe393aab56469f04e432ff851216d3def3436cf5f07e442a240164bf500f \ + --hash=sha256:eceeff0c62419bc78d4b6e70a4762a4d25df3ae8f2d5946e3853ce93e7a57098 \ + --hash=sha256:f2af4a336ea56d6c14f27741a0e1d8294a35dd0b038bcf990d232ebb54eb994b \ + --hash=sha256:f3d6cf93fbe2e7117eb7bedca684216fbe328a41f0843ce34245451d8eb2df1c \ + --hash=sha256:f5e7665f6624e052e5e7f6a36919ab69279decdc976d7b16b4fa15e1897d0513 \ + --hash=sha256:f702e0aeeb6506e57687e881c59e844ebe8f0a6a097ddafe20e3ab25f387be4e + # via mistral-common +tokenizers==0.23.2 \ + --hash=sha256:12f0835dc2ee694746a76adf7b1567d4346a4a502ebe93fb1f5f80ea49799b78 \ + --hash=sha256:2e96f5699d5249c9c64aa8412e044f727aae3a4098cf830f9901ec1afc361cde \ + --hash=sha256:325fee2e0418a9dc6c9ecf736a5f5f0db7875183ace9549ae339da76f7a1fbb7 \ + --hash=sha256:41c2f84d172449b4dadb9cdc508e3e364076613c35b16e76ecfe47a60d1e3305 \ + --hash=sha256:43e4f2071e3cc8d5d86421c874aebc82659bb51a68bcdef5a0da75ee89511ccb \ + --hash=sha256:5c56bda1511921587789163e524d196ed8284174ac23abd7685d5ea8da6c4718 \ + --hash=sha256:7b7e37ba198f24150f523e1242e83c4970de4a525480586be5dcc24d9add32c5 \ + --hash=sha256:7f0f085686b9de0d0079e6f874ae053600db64c5d13049e0bbc0119926d25aac \ + --hash=sha256:85a9a357a3764aecc904ee76bdaf8cf1ad8e5a67a1b929a487c4a39b49ed0e90 \ + --hash=sha256:950d7c9426fa72406a0ffeacdbc0bb9985f5db20eb8b263f29c79aaf83105703 \ + --hash=sha256:986670e43691469dcee610ea0f846f91a8f84e91fc6f7a48d4c064414c0ec2bf \ + --hash=sha256:a37039b5dfc4af84eb3ef0a92f4307e28936c8f9adccba2629d36f652e9bf7a2 \ + --hash=sha256:bef235815a067b2648caf6dcc7a71091b0b0fff9ee8057f6451eb9335fae52ef \ + --hash=sha256:debf978920d93ba9c219bd67cc4bbfaf912c9039e41e7a28b91ec15e3728c95a \ + --hash=sha256:e49c394456dd9985787fec76132438ba3fb8911f857b1bf3d40119f9292d41aa \ + --hash=sha256:eb2f9c8a24da020ea8c11a01a19c1c2547912d92121ae4a01cfbca46125dee40 \ + --hash=sha256:f486f402f6f9abee5bb032553736813af0c710a86b2e0ca592634c55cea1f835 + # via transformers +tqdm==4.70.1 \ + --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 \ + --hash=sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4 + # via + # huggingface-hub + # mlx-audio + # transformers +transformers==5.18.0 \ + --hash=sha256:d79e5a515a572ee3deb33eb0d578be47d91e70c75d54cf21bf6eede6edef14a9 \ + --hash=sha256:d89c206e42e841af7cd314b3d2b21c03e977d819cdb4aef65ada048769a644df + # via mlx-audio +typer==0.27.2 \ + --hash=sha256:269b7eb9d3c202ca84b4bc9618cb04ebb43d3d4d1e567e4c768607232c05f945 \ + --hash=sha256:b3a5fc4342d5fc8fda8fc3010b1cf117e9249aab7fae800c2eff62fd3842d97d + # via transformers +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # huggingface-hub + # mistral-common + # pydantic + # pydantic-core + # pydantic-extra-types + # referencing + # soundfile + # typing-inspection +typing-inspection==0.4.4 \ + --hash=sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47 \ + --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 + # via pydantic +uritemplate==4.2.0 \ + --hash=sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e \ + --hash=sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686 +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via requests +wasabi==1.1.3 \ + --hash=sha256:4bb3008f003809db0c3e28b4daf20906ea871a2bb43f9914197d540f4f2e0878 \ + --hash=sha256:f76e16e8f7e79f8c4c8be49b4024ac725713ab10cd7f19350ad18a8e3f71728c +weasel==1.0.0 \ + --hash=sha256:7b129b44c90cc543b760532974ca1e4eb30dad2aa2026f57bdce66354ae610fc \ + --hash=sha256:89518acee027f49d743126c3502d35e6dd14f5768be5c37c9af47c171b6005cc +wrapt==2.5.1 \ + --hash=sha256:016602dd8827d190280a707c5e67f9a80038f54bac1782cc8ff68a2a16c618bc \ + --hash=sha256:03aa7d2256309b57ddbf317bff2cae5f47e50ea9ae8d582780ebe0b554347b42 \ + --hash=sha256:051220e5071fdfb1a6678707c8abb7bbf4824d40f99758394b2b4d64855fb284 \ + --hash=sha256:0591e6eace0d186c9ef1ecd1244be5a04e98041424cfca425b684ffe4f0d8030 \ + --hash=sha256:05f6138d5833edf68d88f950ea71bd96daf0a9505b53abd48aa002a0b6d05765 \ + --hash=sha256:06740dbf984af8a26d4b63b75a6ee4e88846c068dc865486ad906448079f50d4 \ + --hash=sha256:094b847491b813b6e6c1775e03770930d75078c0821adf929ac712830951ef25 \ + --hash=sha256:09b1893ee4063706574c1813abf479b8b51926633fbdb6f96aab8dc7b0976668 \ + --hash=sha256:0a526227efe17dd94bd16b123d170f879bce42c15f10eb92495a745f54caa943 \ + --hash=sha256:0c9480bdee340a1602cae5a777146ab4be3e384fdcb569fffdf8721032314645 \ + --hash=sha256:129cab3c7b21e68e693c2819a95c47f3b1c41a834b931154688c83b6aef6bdab \ + --hash=sha256:12bee472452019706fa1d4ead093f52a9683b4fe6617953e15bab9acdfdc013f \ + --hash=sha256:12d3d2b9d6553df6e2421ab99e1cc5413509076788f57fcb3169f5ce100a19d1 \ + --hash=sha256:1425fcf0e70b27053bd610d57bae975856e7897e3f6ba1456d2b80b9d7fd15d1 \ + --hash=sha256:183bf0bb893f783c9d22f953cb01fababb9f618e098763f8e66337b575b0647a \ + --hash=sha256:1910be5adc0232cc6e8c0673bf3f41c2ee724547543526bed8d00734458e7bc5 \ + --hash=sha256:1a96e2671c60f9f09ae547b5a815cecb29af16caa68d73693387d0028788cb32 \ + --hash=sha256:22300c5f254627f24ad2197998fde26db6eacbb0f879162944bf7bd79dd5ee5b \ + --hash=sha256:22a9fda6ac53536ec74e3e334f3568af2535a3df1ae70e8f2816f77160c386d9 \ + --hash=sha256:25eb4d928a9abeaf70ca786a35861b46d1ab37cc4ce49ea70a070dacdead4dfe \ + --hash=sha256:25ed8b1b39234140d5b5c6a273130c7595e0abece417c3ca3cb378fcea5cd0fe \ + --hash=sha256:26313f38d18d40a9975123a4ebff9da125ec63ab9ece4f05320a3d8d37d2c1fe \ + --hash=sha256:26d8ea2ec6818aeb656bd8a9e745a6f1fb0edfcd8f54291ccd94f62eb5f5e3bd \ + --hash=sha256:29b62e87fcd6a1893f669abfd02a596a7fc5cfa79fa57e42c4e650a6c170c67b \ + --hash=sha256:2c642a83b6703804b571caa3b8b205aacd341b1b37e2b2d89cd70e03e0e9caa6 \ + --hash=sha256:36d7d0ad593c4f1a651e4032de834db59aee1a929ee396cd483895b673328e51 \ + --hash=sha256:380f72610181883f66b41442cfc7c0f7552b42169efb2113def26e6380013d37 \ + --hash=sha256:3cf273b7e8d2038abb7f0a8c6550aff4f617b9d486a9965c8e8acc96a3a04de9 \ + --hash=sha256:3f93ceb0ac4896de45d5a45a8f4e69474da583440589de10b362ddc1db4691ed \ + --hash=sha256:4b3f410c416752e1dba53d361e2e6562f22c2c3ec855740dfa5836e061b22571 \ + --hash=sha256:521bd5ef2a33171fac08a0a302d51a983c19c3519406c1ee8da7ce29285488da \ + --hash=sha256:5ad562c23e61e626f9d27aa37aa5679f1c29085de1f998466d107854048bba9e \ + --hash=sha256:5b53000b424dc2133eaaf22838a2352d3497f5d7c2e7d9a2acfe675ab7225bb1 \ + --hash=sha256:5be9816d9de88f02fce23cf55f392403411d9bd9c7ae57fdc965a43b22e2de5e \ + --hash=sha256:6201c7e122f40060a9b50696d80deec8f93b1a235ec0443f51d7a8a42f7044a6 \ + --hash=sha256:6405ff2160af9d59132ebb076eda0304db44d9d09809582932412ef7c0788a36 \ + --hash=sha256:69fd0fbb3daf7c8c6f5e062847a0061f880f347374d74cf1daba57220fb64cd0 \ + --hash=sha256:6e3eff05ae616671b40d7ad0a504210329e4adc9fb91415663570aca93c5f5cc \ + --hash=sha256:711e73da3d7983547fc9dd208973b6b0c52640822f5d477910ba24622df6ba64 \ + --hash=sha256:729d644b6acaf4846a4ef81b037857b66a01dea6d227f827c6d71c0b6d656d6c \ + --hash=sha256:736c1de0230c6d24327b14684794214167b2c5ebb6332e28a10f504641b600df \ + --hash=sha256:76f230a9b07e3cb66646d265398f579abb6128b1bb4cb97c74b1ae5d09e96f31 \ + --hash=sha256:7fa321270b40f3e8cdfd954b3a8dcafc6db1d8bbd4d681b92dfa6b9ef91a9a99 \ + --hash=sha256:8078186f719a92693199f1e06c4ec72e1e6d374c2e459da18ed5c39d6966d727 \ + --hash=sha256:859f67bfc31eb7ab55f237b629cd4ab0441b075912446481f910f7d02066811e \ + --hash=sha256:8922821f66ec08a39f72247776c6158db5bfaa09d0c8f607cd854bdf6b2a2c10 \ + --hash=sha256:89d9a8607b7028054bb6fd01d437f205534a5d59d53c3665d15949a99a2fce0d \ + --hash=sha256:8a7c078323e6e1534968cb85488c5eb7ee2b9bbd0f8a291095213a763da40dab \ + --hash=sha256:8bdf4696fb5bb141a7f96710ac6d9a6aa9a57a14c54075f9c7d3946869d457df \ + --hash=sha256:920f700ef41ee774a1e4778c1f4295e117f1ff3435a7e0cd3e997d10da819d32 \ + --hash=sha256:9a34640eb6295f33ca23462977de275fe8f3a50ab339b8918b96d69a7451e2e1 \ + --hash=sha256:9aa7660684d73925c0d1e4f8536ccbaf233cef3897e33a8c2ec462f83b338323 \ + --hash=sha256:9bad4dbb4e61624fcce5f301e37f9e743ecae4f1259a3777b3207eb7eba3dccd \ + --hash=sha256:9bc472825027b276d4bf678d2ac64149db0b122f80ae6f59c423e6d31f0c4bb7 \ + --hash=sha256:9f0750cbc2e29e4f3c9529d3587d4e7ed8f60638ceafb80b87a95833b0c5acd9 \ + --hash=sha256:9f437dd704abc4ee1bd03bb2d796d362d0e75915e8f3113a7900b3b7ec5f8b47 \ + --hash=sha256:a18e63910252eb75d8806b4baefbc3a03612502f63eab042e3741b00b719f043 \ + --hash=sha256:a1e823aecb3746b8f9e0aee2e1413887871ee2f5c502a3e0ef8d466dbd4adde1 \ + --hash=sha256:a424e8a9776c06aef6313af1d0e3fe6e0838af4241d0c09eb0a3b46f2c9a5ff3 \ + --hash=sha256:a88370a7d89fcb1c4953a87673fdd7b4a0eb14a1a4dfce49771f0c827ef44893 \ + --hash=sha256:ab6db7d2a18d366cc57c2228253cf26443190aba0a6dd0939b3c1e8ac6e29e2c \ + --hash=sha256:ad81bf81b0a0b6c6ec74169638202851962843e86749570c463eecc55072f93b \ + --hash=sha256:aed178902c2386d7c5d3d23eb96d32c100e34cb8c2390e7ece0e4901ae43f0e7 \ + --hash=sha256:b0c82c19baca8ddeb4f513f584f53f6d3aa96b1a273f1a507d6d70620b01ba92 \ + --hash=sha256:b238e955ba34ef2b8897f358b7b868b41b9a02ffd338014b62985fa91898cc4a \ + --hash=sha256:b40f814df9e106371fea48911814383284e99df34ec1aa1fdd9b07d2055345d0 \ + --hash=sha256:b40fb47d637df8da7b02d76f242688416c23e53195ea5748895db671c01759d2 \ + --hash=sha256:bc5c0203d383403043fb86c964bd0bab4fcbfb26004ff4bb9c6d02ebc1d608ae \ + --hash=sha256:bde5d1b37101b1e9dd3da1f35072e2e7028e9c5e3511f7d76d3fdd4d071b7663 \ + --hash=sha256:bfaa998ceeea4d0aa72b40cdd0023d19409504e244b439ff2aa9f01729341c5f \ + --hash=sha256:c25c594f58ecb676358d6d6b0ff068b8bbbc506dc831c6d17876460c66ce39c2 \ + --hash=sha256:c39c7130ea0702c4ab0faf12da1df1e02d5174305c17edf02309e2f058c4114f \ + --hash=sha256:c40f3b1cd3ff9dd9f4ae829e4301f0d3a553e3467058b8c3f5528fee2c768a20 \ + --hash=sha256:c44dd9881626da7d621c23805f26726f6b023cf3e9755f48d092bc9cbef4a8e7 \ + --hash=sha256:c4d9c76e9a16a8bae0bdcc57efabad499192565bd9a95258b01fb0b49a62bd63 \ + --hash=sha256:c6e6c226b1ca5402d7ae5fb34a0d21f1b49124fe4200e5884d1e19e53c47ac1d \ + --hash=sha256:ca7b967e96384abdf7e7182c79f71529997981ece8169f8a8ddb31bc5b57cbec \ + --hash=sha256:cab37b82ec328173222e4f9da5eec4f2ec9e8e506f83557c8be8e1bffad351cc \ + --hash=sha256:ce3889e3815f97d46414eb574bffdd9bdb41ff70f503097e2707615a87d4e92c \ + --hash=sha256:cef2a8f006410b6134a0d273ec037fea8cc7a6a914f1bd7555ad9788ad788c6e \ + --hash=sha256:cf63fffcdcd8c60f223d3967bb92cc4fc2e8b46f09e75b67a6a75e6f47c0fc43 \ + --hash=sha256:d5b665a43fe0d3b390cbdd3c003d61c92fa07bd5e3fb1ed3f47920c2d03cd9fd \ + --hash=sha256:d6d274ec50a5b208be75596dc44ea253e65deaa6ee3a600babc86dafbb957dfc \ + --hash=sha256:d800c7689154622b0ba2922ceca44a3cf2ef61c3b9a4c4eeb1d8b3050d7ededa \ + --hash=sha256:d90c91cb4ef83b2ff00db4e0a7bdd9602902504ef9b26d0f9d7ecf6cd05c7554 \ + --hash=sha256:da42395e7add724c1f7caf18a2977b1fbdfd5aab314e5622731f0ed66731eaaf \ + --hash=sha256:da847332447db5505162759a4cd5ac374eb8b74841fe97a98ef3de14edd2586d \ + --hash=sha256:dc401274fcc7b15b3b2c12df2ff34024a11925243a7d3daee91c6d7d14f9addf \ + --hash=sha256:df6e3a36170cda0d313be50fe5065948e7f12f3a181b38cbc262e9f2ee4824e1 \ + --hash=sha256:e089a22ff5af1290b8c759a610830bdb2a829ef9c3d7797e4ee32c2f795ed482 \ + --hash=sha256:e85a9db9e5a5ccc326edb19e35a5106ba16e451d570a2ec8ea9deb1ea52a3c42 \ + --hash=sha256:ea27bcf5c56b13463ba5b9bbfa4d6544997e47ba6db77c59a259b09daa802d4d \ + --hash=sha256:f063c696328408fc4f259b9d7d439398d36b709e12445a904e7b047f0a84c3c5 \ + --hash=sha256:f1630201b0e2a96bb26304b7adfbd91a4ef486abb5a4c48377444a0bed749f37 \ + --hash=sha256:f1c911818fb076910ef509f2298dfcb966a54a6ff068eebd459632102cf589fb \ + --hash=sha256:f280c115ea64eff3dcbd68a668ce3f63476a4ba386bbabb318017e286196ea2c \ + --hash=sha256:f595bb0185aab3e9dc31950c95d914f56ea8278810c3b928f3426e12ed6d27bc \ + --hash=sha256:f98eaf784cd12bc69c77af398084174531007cd81849c962163ccfc6e791f3ea \ + --hash=sha256:fc0eb73b450b53950b7879ac7642889c82918d17bd2d877fd7270348dfd5550c \ + --hash=sha256:fcccaa1484f7dd1091602970988ab741491f9f974013c844f70e45ac1196b80d \ + --hash=sha256:fd3f878a4aac3c262447ddf43c5f4c18fc67dfc3ba69c4fb1c7a4c4af96abe7e +zstandard==0.25.0 \ + --hash=sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64 \ + --hash=sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a \ + --hash=sha256:05353cef599a7b0b98baca9b068dd36810c3ef0f42bf282583f438caf6ddcee3 \ + --hash=sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f \ + --hash=sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6 \ + --hash=sha256:07b527a69c1e1c8b5ab1ab14e2afe0675614a09182213f21a0717b62027b5936 \ + --hash=sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431 \ + --hash=sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250 \ + --hash=sha256:106281ae350e494f4ac8a80470e66d1fe27e497052c8d9c3b95dc4cf1ade81aa \ + --hash=sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f \ + --hash=sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851 \ + --hash=sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3 \ + --hash=sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9 \ + --hash=sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6 \ + --hash=sha256:19796b39075201d51d5f5f790bf849221e58b48a39a5fc74837675d8bafc7362 \ + --hash=sha256:1cd5da4d8e8ee0e88be976c294db744773459d51bb32f707a0f166e5ad5c8649 \ + --hash=sha256:1f3689581a72eaba9131b1d9bdbfe520ccd169999219b41000ede2fca5c1bfdb \ + --hash=sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5 \ + --hash=sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439 \ + --hash=sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137 \ + --hash=sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa \ + --hash=sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd \ + --hash=sha256:25f8f3cd45087d089aef5ba3848cd9efe3ad41163d3400862fb42f81a3a46701 \ + --hash=sha256:2b6bd67528ee8b5c5f10255735abc21aa106931f0dbaf297c7be0c886353c3d0 \ + --hash=sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043 \ + --hash=sha256:3756b3e9da9b83da1796f8809dd57cb024f838b9eeafde28f3cb472012797ac1 \ + --hash=sha256:37daddd452c0ffb65da00620afb8e17abd4adaae6ce6310702841760c2c26860 \ + --hash=sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611 \ + --hash=sha256:3b870ce5a02d4b22286cf4944c628e0f0881b11b3f14667c1d62185a99e04f53 \ + --hash=sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b \ + --hash=sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088 \ + --hash=sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e \ + --hash=sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa \ + --hash=sha256:4b14abacf83dfb5c25eb4e4a79520de9e7e205f72c9ee7702f91233ae57d33a2 \ + --hash=sha256:4b6d83057e713ff235a12e73916b6d356e3084fd3d14ced499d84240f3eecee0 \ + --hash=sha256:4d441506e9b372386a5271c64125f72d5df6d2a8e8a2a45a0ae09b03cb781ef7 \ + --hash=sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf \ + --hash=sha256:51526324f1b23229001eb3735bc8c94f9c578b1bd9e867a0a646a3b17109f388 \ + --hash=sha256:53e08b2445a6bc241261fea89d065536f00a581f02535f8122eba42db9375530 \ + --hash=sha256:53f94448fe5b10ee75d246497168e5825135d54325458c4bfffbaafabcc0a577 \ + --hash=sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902 \ + --hash=sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc \ + --hash=sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98 \ + --hash=sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a \ + --hash=sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097 \ + --hash=sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea \ + --hash=sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09 \ + --hash=sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb \ + --hash=sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7 \ + --hash=sha256:75ffc32a569fb049499e63ce68c743155477610532da1eb38e7f24bf7cd29e74 \ + --hash=sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b \ + --hash=sha256:78228d8a6a1c177a96b94f7e2e8d012c55f9c760761980da16ae7546a15a8e9b \ + --hash=sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b \ + --hash=sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91 \ + --hash=sha256:81dad8d145d8fd981b2962b686b2241d3a1ea07733e76a2f15435dfb7fb60150 \ + --hash=sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049 \ + --hash=sha256:89c4b48479a43f820b749df49cd7ba2dbc2b1b78560ecb5ab52985574fd40b27 \ + --hash=sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a \ + --hash=sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00 \ + --hash=sha256:9174f4ed06f790a6869b41cba05b43eeb9a35f8993c4422ab853b705e8112bbd \ + --hash=sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072 \ + --hash=sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c \ + --hash=sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c \ + --hash=sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065 \ + --hash=sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512 \ + --hash=sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1 \ + --hash=sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f \ + --hash=sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2 \ + --hash=sha256:a51ff14f8017338e2f2e5dab738ce1ec3b5a851f23b18c1ae1359b1eecbee6df \ + --hash=sha256:a5a419712cf88862a45a23def0ae063686db3d324cec7edbe40509d1a79a0aab \ + --hash=sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7 \ + --hash=sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b \ + --hash=sha256:ab85470ab54c2cb96e176f40342d9ed41e58ca5733be6a893b730e7af9c40550 \ + --hash=sha256:b9af1fe743828123e12b41dd8091eca1074d0c1569cc42e6e1eee98027f2bbd0 \ + --hash=sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea \ + --hash=sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277 \ + --hash=sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2 \ + --hash=sha256:c2ba942c94e0691467ab901fc51b6f2085ff48f2eea77b1a48240f011e8247c7 \ + --hash=sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778 \ + --hash=sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859 \ + --hash=sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d \ + --hash=sha256:d8c56bb4e6c795fc77d74d8e8b80846e1fb8292fc0b5060cd8131d522974b751 \ + --hash=sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12 \ + --hash=sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2 \ + --hash=sha256:e05ab82ea7753354bb054b92e2f288afb750e6b439ff6ca78af52939ebbc476d \ + --hash=sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0 \ + --hash=sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3 \ + --hash=sha256:e59fdc271772f6686e01e1b3b74537259800f57e24280be3f29c8a0deb1904dd \ + --hash=sha256:e7360eae90809efd19b886e59a09dad07da4ca9ba096752e61a2e03c8aca188e \ + --hash=sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f \ + --hash=sha256:ea9d54cc3d8064260114a0bbf3479fc4a98b21dffc89b3459edd506b69262f6e \ + --hash=sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94 \ + --hash=sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708 \ + --hash=sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313 \ + --hash=sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4 \ + --hash=sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c \ + --hash=sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344 \ + --hash=sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551 \ + --hash=sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01 + # via mlx-audio diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-embeddings.json b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-embeddings.json new file mode 100644 index 0000000..13d1853 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-embeddings.json @@ -0,0 +1,50 @@ +{ + "schema": "backend/1", + "id": "mlx-embeddings", + "version": "0.1.0", + "builtin": false, + "package": "mlx_embeddings", + "convert": "mlx_embeddings.convert", + "lock": "mlx-embeddings.lock", + "ports": { + "classification": [ + "laya" + ] + }, + "port_bits": { + "laya": [ + 8 + ] + }, + "port_convert": { + "laya": "convert" + }, + "port_signatures": { + "laya": [ + "rl_agent_config.json", + "encoder/config.json", + "tokenizer/tokenizer.json", + "tokenizer/tokenizer_config.json", + "model.safetensors" + ] + }, + "port_quantize": { + "laya": { + "include": [ + "encoder.layers.", + "head.layers." + ], + "group_size": 32 + } + }, + "categories": { + "classification": "classifiers" + }, + "remap_files": {}, + "registry": { + "classification": { + "model_types": [], + "remapping": {} + } + } +} diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-embeddings.lock b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-embeddings.lock new file mode 100644 index 0000000..b9c53b2 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-embeddings.lock @@ -0,0 +1,1576 @@ +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb + # via + # fastapi + # typer +annotated-types==0.8.0 \ + --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 + # via pydantic +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ + --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 + # via + # httpx + # starlette +certifi==2026.10.19 \ + --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b \ + --hash=sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae + # via + # httpcore + # httpx + # requests +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via + # miniaudio + # sounddevice +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f + # via requests +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via + # huggingface-hub + # uvicorn +fastapi==0.142.2 \ + --hash=sha256:06366626f2e70576367714d9ab2fe8472e6c8456dba69b399f9f797ab5e92570 \ + --hash=sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b + # via mlx-vlm +filelock==4.0.9 \ + --hash=sha256:635e7d67fa92654eed444e75e9ca18426d34e77ad9c469bf4373f75a932f7b22 \ + --hash=sha256:9287fd61b99a806e5202be29a83034c0808a1b9830e820537c2f5773981f7eeb + # via huggingface-hub +fsspec==2026.9.0 \ + --hash=sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe \ + --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f + # via huggingface-hub +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via + # httpcore + # uvicorn +hf-xet==1.6.0 \ + --hash=sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7 \ + --hash=sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675 \ + --hash=sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef \ + --hash=sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9 \ + --hash=sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3 \ + --hash=sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb \ + --hash=sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e \ + --hash=sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338 \ + --hash=sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d \ + --hash=sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d \ + --hash=sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765 \ + --hash=sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c \ + --hash=sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522 \ + --hash=sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f \ + --hash=sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a \ + --hash=sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b \ + --hash=sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b + # via huggingface-hub +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via huggingface-hub +huggingface-hub==1.33.0 \ + --hash=sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899 \ + --hash=sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e + # via + # mlx-audio + # mlx-embeddings + # tokenizers + # transformers +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # anyio + # httpx + # requests +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via mlx-vlm +llguidance==1.9.1 \ + --hash=sha256:0ab9d2681c0fcb4389f3c1c79697f1e724771d74d366bb5a22edae155252dfed \ + --hash=sha256:151a615f3d871fc096984b705daa211faccc78ed2f9e3a2a80e57283e16e2898 \ + --hash=sha256:2d7aa419d8b63a528347f73ee57185b4172c8bb09287e5ef9eefe3eb6d709356 \ + --hash=sha256:31b75c9f6e09d000d062ef7e4eb3bcb3759fc1eda5619be50fdd8a830b0e8bdf \ + --hash=sha256:34ad7bdff67219c7bf43ff60eb7b1d97232092fe7a40715ae330f894def04f82 \ + --hash=sha256:367da91f0db6b494e1d4916e6ab3e3db3eb73f178fda6be32681c026af99f775 \ + --hash=sha256:3ba1b37585d07f50bb73e06dca6d45c88c9d5801789c071c8ff784c9dbcc5c46 \ + --hash=sha256:6216861eaeef0f96251d7a9ed1c26a228a4c7879c5f062092691f8f17087ae4e \ + --hash=sha256:8aaa55958adb4871016700e9140543ccbd9d6b5136d0a1756a114af706cc50c1 \ + --hash=sha256:8b29903d5108cefc715e013381c7474846e9b0edb5e2534000f5e3a3d870a293 \ + --hash=sha256:96b52f9d3eccb1d6c1ee4454c3424e8da5171c879e33c294a68e6c731fec25dc \ + --hash=sha256:9892220b2698f58193ef54a886e5f3d5eb2a60d510db49ca4c492b46ebb786fd \ + --hash=sha256:a5dcbdd38bc75f149ed349525f3884160de80900952147a4b7ef40a5b1a4a1a1 \ + --hash=sha256:a942487cd6221b793df3a11450881a6c9d25a45460a900c56d99b682633c45fe \ + --hash=sha256:cc7d1024c3df780a5eb14d30744539e8a1aa6d94c5dabf7605c82c0567d45a51 \ + --hash=sha256:dd5c87df5e60a7d8f54f46bfd5e11abb10f93636712ace09a8c1d2a0c4aeade0 \ + --hash=sha256:dfac6021d9c4c5ef16b3a1d336c05ca5e8208ff30dec29d1c63ebcafbee3a346 \ + --hash=sha256:f824a817a841563d5b068a0dd22563e5f063cdbda382cf3f6e18c1c68c45db38 + # via mlx-vlm +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +markupsafe==3.0.4 \ + --hash=sha256:007e1ffd9bf65bb6ee96df7b258fc632a4868dd5566037986c64781f35a36e98 \ + --hash=sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002 \ + --hash=sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b \ + --hash=sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653 \ + --hash=sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c \ + --hash=sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e \ + --hash=sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc \ + --hash=sha256:0764a13d34cae40db7bbf3a09b7e9b491bf4603e20b263a7a9d6b8e324975d0a \ + --hash=sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92 \ + --hash=sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f \ + --hash=sha256:0cee7cb0f9a1b6892ea482237d9403b3d1b4603aee057d0ff01f0fac2d019a97 \ + --hash=sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4 \ + --hash=sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7 \ + --hash=sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691 \ + --hash=sha256:14bd2d845d62ab678eaf81da89d7b621b51756c72346745c1a594c09d49207a2 \ + --hash=sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc \ + --hash=sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde \ + --hash=sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99 \ + --hash=sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9 \ + --hash=sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df \ + --hash=sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5 \ + --hash=sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17 \ + --hash=sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8 \ + --hash=sha256:2a6ef68ae94aed8721934072b27a3b654ea2100b97e4ab864cf1489c90926fbc \ + --hash=sha256:2b2b1e18af909b448bb3cf9e3433366f7a8726271fc214e8b10e0f62a78c724b \ + --hash=sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea \ + --hash=sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248 \ + --hash=sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741 \ + --hash=sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5 \ + --hash=sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6 \ + --hash=sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7 \ + --hash=sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1 \ + --hash=sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67 \ + --hash=sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f \ + --hash=sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9 \ + --hash=sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c \ + --hash=sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc \ + --hash=sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba \ + --hash=sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17 \ + --hash=sha256:3d23795802fc8bd72534836d64489bbf0f67c088959091bdb22e10735a5107bf \ + --hash=sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6 \ + --hash=sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2 \ + --hash=sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163 \ + --hash=sha256:4a540e2d3192792fc84eced57bef37851ccb2b41f73291bb17408eea77bcd278 \ + --hash=sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d \ + --hash=sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b \ + --hash=sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634 \ + --hash=sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38 \ + --hash=sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed \ + --hash=sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c \ + --hash=sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148 \ + --hash=sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a \ + --hash=sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7 \ + --hash=sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f \ + --hash=sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811 \ + --hash=sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e \ + --hash=sha256:57f9947a7e57a081c1e3e0a2dd0d2dcf290a4531450e6f611e30084c222a7295 \ + --hash=sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2 \ + --hash=sha256:5c22873ad1f0532ba40fa1727f3c0fc1bbbaab6d373d4cbe3f0dc74b2e2521c7 \ + --hash=sha256:5e8b3d0b18fd623afa12ecb2ce8d8becef69f9b5440c6330c7972200e0bb84b0 \ + --hash=sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6 \ + --hash=sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed \ + --hash=sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378 \ + --hash=sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0 \ + --hash=sha256:6768d67d1bce64270e0fdc2e69309d68b9b18ae56ddf6c711d168e9d051c2cac \ + --hash=sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b \ + --hash=sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96 \ + --hash=sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59 \ + --hash=sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808 \ + --hash=sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2 \ + --hash=sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb \ + --hash=sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65 \ + --hash=sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72 \ + --hash=sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8 \ + --hash=sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e \ + --hash=sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91 \ + --hash=sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a \ + --hash=sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2 \ + --hash=sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e \ + --hash=sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707 \ + --hash=sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21 \ + --hash=sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef \ + --hash=sha256:8698d70a8081ee8c090dbb394768b5789a1da8b131b5499f89d071dd3cfaf6be \ + --hash=sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453 \ + --hash=sha256:88d59b473bfb03259722600839af9bbd7fa13a2eb514beefeedb95997882f69a \ + --hash=sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6 \ + --hash=sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977 \ + --hash=sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 \ + --hash=sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692 \ + --hash=sha256:9240187afb63d2f9ddc3e032c670356fe941f6e20662ea168a5dc3f1f317e1b3 \ + --hash=sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369 \ + --hash=sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a \ + --hash=sha256:9388003072b95f2f1e3fd908604194d653ba21330d811961a78b7da1a77e9e36 \ + --hash=sha256:9438a2648b2195980cb2dd8e53ed7b8df91319e2d0b70ae61a9e1d1bc8d3bec9 \ + --hash=sha256:94e4c421742086aeee4c32a506eec8859d7634aad943f7e6aacf70f813478768 \ + --hash=sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916 \ + --hash=sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b \ + --hash=sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f \ + --hash=sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346 \ + --hash=sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c \ + --hash=sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464 \ + --hash=sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9 \ + --hash=sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee \ + --hash=sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300 \ + --hash=sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6 \ + --hash=sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d \ + --hash=sha256:ac0c7c9f1609b0c4c114feb1d7a3409564c7fb77e360bed9e97e5d25dfeaf868 \ + --hash=sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46 \ + --hash=sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97 \ + --hash=sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733 \ + --hash=sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe \ + --hash=sha256:b61687d0828e72bf5cda24a2690188f37170bd31c9359ac97e4e66569f120a16 \ + --hash=sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429 \ + --hash=sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39 \ + --hash=sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894 \ + --hash=sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c \ + --hash=sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c \ + --hash=sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169 \ + --hash=sha256:bf053da3c97a4bc5ecfbb218cdd2983febd91c617be8367d139882aa11e490aa \ + --hash=sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77 \ + --hash=sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe \ + --hash=sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad \ + --hash=sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85 \ + --hash=sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e \ + --hash=sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34 \ + --hash=sha256:cf63c214fe879a65e69a386f915e36104fc84254ab141240f8854602d8e0be2a \ + --hash=sha256:d1aca03ede943eb80ab3d63bb082c84b7aab85ea83bd0fd0c200260945fb49d9 \ + --hash=sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c \ + --hash=sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749 \ + --hash=sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214 \ + --hash=sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932 \ + --hash=sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494 \ + --hash=sha256:dd8ea6ebee7aedbf7c749fa80521d9ccf1ba473e0d1e14805caafbaad281c889 \ + --hash=sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1 \ + --hash=sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0 \ + --hash=sha256:dff05cb7016dff1e9fd68f4122c127b65dfc59de5306cfb7ad92f956f230bee2 \ + --hash=sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786 \ + --hash=sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78 \ + --hash=sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e \ + --hash=sha256:e841068dc0be4cb6dfb5c890eb88cbdcff2f4a332393c7ec94e8e618bd32c1a8 \ + --hash=sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289 \ + --hash=sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c \ + --hash=sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe \ + --hash=sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237 \ + --hash=sha256:f291bcf42ae98eb5107edb162c3c998b4a89648fd8e99ed4cbd12705292788cd \ + --hash=sha256:f61efe1d2fe0de16158a5fe1d1cf3c14bdb6aecd54d8938fd26512c525c1f624 \ + --hash=sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19 \ + --hash=sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977 \ + --hash=sha256:fd9f8797427910198f95bced71ddfed61130d7e349213bfb8466c9c99e2c46a8 \ + --hash=sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3 + # via jinja2 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +miniaudio==1.71 \ + --hash=sha256:06222d80b057ca4beccb6f97a134c2c2bf646ef7890e1759cfc09db7eecec44d \ + --hash=sha256:12bc33e7e61072b4b541c14e10ef76119d5643e6bbb98e2dec0c0738889438fb \ + --hash=sha256:14892ad9b884e637029a22a781dea569b292a1be13682380fd14cefcf80ea4ed \ + --hash=sha256:154b085dd914a0e79e3d93160e1a07aacb27d66c65f9ef6a0d87c1a194f32c04 \ + --hash=sha256:166516449e2bb5f628d89cedbbb8720dceb96a0562c7e08a0e8e3cb10f58647c \ + --hash=sha256:19be6f0a1e601c2237433e579734cfaf6469191b224c20c9e5f73c32ef9ee2b9 \ + --hash=sha256:19dc58e4c50ffc48db2ce988019c28f05ca0eaa7c10b45b5c99b70107e610c8a \ + --hash=sha256:1bf93aeede652926f27f430f0fd69ef0cf8a949c07b537d6a2f295602c747037 \ + --hash=sha256:33986d5d725ebcbc253551e7358689bc81b19b6950b33cec8e8c1142ca4fc0a9 \ + --hash=sha256:3bbeb1e068fe42475e017e8150e9e345182b583d0dd4d9e77ffa20c39935d9ec \ + --hash=sha256:3ef441d139264f8a5dcb9aa6fcd0b1e1e69f58715baae416ff33f045ffba6ad5 \ + --hash=sha256:4c849ccb1349f7b3553a77a66fe7e972315185f5c4c44a0bbda7ebcdd224db37 \ + --hash=sha256:5009b4e29cd43de3631d2d5ab09cc074192c085b4c8dd8a121b856ce1af6bab7 \ + --hash=sha256:50d66729e1dd7a4cf13edc25115ac54f776dd9f67803ba1a7cd1128ebf2e8cfe \ + --hash=sha256:61b86f26d653040db32d9d15b05446321dd10e45beba25b44f841e26935213d5 \ + --hash=sha256:62db602651bc20a2698f36a0d356d7217ed6f4f917550c7ffb3705c8e8be90cf \ + --hash=sha256:70fa2ea5353e6919aca59b8c5768144af009d18c3bca251749d66fb497424563 \ + --hash=sha256:84139a10ef172acd762ccf120142877b037a1aaf71def99d2c75f66329f89d8b \ + --hash=sha256:8a28ff4ad23e55bbde8808ce525d3bb7d249d7612f77646b30e06fc6b7a778ac \ + --hash=sha256:8fc1a4f084cc1b4b25c567d22f54d1e46bfa505c17ed777c8b198e5c53d0f785 \ + --hash=sha256:978cc4d58d8beef1a705e1141dc177a8a357c10ba3a16f7d71482ee722023bbd \ + --hash=sha256:9f379d4995f1fac6dcae65810f6a31cba264339b3e591a14b233f85a6d03d81e \ + --hash=sha256:ab100e5240b104b5326e4ec1be07b6ae461f7d3d4d7a694857fd2f0493d210f9 \ + --hash=sha256:ac4a37ebbbfbfbeca50f4390e50f9952807ca61ac62f0c3bcbbc7dd698531dd3 \ + --hash=sha256:aee8e4eec8d7bde4ee78066561329235a04231a221c9b247f1ffaf850551087d \ + --hash=sha256:d9dc15eff711bcfc62a9d05e0c78e4bc34821a455595e049629f2fea7491a523 \ + --hash=sha256:e6287f15caa808a88aad0700a182bec1ff6d98769717425adf9ebf41259d1936 \ + --hash=sha256:ea86ae04ddbbf2beed20b9970af4a0baca8e6ed0e9625e1ed957be5540c943fd \ + --hash=sha256:f4a44b70b66628b0c307e40ae0ae857695978cae18462179b806d8edc807d416 \ + --hash=sha256:f7042af3a4db5b90e5efaea257b6dfcff9389239ea643e6b0faa80169528e2e6 \ + --hash=sha256:ff51e2887bb673e2e757752b586b3dc924d59aa5fbcae9bbc45f4a111bd3262b + # via + # mlx-audio + # mlx-vlm +mlx==0.32.3 \ + --hash=sha256:0512049c81fded99aa6af2d039ae1fccf113ff1a5f2df937a596578aabd46869 \ + --hash=sha256:097856504e6cf5944b9aaa7c2e1d5be0693428abd9175429cae3893e7ec2f681 \ + --hash=sha256:15e8578c9d10b4fd4728243cd6c926bc46e1da267c6a849cdbceca21d51caa51 \ + --hash=sha256:177bbb598bcb37963568f69b84e7d972f7f9499cdd39d1c6d6dab35a718aeb92 \ + --hash=sha256:19da79adad1aa9653c70819040ac58d65d7a9610a375a30d03c7cce708d4d2bb \ + --hash=sha256:231a278b4fd2804ea16d297960ab169d1749ee1adbf820173e704fbed97bf1b6 \ + --hash=sha256:2dc1085f5c2eba18cdac095d7fb0b68ddfadd51bf695b18a1102eda15e77e76d \ + --hash=sha256:32fa23e57297fdca23e016570229b459f20f645eeb5965527cd890f74143f1d0 \ + --hash=sha256:448fdbc99c5fb0c26f11337e5a182cd6516f8a1a65848184be063d00ad3107dc \ + --hash=sha256:452c621862684e8769be93c1517420c8ba2fd6e8b01a1c970c8ba8069022f0ac \ + --hash=sha256:46df0a5489edcc071cec3d9dd1f7894d492888c9d425effdc8831f4e7021108d \ + --hash=sha256:4e1a8fa78461c6cf1f9473e3bc8a3c4e362f1aaa613b435787f2844fad522bc7 \ + --hash=sha256:50939a8bd54250e6092ef3427a724c7a72c0911dcee09d70f6543431e7f28d3c \ + --hash=sha256:5d12de01f23d2fae739ecfecd9c1d4c07c1f9b778a3729f7c0884ea1168ec14d \ + --hash=sha256:65122a7eeb718df56641d7ac6f7fdb0ada573fd841d1fb0a61a6a443e33f11b5 \ + --hash=sha256:656df5e36b9bfa862fd633d55dee4da072d292def9f27f22134c594e03ff28a7 \ + --hash=sha256:712ca29119d9e2aae04af8f8e416bfb0097538d9ff73e6fab8b028053fb45ff8 \ + --hash=sha256:72711cb23cc6dccc5a460f63f39c11931a00b2c6dcd6fba253921090072e21bc \ + --hash=sha256:75333afef55819afd2a31e87ffbed0bf59f379bca21771e6b13a95a675e4c85b \ + --hash=sha256:78372731c18fadb6d6960c9188cb752d5ea4d87de9be430a311a385e01d7fb42 \ + --hash=sha256:7aecc3d4a904279492dae58e28378a3c4bec4533e7066e64ae5c78c7a5511f22 \ + --hash=sha256:8439baa207db769df1df82a722b0ee0c2362f75c23c0740b7472c3d279253bae \ + --hash=sha256:8564a59943346fac01ab498870a9350d0f79b1cdea95c46c895934ee027d6197 \ + --hash=sha256:8edf996749b331659ab9bb8ecdc895c292f9e7020d02e534147e07cae22bec7c \ + --hash=sha256:8ef541373a94a78fd59ac59368501e976cef5fa07738438362e942315be22ea4 \ + --hash=sha256:936cc82b0c050686c43230c85f3a0bd9f179f650377ce9590f7af3ba659e2fce \ + --hash=sha256:942f0da5328986cdae29ec0eba57b2a08eb9dc6125cfe196853de7498f2c5359 \ + --hash=sha256:951ba2ab75148f61e9a5c222a75fdb0ab481be0b196c21a0d087e9b886d204f0 \ + --hash=sha256:a14387cd679018a32e802fdaa58b1e9845a278cacd309955eddca95757988ce0 \ + --hash=sha256:a5e165179e3d86672cc0d594587bba2b9c3be511ccb99ffdc63e4cc7119eec55 \ + --hash=sha256:a6241ea8f56a08a6c3a06bbaad7d7fc2fc1c05f23757c61fe0a8797325ac6d46 \ + --hash=sha256:ac4472834f38e61d294aa1bf2fdc9f687f3b3162b9729dc9dc15e8c149f8c216 \ + --hash=sha256:b14744a1b3a9dd48e06a62c448710a798ceda9451d7d37a4654a58b5adfa97f8 \ + --hash=sha256:b2acd057e4a27eb1543bff0e9030c34fdc64e18815fe4bc23c2dbcc47b60f0e3 \ + --hash=sha256:b57996fd2b72207a796656bc39508f20089692815297120039284edb296b5fff \ + --hash=sha256:b91012fdd5ec5cbedcec2cae6adfaad9a8f547023e570fc5781cb7eb4bfe2567 \ + --hash=sha256:be5cb6554873d3c86f083fb0b35aedf4a1f12657adee4ac0fc90665ffd7e6722 \ + --hash=sha256:c0764ce1a7a7f19768d6b3473387dcac03cac8b0dc74b7e1c8d05b451ff4786c \ + --hash=sha256:d17f0c33789009a3b3e609a52af34dc687ed39816836e20ab466aab669c14e6c \ + --hash=sha256:d3b9494b45400b803eeed7d05e6e566aa135f1dd2d121c29016ee9492e7ead67 \ + --hash=sha256:d61942263d4603814622cd72bb5991b8a3cb59970bde8a7b40b897a7f088fb84 \ + --hash=sha256:d9ff05b868748b846a02f59913e9aa9d7b576827db92cc358ec1cf660f7f90a5 \ + --hash=sha256:e13f5ce391293dab946edf18e7ae7bb55768beef1140deb509f67a2b14edcf7c \ + --hash=sha256:e32d595df067f19739f1e214e723035ed2a9c216fb1ed3cd118d08f19b9cfabd \ + --hash=sha256:e75fa55161d9477e694f852cad0d383c6b4a5c3b505a5d08339f0e2260c2556c \ + --hash=sha256:f2909f683ffc5a2720f6c8099398a64fd1be53f1b6538e4e728ee55816ce699b \ + --hash=sha256:f348705a72fa236f310c2cbb9c393d3b620a942c9b34ac641fd0f098bdc5be44 \ + --hash=sha256:f5716b51fea3d06352d4d8c851fb6c5141b1a7de5a5164d44de5a6d368d84a7e + # via + # mlx-audio + # mlx-embeddings + # mlx-vlm +mlx-audio==0.5.7 \ + --hash=sha256:2f2b3c666241ff4a45df7e92057d176b3c1079541a5e9062a0d1e8088db5c123 \ + --hash=sha256:4446704bd8305b3957a6e422234118142e32b87a55d7d3e33b0fb8dbca516543 + # via mlx-vlm +mlx-embeddings==0.1.0 \ + --hash=sha256:3fe1feaa786d3b546ccd8909f6b4c22bd3bcce097616fce83173ededd30e6630 \ + --hash=sha256:f80c1e1be26ff7bd22b15c1fba4cc03afd44c86e00a431b5fa75ffd7500affb1 + # via -r req.in +mlx-metal==0.32.3 \ + --hash=sha256:34ae9b83ad2f0ccdd3e5d48ec35176e7119f57069eef187122916dc941a4ae1f \ + --hash=sha256:fdba88ed9131448cf163be2c7ad6374746d8818205f28657aa91b6c69050f691 \ + --hash=sha256:feea99fe4d01f9e9f33ca73d869f988219a1725de1c18870baa7d5cd5aef6cb3 + # via mlx +mlx-vlm==0.7.4 \ + --hash=sha256:7a3e8e288b21115630c4eab75e9ffc38e4b1770f9d2ce4992d29ecffe75a628c \ + --hash=sha256:a35ca38c5be3a061496cc9bb5b3ada20413ceda9317ec67ba0b0fff1c2885e04 + # via mlx-embeddings +numpy==2.5.3 \ + --hash=sha256:012e66aca395d795496446e52aeeb5866312a5d4d3f27da270e5a0b43f70dc5c \ + --hash=sha256:09d5a423c71ad5feb5625844ad58050e35df43871004b52ac9c0ad44a56775be \ + --hash=sha256:09ffa5d903faeaa5c4dd05009cf81c8bab9f2cb37c548b8d39b65b4cfa7c97f7 \ + --hash=sha256:0a59a421a32580a009e8a1751345bf829631b990dc1794b80514ab722b435def \ + --hash=sha256:116f96cadd935c6122e9228d676fe7ede19e741f5c8bb1c3cddbe0c51ccebea2 \ + --hash=sha256:1302b90c0e52281681b2975adfe8a860cb7b12216a27b4b0b4207c44bf7bccf0 \ + --hash=sha256:15aa985ac73a8db02db7663381aa109510449d3819d37206caed27b33a65a8a6 \ + --hash=sha256:1aad64d99730d013cfc6debafed22783b4fc5a7f4b8bc744d2d8cf7dcc880551 \ + --hash=sha256:1c80eabb4035ecf4ca9cd49cde8a9fdd69a729e63e6474887d1523ade7aa277f \ + --hash=sha256:1f3ed25271581281f2fccb1adcedfcde4c07362eec69189b50baf6f90e3ae159 \ + --hash=sha256:1fb6f8fb9ff0b3a69f52c66ce397b0246583e9f28616231b0e32ca49259a5fa6 \ + --hash=sha256:214045a5bf00113a146ab9ee9730c44501af6723cdf1f6830932f7b5ef2e7af0 \ + --hash=sha256:26e15e4aecd8617dfbaecb37d223e365d7b39411fba20454be2670a96aa74cb5 \ + --hash=sha256:2c25dfa72943e4336ddb6b0ee4277b47a0c85bede0807530ec68103bf58e2c10 \ + --hash=sha256:2d8240cb4c16fd831074aa2b2cf9fc54664d826341d61c372245b96a74a49a9a \ + --hash=sha256:350ba9783ce969cf9f7ce6e6a9a58e1a6e2a19ca025b7ee448c4db727706212a \ + --hash=sha256:4c8a6d2ebce6305fd82fbefca827775437147052a976ee7c94b36a0c1b52ac6c \ + --hash=sha256:4f8929ee6c96bfbd7b4ed2032e0c03af86fe1826740ab61ddabf9072d06e57ff \ + --hash=sha256:536f963710a4e63934d80ac0dc4f478804a83e9a84b6828018f25d09953ada33 \ + --hash=sha256:54a115e5a73b8fc44f0cebef486365a1894b5c9760685d4558b72b7c3eb846e0 \ + --hash=sha256:595d020938c84e320bcf40ad71089e108eac0d377cd018e14a8c094f39e98d85 \ + --hash=sha256:66a78fe4556c60aceda5916f9eacd638b18e9e681016ec302dcb4682d6d4d034 \ + --hash=sha256:6b05c171afb3aa07adbd20abc00aea86fe375beb0fdb9ef780ec5b7f63bab1c0 \ + --hash=sha256:6cef4bb1706dfec49243c05d921eefb4e190d41e2528b30d8035ea1f36b4c24a \ + --hash=sha256:6f24021b9f22bc6301c37b196974a92c1c18dccedb6fef3dd252e95f2d6adbe4 \ + --hash=sha256:71b39d9f935b6ec0f8753e3e2afb51e3efba6f2e05b68b32a40754d24bcd4a3c \ + --hash=sha256:71cad2b2a7451ab79d8f5e71b453485b6775963d5cf794179144a7463fe6e8ec \ + --hash=sha256:76c2c1e6bfa5c84adc6434dfbf013aa92096a7985221762c8f11fedfd20fff58 \ + --hash=sha256:8617bbfae4486cf99c9f899966699428d19da931d06ca94ad3da986c76e15997 \ + --hash=sha256:86bff898a431c0fb71f7610b75726e75a54d47b37edc9d537f48de63bb3c0b90 \ + --hash=sha256:8e4dd766076855b5ff7ea52fa5f07ce26286726e0f8bff446b7739d02e6ea204 \ + --hash=sha256:92f30e89b8ee0ecf363033576c422b2f58fed6a80bed0aa48dff6d14c654663e \ + --hash=sha256:93e1f5447e2b1e479d7bd74701e84746b86450cff1fc368b132d195e2b8f8211 \ + --hash=sha256:9a37475425b431b4d060f23b4f52cd2f3aef6bc7c654bd760adf0040eec9d435 \ + --hash=sha256:9deb49575e5b0b94ed72c8a64ec4d033381adc27e9060ae842971f697ba96104 \ + --hash=sha256:a5fa86b80fd24bcd1aff83ad23be44ea323de3f787be8f8b15d4a65621e25321 \ + --hash=sha256:a6391fafaba97500887132cd582abc6e19452b1ac775a47caa7b24490e152058 \ + --hash=sha256:a72f874bc9e10e4b8f80426fb49716d5141f64442a0c8418065093ec8017fbb0 \ + --hash=sha256:ac7bb1c52d445bd4f8f7f97fefe6abc3a084dc4d63df50d79b17fa2b78e89297 \ + --hash=sha256:adc1ada2662f8a5f960b8a10d9986897e7499ef07e06d4cfe7197f8cce923c07 \ + --hash=sha256:b00eefbcf0f292945c4b4dec2ae845389ef5bcdcd596e6e4328051db5b5ba694 \ + --hash=sha256:b0521d0f4aebb6e06189451025fa17a913287b13c03d5fe05c017333b654ea5b \ + --hash=sha256:b5d93cf48f687479941d12b69c873ad2cc76bbd487f0091c2200636497f34034 \ + --hash=sha256:b7e18c623bb5c95acb3b3328861272816ba199fb531921c5d6d0b675f1fde9e3 \ + --hash=sha256:bd4cb9ad3c7889b9b3fe0a9a9fb5d2ed26f9879bff2608d9f01aed147a20d231 \ + --hash=sha256:be5a8381859b6da607c84f4f7d6847725f1cf1853ef8a2c9e115b7d58bef47dc \ + --hash=sha256:befa1ae5bd6030b3f512b43ff3fa5290bbed6b84411a44244b14adf835f5b89d \ + --hash=sha256:bf63afbe037eb5d2fe87fbcc7778e61da53ebaf21d938a4515aa73b62532a5d4 \ + --hash=sha256:c00abe94c1a69d75d827dcf1c025b25c8a45d230b3bcd77a9020883a1b047653 \ + --hash=sha256:c2381f82999704f818e2c987a865050e285ec3621262c66d40f5a96c8f899f8e \ + --hash=sha256:c76d5dde9f445058f83d0c02af00557a4db91de9a9a57c0df87d1535001d654b \ + --hash=sha256:cb189f09db39283b26bfd061ec16189e14f71c6755207f72a0f7540867afe5b9 \ + --hash=sha256:ccb32e0525d29e8b0572eb84c9a57af0e7a4e615726927506f55063c62414034 \ + --hash=sha256:ccbc4665079665c3cf3bab4db9f6b095370cd6437d66be549b6c2a1fd19e1958 \ + --hash=sha256:d1c89973648c85069c5046ad460f7b8a00218b29a2e42359ac8cc63e9ab94832 \ + --hash=sha256:df2d5874ff183595a4ba404edd04f6bd9b5505c1d7708573f6a6c17489a67563 \ + --hash=sha256:e01c918ac3d48e18a927cf7b14a26a3e29ff2bdf2eacb976da0aecd6a43ed034 \ + --hash=sha256:e6ab667ba76450084eb64013762c438ea76d9d29cc676dcd6c2e9892ba37f841 \ + --hash=sha256:e931e4f499e0dc7ef29d269a8e5b35dd722e5d14be07df6240166ea7c6532fae \ + --hash=sha256:f54660b0eb6b0b9f36e7fe1cdfdff472028dd0d14acd9b9b65098efbad059469 \ + --hash=sha256:f59a878c33d6b88122d80d239bb3b845d58708750b0cb06a09aebb9b18ec696c \ + --hash=sha256:f7fabeb6cea87d65f3b926de33d03fb016cfdc29314c90974383b5582ae72891 \ + --hash=sha256:f9579f383d1bf9df80081e72760e84960a7fd4f88cf0c9e535a8597c9bb646f5 \ + --hash=sha256:f9a2353b37a1a9e78fd82b27ad7e2a32a2d036604d18f02b05e3136c62ca3b09 \ + --hash=sha256:fc36dc566135b5eceec4cf89758fcb719266a019ef07dae1754ae7c9f617ef3e \ + --hash=sha256:ffdc76bfcae6b255dff75202c5e7feaf95b40246bc0a17944facc1fecf9f79ab + # via + # mlx-audio + # mlx-vlm + # opencv-python + # scipy + # transformers +opencv-python==5.0.0.93 \ + --hash=sha256:08d5d91d967b58d6db86073b2ad3eaef88ca4ebdfd45c9059bf59f5ded0c7ad2 \ + --hash=sha256:198a75138241810206a17c829dbcc40a7cb1841cda538ca86cbbfc6c7d95f898 \ + --hash=sha256:4b4b1a34c79bf8d3738e3cfe9a9e67b51a79663f6b692cbdad8c31f570da4157 \ + --hash=sha256:66aac3e5b5faa48d4025816592f3af19e4bfc2c68dec067bae2dbb4ca10aa9e2 \ + --hash=sha256:6bbc32f59e1b1a7db7b39c81f63d00625f041d333037fd8702f6da52cc39108b \ + --hash=sha256:c8de2dec111122a02e8beb28e16c31904992dfd6186560b142a92c71403c1039 \ + --hash=sha256:e2b4272e736836f66c2d176e43ab8101f3a00d45654916399f52e150c58981ac \ + --hash=sha256:f8b6d0a212253dd26ad338c812f1f23ca118fdf05a9c8c6b9444f161aa8c5881 \ + --hash=sha256:f90ba04b8f73bc5c3814037699739f0156f597338a98f05956c684e7c3ca10d2 + # via mlx-vlm +opentelemetry-api==1.45.1 \ + --hash=sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75 \ + --hash=sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb + # via fastapi +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # huggingface-hub + # transformers +pillow==13.0.0 \ + --hash=sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27 \ + --hash=sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782 \ + --hash=sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22 \ + --hash=sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15 \ + --hash=sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502 \ + --hash=sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692 \ + --hash=sha256:10b5019883c98f1dd819ca48ffc6f33a91cee04948fe637c605f4345c7cafd6f \ + --hash=sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8 \ + --hash=sha256:12fbb851cf323cc19aac3eea1df5ad17b57e91e9fe3cfd06100dba7bb8208c13 \ + --hash=sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3 \ + --hash=sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369 \ + --hash=sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88 \ + --hash=sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d \ + --hash=sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb \ + --hash=sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f \ + --hash=sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0 \ + --hash=sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40 \ + --hash=sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599 \ + --hash=sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055 \ + --hash=sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033 \ + --hash=sha256:37290fbef0c73e2d51d594eb78fded7dc7170d1c3ad0a890269e6fd8c8c4e061 \ + --hash=sha256:405be6058e206a86fedce2f715ecc2343f94072d7925251632dffa2c05f70dca \ + --hash=sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f \ + --hash=sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca \ + --hash=sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c \ + --hash=sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4 \ + --hash=sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8 \ + --hash=sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620 \ + --hash=sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071 \ + --hash=sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc \ + --hash=sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a \ + --hash=sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6 \ + --hash=sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e \ + --hash=sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582 \ + --hash=sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f \ + --hash=sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79 \ + --hash=sha256:6a0920cb55df5a4d3f6862964908078c0bd59332bdd1bcd20d14e8c70ef27cc5 \ + --hash=sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683 \ + --hash=sha256:6f38f3f885a09cafc93fd667b1173a92f379e43785d635e7983c893f0da319fb \ + --hash=sha256:73ee25cf8cf8b75d5a5aeed7cb6588f5e5dbfbb0fde51707b630bbb23897645c \ + --hash=sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa \ + --hash=sha256:7c88c10dd432f3b39ccdafc9ed81d0c21b8fd634502574da76d1bfa4fe2c76d0 \ + --hash=sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521 \ + --hash=sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e \ + --hash=sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c \ + --hash=sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058 \ + --hash=sha256:83b37abd87b590acd8f47ca8a61f70893322b11834d36e07c2298dacad4d14d2 \ + --hash=sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463 \ + --hash=sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d \ + --hash=sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81 \ + --hash=sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf \ + --hash=sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63 \ + --hash=sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0 \ + --hash=sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42 \ + --hash=sha256:a192360d512305fe92d3053e4f7627bbad1f7708f1f4ad38cccd9336acabe9ad \ + --hash=sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81 \ + --hash=sha256:a625df9a2a5597fa5e5266e01aefc82604bb85d291e3547981e863560d75da6c \ + --hash=sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4 \ + --hash=sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981 \ + --hash=sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43 \ + --hash=sha256:b84be41387fc00db28c13c02e30d42b02d8d8f6a4a741a1aee6012ca7f1f6039 \ + --hash=sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76 \ + --hash=sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51 \ + --hash=sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2 \ + --hash=sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630 \ + --hash=sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac \ + --hash=sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510 \ + --hash=sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6 \ + --hash=sha256:d7710eb676e9bb68965cb045be0d333b4c0d7f9dfbaa7fbf121385b1588683d3 \ + --hash=sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d \ + --hash=sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61 \ + --hash=sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932 \ + --hash=sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1 \ + --hash=sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d \ + --hash=sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00 \ + --hash=sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe \ + --hash=sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313 \ + --hash=sha256:f391dc92a015c478745192fe4f2e120c2af2aba18d9fe0616341cbf537599806 \ + --hash=sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a \ + --hash=sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0 \ + --hash=sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52 \ + --hash=sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b \ + --hash=sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe + # via mlx-vlm +protobuf==7.36.2 \ + --hash=sha256:497d0463ff3316681da6c0b9e8d06cb465d61abce00b613ab42226175644d1bb \ + --hash=sha256:89f23aa53c24553a2416fd4fd1ec06f74fa42b14b546d8883128813f775bbfd2 \ + --hash=sha256:912c1221170e16c08d1f086762f563dd61ff83c18b5fa6652952dfaded66f728 \ + --hash=sha256:a300819d441e078a5608c0d3c709796bb548136058fda017ae51d425b44fd353 \ + --hash=sha256:bdb3a345d48db958e6ce1f18e508beb0cc981d64f24088427549c866cd039f1e \ + --hash=sha256:cbc70b17ee27e28894c7fee8bb04be1abead49e936bc70eb60052531eee2079e \ + --hash=sha256:e11e1f0180583a2af89db6a2ecd9e8dc40aa6d2988ca175bfd0e6d12ea72d74e \ + --hash=sha256:f4fee11ec330d238b34a05c9b675f693c20415d1c5bd7d5320cc2f8a798eb9cf + # via transformers +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pydantic==2.13.5 \ + --hash=sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73 \ + --hash=sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08 + # via fastapi +pydantic-core==2.46.5 \ + --hash=sha256:013d6f3483d81e02e7c328831808f336c8596ee33b4bd4026b9ffb1e960b8942 \ + --hash=sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821 \ + --hash=sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5 \ + --hash=sha256:0bddb4020d8f04175865ccd17eff3040874fc11fb593f424edb452653b4b947c \ + --hash=sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184 \ + --hash=sha256:0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2 \ + --hash=sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc \ + --hash=sha256:15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5 \ + --hash=sha256:18a09e1e1011b462f2e32774f25859ef1223d5c2b0546a633cf56654710721e0 \ + --hash=sha256:193375f3548919d3f0b60936ca113ada3e38f264f91b9b8e0508efaad57be931 \ + --hash=sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e \ + --hash=sha256:1e449def1945a462c464331254e5a44fca7c3b4f9aedf59ec2f50f8066dd8e25 \ + --hash=sha256:1e5aad1220a1192c42341c8fd4a8686657e73ab2a920c970bdc4de334fe3193d \ + --hash=sha256:200aa3dc9f8d54f0754f43247c0bad0999fdcfbfd2488384dd44f37279271fe6 \ + --hash=sha256:2471fd51c61c610e1dcf7de44d7299283661654d11264ab4802b303368d69c47 \ + --hash=sha256:24922243639cbdac66c75fcb6fd6495a9cb52b213d62f9a0d16f0310b1ff8038 \ + --hash=sha256:28a6a556cd3b6066bea827857f9d9cce027c96f776e512f544a581f9e42161f8 \ + --hash=sha256:2bc9419666990c06d7397831f2126a1ecc3594aaa3ff7de5bf2d066802f4e07b \ + --hash=sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a \ + --hash=sha256:2d330aaba8621b1edcec8ae2c4050f63b84ccf6d98723a8f212e9684713abf0e \ + --hash=sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074 \ + --hash=sha256:337639ba62a11acde6ef3aeb08c8ea755f8ef1fe5e513356c0f36a2b0d7568b0 \ + --hash=sha256:347ec774390c87326a2e4929d58d3f7e8763a104d5d35f4cd595a4c952366433 \ + --hash=sha256:356c8368cbc321050b169595683a2e1d63413b1e0e2868b330af9fc14c616d3f \ + --hash=sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034 \ + --hash=sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21 \ + --hash=sha256:3a3e26b6a8274211bddee2d0e4d0d42778f17a34510f49d2ec44b58abfc41736 \ + --hash=sha256:3aa166e99c4f2985407fb8714aebede877ecb5455cf321b606adca926d30d5a0 \ + --hash=sha256:3d2652072b2d774947ba5cf78a9e59644ac62ee572daf6dd2e1dfe905e15b2b7 \ + --hash=sha256:40375c2d05acec10323e45dfe2077ac44bc74659008614af5069034e2cfc781c \ + --hash=sha256:413a717a410d0c817ef5b786a059415550b3794e1d0c2abffd9efb93a3d9f7b4 \ + --hash=sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c \ + --hash=sha256:49776eab08766a08dfff7012f8b422dcd7e25e43b316eedf0477c24fcfa84b7c \ + --hash=sha256:4d44cf99ddebf875f9b68cc267aa684c99b7b44fe63ee1cac4ec163807290069 \ + --hash=sha256:4dedce55295becb61921e386b99d4f2706045306e7fa52249a33004c837379fb \ + --hash=sha256:4f8507560a9284e1370bb048ed4282012fbef4e8d109875b95e884d228552061 \ + --hash=sha256:4fdc8b93a41521988916eeaa271173fcca7fa0803d62f87675aac8dcec1c8e29 \ + --hash=sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3 \ + --hash=sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa \ + --hash=sha256:53feb344243bb9510a9dec7bf3cf1b64d88a98af5dc7872a5160465f8b198c8e \ + --hash=sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38 \ + --hash=sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f \ + --hash=sha256:5cb482e9e84c851f4e623fe4acc1ced89168cf1fe18f7089db4548c8f5bbb65b \ + --hash=sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8 \ + --hash=sha256:5ee239d575f80b08eca11f6e20f90c4c695de7825c67eefe6091fbf20dda648e \ + --hash=sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c \ + --hash=sha256:5f93c5fe914d75fbec9a49209b00da5f08e9e467d69da2b1510c81940cfd10be \ + --hash=sha256:657b40d6240c0a7b6a64b30f22d1e3aa631c7e846c621b0c0f6d1d75e2e15ea6 \ + --hash=sha256:6d30e1a4f138b8951063e9a394752a9179b51da288ffa507b1e659222f4c1793 \ + --hash=sha256:6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1 \ + --hash=sha256:701b2e04b560eeb4bddf7a25ab8ca476176e34fdbd9a0e18196f0d12d4685f0b \ + --hash=sha256:771cf63ae0b1b50dd22e5f3e3549fab5f3f4ff1635d352a9e1a97fe01c7b2e64 \ + --hash=sha256:79bdfa52f843137045b2d081cc05c120ba6665d29b7559c2c47690906f39279f \ + --hash=sha256:7ac031912d54f3d83ef3b3eb98dfabc1608802e2202263d25957eeed40b94761 \ + --hash=sha256:7b0fc826b16c55e561e5d2a0c5c77b051ba1d92808118c4e4b5390f5e0cf191d \ + --hash=sha256:7c6be839a5a8312626b32029a415644a0846b420bc8b52b95b28cd92da162168 \ + --hash=sha256:816ff0a6550ffc06c098ccd2e0698600f9aa7da192a79eaa6f9af504a35db869 \ + --hash=sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111 \ + --hash=sha256:837b396ca3d7b74091ca623f6cbd8351bd42d670a79c2683e79fb089f06a2de5 \ + --hash=sha256:850a08d167dde16db8702c274f320c7be9d7da6f6dff2b58b18f9e815bd94f5b \ + --hash=sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7 \ + --hash=sha256:892a881d5f68c2b9ea304b7a6c2c60d9343df578a311b0f86b94bc8f1ffe8129 \ + --hash=sha256:895395f8918627b04efb1ad2a4cf605387143300ba03304cd1dfa6d03f5e095e \ + --hash=sha256:8b10e3e8fd7ddc2bd915848a2768e44c15b22936f1cc54c462ad1164deb02655 \ + --hash=sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c \ + --hash=sha256:8feeac04b5794e513e710af2f9c87d49f31a6dc47967bb264a1fed61a8989bec \ + --hash=sha256:9432f3598db432cb51c5b37fdbf29a60fcccc79e30d37a05022776a6bc4ab689 \ + --hash=sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f \ + --hash=sha256:978e7b97d4824b5be09c69fb70507cbde3b0323fc147332ca40a94d9a6a0ebbf \ + --hash=sha256:97bf8de4d541598c94a59344eeb988a94c08ff76b5723c41f6567ec18c7892ea \ + --hash=sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9 \ + --hash=sha256:9b68938dd5b0c783d88ff8e2dcc69451b5eb936fe212d516b21b9d5567f6d464 \ + --hash=sha256:9c4b71f10dd532fb7a5cbc8f58707779e64f03a258c2bf8bfbaecfcd9970b519 \ + --hash=sha256:9f47b8a949e60f027f0aa0a6f6c7b7e9c55cbf4380d10b344e282fa4e7ab1e1b \ + --hash=sha256:a1dee1b804ff4d11c663636cf15d2ea47e9f79cd56c033fb1cbf08924842a48f \ + --hash=sha256:a2468d93d181667a7abd66e1b64bb9f76f361b0fef8faddf687456453576f5ee \ + --hash=sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a \ + --hash=sha256:a39ac25a9a2fa4072efdb429833c4a4c8009a51ff9eea3eeae131713cd27991e \ + --hash=sha256:a445486499897b88a7d6c310c88ed64dd37b1b59bfd7ae9107490bbb362f47d6 \ + --hash=sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2 \ + --hash=sha256:ab4b66edffb32d9e951efb3814bd104b8367a7501b81b955cacb5726d897389f \ + --hash=sha256:aca6c767f552b21b10f774aeac128e828eafb796adfa1b666a18bf6321453c3a \ + --hash=sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f \ + --hash=sha256:b10ec717381bdbfafef34607824db4c91de69ff085e4fca3b2af91b4fa17e68a \ + --hash=sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47 \ + --hash=sha256:b6acfb46a814762367fb7ba0828b0a17d441b92ce249a0e007474c9072662dda \ + --hash=sha256:b7ca9034437b6022f941f4857459562ee00a560b97e7cce8a0ec5a74fc6766e0 \ + --hash=sha256:b98134087d9de723658d17a42c7d0da8d6e2ef08015dee7dc93889047315f5e4 \ + --hash=sha256:b9fe6fb92520e3fd61f2e49000b6911b188824f089b75973ea06d6267f0b476d \ + --hash=sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3 \ + --hash=sha256:c14ad3bdc85ee7f318742c457ca3968a92126d144b15721c759033bfb06296c2 \ + --hash=sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355 \ + --hash=sha256:c3471e5c4a949c26ec00a77f01df59096aa9495877de76fd60a980f8ee6be461 \ + --hash=sha256:c583b927a8838dab890706a6fa7573fbb8b70e24000ef9f7238e2d6f6435a5ed \ + --hash=sha256:c76fe65e607be28c7fd4d56fc3c42b1583aa058ce3408b7ad0fd540171d31f9f \ + --hash=sha256:c7ea57fc63aa7da93a1bd2d644e6577befae10c52c4e36377635eea1056a74f5 \ + --hash=sha256:cd5214352ae68f3b5e9af7768bdc5253695ee069675db3480518420b3be881f2 \ + --hash=sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829 \ + --hash=sha256:cdc8b74ecc48c0cb1e9607a05ec4e9e88db60a19ffcc9a1d5f9088ede40c8dc0 \ + --hash=sha256:d0a24b40877af2de4950252be9d21eaf7fb07660f3c2cae1f56c6b599ada5266 \ + --hash=sha256:d22a945598fb91236b4dd793a6e42e4f3dd7740bb5aace5ebd7d4c08d13bb575 \ + --hash=sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290 \ + --hash=sha256:d625a186a65201c23a9e3b8ed9c47e90a026e03256608cc91851c6709096844f \ + --hash=sha256:d925f3d9afd05a8c0fb3a1031463a8d59ebe5e2afad297e29c78be19e13b4e62 \ + --hash=sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f \ + --hash=sha256:e652ab17569c94bff5475520f907b7148b8c24036a8ebbe5cf7cf7493d28579a \ + --hash=sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7 \ + --hash=sha256:e80675d75ae2cd14372cb65cad5400d9347a3d3f6c13000183f22dfd027283ed \ + --hash=sha256:e9c134bb666dd54b778b9fc0d2b50cbb7f979b9e3716f26a88c9ab3b6fc1dd0f \ + --hash=sha256:eb7d8d0e5886a89a55d2eef490e272fa965a9d57c6b29a5b5088a7997ec2cad1 \ + --hash=sha256:ecb42011e12ee19cafbc312887cbf3546959fe02fbad44f272d4be5baa997615 \ + --hash=sha256:ef3fbbf161dc9351a2fe0422e51b129f9e97e42385bd0320b309c15f7d287dd8 \ + --hash=sha256:efd62a42486f1bda5d24cb4f63d15a3c7768375fe83d36f9417b4ad7a2fb20b3 \ + --hash=sha256:f077d0b97ab11fa7dcc633fca53515f290bca8a8a633e966d5b6d1879d9ed01a \ + --hash=sha256:f332f0e72a5a0400141f830744e141bf9f97917878dbe968669e8a7fefea78ff \ + --hash=sha256:f7b0ec93a2893de856652154d73b7ba622f26fa97726487dcac373de5f4c6084 \ + --hash=sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0 \ + --hash=sha256:fc5d783bd4a2387e97b8a2d5ec781cfb92b3d893bf82370548e99db5915935d3 \ + --hash=sha256:fc8515076c11f3cfdf4fb142dcca0fe384b1230a3b5415458ac84f3e0903ec13 \ + --hash=sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9 + # via pydantic +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via rich +python-multipart==0.0.32 \ + --hash=sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e \ + --hash=sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23 + # via mlx-vlm +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # huggingface-hub + # transformers +regex==2026.9.29 \ + --hash=sha256:01000ddf0e3ffef97f2413ceb514f6313040106b6d18a03ee00a4fe35c1eb1db \ + --hash=sha256:0166844493626c5015c6088ee15c9ca2fd060ca15b7641d1657da6a58432ae33 \ + --hash=sha256:044265d77d94f5e3cb2fd72c76723807c429cb8c533e9d4672d0334a6f14f588 \ + --hash=sha256:0476e5bcbe6e1ba3d1c4cc7bbb1c3ba78e3b979b5c8a88d0a6a8cdd4992b8c84 \ + --hash=sha256:066d0e3dbfdd739bce2bf8c2a41dd16f73e3d8adc2eb06dd803a36a307f56075 \ + --hash=sha256:0b65c72739f981377c9c22e0c5c3cd7f42da7bd8a3c9209330fac772c7d893ed \ + --hash=sha256:0c992c19cd45058a4b92f68f139c93db168b48fb1f322c9a7cd620806afb6b51 \ + --hash=sha256:0cc63b5e47c12a48d90c7e9d7de6a035dd14f62868aaedbb4e0ff8ba2b8bfe7b \ + --hash=sha256:0dd8af32e9f7b56b7f95cc1fd79b23054c3bdc172392ae560acc24d57b7ffe71 \ + --hash=sha256:0def9fb6abac55492d6d51cddb7225d07d6f279e774e0adc08569a54a5fc8d46 \ + --hash=sha256:0fd2c901cc307a745ad4bc87f20060d7a0825a3371d1e93488af22e7a387f78f \ + --hash=sha256:1043aedf5917caa861bcb25a9c11460049656bdf0017a90a309fa8f255467725 \ + --hash=sha256:121a76a0985db80ceae9e171c337f8c927868e37d01b54e3ce87bc87f9c6a208 \ + --hash=sha256:143533cc4b6fbc5b95aca0a5b8d541088d374831593def000ec89322c220221d \ + --hash=sha256:14e953ff3607c92d7675bf79c4d4509ef6782aa8c08509f179f9b3d6d0679e86 \ + --hash=sha256:18ae8eed4526e35bdb754d61562b90bf5c00a67fdcf3cc1380dd59597486631b \ + --hash=sha256:19959129885356df0e97556856f77eb2888380dac18bed075a7c05c5128c618d \ + --hash=sha256:1ba8c6a416569ce0d37e83e28a254a61dc99a419084dfb6476cea02d997f74fa \ + --hash=sha256:1c2a0026062abcc321a53db4a185ceba0b59a66b5d37b0808917a88b55a5257f \ + --hash=sha256:1d9fe8091b2e89d470df68a9331111ed008ae8aae6bf1e8e1fba4086a495c84e \ + --hash=sha256:2089fe39c406784d90101c726755ffa1497bb74638fd434300d2b88006186de8 \ + --hash=sha256:23ae6fdad9e63e54038f5ef78aba2933faca61e24d432786589e737bc5522ebb \ + --hash=sha256:26ec4ccce55aa533fbd603d08911b01101a8fcfec987845ac3ae2c7087b2bde3 \ + --hash=sha256:2f7f7aa47b229f2b39a2ae2596d2ad5625d77b5eb9856fac2dab3eb506cdd0a0 \ + --hash=sha256:31b003f9a070335e2a8233ee9b14a3ca8e6d792012ae011f741bf0aaf11744c5 \ + --hash=sha256:32ab11df9677ca80bcbb5fe4eb1da9109a5019239a054836efc6fa1c64e683cf \ + --hash=sha256:33026515aebc0e70d1c89978e53e8d695d35d9e472f8d5b34465ba3c74028650 \ + --hash=sha256:34b6925af9853bf461950e6508910f179fd6e9b1a7ec8548e069606b7e51a26b \ + --hash=sha256:352cf115a810b357caa35193ab656ecf5ef41056855e82f292c99e8514f8d954 \ + --hash=sha256:39ab5894d971f9ac68baa6eca5c50387db579cfcacf36ae8df3feceb1815e6d0 \ + --hash=sha256:3a21a9509d0ee88e7a70e1ad228cd2f0e0fd1e187458db132e8a8d18c97daf9d \ + --hash=sha256:3c5c2ef13797466aa64170cbb66ad98a32351dd4127694cea7199f80f213750d \ + --hash=sha256:3e778bfccd63075167709136afbc251c1f683758d5bf49c803c60ac3f894ce6b \ + --hash=sha256:3f1e6cb402a89457582cd696f982559217d13484a193202c394015297968c86d \ + --hash=sha256:42e82e578c904445d4c8a35b8f28052cf567593215fa5db06266fbc6f77aaa2e \ + --hash=sha256:4408b2b27a95ca8cc48b7411945753773353b5c93b307754781086c99d3a576f \ + --hash=sha256:446654b29bfaa30500d80947eda42cef1449dc8a87f4e3cf061cc8485d3a1f0b \ + --hash=sha256:45010bcfe66df41522d56c9b6114e87ecc597a08970ff6a2ced24415c141ae5f \ + --hash=sha256:49ee178ca31c94621294bf9b8b676a92a2e6bba8af0529591753719e57edb621 \ + --hash=sha256:4d7d93613b01b0199961330e49cfc52d479b3d5776c56c691db31130c0a07d91 \ + --hash=sha256:4fb41211d2333eb930a51e0546a65999761cf1f572a4da56ef9b8a62966c06f2 \ + --hash=sha256:4fe97894d1b306c919b4e50def1e6f6c522f4d03a7283811f4d108f1ce5d3ac2 \ + --hash=sha256:554bffadcbcb6d5f4e5fb10a61cc52084b9a63d1dab5f10bcd2c4343972e8e2c \ + --hash=sha256:59b49507f47479e299a9e1bc41b5cb83a7afda0540625f1dbae886615978acbf \ + --hash=sha256:5eeb8edc6110d9194a4d0d54610f64c37a31c605b5dbb7e407fc6ec7fa34a4a1 \ + --hash=sha256:612b709381c0355b70d89cdb51b7f670591ed5cbbc0e3b5337488019dc667b65 \ + --hash=sha256:61956f074ecd123f55adca68ee3eab46e6a07ad3f8e64e6db95dfacb444f55c4 \ + --hash=sha256:6398d5145689503412cc1748895242598d8846b8967b851133b20dc2ed1e21e8 \ + --hash=sha256:65b408d8fcb273e3499e7ef2ce796810da1becd208c7fb4373692a242d79d461 \ + --hash=sha256:686ac5350fceae63830bb98805fcb8039325bf4c06d9f6f048ff65229d5bffa5 \ + --hash=sha256:6a1a824fbed817e0a891103886b68f063b1e83cc51bc97192a90a60195a9291f \ + --hash=sha256:6abb75ab16bc3281714a5b99548a2225db70dba1f995f6d7f7419b76eb5a8fbe \ + --hash=sha256:6f7121a8914ed13fcfe2099f895341bfb789f004d4c5a0bdece8fa667da10849 \ + --hash=sha256:7020ed44df30b3aa492c00ee3b52d0548c1f30c2c6c5bb13ae897680900d3413 \ + --hash=sha256:720537c7ea6f80dc61913184edb0ce2497a306b39ef19f28505b322553d52bdb \ + --hash=sha256:724184b4aafed865e4f13ca313fdcb43024300c028ec67319cfa16847d84685e \ + --hash=sha256:7c03031610e3e6ed1768a2b7a8fc84637c1257b50c5eacaf094c6e17a84fc563 \ + --hash=sha256:80a5ea3b4fd9d6a5b9a44f7976a9acaaab35aa3c1f6b29e5bd857dfabaded223 \ + --hash=sha256:80c7cadd3fd2bfde5df8aa0787e315812cad0c313a753095d02f4c2b6c01677b \ + --hash=sha256:80ea96f5c1a30bf09007d48466521d9c294bebe197c708c3359096e3e3691632 \ + --hash=sha256:864e9b87ac33c3fb9fb4ad48166d4fdb579c351d5c77deb0d34bccb36a775cd9 \ + --hash=sha256:87fb80cbe3557e27e7b28b995c2b2eedf689b8886f941ab93e0e288f0976518a \ + --hash=sha256:8873c4a11c50b9989168881aeb3f08859f469d809941866aa1feefd8be5431f6 \ + --hash=sha256:888d60953908dcf761aa320c3e390ab8556efbdb551ace63921de90f6ae0848d \ + --hash=sha256:8b5fcc4771732191b2b7d1dd68d8f0353f47f8d90b6150f6dce58bf1112442cb \ + --hash=sha256:8f39588af4731c8923c26810eb3b33f76f17633985e40f59c3cd45a33805a895 \ + --hash=sha256:9173db3be74a35cb6731701094b98120f7ee4876a287882a59cdea1fa7da342f \ + --hash=sha256:92f05c9c42bde5785dc48770bc2194d9f7442544156f951e19cd31b096cec562 \ + --hash=sha256:951733b1bbdb71e377cec567b409f1a7881b47cfcad84121aa74cb575fa425ea \ + --hash=sha256:957bb708e8057ab1649ba566456429d691ec9b90d1c9ad1af1ba7ffbbeaf05f2 \ + --hash=sha256:9916fda742cd4eede63b286f58c06718324265d727ce0856eb1aac86d0d150d6 \ + --hash=sha256:9e1d3a4cb7993b708f0ada8d0c84590efd853f169e7147d2202c9da503180242 \ + --hash=sha256:9e4482589065c8ecd761cff522dcd85f2d39e62f551e37e025d1c7d54772def3 \ + --hash=sha256:a5300757f8a68f5b6cc33f57338d72a0e3589c5cc9ad5f8504ea06f028be582a \ + --hash=sha256:a540abfab208e1b7ef2df231c40ef3b6cbb30a0aad6204e9b6a81c10a6794628 \ + --hash=sha256:a5758353650079898dc1b2b0e95aa51fa23a30d020e06f62c430dd08ee56cdd8 \ + --hash=sha256:a64b85a4760337cfefdb27d42da6ed8b58e8cde3f2d57b6ef43e76ef6ea9ef47 \ + --hash=sha256:a655d34b2a6943af32401f3d94f72e9d731f6ad16285815550bf2b4ee69d420a \ + --hash=sha256:a714befaacbd10092ffe4cea0d3c5f008fb9efe9bc322c715bcdfdee414b9a3d \ + --hash=sha256:a760da040b47767b4b873adfb7c3b691e9ba2fc60f113f9d0b88f1a62f323e85 \ + --hash=sha256:addd736a0547d553283adaf4e05d7104e7f2c7b0b092e9b4d28756825f14531f \ + --hash=sha256:ae4613d7d9dda60fcba95f846cc6f808017f1843f392cf9daad14a6534493d71 \ + --hash=sha256:b11b589e00095ec69cf79841a76360f9b079e95b0368a25b5ebb951ab0c157ff \ + --hash=sha256:b3e445b66c80b4eb4234e855ce94d9adc183eedbd632816228d89930b91b2c5b \ + --hash=sha256:b7b893976e7fe42053da64f2aa27239c24252fd2ec6df471e1be197c0addc3b1 \ + --hash=sha256:b84f186a7f0536fe4ff9a9fa12d06d007b9b71d4b5352ddcc41f59ad6522a312 \ + --hash=sha256:b89efc38431793d28b7cd91227e2f952ad7c48df19132b17f43a5fec3c14143b \ + --hash=sha256:b97a38fb4c732b6832db6bf108963adbcd82ef1268ba2025dce390f45af75efa \ + --hash=sha256:b9d74e4eee9ddb64c2e92d5d61472c59c21684c059eb7b68767be9628e977859 \ + --hash=sha256:bb90e7177944b6684738c1fc36aabd2dd00d1de3be7dbe09f91e196f1bc0dc81 \ + --hash=sha256:bec37990e3d6121f29ecfb594bd8f1bf009e9f7926daba2e50e3b27d3892a783 \ + --hash=sha256:bf3c49863c23a1ad6da9c30351aed6cff8d5ddbeb63c5c8420ae54e98c7d0138 \ + --hash=sha256:bf48516e35cf848390ea68850aba53e7c333720d2945b4d2c25b69fc5171723f \ + --hash=sha256:bfc71e6d970419c1309b3640305298643e2a734cad3f7cfb6d2ddee4175ab53d \ + --hash=sha256:c0094897d7d01f184b2d7fe8c56c66d64efe01b31f4b7d34205b391387df1111 \ + --hash=sha256:c03c6eb6ece86dfdcbb34799efaa339b093132e1aceed491ba5e08fe06cdf699 \ + --hash=sha256:c1a9a6651197fbed6f0212591418b9def774fc3f8324f78d1bf0e6a63e5f8aa1 \ + --hash=sha256:c3589f40749acce747510bf5d589d54e376cb0930ea58b35effac97e5312b0c1 \ + --hash=sha256:c4e38dd8f39c43a91d2410ad2b85610701b0979342c3df1d69eaf8e838c757d8 \ + --hash=sha256:c6c8fabf1dafc1f1ddcbb67896d3f93efb092e8c4b6322d7389b944e76a484e5 \ + --hash=sha256:c90fcf7804ea0a54b896ce0f2b9565350220b8d4890fd0db461a476a4c687963 \ + --hash=sha256:c9b602fae1e00b7c035d661ce85575365719192a7b46784bd71cf64c68053aa0 \ + --hash=sha256:ccb64d887a9db1cd76dbc0f92051a1a478a2a67e7f56c62d915cb881d7734704 \ + --hash=sha256:d06fcdecc10fc7954d7c8f27a03c96055fe525274dc84a7b0dbdc3d6b9e03dab \ + --hash=sha256:d0c3082bf79bcd6a614d55916590ad4b8f93200e10b97f463ea5d9d07c9b5f23 \ + --hash=sha256:d49c18f1ea294cf4adde2e5ac256e98c82ea9d708462ce4bf799dffa7cfe8a2c \ + --hash=sha256:d60030baaa7bfbb02d650c126cdcddcb6e33dbff14d819434c8fa2fdcaeeeba5 \ + --hash=sha256:d7cab119d0df0b9413f106b4d7fc34f2872d3574ed3806fb48959c830b1537da \ + --hash=sha256:d9b77b25b4f395f92de6099ab08e8ae2bc7e51dfe157f22900902243a5cc90c7 \ + --hash=sha256:dabee8f4935e731fb46b2a3091bdda0d3d94b3bbfb907d2b4f12eefce4009619 \ + --hash=sha256:db5e82ba15c142425b8406690032df89e39cca4a2e8afbbb9a3d84edc2373ac3 \ + --hash=sha256:dc79d36d0618752265f0d575915bdc5c5130ecb9c9f6b3bcefeae32e4bdfafcf \ + --hash=sha256:ddfa987262763c3c22a8367d2a49c244b018a74c3a8e3ab1a864119ad45c5633 \ + --hash=sha256:e1172147d28d8fbcf8cb8d26c41506169f5ad8fe9ec969cb116835a19d4d8eca \ + --hash=sha256:e11edba5bc344a32b029a7af9d4b3173982dd79eeafa0b9dbd787364414b0509 \ + --hash=sha256:e2c89e9b762c57f59d5e99ee8b20202adb892e35f8d3485741340999ca55058e \ + --hash=sha256:e31f72490b7c12f7790e1e25c3afffd20503ee1bfb43461d7838b871ff244b19 \ + --hash=sha256:e8c65ef3862a8ad6e86492b6ed9327805dd66904c012bd3649dc67d822ed6c34 \ + --hash=sha256:ebb8912f565b8cdbbf27debfe00df04202c20e2f651b9e32767930c5eace3621 \ + --hash=sha256:ed511a0708e2297e1d6431e7fb217e3402791e491e02da800658ace4973df1bb \ + --hash=sha256:edf06545875f3efa31560d94121e95c7fd70d98b1dfedc0157097d79b13b52ea \ + --hash=sha256:f0fe9834e5aeccaf19a0d8feb296d66a24be1a7c9922002f842a682cd5abb787 \ + --hash=sha256:f1a0d5117230dd46b399a30a38afa44f79c99f3168988fdc4f425c3f928b39df \ + --hash=sha256:f37964e4a5e993d2fd45147741e9dff7f34a2d8c00ab94c4ea0514a4677f959e \ + --hash=sha256:f57dc6b8fef170f105d2cf5cdce254f47b137d7755086cf7050f47e16582abba \ + --hash=sha256:f93bc1c3486ef3747e07c9d7c1d0a147b8fbaab975f80e348aed6f71309dfaca \ + --hash=sha256:fb00027a09a8f9f08028b40dce4c933cf73e4833240ed356583fdc9cfa721566 \ + --hash=sha256:fb99cc9d45f48895d9d67f6a0b8a57f08d39c174d9f25ad97a313e0470267b1c \ + --hash=sha256:fdd88ed5e20b1bcdd234421e454962c971aa44b653bdb7f1ea9ef683e90fb649 \ + --hash=sha256:fe3fa1dd453ed5c7f5ea23a26218329790ed7197a99b90e94330e313959a7f52 + # via transformers +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via mlx-vlm +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via typer +safetensors==0.8.0 \ + --hash=sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358 \ + --hash=sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f \ + --hash=sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d \ + --hash=sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d \ + --hash=sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0 \ + --hash=sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc \ + --hash=sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235 \ + --hash=sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98 \ + --hash=sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4 \ + --hash=sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846 \ + --hash=sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca \ + --hash=sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0 \ + --hash=sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25 \ + --hash=sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452 \ + --hash=sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d \ + --hash=sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78 \ + --hash=sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774 + # via transformers +scipy==1.18.1 \ + --hash=sha256:011413b7426b75012840e35649e00fe0a2c3bae89fed433876e3a99251572efc \ + --hash=sha256:0ac49ea97594532dd44b7136094d35f5440fa06e6d9c6384a74c01764df388c5 \ + --hash=sha256:0e82073ecc7acc6436fac4b31674109c7e1d3e596789767eda01258a8c9e8123 \ + --hash=sha256:0fcb3c93519f27bb4f0c4b0f7802cdcaca7fcf93267b75edda2e9f4e8a55cbd7 \ + --hash=sha256:10ac20c69d880f77f375db44c22e3e6a644f9fefa291d4cd2fb9790a89fc99fd \ + --hash=sha256:11c423f1049c5755ad4409af52a9ada1cff96fe9b50795d4af3619f292901239 \ + --hash=sha256:179ce34a8d0fe273d8883ba59e17e052247d08973dfcb743ca52bb1cce2d60b0 \ + --hash=sha256:1bca3b943fc2567ea49cd02c99abde49da4d5178ec46f624bd8255cda8755beb \ + --hash=sha256:1d73131e358976663dd969e1fb4ed1404b815cd977eaaedc3b3a133ba2d81c35 \ + --hash=sha256:2a0b02f9fc46f8520330c23d45e6560db7e3a0d927232139427637f98943e11d \ + --hash=sha256:2d3ab0e8c69a17dd3559eab8cbb88f258e285c94d572c2719033f90f83290c89 \ + --hash=sha256:30f464bee641fa8e282577c7dce027308403213c6ca8270bba73285c91024bc5 \ + --hash=sha256:33a834464fdabc0f26a45508df31b3cc5d028e04dbf6c5ed398541418e0a12fe \ + --hash=sha256:3ab3523da44749156e1f68b464dc56af11ae4cbc5c739a49d05f32b982eca9f3 \ + --hash=sha256:3c085faa2cfa879c5141df483f836f4d691045a078224a670fa570fa01612d89 \ + --hash=sha256:457fd7a2a8edeb044ab6ffbc0aa03ff6cd18491356e5e0c834d76ce621b916d1 \ + --hash=sha256:49023963c193dacee096301452f223ee24d86ec5807f8df93c0f7221d119e305 \ + --hash=sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307 \ + --hash=sha256:559ed65f60c1af5a03f3912605a1b5114f522c7c32fb23c3376ae8f03219fe28 \ + --hash=sha256:5632e3ae3d09197c446310cd5187de63e28448ce22f0f67b2b93d97503c0c230 \ + --hash=sha256:5e4d44984abc0020154ea81b247adeddcc3ac5527b975ff798bd1ba0adc513c2 \ + --hash=sha256:75b00eb8fb802090aa903f4ea1c7f5a584779f967361e68b7e98e531cc2d7174 \ + --hash=sha256:78a0d7c918e74a232394117160e7e3db503377572a45bcef8826e4ab8a35feba \ + --hash=sha256:78c0665edead396b1abb4897c41a5c1d9bf090c8a637a4c20a61678e0a264e66 \ + --hash=sha256:7bbf207c4453ce1ad2e00b17313852b33310b83090c2311bdaf97f93c0380d12 \ + --hash=sha256:7f4b8bc363b6d65ee2152bec57568e3c52639bb34c46057b09857a307ed5e21d \ + --hash=sha256:82f201b4c878551d48558337aab270d3c6cca5507b8737c8d8a608d234cccde0 \ + --hash=sha256:83de5453a7799afc9048b4616bd085cef126e36412f0ea2f6370c36a2a3a51e7 \ + --hash=sha256:88f0e784020649f88ea48c9f5ddfa403bf9205820667c0914740b392035afb82 \ + --hash=sha256:8bcf3c1ba5d6456e2effd30fcbd3459b044d683fcdac79a2e6830f0bdf7de487 \ + --hash=sha256:911de823097db8b63f034299d12662db93344e6ffa0b881cbb57748974b70168 \ + --hash=sha256:92c14f5bdbfb6216315ce33e78080474082de8b3830122ba97809bfbe65f75c0 \ + --hash=sha256:95298364e251be3e60249facbeeca03631d3bb7584f85879516ec55ac717b81f \ + --hash=sha256:9554bcc6d715ee87a633a3cc8e7703c6628b100dd29cb8a2efc4c0533c7ff729 \ + --hash=sha256:9f2897bf7737392ad0d5213ea7b6add72a4edf5679b3153106aeb88b6507b3b9 \ + --hash=sha256:a1d33a7836f7ddc1993427966a0823468ec41bcbdb1a9f9942d1d7e57f803ba3 \ + --hash=sha256:ac0333bdf38309aa3dcbe7e3fa7ea29e7a2c37c6ea306a757b700ded8e4596ad \ + --hash=sha256:bff0b729edd992766136b34e39cc76bc2fad905aa58897ee72a9cd000a6d8443 \ + --hash=sha256:c24acac1e18912761c4700239bbc1fd32f615af690f1584d49b35859be51324d \ + --hash=sha256:c35d74ce0e193ff740c2f2be2ac913ddc232fe6c1ff40b26cfecb9c670c63314 \ + --hash=sha256:c825cef2f49e46753726a7181a8e199804a912b29519ada542c6ebc654951899 \ + --hash=sha256:c9d18a33309122074ea483dd92dd444189166b8b2ec429fe9ed5ac73c7a0aa23 \ + --hash=sha256:cbf38d043c1aa4ab306e1ada6ab6eddacc3322a20b7af1b30bc93254b366fe09 \ + --hash=sha256:cd479fc04dd9401e3b4f49e76518768ef99c4f517a98c284eb091fd725719adf \ + --hash=sha256:ceb30a00ce7c92d459819443d29ca486d882b83fb6738bdcbb2a1cce94ac5daa \ + --hash=sha256:cfbf154f2ba187f2ed6cce2639efff7d105f1140573642c0161615b6d91d6a87 \ + --hash=sha256:d2924a03db38dc2e848bca2fe9f077dafb891480b91a00a0963a8cf86dfc31c1 \ + --hash=sha256:d416b16cccfd70fbf62400e84d0bb2f4e6af519a45557f1692c749b37f14b315 \ + --hash=sha256:d65d448389b8436493abcf629cc94ad0cf32aecaf06e1acca1de53cc795f2f12 \ + --hash=sha256:d84a09d0dad90ba6525d8ac1c2334b33e64bf3ccfe9e841f02feb867a22681e4 \ + --hash=sha256:ddef79fb382df40104a19bb7151b3b23e57c1778fcf857c71ceecd9bd264513f \ + --hash=sha256:e3b417bf8c2c7c16e8f58ad91db17783ec911ac16e7b50eb6eab6e809b4f5b07 \ + --hash=sha256:e402cf31eb68f453dbb2d36fc6d722b33f24a55d68b2ae1d92fa6305ca71c298 \ + --hash=sha256:e6fb6a55cc0ba97b59a1f288fb86dc6fce8bdfc0fffcbfd015e3a954bf2a2d93 \ + --hash=sha256:e708533e8b2ae2497d65346538a7dcc92814410b25b81432eac66de0f2af8265 \ + --hash=sha256:ea324d9dd34c38bfb9bec8ca4d1b407db97dbb74029f566b8e322b1b6fe56fe6 \ + --hash=sha256:eb0dfcf4e28a99c12c999744a2ff67c9b06200e20401c7c88186e33552a46331 \ + --hash=sha256:eda632a7981f69730d6281f451db9c1c370993a2c0d7ddb43e2a809a2862b83a \ + --hash=sha256:f29633129f9fa7e88a3f0fca835de2d030bfc9643f7799e1a0c46cee24d38fc7 \ + --hash=sha256:f55fa87b6c612ecd6b058f167c53231b1d14e412efe361d3d6e38b3631c73218 \ + --hash=sha256:fdaf5ea890a6183d0565f51a61799d67081bd5b1cf03c5f4b3fd3732108625c9 + # via mlx-audio +sentencepiece==0.2.2 \ + --hash=sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497 \ + --hash=sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e \ + --hash=sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0 \ + --hash=sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719 \ + --hash=sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107 \ + --hash=sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3 \ + --hash=sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b \ + --hash=sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a \ + --hash=sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c \ + --hash=sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838 \ + --hash=sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5 \ + --hash=sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6 \ + --hash=sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d \ + --hash=sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf \ + --hash=sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25 \ + --hash=sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d \ + --hash=sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e \ + --hash=sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b \ + --hash=sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd \ + --hash=sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936 \ + --hash=sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85 \ + --hash=sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b \ + --hash=sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708 \ + --hash=sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb \ + --hash=sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9 \ + --hash=sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0 \ + --hash=sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790 \ + --hash=sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e \ + --hash=sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78 \ + --hash=sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99 \ + --hash=sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9 \ + --hash=sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d \ + --hash=sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8 \ + --hash=sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906 \ + --hash=sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383 \ + --hash=sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b \ + --hash=sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53 \ + --hash=sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b \ + --hash=sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab \ + --hash=sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0 \ + --hash=sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914 \ + --hash=sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91 \ + --hash=sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e \ + --hash=sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a \ + --hash=sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032 \ + --hash=sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711 \ + --hash=sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da \ + --hash=sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c \ + --hash=sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a \ + --hash=sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d \ + --hash=sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151 \ + --hash=sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563 \ + --hash=sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811 \ + --hash=sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b \ + --hash=sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820 \ + --hash=sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090 \ + --hash=sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c + # via + # mlx-vlm + # transformers +shellingham==1.5.4 \ + --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ + --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de + # via typer +sounddevice==0.5.6 \ + --hash=sha256:7f4162f514f007b0bf25a3ccfed3f1705bc2ec311888a90232729eec4f57a4f4 \ + --hash=sha256:8ec9fbfde2e32f020b167e348f3ab3bac6625a5f15af524d790108ac7147a410 \ + --hash=sha256:b36b807eb02abd257198bf84b2af05e4fea199a9d2f0019014169c7136d45e9c \ + --hash=sha256:c8ae19173e5f27f8c12d4b5eee2dbfe542cee125d591e663e0fb4dfb75246d45 \ + --hash=sha256:de099612311ad81e55d31ccbd83f43ea6bf4d87b48f9b6ea55a1fbcde0eee4e0 \ + --hash=sha256:e3aef00ad8b1d1740eb66d9a7671eab88a4d2b8fa4ab33498d742e63b65c309c + # via mlx-audio +starlette==1.7.0 \ + --hash=sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e \ + --hash=sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d + # via + # fastapi + # mlx-vlm +tokenizers==0.23.2 \ + --hash=sha256:12f0835dc2ee694746a76adf7b1567d4346a4a502ebe93fb1f5f80ea49799b78 \ + --hash=sha256:2e96f5699d5249c9c64aa8412e044f727aae3a4098cf830f9901ec1afc361cde \ + --hash=sha256:325fee2e0418a9dc6c9ecf736a5f5f0db7875183ace9549ae339da76f7a1fbb7 \ + --hash=sha256:41c2f84d172449b4dadb9cdc508e3e364076613c35b16e76ecfe47a60d1e3305 \ + --hash=sha256:43e4f2071e3cc8d5d86421c874aebc82659bb51a68bcdef5a0da75ee89511ccb \ + --hash=sha256:5c56bda1511921587789163e524d196ed8284174ac23abd7685d5ea8da6c4718 \ + --hash=sha256:7b7e37ba198f24150f523e1242e83c4970de4a525480586be5dcc24d9add32c5 \ + --hash=sha256:7f0f085686b9de0d0079e6f874ae053600db64c5d13049e0bbc0119926d25aac \ + --hash=sha256:85a9a357a3764aecc904ee76bdaf8cf1ad8e5a67a1b929a487c4a39b49ed0e90 \ + --hash=sha256:950d7c9426fa72406a0ffeacdbc0bb9985f5db20eb8b263f29c79aaf83105703 \ + --hash=sha256:986670e43691469dcee610ea0f846f91a8f84e91fc6f7a48d4c064414c0ec2bf \ + --hash=sha256:a37039b5dfc4af84eb3ef0a92f4307e28936c8f9adccba2629d36f652e9bf7a2 \ + --hash=sha256:bef235815a067b2648caf6dcc7a71091b0b0fff9ee8057f6451eb9335fae52ef \ + --hash=sha256:debf978920d93ba9c219bd67cc4bbfaf912c9039e41e7a28b91ec15e3728c95a \ + --hash=sha256:e49c394456dd9985787fec76132438ba3fb8911f857b1bf3d40119f9292d41aa \ + --hash=sha256:eb2f9c8a24da020ea8c11a01a19c1c2547912d92121ae4a01cfbca46125dee40 \ + --hash=sha256:f486f402f6f9abee5bb032553736813af0c710a86b2e0ca592634c55cea1f835 + # via transformers +tqdm==4.70.1 \ + --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 \ + --hash=sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4 + # via + # huggingface-hub + # mlx-audio + # mlx-vlm + # transformers +transformers==5.18.0 \ + --hash=sha256:d79e5a515a572ee3deb33eb0d578be47d91e70c75d54cf21bf6eede6edef14a9 \ + --hash=sha256:d89c206e42e841af7cd314b3d2b21c03e977d819cdb4aef65ada048769a644df + # via + # mlx-audio + # mlx-embeddings + # mlx-vlm +typer==0.27.2 \ + --hash=sha256:269b7eb9d3c202ca84b4bc9618cb04ebb43d3d4d1e567e4c768607232c05f945 \ + --hash=sha256:b3a5fc4342d5fc8fda8fc3010b1cf117e9249aab7fae800c2eff62fd3842d97d + # via transformers +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # fastapi + # huggingface-hub + # opentelemetry-api + # pydantic + # pydantic-core + # starlette + # typing-inspection +typing-inspection==0.4.4 \ + --hash=sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47 \ + --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 + # via + # fastapi + # pydantic +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via requests +uvicorn==0.54.0 \ + --hash=sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf \ + --hash=sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620 + # via mlx-vlm +websockets==17.1 \ + --hash=sha256:0014eaff8ad5b3b43feda2279f9d34bf2eaae040720b9fbbb55944b10f40b14d \ + --hash=sha256:00679b7468b4c2b12b0757118174e8eabac56bb2f579a928a104d9554a56e098 \ + --hash=sha256:00bf34b64501e3477e81fc281532ff3cbf4da26633c10b63979d5085d46602d3 \ + --hash=sha256:01dcb47deebc40b38fd4a493b9b9f4d0a704b7bec6f35e4d34085b329abce71a \ + --hash=sha256:020e271205f8ab3406d7a59cd00de6dec722315924411c421bd00642f18bad86 \ + --hash=sha256:0340bbef6bfbe16da888b3983d666a4db4954ac3253c38f13bc7aba0c7db5a2f \ + --hash=sha256:054c28db2dcec0e857e3b705d8c28012613e555b38c765d6a4f75340a4fc06a0 \ + --hash=sha256:073c5c3f7e127041fa9d34a9e29ceefee8c3cafbd267ed2927318f425144380d \ + --hash=sha256:07fa3e7c30e2c577928d359b56bf872a3e0cbcc15553eaa0907c1ee86344b56f \ + --hash=sha256:0c84bdef916556cbe1d5a43b423398be4dd3cba6522b463e53d848578b920695 \ + --hash=sha256:0c863507ada5805517ca6dff1c524dcd42942efe6304dacf06700878398d21a6 \ + --hash=sha256:0c9982938980e086da59f70d05f9418cd143401a601a0faac10fa48f7bb1cd3e \ + --hash=sha256:0de501b7f2db11e83739ac20e2d33d46da4604b829f506c24be80e7def069391 \ + --hash=sha256:0f4f50dfe2cc810fc4e2de979b35e83bf8bb4bccdc6fe472d93762ea7b1d5927 \ + --hash=sha256:1036189bd34b0bc1b10a4679321e2c7968af317efe6e8e4c1c5141c4254fb5bb \ + --hash=sha256:10ecb38ffc05e1841b619d99c725307a223ef9ad58e7b1ed33311d472dc43918 \ + --hash=sha256:12c8e2b25df59755954a04dfa09c990b96691025aaf7eafd19ed6da24b09c18d \ + --hash=sha256:12fe8984a32dbfd084e0603f1a8d740c0180cb85b3174585c54a80d2455a8394 \ + --hash=sha256:17aa424ab61620aad21b36b2240efc87b500cc496e7d0e999a5c2ae99395e886 \ + --hash=sha256:1a20136d61f9ca3a31493732762661fafc2c20e8861930214e21afc6a8a692a2 \ + --hash=sha256:1a5cf4e7bbe3ca499e6a289206cb4fcb7444b09919e129bd517f57d5fa192c13 \ + --hash=sha256:1c7ac77401227212dc6e849182feee50d57cf456ec6329ffd6979c94bb136c5c \ + --hash=sha256:1fce0f43e0d41422e0b2cad6561e1970df22f212f4c7e884967df7cf591b031c \ + --hash=sha256:259f45358c76d3b18489e3e80636cdbe807e05ecf1b10fdf1a779106d23d0c8e \ + --hash=sha256:2786cbd273ab69c22612db8a41229ddf2c158060b17b5928884bf388d07887f3 \ + --hash=sha256:29176d8b429cfa0fa443c473878d37a5c06cfd0cb36b71ba4314accc71e05906 \ + --hash=sha256:2a0162a6372110a5601cb5c9fd826635cedf69f3e110c545dd19774e040b970e \ + --hash=sha256:2a4dc6ef83f4559e0d05f313a375cb38f63c986096a9da99fe94fdd779d313e5 \ + --hash=sha256:2afb58c7ba48b329d56769f8dfd89f394efe587b65ef806bae810a484d6d3608 \ + --hash=sha256:2cce251f3e2469b99b6802b55435bcdd07123b41870f54c87b336183af9d7e68 \ + --hash=sha256:2d72879819f5145a342d0030c418702496c65a4b913ef81f5ae944dd91dd50f6 \ + --hash=sha256:32a2a68d989d6e5b74a9d5095415c51189ebae29fceb7cf2b64a1c0318a81256 \ + --hash=sha256:33e45c7ea38428e740a7f233555d71df0b875cef7fc080acebc9654475e35335 \ + --hash=sha256:34879e19bb0a3c44f9317679435aea5327fac993933a704cbf353bf1234b10c7 \ + --hash=sha256:3546ef55b3a074494106508bc6505c73825970d2d9505f7bf53882b3e88b0d1e \ + --hash=sha256:3709a1ab30b4b922027d22f68d2b61a0656a91680ac894a537624e6be7dd7f7c \ + --hash=sha256:4031152769179ab8dcdeafc7b0e58052a49117560a28671700b47b2c7b717aad \ + --hash=sha256:406b8107943a43ef4649b1e0cb0cdc052bbf08fe1c8905a623c4af9586e5cebb \ + --hash=sha256:416b4bc8789a1865a3ff643ec4ee073a5f52402d0dbeafd27b1798d5dd6b6a51 \ + --hash=sha256:43bd0c1ceb924d67f5c1a5254d8361dd9d94246e6331a726064dfa2917880780 \ + --hash=sha256:455ffeea0879d313205df1e745e5883e1feb7f31ecd26be882f5f0babd3db04f \ + --hash=sha256:469355ab1af100b9380f1afb09985019f4a4b94fa1dd0e9396db4361626d7ab8 \ + --hash=sha256:46c0331c9eaaf73a559f3a9e388466be0df96eb83d40f06f1ca6ab6613b35c82 \ + --hash=sha256:46e7a10bf04318c7b0c0273791925ae5e1cbe4a11e34aa934d2ef27862058a80 \ + --hash=sha256:47a62d6045c6eaa0d8f97bc2fb68b8cf90077a0cbfd4e83d6f2d2145611ee134 \ + --hash=sha256:4af784f3e436f65b355c117c6497320f2b5cf6a559295cb1c4c7338e335d45cc \ + --hash=sha256:4e8ffcb486c8490a34a4cef5e4409d8da5a1cb1681e5bf7d786ce5e84aa8540d \ + --hash=sha256:50903d335bfda026c2fa11dd9aed09d8cbee0c451e3a85122a9acb041b7dc69b \ + --hash=sha256:57b39dc8541cf7ed3f639da82bf7451060483967f9e733da1f8173e4095f0642 \ + --hash=sha256:581fa678ef46f4277cc8491312468e582f8ad609dbab907ba6096a08c6a0ff98 \ + --hash=sha256:5a74531ce81af587f906ab42f194032388fcff8fc7938402e5917c9147a39441 \ + --hash=sha256:5aefe78e6a3077fe22b5e64b04666a85a3eb8b934d40e8595a693adcbceb6f11 \ + --hash=sha256:5c27336b1a0ac56569493e858497870347854372395f50483725f8cdacc5a45c \ + --hash=sha256:5d4724255fb8398acd9e583b97eb2279cec20e0bd0f9a94bf75f6056ef9f13da \ + --hash=sha256:5f051f8030a51815dc00e24bd2e5f1435af095c1cc111d747ac6e2a3620d7641 \ + --hash=sha256:615746b12b26a3fd4077bc6fbeb277a1c192a45dd57b531d07ad9ed5c52a9a7a \ + --hash=sha256:617243e19a0992095956f406ee9cd3bc4ba92862d83cb1d83bb59ce574412bec \ + --hash=sha256:655a8e28010f09fd6fa317e857afab3af7647f33e41dee88fa421e92086d1090 \ + --hash=sha256:65be6bda2b537fefa4b3a5ccd6ab386533ce39dd8fe62433ec90901fdc81752d \ + --hash=sha256:67258b00302a5aaf0b267771c7014b13429abd7ea17eebc4c55bd935ff101555 \ + --hash=sha256:677014a073bcb1fbaa7e21144786864f16c08f856d66834f611eceb9006cbab8 \ + --hash=sha256:6a8e768a048c2220697477ce2e67e4345dc9f693d0ee6af53945b5e30227c6a7 \ + --hash=sha256:6da17fc94bd270f5987b10bee113461ac36a36a98b0481ddcc98056e5a90001a \ + --hash=sha256:6de9acef07e3a78e9567fcd26c29011a4da8f050b13004bbf880a0fd82a6eea5 \ + --hash=sha256:6e63c01803be425ff062b7f7fc201a74def1d49fc94a2410dd17375df75936e9 \ + --hash=sha256:6f42912fa9eb4cb7c7ec9fde9b3332ba339eb8a8811981043d4029599f3d950b \ + --hash=sha256:722ec21717eec6477bce582147a28acdfe034e604239466a6a95daedb863e774 \ + --hash=sha256:764cf7bfa149365f32b7a0fd9fed32debdac29dd06295d5635cde1745b446cd8 \ + --hash=sha256:76dd004f59115087c7b700474cb18f01325e37250032e19396c08ae41448e4b3 \ + --hash=sha256:77b37cceca17291897c3c73bd30a7c7c7909593554b5da574ec852af83c1742a \ + --hash=sha256:7a72efa3bf4fa3a6669a54420a472ad056da3973d827f10e3a536da463f926c2 \ + --hash=sha256:7e724f843fa6a0614aece65a7c73e51d0f4412ca41dccac13c3caf98e69536bb \ + --hash=sha256:8196d217eeca52b9235ee1f8a684a09885a5f953d5a31e80ef915bf2c5c94f9d \ + --hash=sha256:829dba1bc049779de9b332088c1a6a9858e96bd67e50b6b644a95e02b67836bc \ + --hash=sha256:84f25e099fdfe3b09f953d84698f729a1f7d1e99101b2787d7a28ed77b323750 \ + --hash=sha256:87f0d5e77548b0c40c8464cdb6108792e7e53f487c6400028a4ec28a8afbe5ab \ + --hash=sha256:880069d21cc33a558dcf180924a546d1ecf8ada5be3e4e70acee87019d706a24 \ + --hash=sha256:882af300d2c6a092b93767d5de03c7bb56dfb06314140c8e872d3f48e09f7b74 \ + --hash=sha256:88381602e379165b66244b2ebc29f9b23ea0851fbe63ae157f91ca324f072d6f \ + --hash=sha256:88b882764ef65147a7a5ae13168dedbe225a04e2ff4858fe543f2c402f093e9c \ + --hash=sha256:88bc5138e53903a85c354e59df7ba73ce306f7b09724cef74dba121e60a88ce2 \ + --hash=sha256:8d1b108bd8f5f6a8b90801f6db3b3858d5deca889acfdb8ac497bbb24e4b0edf \ + --hash=sha256:8f6c38cdcaf98a911d7acc25577f2f9e710f3a2fc2bde1563556784320196b51 \ + --hash=sha256:8fbf28e639544503b7d1c96452a5e5e043e4108d89b1f3fa02910603622d19db \ + --hash=sha256:96abdecbaae746851b87c3a36cb4a661df93ca3d92f114270f79228bf1d00de6 \ + --hash=sha256:98a5b2589a56a4b4f098b0a958099a4356ab904a7844f1da3841efca469af7e9 \ + --hash=sha256:9ae55d24241fc055f22aea3ac924559069848bd0ad4ea065fdd72d2194685fe8 \ + --hash=sha256:9f4a08ff7cb68c27b18e09223cc6304e01d0f82d5a240d251266dfd2e6e44729 \ + --hash=sha256:9f4c0377a83e163a303514fdfab501dbe379bdc13e5b9312a91d112658b29dce \ + --hash=sha256:9fa1ffa08c81a4f809cdab6129f8e55bee4650b9d6d3461019dda73aacd146b6 \ + --hash=sha256:a06f3b5085176763182449559e20391d7ce616a8972a9f7a33deda87ea6d4f3c \ + --hash=sha256:a62d8c424383c9dc769ff3672018df822603117e32686e567d452ed035b6fb2e \ + --hash=sha256:a7b35181a14cbfcae163b4de545d22abfd07d06c2c41ca69cfcd99251d6888ab \ + --hash=sha256:a9fe648abd1d9b89aebfa30407bfdd08a0271ec5dc7d44a4c6ccd1ce22cf562a \ + --hash=sha256:acfea4c20bf54384883ea33b1240fc1db4f52e190823a4e2b334bc3e8bfca96a \ + --hash=sha256:ad68c28a27246fed109a4409393d677b7e1388345cbbd2f5aee5c182d8506110 \ + --hash=sha256:aec00f018d34c67500ff0438dc314b40277be4a1b983cbacbf53ccf7db63e257 \ + --hash=sha256:b1bc819c6db90e8f91a38250a1ab4c058261871aa52d2fe36382eddedf146dee \ + --hash=sha256:b1c323fc3be1dc3f87f6c59458cb7d9e13dcbbf971d6c3f3e2bbaf58d3bfcdfe \ + --hash=sha256:b3b451fd2723ad3191a209afe6f3f4bc86c83e9a85bdc255353b91803ee6aa66 \ + --hash=sha256:b65d5fe48219dc2d5e158de9e6514e75600f379cc7e37108d35f31764c155566 \ + --hash=sha256:bb0efe019480a1c93e168ce96479273aaebd672fc8c350d5eed1e507ababb1b8 \ + --hash=sha256:be3f0129c5654517b2abf07dcb75bb1d9479759a4ccfb569e8293579e9fc029a \ + --hash=sha256:c3241d684a76eaaef8b2dc789afde4343cd3aad55ea81e4e8ab3605b529bae51 \ + --hash=sha256:c65da239a5ad553619804c1f9d65c1a0b3005381c6158ee14da2c7444cbd0c78 \ + --hash=sha256:c89406fa3dcd4aa8662c6406cc5c0de1790e9614d2c3aaf03ca53a8a8ccf3405 \ + --hash=sha256:ccbf3f4a9890d50b3a08ee04029fde30a03bfdeffaa19977628bf17251764e60 \ + --hash=sha256:ce0305b702b20d1e1d60a9aaace6bc89970e1753565543f310d549eab22c2435 \ + --hash=sha256:ce13c7d233239e739600a57d4a73c1192ad8259e655a4d55aa1a454242bc809d \ + --hash=sha256:cec1bb8f22abccc8d20f8ca63df9be41600c26c190f4b97ee86c675fd4a863a6 \ + --hash=sha256:d1e2f5fa2b6d01f0d85b4f223fea7ed1d504be282a02a81bd2be4817ef7a2f03 \ + --hash=sha256:d334d11398086bb5559606cb42d51c013ea7c061c7db701521392373d3c087f5 \ + --hash=sha256:d411ea5ca18ac1b12c0c94be88b60c18ca641ac43bcdfdf1c9f79d46cdbe1603 \ + --hash=sha256:d41ef69d5416fbc1d98cf96c37be6192d10fd101c3e0f8b3ddc36e09432b3c08 \ + --hash=sha256:d58159af7835fde09c462394293c0d7aaf8fb4557d8f8e5699f5e722ccae013d \ + --hash=sha256:d7b349265fad6244013eecd99df8d83c12bf3013943e431f4fadd5bffc37db42 \ + --hash=sha256:d8e83333385cac6030a5167fd18bf96cc6c58b914c308e683f05b0cf94bc8dd0 \ + --hash=sha256:d9d01e8ede41fea4f5a847dad9d628355f74905f437a5b6856d67aa66d193800 \ + --hash=sha256:d9fc873e239c5abeb150bc24dbd1a7af23a9254526383ce0a077f5e20adbeb19 \ + --hash=sha256:db9d7ee47f3ba531e278be539af39e2c7c7d28fb94897b6cd1120d63b0ef5922 \ + --hash=sha256:dc053f9e95a76213c5eb7ed95779f7daf0d2bf0e4e03073629ebfa43a033f151 \ + --hash=sha256:dc2b79afc074d2f3e64b26539350f697fe1b85ea1c49ea24eb588f247b053ce1 \ + --hash=sha256:dc5789e5ea182b77a38881383ada5347202a6c66f4857d054e075290e80b604b \ + --hash=sha256:dcb8d5f7edef7a399d322cf28d4c4e6f98dab64d301c8f50581a1080e5198142 \ + --hash=sha256:e4bd7eacb87d8cf3ed70d6392c770a0d92441f05d7d2a3efafb5bc171d5e3067 \ + --hash=sha256:e552e0037230ac16e5f568de7012041344d1b18c9feed30ec2891b8eba55af81 \ + --hash=sha256:e5f5c7a893507d0e83a80b88aefd6522f7e882cd53f9722c6f23f5a020c9557c \ + --hash=sha256:e74e41f0ad12ff1e8983e349daef79d37cc8280c743ce9d134d6c74c18dab5d6 \ + --hash=sha256:e78fd4b7b2c5086a38671c9c882c1e643385eccea360b5b1fda4a105e590087e \ + --hash=sha256:e8dc3fa6d6b7ead3f9de57895f41b116a28787548e066365d9d90f7356bcaad2 \ + --hash=sha256:ea0ed9373b880115911d9d39634bccc95b8ce590c9c42e8589f5cacc3ef3cee2 \ + --hash=sha256:edadce7a22052056fd4384543019856b34850363c9d387929f677ae01d79709c \ + --hash=sha256:eec113a5b41d124ef42ff56b0d74a6da3fd986400038eab9e58ee42a4024e837 \ + --hash=sha256:f221081107b8c48184d99f7019604486376e7ef826037e70aad6b02540732c23 \ + --hash=sha256:f3a1d577e081667dda7f8e5b4796e6e32f9713c93e2a3d930669519840a3c623 \ + --hash=sha256:f47aafd92aa28b941180e6da8a42b0f711851b14b81a5b6bb28dbbb1fa35152c \ + --hash=sha256:f4c45ee2512d3757b5e6c67c5a34e435143f2ecb7df3324f9fd888688c45c0f4 \ + --hash=sha256:f58f58b4b29bbea2a3635e2c56eff4a3adab011fe383802a9e542e31b97085fc \ + --hash=sha256:f612dc57f00c07cf4aa2673f7cbceabd654ad2457b7e639f061b794d6e11f9fd \ + --hash=sha256:f62114a54117e4948a1e414e89521f7fe1e3c2f83f2a571a06a4fc6718b0900a \ + --hash=sha256:f64e001bb7fa89b9f32cfa600bf8e9ac8ca26759d9b92ae01453ee303d9cd7b4 \ + --hash=sha256:f7233eaf441a345a5943a929fd4b5ea3278f11aed35a9ed0f3106b8cb3ca846a \ + --hash=sha256:f78a3ffb1994304db2c0c4588e4d1a518079b557054fa3bb985a6f5e50ff49a3 \ + --hash=sha256:f8e822efd54137d8cc8310eb64635ab827a4a6c72ff08691f38aa624776d8ecb \ + --hash=sha256:f98bf378d7a5be047a044a1a27c987a8f355e10e3b5754617dbe756248cbc5ce \ + --hash=sha256:fb88076df585b69c5761c387c0081aa87d7b9eb1b205a6535ca4777e25650d81 \ + --hash=sha256:fd8f47dbf2e8adb15c847215f83436de3fdb120b51fdae0fbbdf69fd97a3ad80 \ + --hash=sha256:ff3e2ba7a9f0a110b0555452e9b5a03a34e11662544e01beea15f144b48ba7b7 + # via mlx-vlm diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-lm.json b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-lm.json new file mode 100644 index 0000000..9469bcd --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-lm.json @@ -0,0 +1,123 @@ +{ + "schema": "backend/1", + "id": "mlx-lm", + "version": "0.29.1", + "builtin": true, + "package": "mlx_lm", + "convert": "mlx_lm.convert", + "lock": null, + "categories": { + "text_llm": "models" + }, + "remap_files": { + "text_llm": [ + "utils.py" + ] + }, + "registry": { + "text_llm": { + "model_types": [ + "afm7", + "afmoe", + "apertus", + "baichuan_m1", + "bailing_moe", + "bailing_moe_linear", + "bitnet", + "cohere", + "cohere2", + "dbrx", + "deepseek", + "deepseek_v2", + "deepseek_v3", + "deepseek_v32", + "dots1", + "ernie4_5", + "ernie4_5_moe", + "exaone", + "exaone4", + "falcon_h1", + "gemma", + "gemma2", + "gemma3", + "gemma3_text", + "gemma3n", + "glm", + "glm4", + "glm4_moe", + "gpt2", + "gpt_bigcode", + "gpt_neox", + "gpt_oss", + "granite", + "granitemoe", + "granitemoehybrid", + "helium", + "hunyuan", + "hunyuan_v1_dense", + "internlm2", + "internlm3", + "jamba", + "kimi_linear", + "kimi_vl", + "klear", + "lfm2", + "lfm2_moe", + "lfm2_vl", + "lille_130m", + "llama", + "llama4", + "llama4_text", + "longcat_flash", + "mamba", + "mamba2", + "mimo", + "minicpm", + "minicpm3", + "minimax", + "ministral3", + "mistral3", + "mixtral", + "nanochat", + "nemotron", + "nemotron_h", + "nemotron_nas", + "olmo", + "olmo2", + "olmo3", + "olmoe", + "openelm", + "phi", + "phi3", + "phi3small", + "phimoe", + "phixtral", + "pixtral", + "plamo", + "plamo2", + "qwen", + "qwen2", + "qwen2_moe", + "qwen2_vl", + "qwen3", + "qwen3_moe", + "qwen3_next", + "qwen3_vl", + "qwen3_vl_moe", + "recurrent_gemma", + "seed_oss", + "smollm3", + "stablelm", + "starcoder2" + ], + "remapping": { + "falcon_mamba": "mamba", + "kimi_k2": "deepseek_v3", + "llava": "mistral3", + "mistral": "llama", + "phi_msft": "phixtral", + "qwen2_5_vl": "qwen2_vl" + } + } + } +} diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-video.json b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-video.json new file mode 100644 index 0000000..f61dab7 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-video.json @@ -0,0 +1,27 @@ +{ + "schema": "backend/1", + "id": "mlx-video", + "version": "0.1.37", + "builtin": false, + "package": "mlx_video", + "convert": "mlx_video.convert_wan", + "lock": "mlx-video.lock", + "ports": { + "video_generation": [ + "t2v" + ] + }, + "port_convert": { + "t2v": "convert" + }, + "categories": { + "video_generation": "mlx_agent_ports" + }, + "remap_files": {}, + "registry": { + "video_generation": { + "model_types": [], + "remapping": {} + } + } +} diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-video.lock b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-video.lock new file mode 100644 index 0000000..94ddc3f --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-video.lock @@ -0,0 +1,1635 @@ +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb + # via + # fastapi + # typer +annotated-types==0.8.0 \ + --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 + # via pydantic +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ + --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 + # via + # httpx + # starlette +certifi==2026.10.19 \ + --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b \ + --hash=sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae + # via + # httpcore + # httpx + # requests +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via + # miniaudio + # sounddevice + # soundfile +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f + # via requests +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via + # huggingface-hub + # uvicorn +fastapi==0.142.2 \ + --hash=sha256:06366626f2e70576367714d9ab2fe8472e6c8456dba69b399f9f797ab5e92570 \ + --hash=sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b + # via mlx-vlm +filelock==4.0.9 \ + --hash=sha256:635e7d67fa92654eed444e75e9ca18426d34e77ad9c469bf4373f75a932f7b22 \ + --hash=sha256:9287fd61b99a806e5202be29a83034c0808a1b9830e820537c2f5773981f7eeb + # via + # huggingface-hub + # torch +fsspec==2026.9.0 \ + --hash=sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe \ + --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f + # via + # huggingface-hub + # torch +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via + # httpcore + # uvicorn +hf-xet==1.6.0 \ + --hash=sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7 \ + --hash=sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675 \ + --hash=sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef \ + --hash=sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9 \ + --hash=sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3 \ + --hash=sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb \ + --hash=sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e \ + --hash=sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338 \ + --hash=sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d \ + --hash=sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d \ + --hash=sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765 \ + --hash=sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c \ + --hash=sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522 \ + --hash=sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f \ + --hash=sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a \ + --hash=sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b \ + --hash=sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b + # via huggingface-hub +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via huggingface-hub +huggingface-hub==1.33.0 \ + --hash=sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899 \ + --hash=sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e + # via + # mlx-audio + # mlx-video-with-audio + # tokenizers + # transformers +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # anyio + # httpx + # requests +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via + # mlx-vlm + # torch +llguidance==1.9.1 \ + --hash=sha256:0ab9d2681c0fcb4389f3c1c79697f1e724771d74d366bb5a22edae155252dfed \ + --hash=sha256:151a615f3d871fc096984b705daa211faccc78ed2f9e3a2a80e57283e16e2898 \ + --hash=sha256:2d7aa419d8b63a528347f73ee57185b4172c8bb09287e5ef9eefe3eb6d709356 \ + --hash=sha256:31b75c9f6e09d000d062ef7e4eb3bcb3759fc1eda5619be50fdd8a830b0e8bdf \ + --hash=sha256:34ad7bdff67219c7bf43ff60eb7b1d97232092fe7a40715ae330f894def04f82 \ + --hash=sha256:367da91f0db6b494e1d4916e6ab3e3db3eb73f178fda6be32681c026af99f775 \ + --hash=sha256:3ba1b37585d07f50bb73e06dca6d45c88c9d5801789c071c8ff784c9dbcc5c46 \ + --hash=sha256:6216861eaeef0f96251d7a9ed1c26a228a4c7879c5f062092691f8f17087ae4e \ + --hash=sha256:8aaa55958adb4871016700e9140543ccbd9d6b5136d0a1756a114af706cc50c1 \ + --hash=sha256:8b29903d5108cefc715e013381c7474846e9b0edb5e2534000f5e3a3d870a293 \ + --hash=sha256:96b52f9d3eccb1d6c1ee4454c3424e8da5171c879e33c294a68e6c731fec25dc \ + --hash=sha256:9892220b2698f58193ef54a886e5f3d5eb2a60d510db49ca4c492b46ebb786fd \ + --hash=sha256:a5dcbdd38bc75f149ed349525f3884160de80900952147a4b7ef40a5b1a4a1a1 \ + --hash=sha256:a942487cd6221b793df3a11450881a6c9d25a45460a900c56d99b682633c45fe \ + --hash=sha256:cc7d1024c3df780a5eb14d30744539e8a1aa6d94c5dabf7605c82c0567d45a51 \ + --hash=sha256:dd5c87df5e60a7d8f54f46bfd5e11abb10f93636712ace09a8c1d2a0c4aeade0 \ + --hash=sha256:dfac6021d9c4c5ef16b3a1d336c05ca5e8208ff30dec29d1c63ebcafbee3a346 \ + --hash=sha256:f824a817a841563d5b068a0dd22563e5f063cdbda382cf3f6e18c1c68c45db38 + # via mlx-vlm +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +markupsafe==3.0.4 \ + --hash=sha256:007e1ffd9bf65bb6ee96df7b258fc632a4868dd5566037986c64781f35a36e98 \ + --hash=sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002 \ + --hash=sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b \ + --hash=sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653 \ + --hash=sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c \ + --hash=sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e \ + --hash=sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc \ + --hash=sha256:0764a13d34cae40db7bbf3a09b7e9b491bf4603e20b263a7a9d6b8e324975d0a \ + --hash=sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92 \ + --hash=sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f \ + --hash=sha256:0cee7cb0f9a1b6892ea482237d9403b3d1b4603aee057d0ff01f0fac2d019a97 \ + --hash=sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4 \ + --hash=sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7 \ + --hash=sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691 \ + --hash=sha256:14bd2d845d62ab678eaf81da89d7b621b51756c72346745c1a594c09d49207a2 \ + --hash=sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc \ + --hash=sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde \ + --hash=sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99 \ + --hash=sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9 \ + --hash=sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df \ + --hash=sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5 \ + --hash=sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17 \ + --hash=sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8 \ + --hash=sha256:2a6ef68ae94aed8721934072b27a3b654ea2100b97e4ab864cf1489c90926fbc \ + --hash=sha256:2b2b1e18af909b448bb3cf9e3433366f7a8726271fc214e8b10e0f62a78c724b \ + --hash=sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea \ + --hash=sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248 \ + --hash=sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741 \ + --hash=sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5 \ + --hash=sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6 \ + --hash=sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7 \ + --hash=sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1 \ + --hash=sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67 \ + --hash=sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f \ + --hash=sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9 \ + --hash=sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c \ + --hash=sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc \ + --hash=sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba \ + --hash=sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17 \ + --hash=sha256:3d23795802fc8bd72534836d64489bbf0f67c088959091bdb22e10735a5107bf \ + --hash=sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6 \ + --hash=sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2 \ + --hash=sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163 \ + --hash=sha256:4a540e2d3192792fc84eced57bef37851ccb2b41f73291bb17408eea77bcd278 \ + --hash=sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d \ + --hash=sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b \ + --hash=sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634 \ + --hash=sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38 \ + --hash=sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed \ + --hash=sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c \ + --hash=sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148 \ + --hash=sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a \ + --hash=sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7 \ + --hash=sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f \ + --hash=sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811 \ + --hash=sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e \ + --hash=sha256:57f9947a7e57a081c1e3e0a2dd0d2dcf290a4531450e6f611e30084c222a7295 \ + --hash=sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2 \ + --hash=sha256:5c22873ad1f0532ba40fa1727f3c0fc1bbbaab6d373d4cbe3f0dc74b2e2521c7 \ + --hash=sha256:5e8b3d0b18fd623afa12ecb2ce8d8becef69f9b5440c6330c7972200e0bb84b0 \ + --hash=sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6 \ + --hash=sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed \ + --hash=sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378 \ + --hash=sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0 \ + --hash=sha256:6768d67d1bce64270e0fdc2e69309d68b9b18ae56ddf6c711d168e9d051c2cac \ + --hash=sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b \ + --hash=sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96 \ + --hash=sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59 \ + --hash=sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808 \ + --hash=sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2 \ + --hash=sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb \ + --hash=sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65 \ + --hash=sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72 \ + --hash=sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8 \ + --hash=sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e \ + --hash=sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91 \ + --hash=sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a \ + --hash=sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2 \ + --hash=sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e \ + --hash=sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707 \ + --hash=sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21 \ + --hash=sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef \ + --hash=sha256:8698d70a8081ee8c090dbb394768b5789a1da8b131b5499f89d071dd3cfaf6be \ + --hash=sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453 \ + --hash=sha256:88d59b473bfb03259722600839af9bbd7fa13a2eb514beefeedb95997882f69a \ + --hash=sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6 \ + --hash=sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977 \ + --hash=sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 \ + --hash=sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692 \ + --hash=sha256:9240187afb63d2f9ddc3e032c670356fe941f6e20662ea168a5dc3f1f317e1b3 \ + --hash=sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369 \ + --hash=sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a \ + --hash=sha256:9388003072b95f2f1e3fd908604194d653ba21330d811961a78b7da1a77e9e36 \ + --hash=sha256:9438a2648b2195980cb2dd8e53ed7b8df91319e2d0b70ae61a9e1d1bc8d3bec9 \ + --hash=sha256:94e4c421742086aeee4c32a506eec8859d7634aad943f7e6aacf70f813478768 \ + --hash=sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916 \ + --hash=sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b \ + --hash=sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f \ + --hash=sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346 \ + --hash=sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c \ + --hash=sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464 \ + --hash=sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9 \ + --hash=sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee \ + --hash=sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300 \ + --hash=sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6 \ + --hash=sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d \ + --hash=sha256:ac0c7c9f1609b0c4c114feb1d7a3409564c7fb77e360bed9e97e5d25dfeaf868 \ + --hash=sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46 \ + --hash=sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97 \ + --hash=sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733 \ + --hash=sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe \ + --hash=sha256:b61687d0828e72bf5cda24a2690188f37170bd31c9359ac97e4e66569f120a16 \ + --hash=sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429 \ + --hash=sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39 \ + --hash=sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894 \ + --hash=sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c \ + --hash=sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c \ + --hash=sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169 \ + --hash=sha256:bf053da3c97a4bc5ecfbb218cdd2983febd91c617be8367d139882aa11e490aa \ + --hash=sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77 \ + --hash=sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe \ + --hash=sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad \ + --hash=sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85 \ + --hash=sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e \ + --hash=sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34 \ + --hash=sha256:cf63c214fe879a65e69a386f915e36104fc84254ab141240f8854602d8e0be2a \ + --hash=sha256:d1aca03ede943eb80ab3d63bb082c84b7aab85ea83bd0fd0c200260945fb49d9 \ + --hash=sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c \ + --hash=sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749 \ + --hash=sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214 \ + --hash=sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932 \ + --hash=sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494 \ + --hash=sha256:dd8ea6ebee7aedbf7c749fa80521d9ccf1ba473e0d1e14805caafbaad281c889 \ + --hash=sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1 \ + --hash=sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0 \ + --hash=sha256:dff05cb7016dff1e9fd68f4122c127b65dfc59de5306cfb7ad92f956f230bee2 \ + --hash=sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786 \ + --hash=sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78 \ + --hash=sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e \ + --hash=sha256:e841068dc0be4cb6dfb5c890eb88cbdcff2f4a332393c7ec94e8e618bd32c1a8 \ + --hash=sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289 \ + --hash=sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c \ + --hash=sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe \ + --hash=sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237 \ + --hash=sha256:f291bcf42ae98eb5107edb162c3c998b4a89648fd8e99ed4cbd12705292788cd \ + --hash=sha256:f61efe1d2fe0de16158a5fe1d1cf3c14bdb6aecd54d8938fd26512c525c1f624 \ + --hash=sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19 \ + --hash=sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977 \ + --hash=sha256:fd9f8797427910198f95bced71ddfed61130d7e349213bfb8466c9c99e2c46a8 \ + --hash=sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3 + # via jinja2 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +miniaudio==1.71 \ + --hash=sha256:06222d80b057ca4beccb6f97a134c2c2bf646ef7890e1759cfc09db7eecec44d \ + --hash=sha256:12bc33e7e61072b4b541c14e10ef76119d5643e6bbb98e2dec0c0738889438fb \ + --hash=sha256:14892ad9b884e637029a22a781dea569b292a1be13682380fd14cefcf80ea4ed \ + --hash=sha256:154b085dd914a0e79e3d93160e1a07aacb27d66c65f9ef6a0d87c1a194f32c04 \ + --hash=sha256:166516449e2bb5f628d89cedbbb8720dceb96a0562c7e08a0e8e3cb10f58647c \ + --hash=sha256:19be6f0a1e601c2237433e579734cfaf6469191b224c20c9e5f73c32ef9ee2b9 \ + --hash=sha256:19dc58e4c50ffc48db2ce988019c28f05ca0eaa7c10b45b5c99b70107e610c8a \ + --hash=sha256:1bf93aeede652926f27f430f0fd69ef0cf8a949c07b537d6a2f295602c747037 \ + --hash=sha256:33986d5d725ebcbc253551e7358689bc81b19b6950b33cec8e8c1142ca4fc0a9 \ + --hash=sha256:3bbeb1e068fe42475e017e8150e9e345182b583d0dd4d9e77ffa20c39935d9ec \ + --hash=sha256:3ef441d139264f8a5dcb9aa6fcd0b1e1e69f58715baae416ff33f045ffba6ad5 \ + --hash=sha256:4c849ccb1349f7b3553a77a66fe7e972315185f5c4c44a0bbda7ebcdd224db37 \ + --hash=sha256:5009b4e29cd43de3631d2d5ab09cc074192c085b4c8dd8a121b856ce1af6bab7 \ + --hash=sha256:50d66729e1dd7a4cf13edc25115ac54f776dd9f67803ba1a7cd1128ebf2e8cfe \ + --hash=sha256:61b86f26d653040db32d9d15b05446321dd10e45beba25b44f841e26935213d5 \ + --hash=sha256:62db602651bc20a2698f36a0d356d7217ed6f4f917550c7ffb3705c8e8be90cf \ + --hash=sha256:70fa2ea5353e6919aca59b8c5768144af009d18c3bca251749d66fb497424563 \ + --hash=sha256:84139a10ef172acd762ccf120142877b037a1aaf71def99d2c75f66329f89d8b \ + --hash=sha256:8a28ff4ad23e55bbde8808ce525d3bb7d249d7612f77646b30e06fc6b7a778ac \ + --hash=sha256:8fc1a4f084cc1b4b25c567d22f54d1e46bfa505c17ed777c8b198e5c53d0f785 \ + --hash=sha256:978cc4d58d8beef1a705e1141dc177a8a357c10ba3a16f7d71482ee722023bbd \ + --hash=sha256:9f379d4995f1fac6dcae65810f6a31cba264339b3e591a14b233f85a6d03d81e \ + --hash=sha256:ab100e5240b104b5326e4ec1be07b6ae461f7d3d4d7a694857fd2f0493d210f9 \ + --hash=sha256:ac4a37ebbbfbfbeca50f4390e50f9952807ca61ac62f0c3bcbbc7dd698531dd3 \ + --hash=sha256:aee8e4eec8d7bde4ee78066561329235a04231a221c9b247f1ffaf850551087d \ + --hash=sha256:d9dc15eff711bcfc62a9d05e0c78e4bc34821a455595e049629f2fea7491a523 \ + --hash=sha256:e6287f15caa808a88aad0700a182bec1ff6d98769717425adf9ebf41259d1936 \ + --hash=sha256:ea86ae04ddbbf2beed20b9970af4a0baca8e6ed0e9625e1ed957be5540c943fd \ + --hash=sha256:f4a44b70b66628b0c307e40ae0ae857695978cae18462179b806d8edc807d416 \ + --hash=sha256:f7042af3a4db5b90e5efaea257b6dfcff9389239ea643e6b0faa80169528e2e6 \ + --hash=sha256:ff51e2887bb673e2e757752b586b3dc924d59aa5fbcae9bbc45f4a111bd3262b + # via + # mlx-audio + # mlx-vlm +mlx==0.32.3 \ + --hash=sha256:0512049c81fded99aa6af2d039ae1fccf113ff1a5f2df937a596578aabd46869 \ + --hash=sha256:097856504e6cf5944b9aaa7c2e1d5be0693428abd9175429cae3893e7ec2f681 \ + --hash=sha256:15e8578c9d10b4fd4728243cd6c926bc46e1da267c6a849cdbceca21d51caa51 \ + --hash=sha256:177bbb598bcb37963568f69b84e7d972f7f9499cdd39d1c6d6dab35a718aeb92 \ + --hash=sha256:19da79adad1aa9653c70819040ac58d65d7a9610a375a30d03c7cce708d4d2bb \ + --hash=sha256:231a278b4fd2804ea16d297960ab169d1749ee1adbf820173e704fbed97bf1b6 \ + --hash=sha256:2dc1085f5c2eba18cdac095d7fb0b68ddfadd51bf695b18a1102eda15e77e76d \ + --hash=sha256:32fa23e57297fdca23e016570229b459f20f645eeb5965527cd890f74143f1d0 \ + --hash=sha256:448fdbc99c5fb0c26f11337e5a182cd6516f8a1a65848184be063d00ad3107dc \ + --hash=sha256:452c621862684e8769be93c1517420c8ba2fd6e8b01a1c970c8ba8069022f0ac \ + --hash=sha256:46df0a5489edcc071cec3d9dd1f7894d492888c9d425effdc8831f4e7021108d \ + --hash=sha256:4e1a8fa78461c6cf1f9473e3bc8a3c4e362f1aaa613b435787f2844fad522bc7 \ + --hash=sha256:50939a8bd54250e6092ef3427a724c7a72c0911dcee09d70f6543431e7f28d3c \ + --hash=sha256:5d12de01f23d2fae739ecfecd9c1d4c07c1f9b778a3729f7c0884ea1168ec14d \ + --hash=sha256:65122a7eeb718df56641d7ac6f7fdb0ada573fd841d1fb0a61a6a443e33f11b5 \ + --hash=sha256:656df5e36b9bfa862fd633d55dee4da072d292def9f27f22134c594e03ff28a7 \ + --hash=sha256:712ca29119d9e2aae04af8f8e416bfb0097538d9ff73e6fab8b028053fb45ff8 \ + --hash=sha256:72711cb23cc6dccc5a460f63f39c11931a00b2c6dcd6fba253921090072e21bc \ + --hash=sha256:75333afef55819afd2a31e87ffbed0bf59f379bca21771e6b13a95a675e4c85b \ + --hash=sha256:78372731c18fadb6d6960c9188cb752d5ea4d87de9be430a311a385e01d7fb42 \ + --hash=sha256:7aecc3d4a904279492dae58e28378a3c4bec4533e7066e64ae5c78c7a5511f22 \ + --hash=sha256:8439baa207db769df1df82a722b0ee0c2362f75c23c0740b7472c3d279253bae \ + --hash=sha256:8564a59943346fac01ab498870a9350d0f79b1cdea95c46c895934ee027d6197 \ + --hash=sha256:8edf996749b331659ab9bb8ecdc895c292f9e7020d02e534147e07cae22bec7c \ + --hash=sha256:8ef541373a94a78fd59ac59368501e976cef5fa07738438362e942315be22ea4 \ + --hash=sha256:936cc82b0c050686c43230c85f3a0bd9f179f650377ce9590f7af3ba659e2fce \ + --hash=sha256:942f0da5328986cdae29ec0eba57b2a08eb9dc6125cfe196853de7498f2c5359 \ + --hash=sha256:951ba2ab75148f61e9a5c222a75fdb0ab481be0b196c21a0d087e9b886d204f0 \ + --hash=sha256:a14387cd679018a32e802fdaa58b1e9845a278cacd309955eddca95757988ce0 \ + --hash=sha256:a5e165179e3d86672cc0d594587bba2b9c3be511ccb99ffdc63e4cc7119eec55 \ + --hash=sha256:a6241ea8f56a08a6c3a06bbaad7d7fc2fc1c05f23757c61fe0a8797325ac6d46 \ + --hash=sha256:ac4472834f38e61d294aa1bf2fdc9f687f3b3162b9729dc9dc15e8c149f8c216 \ + --hash=sha256:b14744a1b3a9dd48e06a62c448710a798ceda9451d7d37a4654a58b5adfa97f8 \ + --hash=sha256:b2acd057e4a27eb1543bff0e9030c34fdc64e18815fe4bc23c2dbcc47b60f0e3 \ + --hash=sha256:b57996fd2b72207a796656bc39508f20089692815297120039284edb296b5fff \ + --hash=sha256:b91012fdd5ec5cbedcec2cae6adfaad9a8f547023e570fc5781cb7eb4bfe2567 \ + --hash=sha256:be5cb6554873d3c86f083fb0b35aedf4a1f12657adee4ac0fc90665ffd7e6722 \ + --hash=sha256:c0764ce1a7a7f19768d6b3473387dcac03cac8b0dc74b7e1c8d05b451ff4786c \ + --hash=sha256:d17f0c33789009a3b3e609a52af34dc687ed39816836e20ab466aab669c14e6c \ + --hash=sha256:d3b9494b45400b803eeed7d05e6e566aa135f1dd2d121c29016ee9492e7ead67 \ + --hash=sha256:d61942263d4603814622cd72bb5991b8a3cb59970bde8a7b40b897a7f088fb84 \ + --hash=sha256:d9ff05b868748b846a02f59913e9aa9d7b576827db92cc358ec1cf660f7f90a5 \ + --hash=sha256:e13f5ce391293dab946edf18e7ae7bb55768beef1140deb509f67a2b14edcf7c \ + --hash=sha256:e32d595df067f19739f1e214e723035ed2a9c216fb1ed3cd118d08f19b9cfabd \ + --hash=sha256:e75fa55161d9477e694f852cad0d383c6b4a5c3b505a5d08339f0e2260c2556c \ + --hash=sha256:f2909f683ffc5a2720f6c8099398a64fd1be53f1b6538e4e728ee55816ce699b \ + --hash=sha256:f348705a72fa236f310c2cbb9c393d3b620a942c9b34ac641fd0f098bdc5be44 \ + --hash=sha256:f5716b51fea3d06352d4d8c851fb6c5141b1a7de5a5164d44de5a6d368d84a7e + # via + # mlx-audio + # mlx-video-with-audio + # mlx-vlm +mlx-audio==0.5.7 \ + --hash=sha256:2f2b3c666241ff4a45df7e92057d176b3c1079541a5e9062a0d1e8088db5c123 \ + --hash=sha256:4446704bd8305b3957a6e422234118142e32b87a55d7d3e33b0fb8dbca516543 + # via mlx-vlm +mlx-metal==0.32.3 \ + --hash=sha256:34ae9b83ad2f0ccdd3e5d48ec35176e7119f57069eef187122916dc941a4ae1f \ + --hash=sha256:fdba88ed9131448cf163be2c7ad6374746d8818205f28657aa91b6c69050f691 \ + --hash=sha256:feea99fe4d01f9e9f33ca73d869f988219a1725de1c18870baa7d5cd5aef6cb3 + # via mlx +mlx-video-with-audio==0.1.37 \ + --hash=sha256:b5cb41f0265835a2f9124df4e1056bcf33078ba686757cf72e4818b7ddcbbd14 \ + --hash=sha256:ea560a7b659800fcb0e49457914eb8ae20abe201a0bb66ec421046b401032695 + # via -r mlx-video.in +mlx-vlm==0.7.4 \ + --hash=sha256:7a3e8e288b21115630c4eab75e9ffc38e4b1770f9d2ce4992d29ecffe75a628c \ + --hash=sha256:a35ca38c5be3a061496cc9bb5b3ada20413ceda9317ec67ba0b0fff1c2885e04 + # via mlx-video-with-audio +mpmath==1.3.0 \ + --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ + --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c + # via sympy +networkx==3.7 \ + --hash=sha256:e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0 \ + --hash=sha256:fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a + # via torch +numpy==2.5.3 \ + --hash=sha256:012e66aca395d795496446e52aeeb5866312a5d4d3f27da270e5a0b43f70dc5c \ + --hash=sha256:09d5a423c71ad5feb5625844ad58050e35df43871004b52ac9c0ad44a56775be \ + --hash=sha256:09ffa5d903faeaa5c4dd05009cf81c8bab9f2cb37c548b8d39b65b4cfa7c97f7 \ + --hash=sha256:0a59a421a32580a009e8a1751345bf829631b990dc1794b80514ab722b435def \ + --hash=sha256:116f96cadd935c6122e9228d676fe7ede19e741f5c8bb1c3cddbe0c51ccebea2 \ + --hash=sha256:1302b90c0e52281681b2975adfe8a860cb7b12216a27b4b0b4207c44bf7bccf0 \ + --hash=sha256:15aa985ac73a8db02db7663381aa109510449d3819d37206caed27b33a65a8a6 \ + --hash=sha256:1aad64d99730d013cfc6debafed22783b4fc5a7f4b8bc744d2d8cf7dcc880551 \ + --hash=sha256:1c80eabb4035ecf4ca9cd49cde8a9fdd69a729e63e6474887d1523ade7aa277f \ + --hash=sha256:1f3ed25271581281f2fccb1adcedfcde4c07362eec69189b50baf6f90e3ae159 \ + --hash=sha256:1fb6f8fb9ff0b3a69f52c66ce397b0246583e9f28616231b0e32ca49259a5fa6 \ + --hash=sha256:214045a5bf00113a146ab9ee9730c44501af6723cdf1f6830932f7b5ef2e7af0 \ + --hash=sha256:26e15e4aecd8617dfbaecb37d223e365d7b39411fba20454be2670a96aa74cb5 \ + --hash=sha256:2c25dfa72943e4336ddb6b0ee4277b47a0c85bede0807530ec68103bf58e2c10 \ + --hash=sha256:2d8240cb4c16fd831074aa2b2cf9fc54664d826341d61c372245b96a74a49a9a \ + --hash=sha256:350ba9783ce969cf9f7ce6e6a9a58e1a6e2a19ca025b7ee448c4db727706212a \ + --hash=sha256:4c8a6d2ebce6305fd82fbefca827775437147052a976ee7c94b36a0c1b52ac6c \ + --hash=sha256:4f8929ee6c96bfbd7b4ed2032e0c03af86fe1826740ab61ddabf9072d06e57ff \ + --hash=sha256:536f963710a4e63934d80ac0dc4f478804a83e9a84b6828018f25d09953ada33 \ + --hash=sha256:54a115e5a73b8fc44f0cebef486365a1894b5c9760685d4558b72b7c3eb846e0 \ + --hash=sha256:595d020938c84e320bcf40ad71089e108eac0d377cd018e14a8c094f39e98d85 \ + --hash=sha256:66a78fe4556c60aceda5916f9eacd638b18e9e681016ec302dcb4682d6d4d034 \ + --hash=sha256:6b05c171afb3aa07adbd20abc00aea86fe375beb0fdb9ef780ec5b7f63bab1c0 \ + --hash=sha256:6cef4bb1706dfec49243c05d921eefb4e190d41e2528b30d8035ea1f36b4c24a \ + --hash=sha256:6f24021b9f22bc6301c37b196974a92c1c18dccedb6fef3dd252e95f2d6adbe4 \ + --hash=sha256:71b39d9f935b6ec0f8753e3e2afb51e3efba6f2e05b68b32a40754d24bcd4a3c \ + --hash=sha256:71cad2b2a7451ab79d8f5e71b453485b6775963d5cf794179144a7463fe6e8ec \ + --hash=sha256:76c2c1e6bfa5c84adc6434dfbf013aa92096a7985221762c8f11fedfd20fff58 \ + --hash=sha256:8617bbfae4486cf99c9f899966699428d19da931d06ca94ad3da986c76e15997 \ + --hash=sha256:86bff898a431c0fb71f7610b75726e75a54d47b37edc9d537f48de63bb3c0b90 \ + --hash=sha256:8e4dd766076855b5ff7ea52fa5f07ce26286726e0f8bff446b7739d02e6ea204 \ + --hash=sha256:92f30e89b8ee0ecf363033576c422b2f58fed6a80bed0aa48dff6d14c654663e \ + --hash=sha256:93e1f5447e2b1e479d7bd74701e84746b86450cff1fc368b132d195e2b8f8211 \ + --hash=sha256:9a37475425b431b4d060f23b4f52cd2f3aef6bc7c654bd760adf0040eec9d435 \ + --hash=sha256:9deb49575e5b0b94ed72c8a64ec4d033381adc27e9060ae842971f697ba96104 \ + --hash=sha256:a5fa86b80fd24bcd1aff83ad23be44ea323de3f787be8f8b15d4a65621e25321 \ + --hash=sha256:a6391fafaba97500887132cd582abc6e19452b1ac775a47caa7b24490e152058 \ + --hash=sha256:a72f874bc9e10e4b8f80426fb49716d5141f64442a0c8418065093ec8017fbb0 \ + --hash=sha256:ac7bb1c52d445bd4f8f7f97fefe6abc3a084dc4d63df50d79b17fa2b78e89297 \ + --hash=sha256:adc1ada2662f8a5f960b8a10d9986897e7499ef07e06d4cfe7197f8cce923c07 \ + --hash=sha256:b00eefbcf0f292945c4b4dec2ae845389ef5bcdcd596e6e4328051db5b5ba694 \ + --hash=sha256:b0521d0f4aebb6e06189451025fa17a913287b13c03d5fe05c017333b654ea5b \ + --hash=sha256:b5d93cf48f687479941d12b69c873ad2cc76bbd487f0091c2200636497f34034 \ + --hash=sha256:b7e18c623bb5c95acb3b3328861272816ba199fb531921c5d6d0b675f1fde9e3 \ + --hash=sha256:bd4cb9ad3c7889b9b3fe0a9a9fb5d2ed26f9879bff2608d9f01aed147a20d231 \ + --hash=sha256:be5a8381859b6da607c84f4f7d6847725f1cf1853ef8a2c9e115b7d58bef47dc \ + --hash=sha256:befa1ae5bd6030b3f512b43ff3fa5290bbed6b84411a44244b14adf835f5b89d \ + --hash=sha256:bf63afbe037eb5d2fe87fbcc7778e61da53ebaf21d938a4515aa73b62532a5d4 \ + --hash=sha256:c00abe94c1a69d75d827dcf1c025b25c8a45d230b3bcd77a9020883a1b047653 \ + --hash=sha256:c2381f82999704f818e2c987a865050e285ec3621262c66d40f5a96c8f899f8e \ + --hash=sha256:c76d5dde9f445058f83d0c02af00557a4db91de9a9a57c0df87d1535001d654b \ + --hash=sha256:cb189f09db39283b26bfd061ec16189e14f71c6755207f72a0f7540867afe5b9 \ + --hash=sha256:ccb32e0525d29e8b0572eb84c9a57af0e7a4e615726927506f55063c62414034 \ + --hash=sha256:ccbc4665079665c3cf3bab4db9f6b095370cd6437d66be549b6c2a1fd19e1958 \ + --hash=sha256:d1c89973648c85069c5046ad460f7b8a00218b29a2e42359ac8cc63e9ab94832 \ + --hash=sha256:df2d5874ff183595a4ba404edd04f6bd9b5505c1d7708573f6a6c17489a67563 \ + --hash=sha256:e01c918ac3d48e18a927cf7b14a26a3e29ff2bdf2eacb976da0aecd6a43ed034 \ + --hash=sha256:e6ab667ba76450084eb64013762c438ea76d9d29cc676dcd6c2e9892ba37f841 \ + --hash=sha256:e931e4f499e0dc7ef29d269a8e5b35dd722e5d14be07df6240166ea7c6532fae \ + --hash=sha256:f54660b0eb6b0b9f36e7fe1cdfdff472028dd0d14acd9b9b65098efbad059469 \ + --hash=sha256:f59a878c33d6b88122d80d239bb3b845d58708750b0cb06a09aebb9b18ec696c \ + --hash=sha256:f7fabeb6cea87d65f3b926de33d03fb016cfdc29314c90974383b5582ae72891 \ + --hash=sha256:f9579f383d1bf9df80081e72760e84960a7fd4f88cf0c9e535a8597c9bb646f5 \ + --hash=sha256:f9a2353b37a1a9e78fd82b27ad7e2a32a2d036604d18f02b05e3136c62ca3b09 \ + --hash=sha256:fc36dc566135b5eceec4cf89758fcb719266a019ef07dae1754ae7c9f617ef3e \ + --hash=sha256:ffdc76bfcae6b255dff75202c5e7feaf95b40246bc0a17944facc1fecf9f79ab + # via + # mlx-audio + # mlx-video-with-audio + # mlx-vlm + # opencv-python + # scipy + # soundfile + # transformers +opencv-python==5.0.0.93 \ + --hash=sha256:08d5d91d967b58d6db86073b2ad3eaef88ca4ebdfd45c9059bf59f5ded0c7ad2 \ + --hash=sha256:198a75138241810206a17c829dbcc40a7cb1841cda538ca86cbbfc6c7d95f898 \ + --hash=sha256:4b4b1a34c79bf8d3738e3cfe9a9e67b51a79663f6b692cbdad8c31f570da4157 \ + --hash=sha256:66aac3e5b5faa48d4025816592f3af19e4bfc2c68dec067bae2dbb4ca10aa9e2 \ + --hash=sha256:6bbc32f59e1b1a7db7b39c81f63d00625f041d333037fd8702f6da52cc39108b \ + --hash=sha256:c8de2dec111122a02e8beb28e16c31904992dfd6186560b142a92c71403c1039 \ + --hash=sha256:e2b4272e736836f66c2d176e43ab8101f3a00d45654916399f52e150c58981ac \ + --hash=sha256:f8b6d0a212253dd26ad338c812f1f23ca118fdf05a9c8c6b9444f161aa8c5881 \ + --hash=sha256:f90ba04b8f73bc5c3814037699739f0156f597338a98f05956c684e7c3ca10d2 + # via + # mlx-video-with-audio + # mlx-vlm +opentelemetry-api==1.45.1 \ + --hash=sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75 \ + --hash=sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb + # via fastapi +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # huggingface-hub + # transformers +pillow==13.0.0 \ + --hash=sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27 \ + --hash=sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782 \ + --hash=sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22 \ + --hash=sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15 \ + --hash=sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502 \ + --hash=sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692 \ + --hash=sha256:10b5019883c98f1dd819ca48ffc6f33a91cee04948fe637c605f4345c7cafd6f \ + --hash=sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8 \ + --hash=sha256:12fbb851cf323cc19aac3eea1df5ad17b57e91e9fe3cfd06100dba7bb8208c13 \ + --hash=sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3 \ + --hash=sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369 \ + --hash=sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88 \ + --hash=sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d \ + --hash=sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb \ + --hash=sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f \ + --hash=sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0 \ + --hash=sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40 \ + --hash=sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599 \ + --hash=sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055 \ + --hash=sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033 \ + --hash=sha256:37290fbef0c73e2d51d594eb78fded7dc7170d1c3ad0a890269e6fd8c8c4e061 \ + --hash=sha256:405be6058e206a86fedce2f715ecc2343f94072d7925251632dffa2c05f70dca \ + --hash=sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f \ + --hash=sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca \ + --hash=sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c \ + --hash=sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4 \ + --hash=sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8 \ + --hash=sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620 \ + --hash=sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071 \ + --hash=sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc \ + --hash=sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a \ + --hash=sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6 \ + --hash=sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e \ + --hash=sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582 \ + --hash=sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f \ + --hash=sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79 \ + --hash=sha256:6a0920cb55df5a4d3f6862964908078c0bd59332bdd1bcd20d14e8c70ef27cc5 \ + --hash=sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683 \ + --hash=sha256:6f38f3f885a09cafc93fd667b1173a92f379e43785d635e7983c893f0da319fb \ + --hash=sha256:73ee25cf8cf8b75d5a5aeed7cb6588f5e5dbfbb0fde51707b630bbb23897645c \ + --hash=sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa \ + --hash=sha256:7c88c10dd432f3b39ccdafc9ed81d0c21b8fd634502574da76d1bfa4fe2c76d0 \ + --hash=sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521 \ + --hash=sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e \ + --hash=sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c \ + --hash=sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058 \ + --hash=sha256:83b37abd87b590acd8f47ca8a61f70893322b11834d36e07c2298dacad4d14d2 \ + --hash=sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463 \ + --hash=sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d \ + --hash=sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81 \ + --hash=sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf \ + --hash=sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63 \ + --hash=sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0 \ + --hash=sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42 \ + --hash=sha256:a192360d512305fe92d3053e4f7627bbad1f7708f1f4ad38cccd9336acabe9ad \ + --hash=sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81 \ + --hash=sha256:a625df9a2a5597fa5e5266e01aefc82604bb85d291e3547981e863560d75da6c \ + --hash=sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4 \ + --hash=sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981 \ + --hash=sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43 \ + --hash=sha256:b84be41387fc00db28c13c02e30d42b02d8d8f6a4a741a1aee6012ca7f1f6039 \ + --hash=sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76 \ + --hash=sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51 \ + --hash=sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2 \ + --hash=sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630 \ + --hash=sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac \ + --hash=sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510 \ + --hash=sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6 \ + --hash=sha256:d7710eb676e9bb68965cb045be0d333b4c0d7f9dfbaa7fbf121385b1588683d3 \ + --hash=sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d \ + --hash=sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61 \ + --hash=sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932 \ + --hash=sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1 \ + --hash=sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d \ + --hash=sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00 \ + --hash=sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe \ + --hash=sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313 \ + --hash=sha256:f391dc92a015c478745192fe4f2e120c2af2aba18d9fe0616341cbf537599806 \ + --hash=sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a \ + --hash=sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0 \ + --hash=sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52 \ + --hash=sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b \ + --hash=sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe + # via + # mlx-video-with-audio + # mlx-vlm +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pydantic==2.13.5 \ + --hash=sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73 \ + --hash=sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08 + # via fastapi +pydantic-core==2.46.5 \ + --hash=sha256:013d6f3483d81e02e7c328831808f336c8596ee33b4bd4026b9ffb1e960b8942 \ + --hash=sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821 \ + --hash=sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5 \ + --hash=sha256:0bddb4020d8f04175865ccd17eff3040874fc11fb593f424edb452653b4b947c \ + --hash=sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184 \ + --hash=sha256:0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2 \ + --hash=sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc \ + --hash=sha256:15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5 \ + --hash=sha256:18a09e1e1011b462f2e32774f25859ef1223d5c2b0546a633cf56654710721e0 \ + --hash=sha256:193375f3548919d3f0b60936ca113ada3e38f264f91b9b8e0508efaad57be931 \ + --hash=sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e \ + --hash=sha256:1e449def1945a462c464331254e5a44fca7c3b4f9aedf59ec2f50f8066dd8e25 \ + --hash=sha256:1e5aad1220a1192c42341c8fd4a8686657e73ab2a920c970bdc4de334fe3193d \ + --hash=sha256:200aa3dc9f8d54f0754f43247c0bad0999fdcfbfd2488384dd44f37279271fe6 \ + --hash=sha256:2471fd51c61c610e1dcf7de44d7299283661654d11264ab4802b303368d69c47 \ + --hash=sha256:24922243639cbdac66c75fcb6fd6495a9cb52b213d62f9a0d16f0310b1ff8038 \ + --hash=sha256:28a6a556cd3b6066bea827857f9d9cce027c96f776e512f544a581f9e42161f8 \ + --hash=sha256:2bc9419666990c06d7397831f2126a1ecc3594aaa3ff7de5bf2d066802f4e07b \ + --hash=sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a \ + --hash=sha256:2d330aaba8621b1edcec8ae2c4050f63b84ccf6d98723a8f212e9684713abf0e \ + --hash=sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074 \ + --hash=sha256:337639ba62a11acde6ef3aeb08c8ea755f8ef1fe5e513356c0f36a2b0d7568b0 \ + --hash=sha256:347ec774390c87326a2e4929d58d3f7e8763a104d5d35f4cd595a4c952366433 \ + --hash=sha256:356c8368cbc321050b169595683a2e1d63413b1e0e2868b330af9fc14c616d3f \ + --hash=sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034 \ + --hash=sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21 \ + --hash=sha256:3a3e26b6a8274211bddee2d0e4d0d42778f17a34510f49d2ec44b58abfc41736 \ + --hash=sha256:3aa166e99c4f2985407fb8714aebede877ecb5455cf321b606adca926d30d5a0 \ + --hash=sha256:3d2652072b2d774947ba5cf78a9e59644ac62ee572daf6dd2e1dfe905e15b2b7 \ + --hash=sha256:40375c2d05acec10323e45dfe2077ac44bc74659008614af5069034e2cfc781c \ + --hash=sha256:413a717a410d0c817ef5b786a059415550b3794e1d0c2abffd9efb93a3d9f7b4 \ + --hash=sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c \ + --hash=sha256:49776eab08766a08dfff7012f8b422dcd7e25e43b316eedf0477c24fcfa84b7c \ + --hash=sha256:4d44cf99ddebf875f9b68cc267aa684c99b7b44fe63ee1cac4ec163807290069 \ + --hash=sha256:4dedce55295becb61921e386b99d4f2706045306e7fa52249a33004c837379fb \ + --hash=sha256:4f8507560a9284e1370bb048ed4282012fbef4e8d109875b95e884d228552061 \ + --hash=sha256:4fdc8b93a41521988916eeaa271173fcca7fa0803d62f87675aac8dcec1c8e29 \ + --hash=sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3 \ + --hash=sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa \ + --hash=sha256:53feb344243bb9510a9dec7bf3cf1b64d88a98af5dc7872a5160465f8b198c8e \ + --hash=sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38 \ + --hash=sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f \ + --hash=sha256:5cb482e9e84c851f4e623fe4acc1ced89168cf1fe18f7089db4548c8f5bbb65b \ + --hash=sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8 \ + --hash=sha256:5ee239d575f80b08eca11f6e20f90c4c695de7825c67eefe6091fbf20dda648e \ + --hash=sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c \ + --hash=sha256:5f93c5fe914d75fbec9a49209b00da5f08e9e467d69da2b1510c81940cfd10be \ + --hash=sha256:657b40d6240c0a7b6a64b30f22d1e3aa631c7e846c621b0c0f6d1d75e2e15ea6 \ + --hash=sha256:6d30e1a4f138b8951063e9a394752a9179b51da288ffa507b1e659222f4c1793 \ + --hash=sha256:6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1 \ + --hash=sha256:701b2e04b560eeb4bddf7a25ab8ca476176e34fdbd9a0e18196f0d12d4685f0b \ + --hash=sha256:771cf63ae0b1b50dd22e5f3e3549fab5f3f4ff1635d352a9e1a97fe01c7b2e64 \ + --hash=sha256:79bdfa52f843137045b2d081cc05c120ba6665d29b7559c2c47690906f39279f \ + --hash=sha256:7ac031912d54f3d83ef3b3eb98dfabc1608802e2202263d25957eeed40b94761 \ + --hash=sha256:7b0fc826b16c55e561e5d2a0c5c77b051ba1d92808118c4e4b5390f5e0cf191d \ + --hash=sha256:7c6be839a5a8312626b32029a415644a0846b420bc8b52b95b28cd92da162168 \ + --hash=sha256:816ff0a6550ffc06c098ccd2e0698600f9aa7da192a79eaa6f9af504a35db869 \ + --hash=sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111 \ + --hash=sha256:837b396ca3d7b74091ca623f6cbd8351bd42d670a79c2683e79fb089f06a2de5 \ + --hash=sha256:850a08d167dde16db8702c274f320c7be9d7da6f6dff2b58b18f9e815bd94f5b \ + --hash=sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7 \ + --hash=sha256:892a881d5f68c2b9ea304b7a6c2c60d9343df578a311b0f86b94bc8f1ffe8129 \ + --hash=sha256:895395f8918627b04efb1ad2a4cf605387143300ba03304cd1dfa6d03f5e095e \ + --hash=sha256:8b10e3e8fd7ddc2bd915848a2768e44c15b22936f1cc54c462ad1164deb02655 \ + --hash=sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c \ + --hash=sha256:8feeac04b5794e513e710af2f9c87d49f31a6dc47967bb264a1fed61a8989bec \ + --hash=sha256:9432f3598db432cb51c5b37fdbf29a60fcccc79e30d37a05022776a6bc4ab689 \ + --hash=sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f \ + --hash=sha256:978e7b97d4824b5be09c69fb70507cbde3b0323fc147332ca40a94d9a6a0ebbf \ + --hash=sha256:97bf8de4d541598c94a59344eeb988a94c08ff76b5723c41f6567ec18c7892ea \ + --hash=sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9 \ + --hash=sha256:9b68938dd5b0c783d88ff8e2dcc69451b5eb936fe212d516b21b9d5567f6d464 \ + --hash=sha256:9c4b71f10dd532fb7a5cbc8f58707779e64f03a258c2bf8bfbaecfcd9970b519 \ + --hash=sha256:9f47b8a949e60f027f0aa0a6f6c7b7e9c55cbf4380d10b344e282fa4e7ab1e1b \ + --hash=sha256:a1dee1b804ff4d11c663636cf15d2ea47e9f79cd56c033fb1cbf08924842a48f \ + --hash=sha256:a2468d93d181667a7abd66e1b64bb9f76f361b0fef8faddf687456453576f5ee \ + --hash=sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a \ + --hash=sha256:a39ac25a9a2fa4072efdb429833c4a4c8009a51ff9eea3eeae131713cd27991e \ + --hash=sha256:a445486499897b88a7d6c310c88ed64dd37b1b59bfd7ae9107490bbb362f47d6 \ + --hash=sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2 \ + --hash=sha256:ab4b66edffb32d9e951efb3814bd104b8367a7501b81b955cacb5726d897389f \ + --hash=sha256:aca6c767f552b21b10f774aeac128e828eafb796adfa1b666a18bf6321453c3a \ + --hash=sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f \ + --hash=sha256:b10ec717381bdbfafef34607824db4c91de69ff085e4fca3b2af91b4fa17e68a \ + --hash=sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47 \ + --hash=sha256:b6acfb46a814762367fb7ba0828b0a17d441b92ce249a0e007474c9072662dda \ + --hash=sha256:b7ca9034437b6022f941f4857459562ee00a560b97e7cce8a0ec5a74fc6766e0 \ + --hash=sha256:b98134087d9de723658d17a42c7d0da8d6e2ef08015dee7dc93889047315f5e4 \ + --hash=sha256:b9fe6fb92520e3fd61f2e49000b6911b188824f089b75973ea06d6267f0b476d \ + --hash=sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3 \ + --hash=sha256:c14ad3bdc85ee7f318742c457ca3968a92126d144b15721c759033bfb06296c2 \ + --hash=sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355 \ + --hash=sha256:c3471e5c4a949c26ec00a77f01df59096aa9495877de76fd60a980f8ee6be461 \ + --hash=sha256:c583b927a8838dab890706a6fa7573fbb8b70e24000ef9f7238e2d6f6435a5ed \ + --hash=sha256:c76fe65e607be28c7fd4d56fc3c42b1583aa058ce3408b7ad0fd540171d31f9f \ + --hash=sha256:c7ea57fc63aa7da93a1bd2d644e6577befae10c52c4e36377635eea1056a74f5 \ + --hash=sha256:cd5214352ae68f3b5e9af7768bdc5253695ee069675db3480518420b3be881f2 \ + --hash=sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829 \ + --hash=sha256:cdc8b74ecc48c0cb1e9607a05ec4e9e88db60a19ffcc9a1d5f9088ede40c8dc0 \ + --hash=sha256:d0a24b40877af2de4950252be9d21eaf7fb07660f3c2cae1f56c6b599ada5266 \ + --hash=sha256:d22a945598fb91236b4dd793a6e42e4f3dd7740bb5aace5ebd7d4c08d13bb575 \ + --hash=sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290 \ + --hash=sha256:d625a186a65201c23a9e3b8ed9c47e90a026e03256608cc91851c6709096844f \ + --hash=sha256:d925f3d9afd05a8c0fb3a1031463a8d59ebe5e2afad297e29c78be19e13b4e62 \ + --hash=sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f \ + --hash=sha256:e652ab17569c94bff5475520f907b7148b8c24036a8ebbe5cf7cf7493d28579a \ + --hash=sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7 \ + --hash=sha256:e80675d75ae2cd14372cb65cad5400d9347a3d3f6c13000183f22dfd027283ed \ + --hash=sha256:e9c134bb666dd54b778b9fc0d2b50cbb7f979b9e3716f26a88c9ab3b6fc1dd0f \ + --hash=sha256:eb7d8d0e5886a89a55d2eef490e272fa965a9d57c6b29a5b5088a7997ec2cad1 \ + --hash=sha256:ecb42011e12ee19cafbc312887cbf3546959fe02fbad44f272d4be5baa997615 \ + --hash=sha256:ef3fbbf161dc9351a2fe0422e51b129f9e97e42385bd0320b309c15f7d287dd8 \ + --hash=sha256:efd62a42486f1bda5d24cb4f63d15a3c7768375fe83d36f9417b4ad7a2fb20b3 \ + --hash=sha256:f077d0b97ab11fa7dcc633fca53515f290bca8a8a633e966d5b6d1879d9ed01a \ + --hash=sha256:f332f0e72a5a0400141f830744e141bf9f97917878dbe968669e8a7fefea78ff \ + --hash=sha256:f7b0ec93a2893de856652154d73b7ba622f26fa97726487dcac373de5f4c6084 \ + --hash=sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0 \ + --hash=sha256:fc5d783bd4a2387e97b8a2d5ec781cfb92b3d893bf82370548e99db5915935d3 \ + --hash=sha256:fc8515076c11f3cfdf4fb142dcca0fe384b1230a3b5415458ac84f3e0903ec13 \ + --hash=sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9 + # via pydantic +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via rich +python-multipart==0.0.32 \ + --hash=sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e \ + --hash=sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23 + # via mlx-vlm +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # huggingface-hub + # transformers +regex==2026.9.29 \ + --hash=sha256:01000ddf0e3ffef97f2413ceb514f6313040106b6d18a03ee00a4fe35c1eb1db \ + --hash=sha256:0166844493626c5015c6088ee15c9ca2fd060ca15b7641d1657da6a58432ae33 \ + --hash=sha256:044265d77d94f5e3cb2fd72c76723807c429cb8c533e9d4672d0334a6f14f588 \ + --hash=sha256:0476e5bcbe6e1ba3d1c4cc7bbb1c3ba78e3b979b5c8a88d0a6a8cdd4992b8c84 \ + --hash=sha256:066d0e3dbfdd739bce2bf8c2a41dd16f73e3d8adc2eb06dd803a36a307f56075 \ + --hash=sha256:0b65c72739f981377c9c22e0c5c3cd7f42da7bd8a3c9209330fac772c7d893ed \ + --hash=sha256:0c992c19cd45058a4b92f68f139c93db168b48fb1f322c9a7cd620806afb6b51 \ + --hash=sha256:0cc63b5e47c12a48d90c7e9d7de6a035dd14f62868aaedbb4e0ff8ba2b8bfe7b \ + --hash=sha256:0dd8af32e9f7b56b7f95cc1fd79b23054c3bdc172392ae560acc24d57b7ffe71 \ + --hash=sha256:0def9fb6abac55492d6d51cddb7225d07d6f279e774e0adc08569a54a5fc8d46 \ + --hash=sha256:0fd2c901cc307a745ad4bc87f20060d7a0825a3371d1e93488af22e7a387f78f \ + --hash=sha256:1043aedf5917caa861bcb25a9c11460049656bdf0017a90a309fa8f255467725 \ + --hash=sha256:121a76a0985db80ceae9e171c337f8c927868e37d01b54e3ce87bc87f9c6a208 \ + --hash=sha256:143533cc4b6fbc5b95aca0a5b8d541088d374831593def000ec89322c220221d \ + --hash=sha256:14e953ff3607c92d7675bf79c4d4509ef6782aa8c08509f179f9b3d6d0679e86 \ + --hash=sha256:18ae8eed4526e35bdb754d61562b90bf5c00a67fdcf3cc1380dd59597486631b \ + --hash=sha256:19959129885356df0e97556856f77eb2888380dac18bed075a7c05c5128c618d \ + --hash=sha256:1ba8c6a416569ce0d37e83e28a254a61dc99a419084dfb6476cea02d997f74fa \ + --hash=sha256:1c2a0026062abcc321a53db4a185ceba0b59a66b5d37b0808917a88b55a5257f \ + --hash=sha256:1d9fe8091b2e89d470df68a9331111ed008ae8aae6bf1e8e1fba4086a495c84e \ + --hash=sha256:2089fe39c406784d90101c726755ffa1497bb74638fd434300d2b88006186de8 \ + --hash=sha256:23ae6fdad9e63e54038f5ef78aba2933faca61e24d432786589e737bc5522ebb \ + --hash=sha256:26ec4ccce55aa533fbd603d08911b01101a8fcfec987845ac3ae2c7087b2bde3 \ + --hash=sha256:2f7f7aa47b229f2b39a2ae2596d2ad5625d77b5eb9856fac2dab3eb506cdd0a0 \ + --hash=sha256:31b003f9a070335e2a8233ee9b14a3ca8e6d792012ae011f741bf0aaf11744c5 \ + --hash=sha256:32ab11df9677ca80bcbb5fe4eb1da9109a5019239a054836efc6fa1c64e683cf \ + --hash=sha256:33026515aebc0e70d1c89978e53e8d695d35d9e472f8d5b34465ba3c74028650 \ + --hash=sha256:34b6925af9853bf461950e6508910f179fd6e9b1a7ec8548e069606b7e51a26b \ + --hash=sha256:352cf115a810b357caa35193ab656ecf5ef41056855e82f292c99e8514f8d954 \ + --hash=sha256:39ab5894d971f9ac68baa6eca5c50387db579cfcacf36ae8df3feceb1815e6d0 \ + --hash=sha256:3a21a9509d0ee88e7a70e1ad228cd2f0e0fd1e187458db132e8a8d18c97daf9d \ + --hash=sha256:3c5c2ef13797466aa64170cbb66ad98a32351dd4127694cea7199f80f213750d \ + --hash=sha256:3e778bfccd63075167709136afbc251c1f683758d5bf49c803c60ac3f894ce6b \ + --hash=sha256:3f1e6cb402a89457582cd696f982559217d13484a193202c394015297968c86d \ + --hash=sha256:42e82e578c904445d4c8a35b8f28052cf567593215fa5db06266fbc6f77aaa2e \ + --hash=sha256:4408b2b27a95ca8cc48b7411945753773353b5c93b307754781086c99d3a576f \ + --hash=sha256:446654b29bfaa30500d80947eda42cef1449dc8a87f4e3cf061cc8485d3a1f0b \ + --hash=sha256:45010bcfe66df41522d56c9b6114e87ecc597a08970ff6a2ced24415c141ae5f \ + --hash=sha256:49ee178ca31c94621294bf9b8b676a92a2e6bba8af0529591753719e57edb621 \ + --hash=sha256:4d7d93613b01b0199961330e49cfc52d479b3d5776c56c691db31130c0a07d91 \ + --hash=sha256:4fb41211d2333eb930a51e0546a65999761cf1f572a4da56ef9b8a62966c06f2 \ + --hash=sha256:4fe97894d1b306c919b4e50def1e6f6c522f4d03a7283811f4d108f1ce5d3ac2 \ + --hash=sha256:554bffadcbcb6d5f4e5fb10a61cc52084b9a63d1dab5f10bcd2c4343972e8e2c \ + --hash=sha256:59b49507f47479e299a9e1bc41b5cb83a7afda0540625f1dbae886615978acbf \ + --hash=sha256:5eeb8edc6110d9194a4d0d54610f64c37a31c605b5dbb7e407fc6ec7fa34a4a1 \ + --hash=sha256:612b709381c0355b70d89cdb51b7f670591ed5cbbc0e3b5337488019dc667b65 \ + --hash=sha256:61956f074ecd123f55adca68ee3eab46e6a07ad3f8e64e6db95dfacb444f55c4 \ + --hash=sha256:6398d5145689503412cc1748895242598d8846b8967b851133b20dc2ed1e21e8 \ + --hash=sha256:65b408d8fcb273e3499e7ef2ce796810da1becd208c7fb4373692a242d79d461 \ + --hash=sha256:686ac5350fceae63830bb98805fcb8039325bf4c06d9f6f048ff65229d5bffa5 \ + --hash=sha256:6a1a824fbed817e0a891103886b68f063b1e83cc51bc97192a90a60195a9291f \ + --hash=sha256:6abb75ab16bc3281714a5b99548a2225db70dba1f995f6d7f7419b76eb5a8fbe \ + --hash=sha256:6f7121a8914ed13fcfe2099f895341bfb789f004d4c5a0bdece8fa667da10849 \ + --hash=sha256:7020ed44df30b3aa492c00ee3b52d0548c1f30c2c6c5bb13ae897680900d3413 \ + --hash=sha256:720537c7ea6f80dc61913184edb0ce2497a306b39ef19f28505b322553d52bdb \ + --hash=sha256:724184b4aafed865e4f13ca313fdcb43024300c028ec67319cfa16847d84685e \ + --hash=sha256:7c03031610e3e6ed1768a2b7a8fc84637c1257b50c5eacaf094c6e17a84fc563 \ + --hash=sha256:80a5ea3b4fd9d6a5b9a44f7976a9acaaab35aa3c1f6b29e5bd857dfabaded223 \ + --hash=sha256:80c7cadd3fd2bfde5df8aa0787e315812cad0c313a753095d02f4c2b6c01677b \ + --hash=sha256:80ea96f5c1a30bf09007d48466521d9c294bebe197c708c3359096e3e3691632 \ + --hash=sha256:864e9b87ac33c3fb9fb4ad48166d4fdb579c351d5c77deb0d34bccb36a775cd9 \ + --hash=sha256:87fb80cbe3557e27e7b28b995c2b2eedf689b8886f941ab93e0e288f0976518a \ + --hash=sha256:8873c4a11c50b9989168881aeb3f08859f469d809941866aa1feefd8be5431f6 \ + --hash=sha256:888d60953908dcf761aa320c3e390ab8556efbdb551ace63921de90f6ae0848d \ + --hash=sha256:8b5fcc4771732191b2b7d1dd68d8f0353f47f8d90b6150f6dce58bf1112442cb \ + --hash=sha256:8f39588af4731c8923c26810eb3b33f76f17633985e40f59c3cd45a33805a895 \ + --hash=sha256:9173db3be74a35cb6731701094b98120f7ee4876a287882a59cdea1fa7da342f \ + --hash=sha256:92f05c9c42bde5785dc48770bc2194d9f7442544156f951e19cd31b096cec562 \ + --hash=sha256:951733b1bbdb71e377cec567b409f1a7881b47cfcad84121aa74cb575fa425ea \ + --hash=sha256:957bb708e8057ab1649ba566456429d691ec9b90d1c9ad1af1ba7ffbbeaf05f2 \ + --hash=sha256:9916fda742cd4eede63b286f58c06718324265d727ce0856eb1aac86d0d150d6 \ + --hash=sha256:9e1d3a4cb7993b708f0ada8d0c84590efd853f169e7147d2202c9da503180242 \ + --hash=sha256:9e4482589065c8ecd761cff522dcd85f2d39e62f551e37e025d1c7d54772def3 \ + --hash=sha256:a5300757f8a68f5b6cc33f57338d72a0e3589c5cc9ad5f8504ea06f028be582a \ + --hash=sha256:a540abfab208e1b7ef2df231c40ef3b6cbb30a0aad6204e9b6a81c10a6794628 \ + --hash=sha256:a5758353650079898dc1b2b0e95aa51fa23a30d020e06f62c430dd08ee56cdd8 \ + --hash=sha256:a64b85a4760337cfefdb27d42da6ed8b58e8cde3f2d57b6ef43e76ef6ea9ef47 \ + --hash=sha256:a655d34b2a6943af32401f3d94f72e9d731f6ad16285815550bf2b4ee69d420a \ + --hash=sha256:a714befaacbd10092ffe4cea0d3c5f008fb9efe9bc322c715bcdfdee414b9a3d \ + --hash=sha256:a760da040b47767b4b873adfb7c3b691e9ba2fc60f113f9d0b88f1a62f323e85 \ + --hash=sha256:addd736a0547d553283adaf4e05d7104e7f2c7b0b092e9b4d28756825f14531f \ + --hash=sha256:ae4613d7d9dda60fcba95f846cc6f808017f1843f392cf9daad14a6534493d71 \ + --hash=sha256:b11b589e00095ec69cf79841a76360f9b079e95b0368a25b5ebb951ab0c157ff \ + --hash=sha256:b3e445b66c80b4eb4234e855ce94d9adc183eedbd632816228d89930b91b2c5b \ + --hash=sha256:b7b893976e7fe42053da64f2aa27239c24252fd2ec6df471e1be197c0addc3b1 \ + --hash=sha256:b84f186a7f0536fe4ff9a9fa12d06d007b9b71d4b5352ddcc41f59ad6522a312 \ + --hash=sha256:b89efc38431793d28b7cd91227e2f952ad7c48df19132b17f43a5fec3c14143b \ + --hash=sha256:b97a38fb4c732b6832db6bf108963adbcd82ef1268ba2025dce390f45af75efa \ + --hash=sha256:b9d74e4eee9ddb64c2e92d5d61472c59c21684c059eb7b68767be9628e977859 \ + --hash=sha256:bb90e7177944b6684738c1fc36aabd2dd00d1de3be7dbe09f91e196f1bc0dc81 \ + --hash=sha256:bec37990e3d6121f29ecfb594bd8f1bf009e9f7926daba2e50e3b27d3892a783 \ + --hash=sha256:bf3c49863c23a1ad6da9c30351aed6cff8d5ddbeb63c5c8420ae54e98c7d0138 \ + --hash=sha256:bf48516e35cf848390ea68850aba53e7c333720d2945b4d2c25b69fc5171723f \ + --hash=sha256:bfc71e6d970419c1309b3640305298643e2a734cad3f7cfb6d2ddee4175ab53d \ + --hash=sha256:c0094897d7d01f184b2d7fe8c56c66d64efe01b31f4b7d34205b391387df1111 \ + --hash=sha256:c03c6eb6ece86dfdcbb34799efaa339b093132e1aceed491ba5e08fe06cdf699 \ + --hash=sha256:c1a9a6651197fbed6f0212591418b9def774fc3f8324f78d1bf0e6a63e5f8aa1 \ + --hash=sha256:c3589f40749acce747510bf5d589d54e376cb0930ea58b35effac97e5312b0c1 \ + --hash=sha256:c4e38dd8f39c43a91d2410ad2b85610701b0979342c3df1d69eaf8e838c757d8 \ + --hash=sha256:c6c8fabf1dafc1f1ddcbb67896d3f93efb092e8c4b6322d7389b944e76a484e5 \ + --hash=sha256:c90fcf7804ea0a54b896ce0f2b9565350220b8d4890fd0db461a476a4c687963 \ + --hash=sha256:c9b602fae1e00b7c035d661ce85575365719192a7b46784bd71cf64c68053aa0 \ + --hash=sha256:ccb64d887a9db1cd76dbc0f92051a1a478a2a67e7f56c62d915cb881d7734704 \ + --hash=sha256:d06fcdecc10fc7954d7c8f27a03c96055fe525274dc84a7b0dbdc3d6b9e03dab \ + --hash=sha256:d0c3082bf79bcd6a614d55916590ad4b8f93200e10b97f463ea5d9d07c9b5f23 \ + --hash=sha256:d49c18f1ea294cf4adde2e5ac256e98c82ea9d708462ce4bf799dffa7cfe8a2c \ + --hash=sha256:d60030baaa7bfbb02d650c126cdcddcb6e33dbff14d819434c8fa2fdcaeeeba5 \ + --hash=sha256:d7cab119d0df0b9413f106b4d7fc34f2872d3574ed3806fb48959c830b1537da \ + --hash=sha256:d9b77b25b4f395f92de6099ab08e8ae2bc7e51dfe157f22900902243a5cc90c7 \ + --hash=sha256:dabee8f4935e731fb46b2a3091bdda0d3d94b3bbfb907d2b4f12eefce4009619 \ + --hash=sha256:db5e82ba15c142425b8406690032df89e39cca4a2e8afbbb9a3d84edc2373ac3 \ + --hash=sha256:dc79d36d0618752265f0d575915bdc5c5130ecb9c9f6b3bcefeae32e4bdfafcf \ + --hash=sha256:ddfa987262763c3c22a8367d2a49c244b018a74c3a8e3ab1a864119ad45c5633 \ + --hash=sha256:e1172147d28d8fbcf8cb8d26c41506169f5ad8fe9ec969cb116835a19d4d8eca \ + --hash=sha256:e11edba5bc344a32b029a7af9d4b3173982dd79eeafa0b9dbd787364414b0509 \ + --hash=sha256:e2c89e9b762c57f59d5e99ee8b20202adb892e35f8d3485741340999ca55058e \ + --hash=sha256:e31f72490b7c12f7790e1e25c3afffd20503ee1bfb43461d7838b871ff244b19 \ + --hash=sha256:e8c65ef3862a8ad6e86492b6ed9327805dd66904c012bd3649dc67d822ed6c34 \ + --hash=sha256:ebb8912f565b8cdbbf27debfe00df04202c20e2f651b9e32767930c5eace3621 \ + --hash=sha256:ed511a0708e2297e1d6431e7fb217e3402791e491e02da800658ace4973df1bb \ + --hash=sha256:edf06545875f3efa31560d94121e95c7fd70d98b1dfedc0157097d79b13b52ea \ + --hash=sha256:f0fe9834e5aeccaf19a0d8feb296d66a24be1a7c9922002f842a682cd5abb787 \ + --hash=sha256:f1a0d5117230dd46b399a30a38afa44f79c99f3168988fdc4f425c3f928b39df \ + --hash=sha256:f37964e4a5e993d2fd45147741e9dff7f34a2d8c00ab94c4ea0514a4677f959e \ + --hash=sha256:f57dc6b8fef170f105d2cf5cdce254f47b137d7755086cf7050f47e16582abba \ + --hash=sha256:f93bc1c3486ef3747e07c9d7c1d0a147b8fbaab975f80e348aed6f71309dfaca \ + --hash=sha256:fb00027a09a8f9f08028b40dce4c933cf73e4833240ed356583fdc9cfa721566 \ + --hash=sha256:fb99cc9d45f48895d9d67f6a0b8a57f08d39c174d9f25ad97a313e0470267b1c \ + --hash=sha256:fdd88ed5e20b1bcdd234421e454962c971aa44b653bdb7f1ea9ef683e90fb649 \ + --hash=sha256:fe3fa1dd453ed5c7f5ea23a26218329790ed7197a99b90e94330e313959a7f52 + # via transformers +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via mlx-vlm +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via typer +safetensors==0.8.0 \ + --hash=sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358 \ + --hash=sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f \ + --hash=sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d \ + --hash=sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d \ + --hash=sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0 \ + --hash=sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc \ + --hash=sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235 \ + --hash=sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98 \ + --hash=sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4 \ + --hash=sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846 \ + --hash=sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca \ + --hash=sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0 \ + --hash=sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25 \ + --hash=sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452 \ + --hash=sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d \ + --hash=sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78 \ + --hash=sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774 + # via + # mlx-video-with-audio + # transformers +scipy==1.18.1 \ + --hash=sha256:011413b7426b75012840e35649e00fe0a2c3bae89fed433876e3a99251572efc \ + --hash=sha256:0ac49ea97594532dd44b7136094d35f5440fa06e6d9c6384a74c01764df388c5 \ + --hash=sha256:0e82073ecc7acc6436fac4b31674109c7e1d3e596789767eda01258a8c9e8123 \ + --hash=sha256:0fcb3c93519f27bb4f0c4b0f7802cdcaca7fcf93267b75edda2e9f4e8a55cbd7 \ + --hash=sha256:10ac20c69d880f77f375db44c22e3e6a644f9fefa291d4cd2fb9790a89fc99fd \ + --hash=sha256:11c423f1049c5755ad4409af52a9ada1cff96fe9b50795d4af3619f292901239 \ + --hash=sha256:179ce34a8d0fe273d8883ba59e17e052247d08973dfcb743ca52bb1cce2d60b0 \ + --hash=sha256:1bca3b943fc2567ea49cd02c99abde49da4d5178ec46f624bd8255cda8755beb \ + --hash=sha256:1d73131e358976663dd969e1fb4ed1404b815cd977eaaedc3b3a133ba2d81c35 \ + --hash=sha256:2a0b02f9fc46f8520330c23d45e6560db7e3a0d927232139427637f98943e11d \ + --hash=sha256:2d3ab0e8c69a17dd3559eab8cbb88f258e285c94d572c2719033f90f83290c89 \ + --hash=sha256:30f464bee641fa8e282577c7dce027308403213c6ca8270bba73285c91024bc5 \ + --hash=sha256:33a834464fdabc0f26a45508df31b3cc5d028e04dbf6c5ed398541418e0a12fe \ + --hash=sha256:3ab3523da44749156e1f68b464dc56af11ae4cbc5c739a49d05f32b982eca9f3 \ + --hash=sha256:3c085faa2cfa879c5141df483f836f4d691045a078224a670fa570fa01612d89 \ + --hash=sha256:457fd7a2a8edeb044ab6ffbc0aa03ff6cd18491356e5e0c834d76ce621b916d1 \ + --hash=sha256:49023963c193dacee096301452f223ee24d86ec5807f8df93c0f7221d119e305 \ + --hash=sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307 \ + --hash=sha256:559ed65f60c1af5a03f3912605a1b5114f522c7c32fb23c3376ae8f03219fe28 \ + --hash=sha256:5632e3ae3d09197c446310cd5187de63e28448ce22f0f67b2b93d97503c0c230 \ + --hash=sha256:5e4d44984abc0020154ea81b247adeddcc3ac5527b975ff798bd1ba0adc513c2 \ + --hash=sha256:75b00eb8fb802090aa903f4ea1c7f5a584779f967361e68b7e98e531cc2d7174 \ + --hash=sha256:78a0d7c918e74a232394117160e7e3db503377572a45bcef8826e4ab8a35feba \ + --hash=sha256:78c0665edead396b1abb4897c41a5c1d9bf090c8a637a4c20a61678e0a264e66 \ + --hash=sha256:7bbf207c4453ce1ad2e00b17313852b33310b83090c2311bdaf97f93c0380d12 \ + --hash=sha256:7f4b8bc363b6d65ee2152bec57568e3c52639bb34c46057b09857a307ed5e21d \ + --hash=sha256:82f201b4c878551d48558337aab270d3c6cca5507b8737c8d8a608d234cccde0 \ + --hash=sha256:83de5453a7799afc9048b4616bd085cef126e36412f0ea2f6370c36a2a3a51e7 \ + --hash=sha256:88f0e784020649f88ea48c9f5ddfa403bf9205820667c0914740b392035afb82 \ + --hash=sha256:8bcf3c1ba5d6456e2effd30fcbd3459b044d683fcdac79a2e6830f0bdf7de487 \ + --hash=sha256:911de823097db8b63f034299d12662db93344e6ffa0b881cbb57748974b70168 \ + --hash=sha256:92c14f5bdbfb6216315ce33e78080474082de8b3830122ba97809bfbe65f75c0 \ + --hash=sha256:95298364e251be3e60249facbeeca03631d3bb7584f85879516ec55ac717b81f \ + --hash=sha256:9554bcc6d715ee87a633a3cc8e7703c6628b100dd29cb8a2efc4c0533c7ff729 \ + --hash=sha256:9f2897bf7737392ad0d5213ea7b6add72a4edf5679b3153106aeb88b6507b3b9 \ + --hash=sha256:a1d33a7836f7ddc1993427966a0823468ec41bcbdb1a9f9942d1d7e57f803ba3 \ + --hash=sha256:ac0333bdf38309aa3dcbe7e3fa7ea29e7a2c37c6ea306a757b700ded8e4596ad \ + --hash=sha256:bff0b729edd992766136b34e39cc76bc2fad905aa58897ee72a9cd000a6d8443 \ + --hash=sha256:c24acac1e18912761c4700239bbc1fd32f615af690f1584d49b35859be51324d \ + --hash=sha256:c35d74ce0e193ff740c2f2be2ac913ddc232fe6c1ff40b26cfecb9c670c63314 \ + --hash=sha256:c825cef2f49e46753726a7181a8e199804a912b29519ada542c6ebc654951899 \ + --hash=sha256:c9d18a33309122074ea483dd92dd444189166b8b2ec429fe9ed5ac73c7a0aa23 \ + --hash=sha256:cbf38d043c1aa4ab306e1ada6ab6eddacc3322a20b7af1b30bc93254b366fe09 \ + --hash=sha256:cd479fc04dd9401e3b4f49e76518768ef99c4f517a98c284eb091fd725719adf \ + --hash=sha256:ceb30a00ce7c92d459819443d29ca486d882b83fb6738bdcbb2a1cce94ac5daa \ + --hash=sha256:cfbf154f2ba187f2ed6cce2639efff7d105f1140573642c0161615b6d91d6a87 \ + --hash=sha256:d2924a03db38dc2e848bca2fe9f077dafb891480b91a00a0963a8cf86dfc31c1 \ + --hash=sha256:d416b16cccfd70fbf62400e84d0bb2f4e6af519a45557f1692c749b37f14b315 \ + --hash=sha256:d65d448389b8436493abcf629cc94ad0cf32aecaf06e1acca1de53cc795f2f12 \ + --hash=sha256:d84a09d0dad90ba6525d8ac1c2334b33e64bf3ccfe9e841f02feb867a22681e4 \ + --hash=sha256:ddef79fb382df40104a19bb7151b3b23e57c1778fcf857c71ceecd9bd264513f \ + --hash=sha256:e3b417bf8c2c7c16e8f58ad91db17783ec911ac16e7b50eb6eab6e809b4f5b07 \ + --hash=sha256:e402cf31eb68f453dbb2d36fc6d722b33f24a55d68b2ae1d92fa6305ca71c298 \ + --hash=sha256:e6fb6a55cc0ba97b59a1f288fb86dc6fce8bdfc0fffcbfd015e3a954bf2a2d93 \ + --hash=sha256:e708533e8b2ae2497d65346538a7dcc92814410b25b81432eac66de0f2af8265 \ + --hash=sha256:ea324d9dd34c38bfb9bec8ca4d1b407db97dbb74029f566b8e322b1b6fe56fe6 \ + --hash=sha256:eb0dfcf4e28a99c12c999744a2ff67c9b06200e20401c7c88186e33552a46331 \ + --hash=sha256:eda632a7981f69730d6281f451db9c1c370993a2c0d7ddb43e2a809a2862b83a \ + --hash=sha256:f29633129f9fa7e88a3f0fca835de2d030bfc9643f7799e1a0c46cee24d38fc7 \ + --hash=sha256:f55fa87b6c612ecd6b058f167c53231b1d14e412efe361d3d6e38b3631c73218 \ + --hash=sha256:fdaf5ea890a6183d0565f51a61799d67081bd5b1cf03c5f4b3fd3732108625c9 + # via mlx-audio +sentencepiece==0.2.2 \ + --hash=sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497 \ + --hash=sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e \ + --hash=sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0 \ + --hash=sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719 \ + --hash=sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107 \ + --hash=sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3 \ + --hash=sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b \ + --hash=sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a \ + --hash=sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c \ + --hash=sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838 \ + --hash=sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5 \ + --hash=sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6 \ + --hash=sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d \ + --hash=sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf \ + --hash=sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25 \ + --hash=sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d \ + --hash=sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e \ + --hash=sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b \ + --hash=sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd \ + --hash=sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936 \ + --hash=sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85 \ + --hash=sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b \ + --hash=sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708 \ + --hash=sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb \ + --hash=sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9 \ + --hash=sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0 \ + --hash=sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790 \ + --hash=sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e \ + --hash=sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78 \ + --hash=sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99 \ + --hash=sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9 \ + --hash=sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d \ + --hash=sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8 \ + --hash=sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906 \ + --hash=sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383 \ + --hash=sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b \ + --hash=sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53 \ + --hash=sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b \ + --hash=sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab \ + --hash=sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0 \ + --hash=sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914 \ + --hash=sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91 \ + --hash=sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e \ + --hash=sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a \ + --hash=sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032 \ + --hash=sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711 \ + --hash=sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da \ + --hash=sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c \ + --hash=sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a \ + --hash=sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d \ + --hash=sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151 \ + --hash=sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563 \ + --hash=sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811 \ + --hash=sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b \ + --hash=sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820 \ + --hash=sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090 \ + --hash=sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c + # via mlx-vlm +setuptools==84.0.0 \ + --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \ + --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73 + # via torch +shellingham==1.5.4 \ + --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ + --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de + # via typer +sounddevice==0.5.6 \ + --hash=sha256:7f4162f514f007b0bf25a3ccfed3f1705bc2ec311888a90232729eec4f57a4f4 \ + --hash=sha256:8ec9fbfde2e32f020b167e348f3ab3bac6625a5f15af524d790108ac7147a410 \ + --hash=sha256:b36b807eb02abd257198bf84b2af05e4fea199a9d2f0019014169c7136d45e9c \ + --hash=sha256:c8ae19173e5f27f8c12d4b5eee2dbfe542cee125d591e663e0fb4dfb75246d45 \ + --hash=sha256:de099612311ad81e55d31ccbd83f43ea6bf4d87b48f9b6ea55a1fbcde0eee4e0 \ + --hash=sha256:e3aef00ad8b1d1740eb66d9a7671eab88a4d2b8fa4ab33498d742e63b65c309c + # via mlx-audio +soundfile==0.14.0 \ + --hash=sha256:0a6ae43c50c71b4e020cc55382925cb89451c1ed1a0c3d0f5d802da269226849 \ + --hash=sha256:19be05428da76ed61a4cad29b8e4bcf43a3e5c100089d2ec81dc961eed1b0dd4 \ + --hash=sha256:1e38bac1853412871318e82a1ba69a8be677619b56025bbfcccdb41b6cafe82d \ + --hash=sha256:299491d3499460fb1b74bb4bd78b57ffc2d243a5fafa7b6ec1b264875c78453e \ + --hash=sha256:8ba81ae3a89fd5ab3bef8a8eb481fbbe794e806309675a89b4df48b8d31908a8 \ + --hash=sha256:ba1c1a2d618bca5c406647c83b89f07cc8810fa506a50622a6993ba130c1de11 \ + --hash=sha256:d828d35a059626da52f1415b5faee610aeab393319cb3fc4a9aef47b619fc14c \ + --hash=sha256:e090704718e124e7c844695236f1fce8d18a5e761eaf7c82dfcd124620805f98 \ + --hash=sha256:e85724a90bc99a6e8062c0b4ddf725f53b2a3b70afd4da875e9d2cfc4e92f377 + # via mlx-video-with-audio +starlette==1.7.0 \ + --hash=sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e \ + --hash=sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d + # via + # fastapi + # mlx-vlm +sympy==1.14.0 \ + --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ + --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 + # via torch +tokenizers==0.23.2 \ + --hash=sha256:12f0835dc2ee694746a76adf7b1567d4346a4a502ebe93fb1f5f80ea49799b78 \ + --hash=sha256:2e96f5699d5249c9c64aa8412e044f727aae3a4098cf830f9901ec1afc361cde \ + --hash=sha256:325fee2e0418a9dc6c9ecf736a5f5f0db7875183ace9549ae339da76f7a1fbb7 \ + --hash=sha256:41c2f84d172449b4dadb9cdc508e3e364076613c35b16e76ecfe47a60d1e3305 \ + --hash=sha256:43e4f2071e3cc8d5d86421c874aebc82659bb51a68bcdef5a0da75ee89511ccb \ + --hash=sha256:5c56bda1511921587789163e524d196ed8284174ac23abd7685d5ea8da6c4718 \ + --hash=sha256:7b7e37ba198f24150f523e1242e83c4970de4a525480586be5dcc24d9add32c5 \ + --hash=sha256:7f0f085686b9de0d0079e6f874ae053600db64c5d13049e0bbc0119926d25aac \ + --hash=sha256:85a9a357a3764aecc904ee76bdaf8cf1ad8e5a67a1b929a487c4a39b49ed0e90 \ + --hash=sha256:950d7c9426fa72406a0ffeacdbc0bb9985f5db20eb8b263f29c79aaf83105703 \ + --hash=sha256:986670e43691469dcee610ea0f846f91a8f84e91fc6f7a48d4c064414c0ec2bf \ + --hash=sha256:a37039b5dfc4af84eb3ef0a92f4307e28936c8f9adccba2629d36f652e9bf7a2 \ + --hash=sha256:bef235815a067b2648caf6dcc7a71091b0b0fff9ee8057f6451eb9335fae52ef \ + --hash=sha256:debf978920d93ba9c219bd67cc4bbfaf912c9039e41e7a28b91ec15e3728c95a \ + --hash=sha256:e49c394456dd9985787fec76132438ba3fb8911f857b1bf3d40119f9292d41aa \ + --hash=sha256:eb2f9c8a24da020ea8c11a01a19c1c2547912d92121ae4a01cfbca46125dee40 \ + --hash=sha256:f486f402f6f9abee5bb032553736813af0c710a86b2e0ca592634c55cea1f835 + # via transformers +torch==2.14.1 \ + --hash=sha256:06c3ec25f3b497f9a73dc0c36f293f295cfe8d5584fd446238d9a501b30a66d5 \ + --hash=sha256:0b13acde294b401509830d85a8906cbf2fc97425a12a66cce1d9fa13c5106a2a \ + --hash=sha256:0bd2fb0c4098856bc5af59fe5988bf38bdbcdb6550b44273c12ae425cb8ec17e \ + --hash=sha256:1d4df07be7338bbcc4d54085adee21363c91314702b9bd4d78ef72ffac9465ba \ + --hash=sha256:1fee6b20869a90ccec05e35559a706338ac8aab40a9c4fb58f16667e93af36ae \ + --hash=sha256:23011fe29a99b591eabb3a31a25080c62e2c2f0690a3c798744e5489dae50451 \ + --hash=sha256:305a61f61f35f128579f299c5bd33d475f6a01c6307336139632e30856c4854d \ + --hash=sha256:38bee9f2a2ccfc6898172a5075e4ba52522fbe143eeb099674fc67ab390e858d \ + --hash=sha256:420dbf314c180ee4b86e9bc00aee5746a7d6e5bacdd7df925af671ed393f0b2e \ + --hash=sha256:6d530bd11479fb574567af3a1f384af9a77bf5bb9550cde8080b356ca7220d5a \ + --hash=sha256:711713391d26a1ce5e9fbc6c996d954a8c12e8825374cc77b809a6af29539b8c \ + --hash=sha256:9113f94f70429f9f302bf55b090b5411269083e8a72a5d015ffcc5a7f83f2c69 \ + --hash=sha256:957df236525bcd940ccd9ca59074e9821182171253417c0d98e7b82647254799 \ + --hash=sha256:9ee7d50ec80ae5b4258cafc0ea108711988d961f21d0a68cf685650da2812e5f \ + --hash=sha256:b6074b130fd26bc50f5d171f07dfed70db22dd0c354ab34767729249f9e0f042 \ + --hash=sha256:c8f71aabc67bcbfc9373dc131537a5968d04edce73e88add21354a7cd0a76985 \ + --hash=sha256:cee091caf2a6229e248daf41d18ceebf590ba02d9179205610062c64ee5fef03 \ + --hash=sha256:d02a4c48a2ca5fb7654e36e71f710f74494d83f1f10aaff8e059dd554adca956 \ + --hash=sha256:dbe359d705f4d67236743794c296c6dee93a922fd8117eff8c3e880d7d0fb2b9 \ + --hash=sha256:e07306caa1de2a4ac1467e11ecfc92fc44f523dd6a521145039aef46d913963c \ + --hash=sha256:e24b9815001d18e73e76eae1ea25f8ef908c6103e857378ad20f8f6cf5c954f5 \ + --hash=sha256:e65d5fe136e533b23c2d134377f7f126721c7c78dd75b3dbb735af082a8aeb85 \ + --hash=sha256:ef1276fd07adb8d463ac402a24ea7482e45c9a42df8ebdac211aa757779200ba \ + --hash=sha256:f68f5476e2bc0e8f60b74f7ca21acda885c4477af6392f8977e4f0d1ea1aa162 + # via -r mlx-video.in +tqdm==4.70.1 \ + --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 \ + --hash=sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4 + # via + # huggingface-hub + # mlx-audio + # mlx-video-with-audio + # mlx-vlm + # transformers +transformers==5.18.0 \ + --hash=sha256:d79e5a515a572ee3deb33eb0d578be47d91e70c75d54cf21bf6eede6edef14a9 \ + --hash=sha256:d89c206e42e841af7cd314b3d2b21c03e977d819cdb4aef65ada048769a644df + # via + # mlx-audio + # mlx-video-with-audio + # mlx-vlm +typer==0.27.2 \ + --hash=sha256:269b7eb9d3c202ca84b4bc9618cb04ebb43d3d4d1e567e4c768607232c05f945 \ + --hash=sha256:b3a5fc4342d5fc8fda8fc3010b1cf117e9249aab7fae800c2eff62fd3842d97d + # via transformers +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # fastapi + # huggingface-hub + # opentelemetry-api + # pydantic + # pydantic-core + # soundfile + # starlette + # torch + # typing-inspection +typing-inspection==0.4.4 \ + --hash=sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47 \ + --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 + # via + # fastapi + # pydantic +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via requests +uvicorn==0.54.0 \ + --hash=sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf \ + --hash=sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620 + # via mlx-vlm +websockets==17.1 \ + --hash=sha256:0014eaff8ad5b3b43feda2279f9d34bf2eaae040720b9fbbb55944b10f40b14d \ + --hash=sha256:00679b7468b4c2b12b0757118174e8eabac56bb2f579a928a104d9554a56e098 \ + --hash=sha256:00bf34b64501e3477e81fc281532ff3cbf4da26633c10b63979d5085d46602d3 \ + --hash=sha256:01dcb47deebc40b38fd4a493b9b9f4d0a704b7bec6f35e4d34085b329abce71a \ + --hash=sha256:020e271205f8ab3406d7a59cd00de6dec722315924411c421bd00642f18bad86 \ + --hash=sha256:0340bbef6bfbe16da888b3983d666a4db4954ac3253c38f13bc7aba0c7db5a2f \ + --hash=sha256:054c28db2dcec0e857e3b705d8c28012613e555b38c765d6a4f75340a4fc06a0 \ + --hash=sha256:073c5c3f7e127041fa9d34a9e29ceefee8c3cafbd267ed2927318f425144380d \ + --hash=sha256:07fa3e7c30e2c577928d359b56bf872a3e0cbcc15553eaa0907c1ee86344b56f \ + --hash=sha256:0c84bdef916556cbe1d5a43b423398be4dd3cba6522b463e53d848578b920695 \ + --hash=sha256:0c863507ada5805517ca6dff1c524dcd42942efe6304dacf06700878398d21a6 \ + --hash=sha256:0c9982938980e086da59f70d05f9418cd143401a601a0faac10fa48f7bb1cd3e \ + --hash=sha256:0de501b7f2db11e83739ac20e2d33d46da4604b829f506c24be80e7def069391 \ + --hash=sha256:0f4f50dfe2cc810fc4e2de979b35e83bf8bb4bccdc6fe472d93762ea7b1d5927 \ + --hash=sha256:1036189bd34b0bc1b10a4679321e2c7968af317efe6e8e4c1c5141c4254fb5bb \ + --hash=sha256:10ecb38ffc05e1841b619d99c725307a223ef9ad58e7b1ed33311d472dc43918 \ + --hash=sha256:12c8e2b25df59755954a04dfa09c990b96691025aaf7eafd19ed6da24b09c18d \ + --hash=sha256:12fe8984a32dbfd084e0603f1a8d740c0180cb85b3174585c54a80d2455a8394 \ + --hash=sha256:17aa424ab61620aad21b36b2240efc87b500cc496e7d0e999a5c2ae99395e886 \ + --hash=sha256:1a20136d61f9ca3a31493732762661fafc2c20e8861930214e21afc6a8a692a2 \ + --hash=sha256:1a5cf4e7bbe3ca499e6a289206cb4fcb7444b09919e129bd517f57d5fa192c13 \ + --hash=sha256:1c7ac77401227212dc6e849182feee50d57cf456ec6329ffd6979c94bb136c5c \ + --hash=sha256:1fce0f43e0d41422e0b2cad6561e1970df22f212f4c7e884967df7cf591b031c \ + --hash=sha256:259f45358c76d3b18489e3e80636cdbe807e05ecf1b10fdf1a779106d23d0c8e \ + --hash=sha256:2786cbd273ab69c22612db8a41229ddf2c158060b17b5928884bf388d07887f3 \ + --hash=sha256:29176d8b429cfa0fa443c473878d37a5c06cfd0cb36b71ba4314accc71e05906 \ + --hash=sha256:2a0162a6372110a5601cb5c9fd826635cedf69f3e110c545dd19774e040b970e \ + --hash=sha256:2a4dc6ef83f4559e0d05f313a375cb38f63c986096a9da99fe94fdd779d313e5 \ + --hash=sha256:2afb58c7ba48b329d56769f8dfd89f394efe587b65ef806bae810a484d6d3608 \ + --hash=sha256:2cce251f3e2469b99b6802b55435bcdd07123b41870f54c87b336183af9d7e68 \ + --hash=sha256:2d72879819f5145a342d0030c418702496c65a4b913ef81f5ae944dd91dd50f6 \ + --hash=sha256:32a2a68d989d6e5b74a9d5095415c51189ebae29fceb7cf2b64a1c0318a81256 \ + --hash=sha256:33e45c7ea38428e740a7f233555d71df0b875cef7fc080acebc9654475e35335 \ + --hash=sha256:34879e19bb0a3c44f9317679435aea5327fac993933a704cbf353bf1234b10c7 \ + --hash=sha256:3546ef55b3a074494106508bc6505c73825970d2d9505f7bf53882b3e88b0d1e \ + --hash=sha256:3709a1ab30b4b922027d22f68d2b61a0656a91680ac894a537624e6be7dd7f7c \ + --hash=sha256:4031152769179ab8dcdeafc7b0e58052a49117560a28671700b47b2c7b717aad \ + --hash=sha256:406b8107943a43ef4649b1e0cb0cdc052bbf08fe1c8905a623c4af9586e5cebb \ + --hash=sha256:416b4bc8789a1865a3ff643ec4ee073a5f52402d0dbeafd27b1798d5dd6b6a51 \ + --hash=sha256:43bd0c1ceb924d67f5c1a5254d8361dd9d94246e6331a726064dfa2917880780 \ + --hash=sha256:455ffeea0879d313205df1e745e5883e1feb7f31ecd26be882f5f0babd3db04f \ + --hash=sha256:469355ab1af100b9380f1afb09985019f4a4b94fa1dd0e9396db4361626d7ab8 \ + --hash=sha256:46c0331c9eaaf73a559f3a9e388466be0df96eb83d40f06f1ca6ab6613b35c82 \ + --hash=sha256:46e7a10bf04318c7b0c0273791925ae5e1cbe4a11e34aa934d2ef27862058a80 \ + --hash=sha256:47a62d6045c6eaa0d8f97bc2fb68b8cf90077a0cbfd4e83d6f2d2145611ee134 \ + --hash=sha256:4af784f3e436f65b355c117c6497320f2b5cf6a559295cb1c4c7338e335d45cc \ + --hash=sha256:4e8ffcb486c8490a34a4cef5e4409d8da5a1cb1681e5bf7d786ce5e84aa8540d \ + --hash=sha256:50903d335bfda026c2fa11dd9aed09d8cbee0c451e3a85122a9acb041b7dc69b \ + --hash=sha256:57b39dc8541cf7ed3f639da82bf7451060483967f9e733da1f8173e4095f0642 \ + --hash=sha256:581fa678ef46f4277cc8491312468e582f8ad609dbab907ba6096a08c6a0ff98 \ + --hash=sha256:5a74531ce81af587f906ab42f194032388fcff8fc7938402e5917c9147a39441 \ + --hash=sha256:5aefe78e6a3077fe22b5e64b04666a85a3eb8b934d40e8595a693adcbceb6f11 \ + --hash=sha256:5c27336b1a0ac56569493e858497870347854372395f50483725f8cdacc5a45c \ + --hash=sha256:5d4724255fb8398acd9e583b97eb2279cec20e0bd0f9a94bf75f6056ef9f13da \ + --hash=sha256:5f051f8030a51815dc00e24bd2e5f1435af095c1cc111d747ac6e2a3620d7641 \ + --hash=sha256:615746b12b26a3fd4077bc6fbeb277a1c192a45dd57b531d07ad9ed5c52a9a7a \ + --hash=sha256:617243e19a0992095956f406ee9cd3bc4ba92862d83cb1d83bb59ce574412bec \ + --hash=sha256:655a8e28010f09fd6fa317e857afab3af7647f33e41dee88fa421e92086d1090 \ + --hash=sha256:65be6bda2b537fefa4b3a5ccd6ab386533ce39dd8fe62433ec90901fdc81752d \ + --hash=sha256:67258b00302a5aaf0b267771c7014b13429abd7ea17eebc4c55bd935ff101555 \ + --hash=sha256:677014a073bcb1fbaa7e21144786864f16c08f856d66834f611eceb9006cbab8 \ + --hash=sha256:6a8e768a048c2220697477ce2e67e4345dc9f693d0ee6af53945b5e30227c6a7 \ + --hash=sha256:6da17fc94bd270f5987b10bee113461ac36a36a98b0481ddcc98056e5a90001a \ + --hash=sha256:6de9acef07e3a78e9567fcd26c29011a4da8f050b13004bbf880a0fd82a6eea5 \ + --hash=sha256:6e63c01803be425ff062b7f7fc201a74def1d49fc94a2410dd17375df75936e9 \ + --hash=sha256:6f42912fa9eb4cb7c7ec9fde9b3332ba339eb8a8811981043d4029599f3d950b \ + --hash=sha256:722ec21717eec6477bce582147a28acdfe034e604239466a6a95daedb863e774 \ + --hash=sha256:764cf7bfa149365f32b7a0fd9fed32debdac29dd06295d5635cde1745b446cd8 \ + --hash=sha256:76dd004f59115087c7b700474cb18f01325e37250032e19396c08ae41448e4b3 \ + --hash=sha256:77b37cceca17291897c3c73bd30a7c7c7909593554b5da574ec852af83c1742a \ + --hash=sha256:7a72efa3bf4fa3a6669a54420a472ad056da3973d827f10e3a536da463f926c2 \ + --hash=sha256:7e724f843fa6a0614aece65a7c73e51d0f4412ca41dccac13c3caf98e69536bb \ + --hash=sha256:8196d217eeca52b9235ee1f8a684a09885a5f953d5a31e80ef915bf2c5c94f9d \ + --hash=sha256:829dba1bc049779de9b332088c1a6a9858e96bd67e50b6b644a95e02b67836bc \ + --hash=sha256:84f25e099fdfe3b09f953d84698f729a1f7d1e99101b2787d7a28ed77b323750 \ + --hash=sha256:87f0d5e77548b0c40c8464cdb6108792e7e53f487c6400028a4ec28a8afbe5ab \ + --hash=sha256:880069d21cc33a558dcf180924a546d1ecf8ada5be3e4e70acee87019d706a24 \ + --hash=sha256:882af300d2c6a092b93767d5de03c7bb56dfb06314140c8e872d3f48e09f7b74 \ + --hash=sha256:88381602e379165b66244b2ebc29f9b23ea0851fbe63ae157f91ca324f072d6f \ + --hash=sha256:88b882764ef65147a7a5ae13168dedbe225a04e2ff4858fe543f2c402f093e9c \ + --hash=sha256:88bc5138e53903a85c354e59df7ba73ce306f7b09724cef74dba121e60a88ce2 \ + --hash=sha256:8d1b108bd8f5f6a8b90801f6db3b3858d5deca889acfdb8ac497bbb24e4b0edf \ + --hash=sha256:8f6c38cdcaf98a911d7acc25577f2f9e710f3a2fc2bde1563556784320196b51 \ + --hash=sha256:8fbf28e639544503b7d1c96452a5e5e043e4108d89b1f3fa02910603622d19db \ + --hash=sha256:96abdecbaae746851b87c3a36cb4a661df93ca3d92f114270f79228bf1d00de6 \ + --hash=sha256:98a5b2589a56a4b4f098b0a958099a4356ab904a7844f1da3841efca469af7e9 \ + --hash=sha256:9ae55d24241fc055f22aea3ac924559069848bd0ad4ea065fdd72d2194685fe8 \ + --hash=sha256:9f4a08ff7cb68c27b18e09223cc6304e01d0f82d5a240d251266dfd2e6e44729 \ + --hash=sha256:9f4c0377a83e163a303514fdfab501dbe379bdc13e5b9312a91d112658b29dce \ + --hash=sha256:9fa1ffa08c81a4f809cdab6129f8e55bee4650b9d6d3461019dda73aacd146b6 \ + --hash=sha256:a06f3b5085176763182449559e20391d7ce616a8972a9f7a33deda87ea6d4f3c \ + --hash=sha256:a62d8c424383c9dc769ff3672018df822603117e32686e567d452ed035b6fb2e \ + --hash=sha256:a7b35181a14cbfcae163b4de545d22abfd07d06c2c41ca69cfcd99251d6888ab \ + --hash=sha256:a9fe648abd1d9b89aebfa30407bfdd08a0271ec5dc7d44a4c6ccd1ce22cf562a \ + --hash=sha256:acfea4c20bf54384883ea33b1240fc1db4f52e190823a4e2b334bc3e8bfca96a \ + --hash=sha256:ad68c28a27246fed109a4409393d677b7e1388345cbbd2f5aee5c182d8506110 \ + --hash=sha256:aec00f018d34c67500ff0438dc314b40277be4a1b983cbacbf53ccf7db63e257 \ + --hash=sha256:b1bc819c6db90e8f91a38250a1ab4c058261871aa52d2fe36382eddedf146dee \ + --hash=sha256:b1c323fc3be1dc3f87f6c59458cb7d9e13dcbbf971d6c3f3e2bbaf58d3bfcdfe \ + --hash=sha256:b3b451fd2723ad3191a209afe6f3f4bc86c83e9a85bdc255353b91803ee6aa66 \ + --hash=sha256:b65d5fe48219dc2d5e158de9e6514e75600f379cc7e37108d35f31764c155566 \ + --hash=sha256:bb0efe019480a1c93e168ce96479273aaebd672fc8c350d5eed1e507ababb1b8 \ + --hash=sha256:be3f0129c5654517b2abf07dcb75bb1d9479759a4ccfb569e8293579e9fc029a \ + --hash=sha256:c3241d684a76eaaef8b2dc789afde4343cd3aad55ea81e4e8ab3605b529bae51 \ + --hash=sha256:c65da239a5ad553619804c1f9d65c1a0b3005381c6158ee14da2c7444cbd0c78 \ + --hash=sha256:c89406fa3dcd4aa8662c6406cc5c0de1790e9614d2c3aaf03ca53a8a8ccf3405 \ + --hash=sha256:ccbf3f4a9890d50b3a08ee04029fde30a03bfdeffaa19977628bf17251764e60 \ + --hash=sha256:ce0305b702b20d1e1d60a9aaace6bc89970e1753565543f310d549eab22c2435 \ + --hash=sha256:ce13c7d233239e739600a57d4a73c1192ad8259e655a4d55aa1a454242bc809d \ + --hash=sha256:cec1bb8f22abccc8d20f8ca63df9be41600c26c190f4b97ee86c675fd4a863a6 \ + --hash=sha256:d1e2f5fa2b6d01f0d85b4f223fea7ed1d504be282a02a81bd2be4817ef7a2f03 \ + --hash=sha256:d334d11398086bb5559606cb42d51c013ea7c061c7db701521392373d3c087f5 \ + --hash=sha256:d411ea5ca18ac1b12c0c94be88b60c18ca641ac43bcdfdf1c9f79d46cdbe1603 \ + --hash=sha256:d41ef69d5416fbc1d98cf96c37be6192d10fd101c3e0f8b3ddc36e09432b3c08 \ + --hash=sha256:d58159af7835fde09c462394293c0d7aaf8fb4557d8f8e5699f5e722ccae013d \ + --hash=sha256:d7b349265fad6244013eecd99df8d83c12bf3013943e431f4fadd5bffc37db42 \ + --hash=sha256:d8e83333385cac6030a5167fd18bf96cc6c58b914c308e683f05b0cf94bc8dd0 \ + --hash=sha256:d9d01e8ede41fea4f5a847dad9d628355f74905f437a5b6856d67aa66d193800 \ + --hash=sha256:d9fc873e239c5abeb150bc24dbd1a7af23a9254526383ce0a077f5e20adbeb19 \ + --hash=sha256:db9d7ee47f3ba531e278be539af39e2c7c7d28fb94897b6cd1120d63b0ef5922 \ + --hash=sha256:dc053f9e95a76213c5eb7ed95779f7daf0d2bf0e4e03073629ebfa43a033f151 \ + --hash=sha256:dc2b79afc074d2f3e64b26539350f697fe1b85ea1c49ea24eb588f247b053ce1 \ + --hash=sha256:dc5789e5ea182b77a38881383ada5347202a6c66f4857d054e075290e80b604b \ + --hash=sha256:dcb8d5f7edef7a399d322cf28d4c4e6f98dab64d301c8f50581a1080e5198142 \ + --hash=sha256:e4bd7eacb87d8cf3ed70d6392c770a0d92441f05d7d2a3efafb5bc171d5e3067 \ + --hash=sha256:e552e0037230ac16e5f568de7012041344d1b18c9feed30ec2891b8eba55af81 \ + --hash=sha256:e5f5c7a893507d0e83a80b88aefd6522f7e882cd53f9722c6f23f5a020c9557c \ + --hash=sha256:e74e41f0ad12ff1e8983e349daef79d37cc8280c743ce9d134d6c74c18dab5d6 \ + --hash=sha256:e78fd4b7b2c5086a38671c9c882c1e643385eccea360b5b1fda4a105e590087e \ + --hash=sha256:e8dc3fa6d6b7ead3f9de57895f41b116a28787548e066365d9d90f7356bcaad2 \ + --hash=sha256:ea0ed9373b880115911d9d39634bccc95b8ce590c9c42e8589f5cacc3ef3cee2 \ + --hash=sha256:edadce7a22052056fd4384543019856b34850363c9d387929f677ae01d79709c \ + --hash=sha256:eec113a5b41d124ef42ff56b0d74a6da3fd986400038eab9e58ee42a4024e837 \ + --hash=sha256:f221081107b8c48184d99f7019604486376e7ef826037e70aad6b02540732c23 \ + --hash=sha256:f3a1d577e081667dda7f8e5b4796e6e32f9713c93e2a3d930669519840a3c623 \ + --hash=sha256:f47aafd92aa28b941180e6da8a42b0f711851b14b81a5b6bb28dbbb1fa35152c \ + --hash=sha256:f4c45ee2512d3757b5e6c67c5a34e435143f2ecb7df3324f9fd888688c45c0f4 \ + --hash=sha256:f58f58b4b29bbea2a3635e2c56eff4a3adab011fe383802a9e542e31b97085fc \ + --hash=sha256:f612dc57f00c07cf4aa2673f7cbceabd654ad2457b7e639f061b794d6e11f9fd \ + --hash=sha256:f62114a54117e4948a1e414e89521f7fe1e3c2f83f2a571a06a4fc6718b0900a \ + --hash=sha256:f64e001bb7fa89b9f32cfa600bf8e9ac8ca26759d9b92ae01453ee303d9cd7b4 \ + --hash=sha256:f7233eaf441a345a5943a929fd4b5ea3278f11aed35a9ed0f3106b8cb3ca846a \ + --hash=sha256:f78a3ffb1994304db2c0c4588e4d1a518079b557054fa3bb985a6f5e50ff49a3 \ + --hash=sha256:f8e822efd54137d8cc8310eb64635ab827a4a6c72ff08691f38aa624776d8ecb \ + --hash=sha256:f98bf378d7a5be047a044a1a27c987a8f355e10e3b5754617dbe756248cbc5ce \ + --hash=sha256:fb88076df585b69c5761c387c0081aa87d7b9eb1b205a6535ca4777e25650d81 \ + --hash=sha256:fd8f47dbf2e8adb15c847215f83436de3fdb120b51fdae0fbbdf69fd97a3ad80 \ + --hash=sha256:ff3e2ba7a9f0a110b0555452e9b5a03a34e11662544e01beea15f144b48ba7b7 + # via mlx-vlm diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-vlm.json b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-vlm.json new file mode 100644 index 0000000..2e64419 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-vlm.json @@ -0,0 +1,356 @@ +{ + "schema": "backend/1", + "id": "mlx-vlm", + "version": "0.7.4", + "builtin": false, + "package": "mlx_vlm", + "convert": "mlx_vlm.convert", + "lock": "mlx-vlm.lock", + "categories": { + "vision_language": "models" + }, + "remap_files": { + "vision_language": [ + "utils.py" + ] + }, + "registry": { + "vision_language": { + "model_types": [ + "afm7", + "afmoe", + "apertus", + "aya_vision", + "baichuan_m1", + "bailing_moe", + "bailing_moe_linear", + "bert", + "bitnet", + "cohere", + "cohere2", + "cohere2_moe", + "cohere_compass", + "colidefics3", + "colqwen2_5", + "dbrx", + "deepseek", + "deepseek_v2", + "deepseek_v3", + "deepseek_v32", + "deepseek_v4", + "deepseek_v41", + "deepseek_vl_v2", + "deepseekocr", + "deepseekocr_2", + "diffusion_gemma", + "dinov2", + "dots1", + "dots_ocr", + "ernie4_5", + "ernie4_5_moe", + "ernie4_5_moe_vl", + "exaone", + "exaone4", + "exaone_moe", + "falcon_h1", + "falcon_ocr", + "falcon_perception", + "fastvlm", + "florence2", + "gemma", + "gemma2", + "gemma3", + "gemma3_embedding", + "gemma3_text", + "gemma3n", + "gemma4", + "gemma4_text", + "gemma4_unified", + "gliner2_5", + "glm", + "glm4", + "glm4_moe", + "glm4_moe_lite", + "glm4v", + "glm4v_moe", + "glm5_next", + "glm_moe_dsa", + "glm_ocr", + "got", + "gpt2", + "gpt_bigcode", + "gpt_neox", + "gpt_oss", + "granite", + "granite4_vision", + "granite_vision", + "granitemoe", + "granitemoehybrid", + "helium", + "hrm_text", + "hunyuan", + "hunyuan_v1_dense", + "hunyuan_vl", + "hy_v4", + "idefics2", + "idefics3", + "indic_ocr", + "inkling", + "internlm2", + "internlm3", + "internvl", + "internvl_chat", + "iquestloopcoder", + "jamba", + "jina_vlm", + "k2_horizon", + "kimi_k25", + "kimi_k3", + "kimi_linear", + "kimi_vl", + "klear", + "laguna", + "lfm2", + "lfm2_colbert", + "lfm2_embedding", + "lfm2_encoder", + "lfm2_moe", + "lfm2_vl", + "lille_130m", + "limite", + "llada2_moe", + "llama", + "llama4", + "llama4_text", + "llama_bidirec", + "llama_nemotron_vl_embedding", + "llava", + "llava_bunny", + "llava_next", + "llava_onevision", + "llmjpvl", + "locateanything", + "longcat_flash", + "longcat_flash_ngram", + "longcat_flash_sparse", + "mage_vl", + "mamba", + "mamba2", + "mellum", + "mimo", + "mimo_v2", + "mimo_v2_flash", + "minicpm", + "minicpm3", + "minicpmo", + "minicpmv4_6", + "minimax", + "minimax_m3", + "minimax_m3_vl", + "ministral3", + "ministral3_embedding", + "mistral3", + "mistral_large3", + "mixtral", + "mllama", + "modernbert", + "moge3", + "molmo", + "molmo2", + "molmo_point", + "moondream2", + "moondream3", + "multi_modality", + "muse_glimmer", + "nanochat", + "nemotron", + "nemotron_h", + "nemotron_h_nano_omni", + "nemotron_labs_diffusion", + "nemotron_nas", + "nemotron_parse", + "nemotron_voicechat", + "olmo", + "olmo2", + "olmo3", + "olmoe", + "openai_privacy_filter", + "openelm", + "paddleocr_vl", + "paligemma", + "phi", + "phi3", + "phi3_v", + "phi3small", + "phi4_siglip", + "phi4mm", + "phimoe", + "phixtral", + "pixtral", + "plamo", + "plamo2", + "plamo2vl", + "pp_doclayout_v3", + "prism_hadamard_qwen35", + "qwen", + "qwen2", + "qwen2_5_vl", + "qwen2_moe", + "qwen2_vl", + "qwen3", + "qwen3_5", + "qwen3_5_moe", + "qwen3_5_moe_text", + "qwen3_5_text", + "qwen3_embedding", + "qwen3_moe", + "qwen3_next", + "qwen3_omni_moe", + "qwen3_vl", + "qwen3_vl_embedding", + "qwen3_vl_moe", + "qwen4_exp", + "recurrent_gemma", + "rfdetr", + "rt_detr_v2", + "rwkv7", + "sam3", + "sam3_1", + "sam3d_body", + "sam3d_objects", + "sapiens2", + "seed_oss", + "siglip", + "smollm3", + "smolvlm", + "solar_open", + "spark2_5", + "stablelm", + "starcoder2", + "step3p5", + "step3p7", + "telechat3", + "unlimited_ocr", + "video_depth_anything", + "xlm_roberta", + "youtu_llm", + "youtu_vl", + "z1t", + "zaya1_vl" + ], + "remapping": { + "bunny_llama": "llava_bunny", + "cohere2_vision": "aya_vision", + "cohere2moe": "cohere2_moe", + "dinov2_with_registers": "dinov2", + "falcon_mamba": "mamba", + "falcon_perception": "falcon_perception", + "granite4_vision": "granite4_vision", + "granite_vision": "granite_vision", + "inkling_mm_model": "inkling", + "iquestcoder": "llama", + "joyai_llm_flash": "deepseek_v3", + "jvlm": "jina_vlm", + "kimi_k2": "deepseek_v3", + "lfm2_vl": "lfm2_vl", + "lille_130m": "lille_130m", + "llava_qwen2": "llava_bunny", + "minimax_m2": "minimax", + "mistral": "llama", + "moondream1": "moondream2", + "nemotron_nas": "nemotron_nas", + "nemotronh_nano_omni_reasoning_v3": "nemotron_h_nano_omni", + "phi4_siglip": "phi4_siglip", + "phi_msft": "phixtral", + "rf_detr": "rfdetr", + "sam3.1_video": "sam3_1", + "sam3_video": "sam3", + "unlimited_ocr": "unlimited_ocr" + }, + "vision_types": [ + "aya_vision", + "cohere_compass", + "deepseek_v4", + "deepseek_v41", + "deepseek_vl_v2", + "deepseekocr", + "deepseekocr_2", + "diffusion_gemma", + "dots_ocr", + "ernie4_5_moe_vl", + "fastvlm", + "florence2", + "gemma3", + "gemma3n", + "gemma4", + "glm4v", + "glm4v_moe", + "glm5_next", + "glm_ocr", + "got", + "granite4_vision", + "granite_vision", + "hunyuan_vl", + "idefics2", + "idefics3", + "inkling", + "internvl", + "internvl_chat", + "jina_vlm", + "kimi_k25", + "kimi_k3", + "kimi_vl", + "lfm2_vl", + "llama4", + "llava", + "llava_bunny", + "llava_next", + "llava_onevision", + "llmjpvl", + "locateanything", + "mage_vl", + "mimo_v2", + "minicpmo", + "minicpmv4_6", + "minimax_m3_vl", + "mllama", + "moge3", + "molmo", + "molmo2", + "molmo_point", + "moondream2", + "moondream3", + "multi_modality", + "muse_glimmer", + "nemotron_h_nano_omni", + "nemotron_parse", + "paddleocr_vl", + "paligemma", + "phi3_v", + "phi4_siglip", + "phi4mm", + "pixtral", + "plamo2vl", + "qwen2_5_vl", + "qwen2_vl", + "qwen3_5", + "qwen3_5_moe", + "qwen3_omni_moe", + "qwen3_vl", + "qwen3_vl_moe", + "qwen4_exp", + "rfdetr", + "rt_detr_v2", + "sam3", + "sam3_1", + "sam3d_body", + "sam3d_objects", + "sapiens2", + "siglip", + "step3p7", + "youtu_vl", + "zaya1_vl" + ] + } + } +} diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-vlm.lock b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-vlm.lock new file mode 100644 index 0000000..ab7d7dd --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/backends/mlx-vlm.lock @@ -0,0 +1,1495 @@ +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb + # via + # fastapi + # typer +annotated-types==0.8.0 \ + --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 + # via pydantic +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ + --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 + # via + # httpx + # starlette +certifi==2026.10.19 \ + --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b \ + --hash=sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae + # via + # httpcore + # httpx + # requests +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via + # miniaudio + # sounddevice +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f + # via requests +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via + # huggingface-hub + # uvicorn +fastapi==0.142.2 \ + --hash=sha256:06366626f2e70576367714d9ab2fe8472e6c8456dba69b399f9f797ab5e92570 \ + --hash=sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b + # via mlx-vlm +filelock==4.0.8 \ + --hash=sha256:325ff22f358c18443b1fcdfa0a7aa3faec4b500c2807c554719da4567b533d31 \ + --hash=sha256:733d9b6b153fc63672f86104324186818b6bbe9dd7db84e9bb9887b6a04a2775 + # via huggingface-hub +fsspec==2026.9.0 \ + --hash=sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe \ + --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f + # via huggingface-hub +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via + # httpcore + # uvicorn +hf-xet==1.6.0 \ + --hash=sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7 \ + --hash=sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675 \ + --hash=sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef \ + --hash=sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9 \ + --hash=sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3 \ + --hash=sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb \ + --hash=sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e \ + --hash=sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338 \ + --hash=sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d \ + --hash=sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d \ + --hash=sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765 \ + --hash=sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c \ + --hash=sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522 \ + --hash=sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f \ + --hash=sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a \ + --hash=sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b \ + --hash=sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b + # via huggingface-hub +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via huggingface-hub +huggingface-hub==1.33.0 \ + --hash=sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899 \ + --hash=sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e + # via + # mlx-audio + # tokenizers + # transformers +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # anyio + # httpx + # requests +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via mlx-vlm +llguidance==1.9.1 \ + --hash=sha256:0ab9d2681c0fcb4389f3c1c79697f1e724771d74d366bb5a22edae155252dfed \ + --hash=sha256:151a615f3d871fc096984b705daa211faccc78ed2f9e3a2a80e57283e16e2898 \ + --hash=sha256:2d7aa419d8b63a528347f73ee57185b4172c8bb09287e5ef9eefe3eb6d709356 \ + --hash=sha256:31b75c9f6e09d000d062ef7e4eb3bcb3759fc1eda5619be50fdd8a830b0e8bdf \ + --hash=sha256:34ad7bdff67219c7bf43ff60eb7b1d97232092fe7a40715ae330f894def04f82 \ + --hash=sha256:367da91f0db6b494e1d4916e6ab3e3db3eb73f178fda6be32681c026af99f775 \ + --hash=sha256:3ba1b37585d07f50bb73e06dca6d45c88c9d5801789c071c8ff784c9dbcc5c46 \ + --hash=sha256:6216861eaeef0f96251d7a9ed1c26a228a4c7879c5f062092691f8f17087ae4e \ + --hash=sha256:8aaa55958adb4871016700e9140543ccbd9d6b5136d0a1756a114af706cc50c1 \ + --hash=sha256:8b29903d5108cefc715e013381c7474846e9b0edb5e2534000f5e3a3d870a293 \ + --hash=sha256:96b52f9d3eccb1d6c1ee4454c3424e8da5171c879e33c294a68e6c731fec25dc \ + --hash=sha256:9892220b2698f58193ef54a886e5f3d5eb2a60d510db49ca4c492b46ebb786fd \ + --hash=sha256:a5dcbdd38bc75f149ed349525f3884160de80900952147a4b7ef40a5b1a4a1a1 \ + --hash=sha256:a942487cd6221b793df3a11450881a6c9d25a45460a900c56d99b682633c45fe \ + --hash=sha256:cc7d1024c3df780a5eb14d30744539e8a1aa6d94c5dabf7605c82c0567d45a51 \ + --hash=sha256:dd5c87df5e60a7d8f54f46bfd5e11abb10f93636712ace09a8c1d2a0c4aeade0 \ + --hash=sha256:dfac6021d9c4c5ef16b3a1d336c05ca5e8208ff30dec29d1c63ebcafbee3a346 \ + --hash=sha256:f824a817a841563d5b068a0dd22563e5f063cdbda382cf3f6e18c1c68c45db38 + # via mlx-vlm +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via jinja2 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +miniaudio==1.71 \ + --hash=sha256:06222d80b057ca4beccb6f97a134c2c2bf646ef7890e1759cfc09db7eecec44d \ + --hash=sha256:12bc33e7e61072b4b541c14e10ef76119d5643e6bbb98e2dec0c0738889438fb \ + --hash=sha256:14892ad9b884e637029a22a781dea569b292a1be13682380fd14cefcf80ea4ed \ + --hash=sha256:154b085dd914a0e79e3d93160e1a07aacb27d66c65f9ef6a0d87c1a194f32c04 \ + --hash=sha256:166516449e2bb5f628d89cedbbb8720dceb96a0562c7e08a0e8e3cb10f58647c \ + --hash=sha256:19be6f0a1e601c2237433e579734cfaf6469191b224c20c9e5f73c32ef9ee2b9 \ + --hash=sha256:19dc58e4c50ffc48db2ce988019c28f05ca0eaa7c10b45b5c99b70107e610c8a \ + --hash=sha256:1bf93aeede652926f27f430f0fd69ef0cf8a949c07b537d6a2f295602c747037 \ + --hash=sha256:33986d5d725ebcbc253551e7358689bc81b19b6950b33cec8e8c1142ca4fc0a9 \ + --hash=sha256:3bbeb1e068fe42475e017e8150e9e345182b583d0dd4d9e77ffa20c39935d9ec \ + --hash=sha256:3ef441d139264f8a5dcb9aa6fcd0b1e1e69f58715baae416ff33f045ffba6ad5 \ + --hash=sha256:4c849ccb1349f7b3553a77a66fe7e972315185f5c4c44a0bbda7ebcdd224db37 \ + --hash=sha256:5009b4e29cd43de3631d2d5ab09cc074192c085b4c8dd8a121b856ce1af6bab7 \ + --hash=sha256:50d66729e1dd7a4cf13edc25115ac54f776dd9f67803ba1a7cd1128ebf2e8cfe \ + --hash=sha256:61b86f26d653040db32d9d15b05446321dd10e45beba25b44f841e26935213d5 \ + --hash=sha256:62db602651bc20a2698f36a0d356d7217ed6f4f917550c7ffb3705c8e8be90cf \ + --hash=sha256:70fa2ea5353e6919aca59b8c5768144af009d18c3bca251749d66fb497424563 \ + --hash=sha256:84139a10ef172acd762ccf120142877b037a1aaf71def99d2c75f66329f89d8b \ + --hash=sha256:8a28ff4ad23e55bbde8808ce525d3bb7d249d7612f77646b30e06fc6b7a778ac \ + --hash=sha256:8fc1a4f084cc1b4b25c567d22f54d1e46bfa505c17ed777c8b198e5c53d0f785 \ + --hash=sha256:978cc4d58d8beef1a705e1141dc177a8a357c10ba3a16f7d71482ee722023bbd \ + --hash=sha256:9f379d4995f1fac6dcae65810f6a31cba264339b3e591a14b233f85a6d03d81e \ + --hash=sha256:ab100e5240b104b5326e4ec1be07b6ae461f7d3d4d7a694857fd2f0493d210f9 \ + --hash=sha256:ac4a37ebbbfbfbeca50f4390e50f9952807ca61ac62f0c3bcbbc7dd698531dd3 \ + --hash=sha256:aee8e4eec8d7bde4ee78066561329235a04231a221c9b247f1ffaf850551087d \ + --hash=sha256:d9dc15eff711bcfc62a9d05e0c78e4bc34821a455595e049629f2fea7491a523 \ + --hash=sha256:e6287f15caa808a88aad0700a182bec1ff6d98769717425adf9ebf41259d1936 \ + --hash=sha256:ea86ae04ddbbf2beed20b9970af4a0baca8e6ed0e9625e1ed957be5540c943fd \ + --hash=sha256:f4a44b70b66628b0c307e40ae0ae857695978cae18462179b806d8edc807d416 \ + --hash=sha256:f7042af3a4db5b90e5efaea257b6dfcff9389239ea643e6b0faa80169528e2e6 \ + --hash=sha256:ff51e2887bb673e2e757752b586b3dc924d59aa5fbcae9bbc45f4a111bd3262b + # via + # mlx-audio + # mlx-vlm +mlx==0.32.3 \ + --hash=sha256:0512049c81fded99aa6af2d039ae1fccf113ff1a5f2df937a596578aabd46869 \ + --hash=sha256:097856504e6cf5944b9aaa7c2e1d5be0693428abd9175429cae3893e7ec2f681 \ + --hash=sha256:15e8578c9d10b4fd4728243cd6c926bc46e1da267c6a849cdbceca21d51caa51 \ + --hash=sha256:177bbb598bcb37963568f69b84e7d972f7f9499cdd39d1c6d6dab35a718aeb92 \ + --hash=sha256:19da79adad1aa9653c70819040ac58d65d7a9610a375a30d03c7cce708d4d2bb \ + --hash=sha256:231a278b4fd2804ea16d297960ab169d1749ee1adbf820173e704fbed97bf1b6 \ + --hash=sha256:2dc1085f5c2eba18cdac095d7fb0b68ddfadd51bf695b18a1102eda15e77e76d \ + --hash=sha256:32fa23e57297fdca23e016570229b459f20f645eeb5965527cd890f74143f1d0 \ + --hash=sha256:448fdbc99c5fb0c26f11337e5a182cd6516f8a1a65848184be063d00ad3107dc \ + --hash=sha256:452c621862684e8769be93c1517420c8ba2fd6e8b01a1c970c8ba8069022f0ac \ + --hash=sha256:46df0a5489edcc071cec3d9dd1f7894d492888c9d425effdc8831f4e7021108d \ + --hash=sha256:4e1a8fa78461c6cf1f9473e3bc8a3c4e362f1aaa613b435787f2844fad522bc7 \ + --hash=sha256:50939a8bd54250e6092ef3427a724c7a72c0911dcee09d70f6543431e7f28d3c \ + --hash=sha256:5d12de01f23d2fae739ecfecd9c1d4c07c1f9b778a3729f7c0884ea1168ec14d \ + --hash=sha256:65122a7eeb718df56641d7ac6f7fdb0ada573fd841d1fb0a61a6a443e33f11b5 \ + --hash=sha256:656df5e36b9bfa862fd633d55dee4da072d292def9f27f22134c594e03ff28a7 \ + --hash=sha256:712ca29119d9e2aae04af8f8e416bfb0097538d9ff73e6fab8b028053fb45ff8 \ + --hash=sha256:72711cb23cc6dccc5a460f63f39c11931a00b2c6dcd6fba253921090072e21bc \ + --hash=sha256:75333afef55819afd2a31e87ffbed0bf59f379bca21771e6b13a95a675e4c85b \ + --hash=sha256:78372731c18fadb6d6960c9188cb752d5ea4d87de9be430a311a385e01d7fb42 \ + --hash=sha256:7aecc3d4a904279492dae58e28378a3c4bec4533e7066e64ae5c78c7a5511f22 \ + --hash=sha256:8439baa207db769df1df82a722b0ee0c2362f75c23c0740b7472c3d279253bae \ + --hash=sha256:8564a59943346fac01ab498870a9350d0f79b1cdea95c46c895934ee027d6197 \ + --hash=sha256:8edf996749b331659ab9bb8ecdc895c292f9e7020d02e534147e07cae22bec7c \ + --hash=sha256:8ef541373a94a78fd59ac59368501e976cef5fa07738438362e942315be22ea4 \ + --hash=sha256:936cc82b0c050686c43230c85f3a0bd9f179f650377ce9590f7af3ba659e2fce \ + --hash=sha256:942f0da5328986cdae29ec0eba57b2a08eb9dc6125cfe196853de7498f2c5359 \ + --hash=sha256:951ba2ab75148f61e9a5c222a75fdb0ab481be0b196c21a0d087e9b886d204f0 \ + --hash=sha256:a14387cd679018a32e802fdaa58b1e9845a278cacd309955eddca95757988ce0 \ + --hash=sha256:a5e165179e3d86672cc0d594587bba2b9c3be511ccb99ffdc63e4cc7119eec55 \ + --hash=sha256:a6241ea8f56a08a6c3a06bbaad7d7fc2fc1c05f23757c61fe0a8797325ac6d46 \ + --hash=sha256:ac4472834f38e61d294aa1bf2fdc9f687f3b3162b9729dc9dc15e8c149f8c216 \ + --hash=sha256:b14744a1b3a9dd48e06a62c448710a798ceda9451d7d37a4654a58b5adfa97f8 \ + --hash=sha256:b2acd057e4a27eb1543bff0e9030c34fdc64e18815fe4bc23c2dbcc47b60f0e3 \ + --hash=sha256:b57996fd2b72207a796656bc39508f20089692815297120039284edb296b5fff \ + --hash=sha256:b91012fdd5ec5cbedcec2cae6adfaad9a8f547023e570fc5781cb7eb4bfe2567 \ + --hash=sha256:be5cb6554873d3c86f083fb0b35aedf4a1f12657adee4ac0fc90665ffd7e6722 \ + --hash=sha256:c0764ce1a7a7f19768d6b3473387dcac03cac8b0dc74b7e1c8d05b451ff4786c \ + --hash=sha256:d17f0c33789009a3b3e609a52af34dc687ed39816836e20ab466aab669c14e6c \ + --hash=sha256:d3b9494b45400b803eeed7d05e6e566aa135f1dd2d121c29016ee9492e7ead67 \ + --hash=sha256:d61942263d4603814622cd72bb5991b8a3cb59970bde8a7b40b897a7f088fb84 \ + --hash=sha256:d9ff05b868748b846a02f59913e9aa9d7b576827db92cc358ec1cf660f7f90a5 \ + --hash=sha256:e13f5ce391293dab946edf18e7ae7bb55768beef1140deb509f67a2b14edcf7c \ + --hash=sha256:e32d595df067f19739f1e214e723035ed2a9c216fb1ed3cd118d08f19b9cfabd \ + --hash=sha256:e75fa55161d9477e694f852cad0d383c6b4a5c3b505a5d08339f0e2260c2556c \ + --hash=sha256:f2909f683ffc5a2720f6c8099398a64fd1be53f1b6538e4e728ee55816ce699b \ + --hash=sha256:f348705a72fa236f310c2cbb9c393d3b620a942c9b34ac641fd0f098bdc5be44 \ + --hash=sha256:f5716b51fea3d06352d4d8c851fb6c5141b1a7de5a5164d44de5a6d368d84a7e + # via + # mlx-audio + # mlx-vlm +mlx-audio==0.5.7 \ + --hash=sha256:2f2b3c666241ff4a45df7e92057d176b3c1079541a5e9062a0d1e8088db5c123 \ + --hash=sha256:4446704bd8305b3957a6e422234118142e32b87a55d7d3e33b0fb8dbca516543 + # via mlx-vlm +mlx-metal==0.32.3 \ + --hash=sha256:34ae9b83ad2f0ccdd3e5d48ec35176e7119f57069eef187122916dc941a4ae1f \ + --hash=sha256:fdba88ed9131448cf163be2c7ad6374746d8818205f28657aa91b6c69050f691 \ + --hash=sha256:feea99fe4d01f9e9f33ca73d869f988219a1725de1c18870baa7d5cd5aef6cb3 + # via mlx +mlx-vlm==0.7.4 \ + --hash=sha256:7a3e8e288b21115630c4eab75e9ffc38e4b1770f9d2ce4992d29ecffe75a628c \ + --hash=sha256:a35ca38c5be3a061496cc9bb5b3ada20413ceda9317ec67ba0b0fff1c2885e04 + # via -r .tmp/hf-intake/locks/mlx-vlm.in +numpy==2.5.3 \ + --hash=sha256:012e66aca395d795496446e52aeeb5866312a5d4d3f27da270e5a0b43f70dc5c \ + --hash=sha256:09d5a423c71ad5feb5625844ad58050e35df43871004b52ac9c0ad44a56775be \ + --hash=sha256:09ffa5d903faeaa5c4dd05009cf81c8bab9f2cb37c548b8d39b65b4cfa7c97f7 \ + --hash=sha256:0a59a421a32580a009e8a1751345bf829631b990dc1794b80514ab722b435def \ + --hash=sha256:116f96cadd935c6122e9228d676fe7ede19e741f5c8bb1c3cddbe0c51ccebea2 \ + --hash=sha256:1302b90c0e52281681b2975adfe8a860cb7b12216a27b4b0b4207c44bf7bccf0 \ + --hash=sha256:15aa985ac73a8db02db7663381aa109510449d3819d37206caed27b33a65a8a6 \ + --hash=sha256:1aad64d99730d013cfc6debafed22783b4fc5a7f4b8bc744d2d8cf7dcc880551 \ + --hash=sha256:1c80eabb4035ecf4ca9cd49cde8a9fdd69a729e63e6474887d1523ade7aa277f \ + --hash=sha256:1f3ed25271581281f2fccb1adcedfcde4c07362eec69189b50baf6f90e3ae159 \ + --hash=sha256:1fb6f8fb9ff0b3a69f52c66ce397b0246583e9f28616231b0e32ca49259a5fa6 \ + --hash=sha256:214045a5bf00113a146ab9ee9730c44501af6723cdf1f6830932f7b5ef2e7af0 \ + --hash=sha256:26e15e4aecd8617dfbaecb37d223e365d7b39411fba20454be2670a96aa74cb5 \ + --hash=sha256:2c25dfa72943e4336ddb6b0ee4277b47a0c85bede0807530ec68103bf58e2c10 \ + --hash=sha256:2d8240cb4c16fd831074aa2b2cf9fc54664d826341d61c372245b96a74a49a9a \ + --hash=sha256:350ba9783ce969cf9f7ce6e6a9a58e1a6e2a19ca025b7ee448c4db727706212a \ + --hash=sha256:4c8a6d2ebce6305fd82fbefca827775437147052a976ee7c94b36a0c1b52ac6c \ + --hash=sha256:4f8929ee6c96bfbd7b4ed2032e0c03af86fe1826740ab61ddabf9072d06e57ff \ + --hash=sha256:536f963710a4e63934d80ac0dc4f478804a83e9a84b6828018f25d09953ada33 \ + --hash=sha256:54a115e5a73b8fc44f0cebef486365a1894b5c9760685d4558b72b7c3eb846e0 \ + --hash=sha256:595d020938c84e320bcf40ad71089e108eac0d377cd018e14a8c094f39e98d85 \ + --hash=sha256:66a78fe4556c60aceda5916f9eacd638b18e9e681016ec302dcb4682d6d4d034 \ + --hash=sha256:6b05c171afb3aa07adbd20abc00aea86fe375beb0fdb9ef780ec5b7f63bab1c0 \ + --hash=sha256:6cef4bb1706dfec49243c05d921eefb4e190d41e2528b30d8035ea1f36b4c24a \ + --hash=sha256:6f24021b9f22bc6301c37b196974a92c1c18dccedb6fef3dd252e95f2d6adbe4 \ + --hash=sha256:71b39d9f935b6ec0f8753e3e2afb51e3efba6f2e05b68b32a40754d24bcd4a3c \ + --hash=sha256:71cad2b2a7451ab79d8f5e71b453485b6775963d5cf794179144a7463fe6e8ec \ + --hash=sha256:76c2c1e6bfa5c84adc6434dfbf013aa92096a7985221762c8f11fedfd20fff58 \ + --hash=sha256:8617bbfae4486cf99c9f899966699428d19da931d06ca94ad3da986c76e15997 \ + --hash=sha256:86bff898a431c0fb71f7610b75726e75a54d47b37edc9d537f48de63bb3c0b90 \ + --hash=sha256:8e4dd766076855b5ff7ea52fa5f07ce26286726e0f8bff446b7739d02e6ea204 \ + --hash=sha256:92f30e89b8ee0ecf363033576c422b2f58fed6a80bed0aa48dff6d14c654663e \ + --hash=sha256:93e1f5447e2b1e479d7bd74701e84746b86450cff1fc368b132d195e2b8f8211 \ + --hash=sha256:9a37475425b431b4d060f23b4f52cd2f3aef6bc7c654bd760adf0040eec9d435 \ + --hash=sha256:9deb49575e5b0b94ed72c8a64ec4d033381adc27e9060ae842971f697ba96104 \ + --hash=sha256:a5fa86b80fd24bcd1aff83ad23be44ea323de3f787be8f8b15d4a65621e25321 \ + --hash=sha256:a6391fafaba97500887132cd582abc6e19452b1ac775a47caa7b24490e152058 \ + --hash=sha256:a72f874bc9e10e4b8f80426fb49716d5141f64442a0c8418065093ec8017fbb0 \ + --hash=sha256:ac7bb1c52d445bd4f8f7f97fefe6abc3a084dc4d63df50d79b17fa2b78e89297 \ + --hash=sha256:adc1ada2662f8a5f960b8a10d9986897e7499ef07e06d4cfe7197f8cce923c07 \ + --hash=sha256:b00eefbcf0f292945c4b4dec2ae845389ef5bcdcd596e6e4328051db5b5ba694 \ + --hash=sha256:b0521d0f4aebb6e06189451025fa17a913287b13c03d5fe05c017333b654ea5b \ + --hash=sha256:b5d93cf48f687479941d12b69c873ad2cc76bbd487f0091c2200636497f34034 \ + --hash=sha256:b7e18c623bb5c95acb3b3328861272816ba199fb531921c5d6d0b675f1fde9e3 \ + --hash=sha256:bd4cb9ad3c7889b9b3fe0a9a9fb5d2ed26f9879bff2608d9f01aed147a20d231 \ + --hash=sha256:be5a8381859b6da607c84f4f7d6847725f1cf1853ef8a2c9e115b7d58bef47dc \ + --hash=sha256:befa1ae5bd6030b3f512b43ff3fa5290bbed6b84411a44244b14adf835f5b89d \ + --hash=sha256:bf63afbe037eb5d2fe87fbcc7778e61da53ebaf21d938a4515aa73b62532a5d4 \ + --hash=sha256:c00abe94c1a69d75d827dcf1c025b25c8a45d230b3bcd77a9020883a1b047653 \ + --hash=sha256:c2381f82999704f818e2c987a865050e285ec3621262c66d40f5a96c8f899f8e \ + --hash=sha256:c76d5dde9f445058f83d0c02af00557a4db91de9a9a57c0df87d1535001d654b \ + --hash=sha256:cb189f09db39283b26bfd061ec16189e14f71c6755207f72a0f7540867afe5b9 \ + --hash=sha256:ccb32e0525d29e8b0572eb84c9a57af0e7a4e615726927506f55063c62414034 \ + --hash=sha256:ccbc4665079665c3cf3bab4db9f6b095370cd6437d66be549b6c2a1fd19e1958 \ + --hash=sha256:d1c89973648c85069c5046ad460f7b8a00218b29a2e42359ac8cc63e9ab94832 \ + --hash=sha256:df2d5874ff183595a4ba404edd04f6bd9b5505c1d7708573f6a6c17489a67563 \ + --hash=sha256:e01c918ac3d48e18a927cf7b14a26a3e29ff2bdf2eacb976da0aecd6a43ed034 \ + --hash=sha256:e6ab667ba76450084eb64013762c438ea76d9d29cc676dcd6c2e9892ba37f841 \ + --hash=sha256:e931e4f499e0dc7ef29d269a8e5b35dd722e5d14be07df6240166ea7c6532fae \ + --hash=sha256:f54660b0eb6b0b9f36e7fe1cdfdff472028dd0d14acd9b9b65098efbad059469 \ + --hash=sha256:f59a878c33d6b88122d80d239bb3b845d58708750b0cb06a09aebb9b18ec696c \ + --hash=sha256:f7fabeb6cea87d65f3b926de33d03fb016cfdc29314c90974383b5582ae72891 \ + --hash=sha256:f9579f383d1bf9df80081e72760e84960a7fd4f88cf0c9e535a8597c9bb646f5 \ + --hash=sha256:f9a2353b37a1a9e78fd82b27ad7e2a32a2d036604d18f02b05e3136c62ca3b09 \ + --hash=sha256:fc36dc566135b5eceec4cf89758fcb719266a019ef07dae1754ae7c9f617ef3e \ + --hash=sha256:ffdc76bfcae6b255dff75202c5e7feaf95b40246bc0a17944facc1fecf9f79ab + # via + # mlx-audio + # mlx-vlm + # opencv-python + # scipy + # transformers +opencv-python==5.0.0.93 \ + --hash=sha256:08d5d91d967b58d6db86073b2ad3eaef88ca4ebdfd45c9059bf59f5ded0c7ad2 \ + --hash=sha256:198a75138241810206a17c829dbcc40a7cb1841cda538ca86cbbfc6c7d95f898 \ + --hash=sha256:4b4b1a34c79bf8d3738e3cfe9a9e67b51a79663f6b692cbdad8c31f570da4157 \ + --hash=sha256:66aac3e5b5faa48d4025816592f3af19e4bfc2c68dec067bae2dbb4ca10aa9e2 \ + --hash=sha256:6bbc32f59e1b1a7db7b39c81f63d00625f041d333037fd8702f6da52cc39108b \ + --hash=sha256:c8de2dec111122a02e8beb28e16c31904992dfd6186560b142a92c71403c1039 \ + --hash=sha256:e2b4272e736836f66c2d176e43ab8101f3a00d45654916399f52e150c58981ac \ + --hash=sha256:f8b6d0a212253dd26ad338c812f1f23ca118fdf05a9c8c6b9444f161aa8c5881 \ + --hash=sha256:f90ba04b8f73bc5c3814037699739f0156f597338a98f05956c684e7c3ca10d2 + # via mlx-vlm +opentelemetry-api==1.45.1 \ + --hash=sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75 \ + --hash=sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb + # via fastapi +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # huggingface-hub + # transformers +pillow==13.0.0 \ + --hash=sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27 \ + --hash=sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782 \ + --hash=sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22 \ + --hash=sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15 \ + --hash=sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502 \ + --hash=sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692 \ + --hash=sha256:10b5019883c98f1dd819ca48ffc6f33a91cee04948fe637c605f4345c7cafd6f \ + --hash=sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8 \ + --hash=sha256:12fbb851cf323cc19aac3eea1df5ad17b57e91e9fe3cfd06100dba7bb8208c13 \ + --hash=sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3 \ + --hash=sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369 \ + --hash=sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88 \ + --hash=sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d \ + --hash=sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb \ + --hash=sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f \ + --hash=sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0 \ + --hash=sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40 \ + --hash=sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599 \ + --hash=sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055 \ + --hash=sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033 \ + --hash=sha256:37290fbef0c73e2d51d594eb78fded7dc7170d1c3ad0a890269e6fd8c8c4e061 \ + --hash=sha256:405be6058e206a86fedce2f715ecc2343f94072d7925251632dffa2c05f70dca \ + --hash=sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f \ + --hash=sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca \ + --hash=sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c \ + --hash=sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4 \ + --hash=sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8 \ + --hash=sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620 \ + --hash=sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071 \ + --hash=sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc \ + --hash=sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a \ + --hash=sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6 \ + --hash=sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e \ + --hash=sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582 \ + --hash=sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f \ + --hash=sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79 \ + --hash=sha256:6a0920cb55df5a4d3f6862964908078c0bd59332bdd1bcd20d14e8c70ef27cc5 \ + --hash=sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683 \ + --hash=sha256:6f38f3f885a09cafc93fd667b1173a92f379e43785d635e7983c893f0da319fb \ + --hash=sha256:73ee25cf8cf8b75d5a5aeed7cb6588f5e5dbfbb0fde51707b630bbb23897645c \ + --hash=sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa \ + --hash=sha256:7c88c10dd432f3b39ccdafc9ed81d0c21b8fd634502574da76d1bfa4fe2c76d0 \ + --hash=sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521 \ + --hash=sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e \ + --hash=sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c \ + --hash=sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058 \ + --hash=sha256:83b37abd87b590acd8f47ca8a61f70893322b11834d36e07c2298dacad4d14d2 \ + --hash=sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463 \ + --hash=sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d \ + --hash=sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81 \ + --hash=sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf \ + --hash=sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63 \ + --hash=sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0 \ + --hash=sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42 \ + --hash=sha256:a192360d512305fe92d3053e4f7627bbad1f7708f1f4ad38cccd9336acabe9ad \ + --hash=sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81 \ + --hash=sha256:a625df9a2a5597fa5e5266e01aefc82604bb85d291e3547981e863560d75da6c \ + --hash=sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4 \ + --hash=sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981 \ + --hash=sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43 \ + --hash=sha256:b84be41387fc00db28c13c02e30d42b02d8d8f6a4a741a1aee6012ca7f1f6039 \ + --hash=sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76 \ + --hash=sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51 \ + --hash=sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2 \ + --hash=sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630 \ + --hash=sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac \ + --hash=sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510 \ + --hash=sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6 \ + --hash=sha256:d7710eb676e9bb68965cb045be0d333b4c0d7f9dfbaa7fbf121385b1588683d3 \ + --hash=sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d \ + --hash=sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61 \ + --hash=sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932 \ + --hash=sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1 \ + --hash=sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d \ + --hash=sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00 \ + --hash=sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe \ + --hash=sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313 \ + --hash=sha256:f391dc92a015c478745192fe4f2e120c2af2aba18d9fe0616341cbf537599806 \ + --hash=sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a \ + --hash=sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0 \ + --hash=sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52 \ + --hash=sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b \ + --hash=sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe + # via mlx-vlm +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pydantic==2.13.5 \ + --hash=sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73 \ + --hash=sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08 + # via fastapi +pydantic-core==2.46.5 \ + --hash=sha256:013d6f3483d81e02e7c328831808f336c8596ee33b4bd4026b9ffb1e960b8942 \ + --hash=sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821 \ + --hash=sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5 \ + --hash=sha256:0bddb4020d8f04175865ccd17eff3040874fc11fb593f424edb452653b4b947c \ + --hash=sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184 \ + --hash=sha256:0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2 \ + --hash=sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc \ + --hash=sha256:15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5 \ + --hash=sha256:18a09e1e1011b462f2e32774f25859ef1223d5c2b0546a633cf56654710721e0 \ + --hash=sha256:193375f3548919d3f0b60936ca113ada3e38f264f91b9b8e0508efaad57be931 \ + --hash=sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e \ + --hash=sha256:1e449def1945a462c464331254e5a44fca7c3b4f9aedf59ec2f50f8066dd8e25 \ + --hash=sha256:1e5aad1220a1192c42341c8fd4a8686657e73ab2a920c970bdc4de334fe3193d \ + --hash=sha256:200aa3dc9f8d54f0754f43247c0bad0999fdcfbfd2488384dd44f37279271fe6 \ + --hash=sha256:2471fd51c61c610e1dcf7de44d7299283661654d11264ab4802b303368d69c47 \ + --hash=sha256:24922243639cbdac66c75fcb6fd6495a9cb52b213d62f9a0d16f0310b1ff8038 \ + --hash=sha256:28a6a556cd3b6066bea827857f9d9cce027c96f776e512f544a581f9e42161f8 \ + --hash=sha256:2bc9419666990c06d7397831f2126a1ecc3594aaa3ff7de5bf2d066802f4e07b \ + --hash=sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a \ + --hash=sha256:2d330aaba8621b1edcec8ae2c4050f63b84ccf6d98723a8f212e9684713abf0e \ + --hash=sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074 \ + --hash=sha256:337639ba62a11acde6ef3aeb08c8ea755f8ef1fe5e513356c0f36a2b0d7568b0 \ + --hash=sha256:347ec774390c87326a2e4929d58d3f7e8763a104d5d35f4cd595a4c952366433 \ + --hash=sha256:356c8368cbc321050b169595683a2e1d63413b1e0e2868b330af9fc14c616d3f \ + --hash=sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034 \ + --hash=sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21 \ + --hash=sha256:3a3e26b6a8274211bddee2d0e4d0d42778f17a34510f49d2ec44b58abfc41736 \ + --hash=sha256:3aa166e99c4f2985407fb8714aebede877ecb5455cf321b606adca926d30d5a0 \ + --hash=sha256:3d2652072b2d774947ba5cf78a9e59644ac62ee572daf6dd2e1dfe905e15b2b7 \ + --hash=sha256:40375c2d05acec10323e45dfe2077ac44bc74659008614af5069034e2cfc781c \ + --hash=sha256:413a717a410d0c817ef5b786a059415550b3794e1d0c2abffd9efb93a3d9f7b4 \ + --hash=sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c \ + --hash=sha256:49776eab08766a08dfff7012f8b422dcd7e25e43b316eedf0477c24fcfa84b7c \ + --hash=sha256:4d44cf99ddebf875f9b68cc267aa684c99b7b44fe63ee1cac4ec163807290069 \ + --hash=sha256:4dedce55295becb61921e386b99d4f2706045306e7fa52249a33004c837379fb \ + --hash=sha256:4f8507560a9284e1370bb048ed4282012fbef4e8d109875b95e884d228552061 \ + --hash=sha256:4fdc8b93a41521988916eeaa271173fcca7fa0803d62f87675aac8dcec1c8e29 \ + --hash=sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3 \ + --hash=sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa \ + --hash=sha256:53feb344243bb9510a9dec7bf3cf1b64d88a98af5dc7872a5160465f8b198c8e \ + --hash=sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38 \ + --hash=sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f \ + --hash=sha256:5cb482e9e84c851f4e623fe4acc1ced89168cf1fe18f7089db4548c8f5bbb65b \ + --hash=sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8 \ + --hash=sha256:5ee239d575f80b08eca11f6e20f90c4c695de7825c67eefe6091fbf20dda648e \ + --hash=sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c \ + --hash=sha256:5f93c5fe914d75fbec9a49209b00da5f08e9e467d69da2b1510c81940cfd10be \ + --hash=sha256:657b40d6240c0a7b6a64b30f22d1e3aa631c7e846c621b0c0f6d1d75e2e15ea6 \ + --hash=sha256:6d30e1a4f138b8951063e9a394752a9179b51da288ffa507b1e659222f4c1793 \ + --hash=sha256:6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1 \ + --hash=sha256:701b2e04b560eeb4bddf7a25ab8ca476176e34fdbd9a0e18196f0d12d4685f0b \ + --hash=sha256:771cf63ae0b1b50dd22e5f3e3549fab5f3f4ff1635d352a9e1a97fe01c7b2e64 \ + --hash=sha256:79bdfa52f843137045b2d081cc05c120ba6665d29b7559c2c47690906f39279f \ + --hash=sha256:7ac031912d54f3d83ef3b3eb98dfabc1608802e2202263d25957eeed40b94761 \ + --hash=sha256:7b0fc826b16c55e561e5d2a0c5c77b051ba1d92808118c4e4b5390f5e0cf191d \ + --hash=sha256:7c6be839a5a8312626b32029a415644a0846b420bc8b52b95b28cd92da162168 \ + --hash=sha256:816ff0a6550ffc06c098ccd2e0698600f9aa7da192a79eaa6f9af504a35db869 \ + --hash=sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111 \ + --hash=sha256:837b396ca3d7b74091ca623f6cbd8351bd42d670a79c2683e79fb089f06a2de5 \ + --hash=sha256:850a08d167dde16db8702c274f320c7be9d7da6f6dff2b58b18f9e815bd94f5b \ + --hash=sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7 \ + --hash=sha256:892a881d5f68c2b9ea304b7a6c2c60d9343df578a311b0f86b94bc8f1ffe8129 \ + --hash=sha256:895395f8918627b04efb1ad2a4cf605387143300ba03304cd1dfa6d03f5e095e \ + --hash=sha256:8b10e3e8fd7ddc2bd915848a2768e44c15b22936f1cc54c462ad1164deb02655 \ + --hash=sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c \ + --hash=sha256:8feeac04b5794e513e710af2f9c87d49f31a6dc47967bb264a1fed61a8989bec \ + --hash=sha256:9432f3598db432cb51c5b37fdbf29a60fcccc79e30d37a05022776a6bc4ab689 \ + --hash=sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f \ + --hash=sha256:978e7b97d4824b5be09c69fb70507cbde3b0323fc147332ca40a94d9a6a0ebbf \ + --hash=sha256:97bf8de4d541598c94a59344eeb988a94c08ff76b5723c41f6567ec18c7892ea \ + --hash=sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9 \ + --hash=sha256:9b68938dd5b0c783d88ff8e2dcc69451b5eb936fe212d516b21b9d5567f6d464 \ + --hash=sha256:9c4b71f10dd532fb7a5cbc8f58707779e64f03a258c2bf8bfbaecfcd9970b519 \ + --hash=sha256:9f47b8a949e60f027f0aa0a6f6c7b7e9c55cbf4380d10b344e282fa4e7ab1e1b \ + --hash=sha256:a1dee1b804ff4d11c663636cf15d2ea47e9f79cd56c033fb1cbf08924842a48f \ + --hash=sha256:a2468d93d181667a7abd66e1b64bb9f76f361b0fef8faddf687456453576f5ee \ + --hash=sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a \ + --hash=sha256:a39ac25a9a2fa4072efdb429833c4a4c8009a51ff9eea3eeae131713cd27991e \ + --hash=sha256:a445486499897b88a7d6c310c88ed64dd37b1b59bfd7ae9107490bbb362f47d6 \ + --hash=sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2 \ + --hash=sha256:ab4b66edffb32d9e951efb3814bd104b8367a7501b81b955cacb5726d897389f \ + --hash=sha256:aca6c767f552b21b10f774aeac128e828eafb796adfa1b666a18bf6321453c3a \ + --hash=sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f \ + --hash=sha256:b10ec717381bdbfafef34607824db4c91de69ff085e4fca3b2af91b4fa17e68a \ + --hash=sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47 \ + --hash=sha256:b6acfb46a814762367fb7ba0828b0a17d441b92ce249a0e007474c9072662dda \ + --hash=sha256:b7ca9034437b6022f941f4857459562ee00a560b97e7cce8a0ec5a74fc6766e0 \ + --hash=sha256:b98134087d9de723658d17a42c7d0da8d6e2ef08015dee7dc93889047315f5e4 \ + --hash=sha256:b9fe6fb92520e3fd61f2e49000b6911b188824f089b75973ea06d6267f0b476d \ + --hash=sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3 \ + --hash=sha256:c14ad3bdc85ee7f318742c457ca3968a92126d144b15721c759033bfb06296c2 \ + --hash=sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355 \ + --hash=sha256:c3471e5c4a949c26ec00a77f01df59096aa9495877de76fd60a980f8ee6be461 \ + --hash=sha256:c583b927a8838dab890706a6fa7573fbb8b70e24000ef9f7238e2d6f6435a5ed \ + --hash=sha256:c76fe65e607be28c7fd4d56fc3c42b1583aa058ce3408b7ad0fd540171d31f9f \ + --hash=sha256:c7ea57fc63aa7da93a1bd2d644e6577befae10c52c4e36377635eea1056a74f5 \ + --hash=sha256:cd5214352ae68f3b5e9af7768bdc5253695ee069675db3480518420b3be881f2 \ + --hash=sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829 \ + --hash=sha256:cdc8b74ecc48c0cb1e9607a05ec4e9e88db60a19ffcc9a1d5f9088ede40c8dc0 \ + --hash=sha256:d0a24b40877af2de4950252be9d21eaf7fb07660f3c2cae1f56c6b599ada5266 \ + --hash=sha256:d22a945598fb91236b4dd793a6e42e4f3dd7740bb5aace5ebd7d4c08d13bb575 \ + --hash=sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290 \ + --hash=sha256:d625a186a65201c23a9e3b8ed9c47e90a026e03256608cc91851c6709096844f \ + --hash=sha256:d925f3d9afd05a8c0fb3a1031463a8d59ebe5e2afad297e29c78be19e13b4e62 \ + --hash=sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f \ + --hash=sha256:e652ab17569c94bff5475520f907b7148b8c24036a8ebbe5cf7cf7493d28579a \ + --hash=sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7 \ + --hash=sha256:e80675d75ae2cd14372cb65cad5400d9347a3d3f6c13000183f22dfd027283ed \ + --hash=sha256:e9c134bb666dd54b778b9fc0d2b50cbb7f979b9e3716f26a88c9ab3b6fc1dd0f \ + --hash=sha256:eb7d8d0e5886a89a55d2eef490e272fa965a9d57c6b29a5b5088a7997ec2cad1 \ + --hash=sha256:ecb42011e12ee19cafbc312887cbf3546959fe02fbad44f272d4be5baa997615 \ + --hash=sha256:ef3fbbf161dc9351a2fe0422e51b129f9e97e42385bd0320b309c15f7d287dd8 \ + --hash=sha256:efd62a42486f1bda5d24cb4f63d15a3c7768375fe83d36f9417b4ad7a2fb20b3 \ + --hash=sha256:f077d0b97ab11fa7dcc633fca53515f290bca8a8a633e966d5b6d1879d9ed01a \ + --hash=sha256:f332f0e72a5a0400141f830744e141bf9f97917878dbe968669e8a7fefea78ff \ + --hash=sha256:f7b0ec93a2893de856652154d73b7ba622f26fa97726487dcac373de5f4c6084 \ + --hash=sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0 \ + --hash=sha256:fc5d783bd4a2387e97b8a2d5ec781cfb92b3d893bf82370548e99db5915935d3 \ + --hash=sha256:fc8515076c11f3cfdf4fb142dcca0fe384b1230a3b5415458ac84f3e0903ec13 \ + --hash=sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9 + # via pydantic +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via rich +python-multipart==0.0.32 \ + --hash=sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e \ + --hash=sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23 + # via mlx-vlm +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # huggingface-hub + # transformers +regex==2026.9.29 \ + --hash=sha256:01000ddf0e3ffef97f2413ceb514f6313040106b6d18a03ee00a4fe35c1eb1db \ + --hash=sha256:0166844493626c5015c6088ee15c9ca2fd060ca15b7641d1657da6a58432ae33 \ + --hash=sha256:044265d77d94f5e3cb2fd72c76723807c429cb8c533e9d4672d0334a6f14f588 \ + --hash=sha256:0476e5bcbe6e1ba3d1c4cc7bbb1c3ba78e3b979b5c8a88d0a6a8cdd4992b8c84 \ + --hash=sha256:066d0e3dbfdd739bce2bf8c2a41dd16f73e3d8adc2eb06dd803a36a307f56075 \ + --hash=sha256:0b65c72739f981377c9c22e0c5c3cd7f42da7bd8a3c9209330fac772c7d893ed \ + --hash=sha256:0c992c19cd45058a4b92f68f139c93db168b48fb1f322c9a7cd620806afb6b51 \ + --hash=sha256:0cc63b5e47c12a48d90c7e9d7de6a035dd14f62868aaedbb4e0ff8ba2b8bfe7b \ + --hash=sha256:0dd8af32e9f7b56b7f95cc1fd79b23054c3bdc172392ae560acc24d57b7ffe71 \ + --hash=sha256:0def9fb6abac55492d6d51cddb7225d07d6f279e774e0adc08569a54a5fc8d46 \ + --hash=sha256:0fd2c901cc307a745ad4bc87f20060d7a0825a3371d1e93488af22e7a387f78f \ + --hash=sha256:1043aedf5917caa861bcb25a9c11460049656bdf0017a90a309fa8f255467725 \ + --hash=sha256:121a76a0985db80ceae9e171c337f8c927868e37d01b54e3ce87bc87f9c6a208 \ + --hash=sha256:143533cc4b6fbc5b95aca0a5b8d541088d374831593def000ec89322c220221d \ + --hash=sha256:14e953ff3607c92d7675bf79c4d4509ef6782aa8c08509f179f9b3d6d0679e86 \ + --hash=sha256:18ae8eed4526e35bdb754d61562b90bf5c00a67fdcf3cc1380dd59597486631b \ + --hash=sha256:19959129885356df0e97556856f77eb2888380dac18bed075a7c05c5128c618d \ + --hash=sha256:1ba8c6a416569ce0d37e83e28a254a61dc99a419084dfb6476cea02d997f74fa \ + --hash=sha256:1c2a0026062abcc321a53db4a185ceba0b59a66b5d37b0808917a88b55a5257f \ + --hash=sha256:1d9fe8091b2e89d470df68a9331111ed008ae8aae6bf1e8e1fba4086a495c84e \ + --hash=sha256:2089fe39c406784d90101c726755ffa1497bb74638fd434300d2b88006186de8 \ + --hash=sha256:23ae6fdad9e63e54038f5ef78aba2933faca61e24d432786589e737bc5522ebb \ + --hash=sha256:26ec4ccce55aa533fbd603d08911b01101a8fcfec987845ac3ae2c7087b2bde3 \ + --hash=sha256:2f7f7aa47b229f2b39a2ae2596d2ad5625d77b5eb9856fac2dab3eb506cdd0a0 \ + --hash=sha256:31b003f9a070335e2a8233ee9b14a3ca8e6d792012ae011f741bf0aaf11744c5 \ + --hash=sha256:32ab11df9677ca80bcbb5fe4eb1da9109a5019239a054836efc6fa1c64e683cf \ + --hash=sha256:33026515aebc0e70d1c89978e53e8d695d35d9e472f8d5b34465ba3c74028650 \ + --hash=sha256:34b6925af9853bf461950e6508910f179fd6e9b1a7ec8548e069606b7e51a26b \ + --hash=sha256:352cf115a810b357caa35193ab656ecf5ef41056855e82f292c99e8514f8d954 \ + --hash=sha256:39ab5894d971f9ac68baa6eca5c50387db579cfcacf36ae8df3feceb1815e6d0 \ + --hash=sha256:3a21a9509d0ee88e7a70e1ad228cd2f0e0fd1e187458db132e8a8d18c97daf9d \ + --hash=sha256:3c5c2ef13797466aa64170cbb66ad98a32351dd4127694cea7199f80f213750d \ + --hash=sha256:3e778bfccd63075167709136afbc251c1f683758d5bf49c803c60ac3f894ce6b \ + --hash=sha256:3f1e6cb402a89457582cd696f982559217d13484a193202c394015297968c86d \ + --hash=sha256:42e82e578c904445d4c8a35b8f28052cf567593215fa5db06266fbc6f77aaa2e \ + --hash=sha256:4408b2b27a95ca8cc48b7411945753773353b5c93b307754781086c99d3a576f \ + --hash=sha256:446654b29bfaa30500d80947eda42cef1449dc8a87f4e3cf061cc8485d3a1f0b \ + --hash=sha256:45010bcfe66df41522d56c9b6114e87ecc597a08970ff6a2ced24415c141ae5f \ + --hash=sha256:49ee178ca31c94621294bf9b8b676a92a2e6bba8af0529591753719e57edb621 \ + --hash=sha256:4d7d93613b01b0199961330e49cfc52d479b3d5776c56c691db31130c0a07d91 \ + --hash=sha256:4fb41211d2333eb930a51e0546a65999761cf1f572a4da56ef9b8a62966c06f2 \ + --hash=sha256:4fe97894d1b306c919b4e50def1e6f6c522f4d03a7283811f4d108f1ce5d3ac2 \ + --hash=sha256:554bffadcbcb6d5f4e5fb10a61cc52084b9a63d1dab5f10bcd2c4343972e8e2c \ + --hash=sha256:59b49507f47479e299a9e1bc41b5cb83a7afda0540625f1dbae886615978acbf \ + --hash=sha256:5eeb8edc6110d9194a4d0d54610f64c37a31c605b5dbb7e407fc6ec7fa34a4a1 \ + --hash=sha256:612b709381c0355b70d89cdb51b7f670591ed5cbbc0e3b5337488019dc667b65 \ + --hash=sha256:61956f074ecd123f55adca68ee3eab46e6a07ad3f8e64e6db95dfacb444f55c4 \ + --hash=sha256:6398d5145689503412cc1748895242598d8846b8967b851133b20dc2ed1e21e8 \ + --hash=sha256:65b408d8fcb273e3499e7ef2ce796810da1becd208c7fb4373692a242d79d461 \ + --hash=sha256:686ac5350fceae63830bb98805fcb8039325bf4c06d9f6f048ff65229d5bffa5 \ + --hash=sha256:6a1a824fbed817e0a891103886b68f063b1e83cc51bc97192a90a60195a9291f \ + --hash=sha256:6abb75ab16bc3281714a5b99548a2225db70dba1f995f6d7f7419b76eb5a8fbe \ + --hash=sha256:6f7121a8914ed13fcfe2099f895341bfb789f004d4c5a0bdece8fa667da10849 \ + --hash=sha256:7020ed44df30b3aa492c00ee3b52d0548c1f30c2c6c5bb13ae897680900d3413 \ + --hash=sha256:720537c7ea6f80dc61913184edb0ce2497a306b39ef19f28505b322553d52bdb \ + --hash=sha256:724184b4aafed865e4f13ca313fdcb43024300c028ec67319cfa16847d84685e \ + --hash=sha256:7c03031610e3e6ed1768a2b7a8fc84637c1257b50c5eacaf094c6e17a84fc563 \ + --hash=sha256:80a5ea3b4fd9d6a5b9a44f7976a9acaaab35aa3c1f6b29e5bd857dfabaded223 \ + --hash=sha256:80c7cadd3fd2bfde5df8aa0787e315812cad0c313a753095d02f4c2b6c01677b \ + --hash=sha256:80ea96f5c1a30bf09007d48466521d9c294bebe197c708c3359096e3e3691632 \ + --hash=sha256:864e9b87ac33c3fb9fb4ad48166d4fdb579c351d5c77deb0d34bccb36a775cd9 \ + --hash=sha256:87fb80cbe3557e27e7b28b995c2b2eedf689b8886f941ab93e0e288f0976518a \ + --hash=sha256:8873c4a11c50b9989168881aeb3f08859f469d809941866aa1feefd8be5431f6 \ + --hash=sha256:888d60953908dcf761aa320c3e390ab8556efbdb551ace63921de90f6ae0848d \ + --hash=sha256:8b5fcc4771732191b2b7d1dd68d8f0353f47f8d90b6150f6dce58bf1112442cb \ + --hash=sha256:8f39588af4731c8923c26810eb3b33f76f17633985e40f59c3cd45a33805a895 \ + --hash=sha256:9173db3be74a35cb6731701094b98120f7ee4876a287882a59cdea1fa7da342f \ + --hash=sha256:92f05c9c42bde5785dc48770bc2194d9f7442544156f951e19cd31b096cec562 \ + --hash=sha256:951733b1bbdb71e377cec567b409f1a7881b47cfcad84121aa74cb575fa425ea \ + --hash=sha256:957bb708e8057ab1649ba566456429d691ec9b90d1c9ad1af1ba7ffbbeaf05f2 \ + --hash=sha256:9916fda742cd4eede63b286f58c06718324265d727ce0856eb1aac86d0d150d6 \ + --hash=sha256:9e1d3a4cb7993b708f0ada8d0c84590efd853f169e7147d2202c9da503180242 \ + --hash=sha256:9e4482589065c8ecd761cff522dcd85f2d39e62f551e37e025d1c7d54772def3 \ + --hash=sha256:a5300757f8a68f5b6cc33f57338d72a0e3589c5cc9ad5f8504ea06f028be582a \ + --hash=sha256:a540abfab208e1b7ef2df231c40ef3b6cbb30a0aad6204e9b6a81c10a6794628 \ + --hash=sha256:a5758353650079898dc1b2b0e95aa51fa23a30d020e06f62c430dd08ee56cdd8 \ + --hash=sha256:a64b85a4760337cfefdb27d42da6ed8b58e8cde3f2d57b6ef43e76ef6ea9ef47 \ + --hash=sha256:a655d34b2a6943af32401f3d94f72e9d731f6ad16285815550bf2b4ee69d420a \ + --hash=sha256:a714befaacbd10092ffe4cea0d3c5f008fb9efe9bc322c715bcdfdee414b9a3d \ + --hash=sha256:a760da040b47767b4b873adfb7c3b691e9ba2fc60f113f9d0b88f1a62f323e85 \ + --hash=sha256:addd736a0547d553283adaf4e05d7104e7f2c7b0b092e9b4d28756825f14531f \ + --hash=sha256:ae4613d7d9dda60fcba95f846cc6f808017f1843f392cf9daad14a6534493d71 \ + --hash=sha256:b11b589e00095ec69cf79841a76360f9b079e95b0368a25b5ebb951ab0c157ff \ + --hash=sha256:b3e445b66c80b4eb4234e855ce94d9adc183eedbd632816228d89930b91b2c5b \ + --hash=sha256:b7b893976e7fe42053da64f2aa27239c24252fd2ec6df471e1be197c0addc3b1 \ + --hash=sha256:b84f186a7f0536fe4ff9a9fa12d06d007b9b71d4b5352ddcc41f59ad6522a312 \ + --hash=sha256:b89efc38431793d28b7cd91227e2f952ad7c48df19132b17f43a5fec3c14143b \ + --hash=sha256:b97a38fb4c732b6832db6bf108963adbcd82ef1268ba2025dce390f45af75efa \ + --hash=sha256:b9d74e4eee9ddb64c2e92d5d61472c59c21684c059eb7b68767be9628e977859 \ + --hash=sha256:bb90e7177944b6684738c1fc36aabd2dd00d1de3be7dbe09f91e196f1bc0dc81 \ + --hash=sha256:bec37990e3d6121f29ecfb594bd8f1bf009e9f7926daba2e50e3b27d3892a783 \ + --hash=sha256:bf3c49863c23a1ad6da9c30351aed6cff8d5ddbeb63c5c8420ae54e98c7d0138 \ + --hash=sha256:bf48516e35cf848390ea68850aba53e7c333720d2945b4d2c25b69fc5171723f \ + --hash=sha256:bfc71e6d970419c1309b3640305298643e2a734cad3f7cfb6d2ddee4175ab53d \ + --hash=sha256:c0094897d7d01f184b2d7fe8c56c66d64efe01b31f4b7d34205b391387df1111 \ + --hash=sha256:c03c6eb6ece86dfdcbb34799efaa339b093132e1aceed491ba5e08fe06cdf699 \ + --hash=sha256:c1a9a6651197fbed6f0212591418b9def774fc3f8324f78d1bf0e6a63e5f8aa1 \ + --hash=sha256:c3589f40749acce747510bf5d589d54e376cb0930ea58b35effac97e5312b0c1 \ + --hash=sha256:c4e38dd8f39c43a91d2410ad2b85610701b0979342c3df1d69eaf8e838c757d8 \ + --hash=sha256:c6c8fabf1dafc1f1ddcbb67896d3f93efb092e8c4b6322d7389b944e76a484e5 \ + --hash=sha256:c90fcf7804ea0a54b896ce0f2b9565350220b8d4890fd0db461a476a4c687963 \ + --hash=sha256:c9b602fae1e00b7c035d661ce85575365719192a7b46784bd71cf64c68053aa0 \ + --hash=sha256:ccb64d887a9db1cd76dbc0f92051a1a478a2a67e7f56c62d915cb881d7734704 \ + --hash=sha256:d06fcdecc10fc7954d7c8f27a03c96055fe525274dc84a7b0dbdc3d6b9e03dab \ + --hash=sha256:d0c3082bf79bcd6a614d55916590ad4b8f93200e10b97f463ea5d9d07c9b5f23 \ + --hash=sha256:d49c18f1ea294cf4adde2e5ac256e98c82ea9d708462ce4bf799dffa7cfe8a2c \ + --hash=sha256:d60030baaa7bfbb02d650c126cdcddcb6e33dbff14d819434c8fa2fdcaeeeba5 \ + --hash=sha256:d7cab119d0df0b9413f106b4d7fc34f2872d3574ed3806fb48959c830b1537da \ + --hash=sha256:d9b77b25b4f395f92de6099ab08e8ae2bc7e51dfe157f22900902243a5cc90c7 \ + --hash=sha256:dabee8f4935e731fb46b2a3091bdda0d3d94b3bbfb907d2b4f12eefce4009619 \ + --hash=sha256:db5e82ba15c142425b8406690032df89e39cca4a2e8afbbb9a3d84edc2373ac3 \ + --hash=sha256:dc79d36d0618752265f0d575915bdc5c5130ecb9c9f6b3bcefeae32e4bdfafcf \ + --hash=sha256:ddfa987262763c3c22a8367d2a49c244b018a74c3a8e3ab1a864119ad45c5633 \ + --hash=sha256:e1172147d28d8fbcf8cb8d26c41506169f5ad8fe9ec969cb116835a19d4d8eca \ + --hash=sha256:e11edba5bc344a32b029a7af9d4b3173982dd79eeafa0b9dbd787364414b0509 \ + --hash=sha256:e2c89e9b762c57f59d5e99ee8b20202adb892e35f8d3485741340999ca55058e \ + --hash=sha256:e31f72490b7c12f7790e1e25c3afffd20503ee1bfb43461d7838b871ff244b19 \ + --hash=sha256:e8c65ef3862a8ad6e86492b6ed9327805dd66904c012bd3649dc67d822ed6c34 \ + --hash=sha256:ebb8912f565b8cdbbf27debfe00df04202c20e2f651b9e32767930c5eace3621 \ + --hash=sha256:ed511a0708e2297e1d6431e7fb217e3402791e491e02da800658ace4973df1bb \ + --hash=sha256:edf06545875f3efa31560d94121e95c7fd70d98b1dfedc0157097d79b13b52ea \ + --hash=sha256:f0fe9834e5aeccaf19a0d8feb296d66a24be1a7c9922002f842a682cd5abb787 \ + --hash=sha256:f1a0d5117230dd46b399a30a38afa44f79c99f3168988fdc4f425c3f928b39df \ + --hash=sha256:f37964e4a5e993d2fd45147741e9dff7f34a2d8c00ab94c4ea0514a4677f959e \ + --hash=sha256:f57dc6b8fef170f105d2cf5cdce254f47b137d7755086cf7050f47e16582abba \ + --hash=sha256:f93bc1c3486ef3747e07c9d7c1d0a147b8fbaab975f80e348aed6f71309dfaca \ + --hash=sha256:fb00027a09a8f9f08028b40dce4c933cf73e4833240ed356583fdc9cfa721566 \ + --hash=sha256:fb99cc9d45f48895d9d67f6a0b8a57f08d39c174d9f25ad97a313e0470267b1c \ + --hash=sha256:fdd88ed5e20b1bcdd234421e454962c971aa44b653bdb7f1ea9ef683e90fb649 \ + --hash=sha256:fe3fa1dd453ed5c7f5ea23a26218329790ed7197a99b90e94330e313959a7f52 + # via transformers +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via mlx-vlm +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via typer +safetensors==0.8.0 \ + --hash=sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358 \ + --hash=sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f \ + --hash=sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d \ + --hash=sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d \ + --hash=sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0 \ + --hash=sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc \ + --hash=sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235 \ + --hash=sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98 \ + --hash=sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4 \ + --hash=sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846 \ + --hash=sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca \ + --hash=sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0 \ + --hash=sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25 \ + --hash=sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452 \ + --hash=sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d \ + --hash=sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78 \ + --hash=sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774 + # via transformers +scipy==1.18.1 \ + --hash=sha256:011413b7426b75012840e35649e00fe0a2c3bae89fed433876e3a99251572efc \ + --hash=sha256:0ac49ea97594532dd44b7136094d35f5440fa06e6d9c6384a74c01764df388c5 \ + --hash=sha256:0e82073ecc7acc6436fac4b31674109c7e1d3e596789767eda01258a8c9e8123 \ + --hash=sha256:0fcb3c93519f27bb4f0c4b0f7802cdcaca7fcf93267b75edda2e9f4e8a55cbd7 \ + --hash=sha256:10ac20c69d880f77f375db44c22e3e6a644f9fefa291d4cd2fb9790a89fc99fd \ + --hash=sha256:11c423f1049c5755ad4409af52a9ada1cff96fe9b50795d4af3619f292901239 \ + --hash=sha256:179ce34a8d0fe273d8883ba59e17e052247d08973dfcb743ca52bb1cce2d60b0 \ + --hash=sha256:1bca3b943fc2567ea49cd02c99abde49da4d5178ec46f624bd8255cda8755beb \ + --hash=sha256:1d73131e358976663dd969e1fb4ed1404b815cd977eaaedc3b3a133ba2d81c35 \ + --hash=sha256:2a0b02f9fc46f8520330c23d45e6560db7e3a0d927232139427637f98943e11d \ + --hash=sha256:2d3ab0e8c69a17dd3559eab8cbb88f258e285c94d572c2719033f90f83290c89 \ + --hash=sha256:30f464bee641fa8e282577c7dce027308403213c6ca8270bba73285c91024bc5 \ + --hash=sha256:33a834464fdabc0f26a45508df31b3cc5d028e04dbf6c5ed398541418e0a12fe \ + --hash=sha256:3ab3523da44749156e1f68b464dc56af11ae4cbc5c739a49d05f32b982eca9f3 \ + --hash=sha256:3c085faa2cfa879c5141df483f836f4d691045a078224a670fa570fa01612d89 \ + --hash=sha256:457fd7a2a8edeb044ab6ffbc0aa03ff6cd18491356e5e0c834d76ce621b916d1 \ + --hash=sha256:49023963c193dacee096301452f223ee24d86ec5807f8df93c0f7221d119e305 \ + --hash=sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307 \ + --hash=sha256:559ed65f60c1af5a03f3912605a1b5114f522c7c32fb23c3376ae8f03219fe28 \ + --hash=sha256:5632e3ae3d09197c446310cd5187de63e28448ce22f0f67b2b93d97503c0c230 \ + --hash=sha256:5e4d44984abc0020154ea81b247adeddcc3ac5527b975ff798bd1ba0adc513c2 \ + --hash=sha256:75b00eb8fb802090aa903f4ea1c7f5a584779f967361e68b7e98e531cc2d7174 \ + --hash=sha256:78a0d7c918e74a232394117160e7e3db503377572a45bcef8826e4ab8a35feba \ + --hash=sha256:78c0665edead396b1abb4897c41a5c1d9bf090c8a637a4c20a61678e0a264e66 \ + --hash=sha256:7bbf207c4453ce1ad2e00b17313852b33310b83090c2311bdaf97f93c0380d12 \ + --hash=sha256:7f4b8bc363b6d65ee2152bec57568e3c52639bb34c46057b09857a307ed5e21d \ + --hash=sha256:82f201b4c878551d48558337aab270d3c6cca5507b8737c8d8a608d234cccde0 \ + --hash=sha256:83de5453a7799afc9048b4616bd085cef126e36412f0ea2f6370c36a2a3a51e7 \ + --hash=sha256:88f0e784020649f88ea48c9f5ddfa403bf9205820667c0914740b392035afb82 \ + --hash=sha256:8bcf3c1ba5d6456e2effd30fcbd3459b044d683fcdac79a2e6830f0bdf7de487 \ + --hash=sha256:911de823097db8b63f034299d12662db93344e6ffa0b881cbb57748974b70168 \ + --hash=sha256:92c14f5bdbfb6216315ce33e78080474082de8b3830122ba97809bfbe65f75c0 \ + --hash=sha256:95298364e251be3e60249facbeeca03631d3bb7584f85879516ec55ac717b81f \ + --hash=sha256:9554bcc6d715ee87a633a3cc8e7703c6628b100dd29cb8a2efc4c0533c7ff729 \ + --hash=sha256:9f2897bf7737392ad0d5213ea7b6add72a4edf5679b3153106aeb88b6507b3b9 \ + --hash=sha256:a1d33a7836f7ddc1993427966a0823468ec41bcbdb1a9f9942d1d7e57f803ba3 \ + --hash=sha256:ac0333bdf38309aa3dcbe7e3fa7ea29e7a2c37c6ea306a757b700ded8e4596ad \ + --hash=sha256:bff0b729edd992766136b34e39cc76bc2fad905aa58897ee72a9cd000a6d8443 \ + --hash=sha256:c24acac1e18912761c4700239bbc1fd32f615af690f1584d49b35859be51324d \ + --hash=sha256:c35d74ce0e193ff740c2f2be2ac913ddc232fe6c1ff40b26cfecb9c670c63314 \ + --hash=sha256:c825cef2f49e46753726a7181a8e199804a912b29519ada542c6ebc654951899 \ + --hash=sha256:c9d18a33309122074ea483dd92dd444189166b8b2ec429fe9ed5ac73c7a0aa23 \ + --hash=sha256:cbf38d043c1aa4ab306e1ada6ab6eddacc3322a20b7af1b30bc93254b366fe09 \ + --hash=sha256:cd479fc04dd9401e3b4f49e76518768ef99c4f517a98c284eb091fd725719adf \ + --hash=sha256:ceb30a00ce7c92d459819443d29ca486d882b83fb6738bdcbb2a1cce94ac5daa \ + --hash=sha256:cfbf154f2ba187f2ed6cce2639efff7d105f1140573642c0161615b6d91d6a87 \ + --hash=sha256:d2924a03db38dc2e848bca2fe9f077dafb891480b91a00a0963a8cf86dfc31c1 \ + --hash=sha256:d416b16cccfd70fbf62400e84d0bb2f4e6af519a45557f1692c749b37f14b315 \ + --hash=sha256:d65d448389b8436493abcf629cc94ad0cf32aecaf06e1acca1de53cc795f2f12 \ + --hash=sha256:d84a09d0dad90ba6525d8ac1c2334b33e64bf3ccfe9e841f02feb867a22681e4 \ + --hash=sha256:ddef79fb382df40104a19bb7151b3b23e57c1778fcf857c71ceecd9bd264513f \ + --hash=sha256:e3b417bf8c2c7c16e8f58ad91db17783ec911ac16e7b50eb6eab6e809b4f5b07 \ + --hash=sha256:e402cf31eb68f453dbb2d36fc6d722b33f24a55d68b2ae1d92fa6305ca71c298 \ + --hash=sha256:e6fb6a55cc0ba97b59a1f288fb86dc6fce8bdfc0fffcbfd015e3a954bf2a2d93 \ + --hash=sha256:e708533e8b2ae2497d65346538a7dcc92814410b25b81432eac66de0f2af8265 \ + --hash=sha256:ea324d9dd34c38bfb9bec8ca4d1b407db97dbb74029f566b8e322b1b6fe56fe6 \ + --hash=sha256:eb0dfcf4e28a99c12c999744a2ff67c9b06200e20401c7c88186e33552a46331 \ + --hash=sha256:eda632a7981f69730d6281f451db9c1c370993a2c0d7ddb43e2a809a2862b83a \ + --hash=sha256:f29633129f9fa7e88a3f0fca835de2d030bfc9643f7799e1a0c46cee24d38fc7 \ + --hash=sha256:f55fa87b6c612ecd6b058f167c53231b1d14e412efe361d3d6e38b3631c73218 \ + --hash=sha256:fdaf5ea890a6183d0565f51a61799d67081bd5b1cf03c5f4b3fd3732108625c9 + # via mlx-audio +sentencepiece==0.2.2 \ + --hash=sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497 \ + --hash=sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e \ + --hash=sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0 \ + --hash=sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719 \ + --hash=sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107 \ + --hash=sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3 \ + --hash=sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b \ + --hash=sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a \ + --hash=sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c \ + --hash=sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838 \ + --hash=sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5 \ + --hash=sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6 \ + --hash=sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d \ + --hash=sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf \ + --hash=sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25 \ + --hash=sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d \ + --hash=sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e \ + --hash=sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b \ + --hash=sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd \ + --hash=sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936 \ + --hash=sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85 \ + --hash=sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b \ + --hash=sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708 \ + --hash=sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb \ + --hash=sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9 \ + --hash=sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0 \ + --hash=sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790 \ + --hash=sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e \ + --hash=sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78 \ + --hash=sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99 \ + --hash=sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9 \ + --hash=sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d \ + --hash=sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8 \ + --hash=sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906 \ + --hash=sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383 \ + --hash=sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b \ + --hash=sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53 \ + --hash=sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b \ + --hash=sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab \ + --hash=sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0 \ + --hash=sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914 \ + --hash=sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91 \ + --hash=sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e \ + --hash=sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a \ + --hash=sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032 \ + --hash=sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711 \ + --hash=sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da \ + --hash=sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c \ + --hash=sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a \ + --hash=sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d \ + --hash=sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151 \ + --hash=sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563 \ + --hash=sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811 \ + --hash=sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b \ + --hash=sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820 \ + --hash=sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090 \ + --hash=sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c + # via mlx-vlm +shellingham==1.5.4 \ + --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ + --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de + # via typer +sounddevice==0.5.6 \ + --hash=sha256:7f4162f514f007b0bf25a3ccfed3f1705bc2ec311888a90232729eec4f57a4f4 \ + --hash=sha256:8ec9fbfde2e32f020b167e348f3ab3bac6625a5f15af524d790108ac7147a410 \ + --hash=sha256:b36b807eb02abd257198bf84b2af05e4fea199a9d2f0019014169c7136d45e9c \ + --hash=sha256:c8ae19173e5f27f8c12d4b5eee2dbfe542cee125d591e663e0fb4dfb75246d45 \ + --hash=sha256:de099612311ad81e55d31ccbd83f43ea6bf4d87b48f9b6ea55a1fbcde0eee4e0 \ + --hash=sha256:e3aef00ad8b1d1740eb66d9a7671eab88a4d2b8fa4ab33498d742e63b65c309c + # via mlx-audio +starlette==1.7.0 \ + --hash=sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e \ + --hash=sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d + # via + # fastapi + # mlx-vlm +tokenizers==0.23.2 \ + --hash=sha256:12f0835dc2ee694746a76adf7b1567d4346a4a502ebe93fb1f5f80ea49799b78 \ + --hash=sha256:2e96f5699d5249c9c64aa8412e044f727aae3a4098cf830f9901ec1afc361cde \ + --hash=sha256:325fee2e0418a9dc6c9ecf736a5f5f0db7875183ace9549ae339da76f7a1fbb7 \ + --hash=sha256:41c2f84d172449b4dadb9cdc508e3e364076613c35b16e76ecfe47a60d1e3305 \ + --hash=sha256:43e4f2071e3cc8d5d86421c874aebc82659bb51a68bcdef5a0da75ee89511ccb \ + --hash=sha256:5c56bda1511921587789163e524d196ed8284174ac23abd7685d5ea8da6c4718 \ + --hash=sha256:7b7e37ba198f24150f523e1242e83c4970de4a525480586be5dcc24d9add32c5 \ + --hash=sha256:7f0f085686b9de0d0079e6f874ae053600db64c5d13049e0bbc0119926d25aac \ + --hash=sha256:85a9a357a3764aecc904ee76bdaf8cf1ad8e5a67a1b929a487c4a39b49ed0e90 \ + --hash=sha256:950d7c9426fa72406a0ffeacdbc0bb9985f5db20eb8b263f29c79aaf83105703 \ + --hash=sha256:986670e43691469dcee610ea0f846f91a8f84e91fc6f7a48d4c064414c0ec2bf \ + --hash=sha256:a37039b5dfc4af84eb3ef0a92f4307e28936c8f9adccba2629d36f652e9bf7a2 \ + --hash=sha256:bef235815a067b2648caf6dcc7a71091b0b0fff9ee8057f6451eb9335fae52ef \ + --hash=sha256:debf978920d93ba9c219bd67cc4bbfaf912c9039e41e7a28b91ec15e3728c95a \ + --hash=sha256:e49c394456dd9985787fec76132438ba3fb8911f857b1bf3d40119f9292d41aa \ + --hash=sha256:eb2f9c8a24da020ea8c11a01a19c1c2547912d92121ae4a01cfbca46125dee40 \ + --hash=sha256:f486f402f6f9abee5bb032553736813af0c710a86b2e0ca592634c55cea1f835 + # via transformers +tqdm==4.70.1 \ + --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 \ + --hash=sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4 + # via + # huggingface-hub + # mlx-audio + # mlx-vlm + # transformers +transformers==5.18.0 \ + --hash=sha256:d79e5a515a572ee3deb33eb0d578be47d91e70c75d54cf21bf6eede6edef14a9 \ + --hash=sha256:d89c206e42e841af7cd314b3d2b21c03e977d819cdb4aef65ada048769a644df + # via + # mlx-audio + # mlx-vlm +typer==0.27.2 \ + --hash=sha256:269b7eb9d3c202ca84b4bc9618cb04ebb43d3d4d1e567e4c768607232c05f945 \ + --hash=sha256:b3a5fc4342d5fc8fda8fc3010b1cf117e9249aab7fae800c2eff62fd3842d97d + # via transformers +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # fastapi + # huggingface-hub + # opentelemetry-api + # pydantic + # pydantic-core + # starlette + # typing-inspection +typing-inspection==0.4.4 \ + --hash=sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47 \ + --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 + # via + # fastapi + # pydantic +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via requests +uvicorn==0.54.0 \ + --hash=sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf \ + --hash=sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620 + # via mlx-vlm +websockets==17.1 \ + --hash=sha256:0014eaff8ad5b3b43feda2279f9d34bf2eaae040720b9fbbb55944b10f40b14d \ + --hash=sha256:00679b7468b4c2b12b0757118174e8eabac56bb2f579a928a104d9554a56e098 \ + --hash=sha256:00bf34b64501e3477e81fc281532ff3cbf4da26633c10b63979d5085d46602d3 \ + --hash=sha256:01dcb47deebc40b38fd4a493b9b9f4d0a704b7bec6f35e4d34085b329abce71a \ + --hash=sha256:020e271205f8ab3406d7a59cd00de6dec722315924411c421bd00642f18bad86 \ + --hash=sha256:0340bbef6bfbe16da888b3983d666a4db4954ac3253c38f13bc7aba0c7db5a2f \ + --hash=sha256:054c28db2dcec0e857e3b705d8c28012613e555b38c765d6a4f75340a4fc06a0 \ + --hash=sha256:073c5c3f7e127041fa9d34a9e29ceefee8c3cafbd267ed2927318f425144380d \ + --hash=sha256:07fa3e7c30e2c577928d359b56bf872a3e0cbcc15553eaa0907c1ee86344b56f \ + --hash=sha256:0c84bdef916556cbe1d5a43b423398be4dd3cba6522b463e53d848578b920695 \ + --hash=sha256:0c863507ada5805517ca6dff1c524dcd42942efe6304dacf06700878398d21a6 \ + --hash=sha256:0c9982938980e086da59f70d05f9418cd143401a601a0faac10fa48f7bb1cd3e \ + --hash=sha256:0de501b7f2db11e83739ac20e2d33d46da4604b829f506c24be80e7def069391 \ + --hash=sha256:0f4f50dfe2cc810fc4e2de979b35e83bf8bb4bccdc6fe472d93762ea7b1d5927 \ + --hash=sha256:1036189bd34b0bc1b10a4679321e2c7968af317efe6e8e4c1c5141c4254fb5bb \ + --hash=sha256:10ecb38ffc05e1841b619d99c725307a223ef9ad58e7b1ed33311d472dc43918 \ + --hash=sha256:12c8e2b25df59755954a04dfa09c990b96691025aaf7eafd19ed6da24b09c18d \ + --hash=sha256:12fe8984a32dbfd084e0603f1a8d740c0180cb85b3174585c54a80d2455a8394 \ + --hash=sha256:17aa424ab61620aad21b36b2240efc87b500cc496e7d0e999a5c2ae99395e886 \ + --hash=sha256:1a20136d61f9ca3a31493732762661fafc2c20e8861930214e21afc6a8a692a2 \ + --hash=sha256:1a5cf4e7bbe3ca499e6a289206cb4fcb7444b09919e129bd517f57d5fa192c13 \ + --hash=sha256:1c7ac77401227212dc6e849182feee50d57cf456ec6329ffd6979c94bb136c5c \ + --hash=sha256:1fce0f43e0d41422e0b2cad6561e1970df22f212f4c7e884967df7cf591b031c \ + --hash=sha256:259f45358c76d3b18489e3e80636cdbe807e05ecf1b10fdf1a779106d23d0c8e \ + --hash=sha256:2786cbd273ab69c22612db8a41229ddf2c158060b17b5928884bf388d07887f3 \ + --hash=sha256:29176d8b429cfa0fa443c473878d37a5c06cfd0cb36b71ba4314accc71e05906 \ + --hash=sha256:2a0162a6372110a5601cb5c9fd826635cedf69f3e110c545dd19774e040b970e \ + --hash=sha256:2a4dc6ef83f4559e0d05f313a375cb38f63c986096a9da99fe94fdd779d313e5 \ + --hash=sha256:2afb58c7ba48b329d56769f8dfd89f394efe587b65ef806bae810a484d6d3608 \ + --hash=sha256:2cce251f3e2469b99b6802b55435bcdd07123b41870f54c87b336183af9d7e68 \ + --hash=sha256:2d72879819f5145a342d0030c418702496c65a4b913ef81f5ae944dd91dd50f6 \ + --hash=sha256:32a2a68d989d6e5b74a9d5095415c51189ebae29fceb7cf2b64a1c0318a81256 \ + --hash=sha256:33e45c7ea38428e740a7f233555d71df0b875cef7fc080acebc9654475e35335 \ + --hash=sha256:34879e19bb0a3c44f9317679435aea5327fac993933a704cbf353bf1234b10c7 \ + --hash=sha256:3546ef55b3a074494106508bc6505c73825970d2d9505f7bf53882b3e88b0d1e \ + --hash=sha256:3709a1ab30b4b922027d22f68d2b61a0656a91680ac894a537624e6be7dd7f7c \ + --hash=sha256:4031152769179ab8dcdeafc7b0e58052a49117560a28671700b47b2c7b717aad \ + --hash=sha256:406b8107943a43ef4649b1e0cb0cdc052bbf08fe1c8905a623c4af9586e5cebb \ + --hash=sha256:416b4bc8789a1865a3ff643ec4ee073a5f52402d0dbeafd27b1798d5dd6b6a51 \ + --hash=sha256:43bd0c1ceb924d67f5c1a5254d8361dd9d94246e6331a726064dfa2917880780 \ + --hash=sha256:455ffeea0879d313205df1e745e5883e1feb7f31ecd26be882f5f0babd3db04f \ + --hash=sha256:469355ab1af100b9380f1afb09985019f4a4b94fa1dd0e9396db4361626d7ab8 \ + --hash=sha256:46c0331c9eaaf73a559f3a9e388466be0df96eb83d40f06f1ca6ab6613b35c82 \ + --hash=sha256:46e7a10bf04318c7b0c0273791925ae5e1cbe4a11e34aa934d2ef27862058a80 \ + --hash=sha256:47a62d6045c6eaa0d8f97bc2fb68b8cf90077a0cbfd4e83d6f2d2145611ee134 \ + --hash=sha256:4af784f3e436f65b355c117c6497320f2b5cf6a559295cb1c4c7338e335d45cc \ + --hash=sha256:4e8ffcb486c8490a34a4cef5e4409d8da5a1cb1681e5bf7d786ce5e84aa8540d \ + --hash=sha256:50903d335bfda026c2fa11dd9aed09d8cbee0c451e3a85122a9acb041b7dc69b \ + --hash=sha256:57b39dc8541cf7ed3f639da82bf7451060483967f9e733da1f8173e4095f0642 \ + --hash=sha256:581fa678ef46f4277cc8491312468e582f8ad609dbab907ba6096a08c6a0ff98 \ + --hash=sha256:5a74531ce81af587f906ab42f194032388fcff8fc7938402e5917c9147a39441 \ + --hash=sha256:5aefe78e6a3077fe22b5e64b04666a85a3eb8b934d40e8595a693adcbceb6f11 \ + --hash=sha256:5c27336b1a0ac56569493e858497870347854372395f50483725f8cdacc5a45c \ + --hash=sha256:5d4724255fb8398acd9e583b97eb2279cec20e0bd0f9a94bf75f6056ef9f13da \ + --hash=sha256:5f051f8030a51815dc00e24bd2e5f1435af095c1cc111d747ac6e2a3620d7641 \ + --hash=sha256:615746b12b26a3fd4077bc6fbeb277a1c192a45dd57b531d07ad9ed5c52a9a7a \ + --hash=sha256:617243e19a0992095956f406ee9cd3bc4ba92862d83cb1d83bb59ce574412bec \ + --hash=sha256:655a8e28010f09fd6fa317e857afab3af7647f33e41dee88fa421e92086d1090 \ + --hash=sha256:65be6bda2b537fefa4b3a5ccd6ab386533ce39dd8fe62433ec90901fdc81752d \ + --hash=sha256:67258b00302a5aaf0b267771c7014b13429abd7ea17eebc4c55bd935ff101555 \ + --hash=sha256:677014a073bcb1fbaa7e21144786864f16c08f856d66834f611eceb9006cbab8 \ + --hash=sha256:6a8e768a048c2220697477ce2e67e4345dc9f693d0ee6af53945b5e30227c6a7 \ + --hash=sha256:6da17fc94bd270f5987b10bee113461ac36a36a98b0481ddcc98056e5a90001a \ + --hash=sha256:6de9acef07e3a78e9567fcd26c29011a4da8f050b13004bbf880a0fd82a6eea5 \ + --hash=sha256:6e63c01803be425ff062b7f7fc201a74def1d49fc94a2410dd17375df75936e9 \ + --hash=sha256:6f42912fa9eb4cb7c7ec9fde9b3332ba339eb8a8811981043d4029599f3d950b \ + --hash=sha256:722ec21717eec6477bce582147a28acdfe034e604239466a6a95daedb863e774 \ + --hash=sha256:764cf7bfa149365f32b7a0fd9fed32debdac29dd06295d5635cde1745b446cd8 \ + --hash=sha256:76dd004f59115087c7b700474cb18f01325e37250032e19396c08ae41448e4b3 \ + --hash=sha256:77b37cceca17291897c3c73bd30a7c7c7909593554b5da574ec852af83c1742a \ + --hash=sha256:7a72efa3bf4fa3a6669a54420a472ad056da3973d827f10e3a536da463f926c2 \ + --hash=sha256:7e724f843fa6a0614aece65a7c73e51d0f4412ca41dccac13c3caf98e69536bb \ + --hash=sha256:8196d217eeca52b9235ee1f8a684a09885a5f953d5a31e80ef915bf2c5c94f9d \ + --hash=sha256:829dba1bc049779de9b332088c1a6a9858e96bd67e50b6b644a95e02b67836bc \ + --hash=sha256:84f25e099fdfe3b09f953d84698f729a1f7d1e99101b2787d7a28ed77b323750 \ + --hash=sha256:87f0d5e77548b0c40c8464cdb6108792e7e53f487c6400028a4ec28a8afbe5ab \ + --hash=sha256:880069d21cc33a558dcf180924a546d1ecf8ada5be3e4e70acee87019d706a24 \ + --hash=sha256:882af300d2c6a092b93767d5de03c7bb56dfb06314140c8e872d3f48e09f7b74 \ + --hash=sha256:88381602e379165b66244b2ebc29f9b23ea0851fbe63ae157f91ca324f072d6f \ + --hash=sha256:88b882764ef65147a7a5ae13168dedbe225a04e2ff4858fe543f2c402f093e9c \ + --hash=sha256:88bc5138e53903a85c354e59df7ba73ce306f7b09724cef74dba121e60a88ce2 \ + --hash=sha256:8d1b108bd8f5f6a8b90801f6db3b3858d5deca889acfdb8ac497bbb24e4b0edf \ + --hash=sha256:8f6c38cdcaf98a911d7acc25577f2f9e710f3a2fc2bde1563556784320196b51 \ + --hash=sha256:8fbf28e639544503b7d1c96452a5e5e043e4108d89b1f3fa02910603622d19db \ + --hash=sha256:96abdecbaae746851b87c3a36cb4a661df93ca3d92f114270f79228bf1d00de6 \ + --hash=sha256:98a5b2589a56a4b4f098b0a958099a4356ab904a7844f1da3841efca469af7e9 \ + --hash=sha256:9ae55d24241fc055f22aea3ac924559069848bd0ad4ea065fdd72d2194685fe8 \ + --hash=sha256:9f4a08ff7cb68c27b18e09223cc6304e01d0f82d5a240d251266dfd2e6e44729 \ + --hash=sha256:9f4c0377a83e163a303514fdfab501dbe379bdc13e5b9312a91d112658b29dce \ + --hash=sha256:9fa1ffa08c81a4f809cdab6129f8e55bee4650b9d6d3461019dda73aacd146b6 \ + --hash=sha256:a06f3b5085176763182449559e20391d7ce616a8972a9f7a33deda87ea6d4f3c \ + --hash=sha256:a62d8c424383c9dc769ff3672018df822603117e32686e567d452ed035b6fb2e \ + --hash=sha256:a7b35181a14cbfcae163b4de545d22abfd07d06c2c41ca69cfcd99251d6888ab \ + --hash=sha256:a9fe648abd1d9b89aebfa30407bfdd08a0271ec5dc7d44a4c6ccd1ce22cf562a \ + --hash=sha256:acfea4c20bf54384883ea33b1240fc1db4f52e190823a4e2b334bc3e8bfca96a \ + --hash=sha256:ad68c28a27246fed109a4409393d677b7e1388345cbbd2f5aee5c182d8506110 \ + --hash=sha256:aec00f018d34c67500ff0438dc314b40277be4a1b983cbacbf53ccf7db63e257 \ + --hash=sha256:b1bc819c6db90e8f91a38250a1ab4c058261871aa52d2fe36382eddedf146dee \ + --hash=sha256:b1c323fc3be1dc3f87f6c59458cb7d9e13dcbbf971d6c3f3e2bbaf58d3bfcdfe \ + --hash=sha256:b3b451fd2723ad3191a209afe6f3f4bc86c83e9a85bdc255353b91803ee6aa66 \ + --hash=sha256:b65d5fe48219dc2d5e158de9e6514e75600f379cc7e37108d35f31764c155566 \ + --hash=sha256:bb0efe019480a1c93e168ce96479273aaebd672fc8c350d5eed1e507ababb1b8 \ + --hash=sha256:be3f0129c5654517b2abf07dcb75bb1d9479759a4ccfb569e8293579e9fc029a \ + --hash=sha256:c3241d684a76eaaef8b2dc789afde4343cd3aad55ea81e4e8ab3605b529bae51 \ + --hash=sha256:c65da239a5ad553619804c1f9d65c1a0b3005381c6158ee14da2c7444cbd0c78 \ + --hash=sha256:c89406fa3dcd4aa8662c6406cc5c0de1790e9614d2c3aaf03ca53a8a8ccf3405 \ + --hash=sha256:ccbf3f4a9890d50b3a08ee04029fde30a03bfdeffaa19977628bf17251764e60 \ + --hash=sha256:ce0305b702b20d1e1d60a9aaace6bc89970e1753565543f310d549eab22c2435 \ + --hash=sha256:ce13c7d233239e739600a57d4a73c1192ad8259e655a4d55aa1a454242bc809d \ + --hash=sha256:cec1bb8f22abccc8d20f8ca63df9be41600c26c190f4b97ee86c675fd4a863a6 \ + --hash=sha256:d1e2f5fa2b6d01f0d85b4f223fea7ed1d504be282a02a81bd2be4817ef7a2f03 \ + --hash=sha256:d334d11398086bb5559606cb42d51c013ea7c061c7db701521392373d3c087f5 \ + --hash=sha256:d411ea5ca18ac1b12c0c94be88b60c18ca641ac43bcdfdf1c9f79d46cdbe1603 \ + --hash=sha256:d41ef69d5416fbc1d98cf96c37be6192d10fd101c3e0f8b3ddc36e09432b3c08 \ + --hash=sha256:d58159af7835fde09c462394293c0d7aaf8fb4557d8f8e5699f5e722ccae013d \ + --hash=sha256:d7b349265fad6244013eecd99df8d83c12bf3013943e431f4fadd5bffc37db42 \ + --hash=sha256:d8e83333385cac6030a5167fd18bf96cc6c58b914c308e683f05b0cf94bc8dd0 \ + --hash=sha256:d9d01e8ede41fea4f5a847dad9d628355f74905f437a5b6856d67aa66d193800 \ + --hash=sha256:d9fc873e239c5abeb150bc24dbd1a7af23a9254526383ce0a077f5e20adbeb19 \ + --hash=sha256:db9d7ee47f3ba531e278be539af39e2c7c7d28fb94897b6cd1120d63b0ef5922 \ + --hash=sha256:dc053f9e95a76213c5eb7ed95779f7daf0d2bf0e4e03073629ebfa43a033f151 \ + --hash=sha256:dc2b79afc074d2f3e64b26539350f697fe1b85ea1c49ea24eb588f247b053ce1 \ + --hash=sha256:dc5789e5ea182b77a38881383ada5347202a6c66f4857d054e075290e80b604b \ + --hash=sha256:dcb8d5f7edef7a399d322cf28d4c4e6f98dab64d301c8f50581a1080e5198142 \ + --hash=sha256:e4bd7eacb87d8cf3ed70d6392c770a0d92441f05d7d2a3efafb5bc171d5e3067 \ + --hash=sha256:e552e0037230ac16e5f568de7012041344d1b18c9feed30ec2891b8eba55af81 \ + --hash=sha256:e5f5c7a893507d0e83a80b88aefd6522f7e882cd53f9722c6f23f5a020c9557c \ + --hash=sha256:e74e41f0ad12ff1e8983e349daef79d37cc8280c743ce9d134d6c74c18dab5d6 \ + --hash=sha256:e78fd4b7b2c5086a38671c9c882c1e643385eccea360b5b1fda4a105e590087e \ + --hash=sha256:e8dc3fa6d6b7ead3f9de57895f41b116a28787548e066365d9d90f7356bcaad2 \ + --hash=sha256:ea0ed9373b880115911d9d39634bccc95b8ce590c9c42e8589f5cacc3ef3cee2 \ + --hash=sha256:edadce7a22052056fd4384543019856b34850363c9d387929f677ae01d79709c \ + --hash=sha256:eec113a5b41d124ef42ff56b0d74a6da3fd986400038eab9e58ee42a4024e837 \ + --hash=sha256:f221081107b8c48184d99f7019604486376e7ef826037e70aad6b02540732c23 \ + --hash=sha256:f3a1d577e081667dda7f8e5b4796e6e32f9713c93e2a3d930669519840a3c623 \ + --hash=sha256:f47aafd92aa28b941180e6da8a42b0f711851b14b81a5b6bb28dbbb1fa35152c \ + --hash=sha256:f4c45ee2512d3757b5e6c67c5a34e435143f2ecb7df3324f9fd888688c45c0f4 \ + --hash=sha256:f58f58b4b29bbea2a3635e2c56eff4a3adab011fe383802a9e542e31b97085fc \ + --hash=sha256:f612dc57f00c07cf4aa2673f7cbceabd654ad2457b7e639f061b794d6e11f9fd \ + --hash=sha256:f62114a54117e4948a1e414e89521f7fe1e3c2f83f2a571a06a4fc6718b0900a \ + --hash=sha256:f64e001bb7fa89b9f32cfa600bf8e9ac8ca26759d9b92ae01453ee303d9cd7b4 \ + --hash=sha256:f7233eaf441a345a5943a929fd4b5ea3278f11aed35a9ed0f3106b8cb3ca846a \ + --hash=sha256:f78a3ffb1994304db2c0c4588e4d1a518079b557054fa3bb985a6f5e50ff49a3 \ + --hash=sha256:f8e822efd54137d8cc8310eb64635ab827a4a6c72ff08691f38aa624776d8ecb \ + --hash=sha256:f98bf378d7a5be047a044a1a27c987a8f355e10e3b5754617dbe756248cbc5ce \ + --hash=sha256:fb88076df585b69c5761c387c0081aa87d7b9eb1b205a6535ca4777e25650d81 \ + --hash=sha256:fd8f47dbf2e8adb15c847215f83436de3fdb120b51fdae0fbbdf69fd97a3ad80 \ + --hash=sha256:ff3e2ba7a9f0a110b0555452e9b5a03a34e11662544e01beea15f144b48ba7b7 + # via mlx-vlm diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/resources/bench-community.json b/packages/codex/mlx-agent/src/mlx_agent/resources/bench-community.json similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/resources/bench-community.json rename to packages/codex/mlx-agent/src/mlx_agent/resources/bench-community.json diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/freeze.mjs b/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/freeze.mjs new file mode 100644 index 0000000..19250dd --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/freeze.mjs @@ -0,0 +1,99 @@ +// This entrypoint runs with BUN_BE_BUN, --no-install and --no-macros. +// Compilation observes source bytes; it never imports an extension for execution. +import {readFileSync, lstatSync, realpathSync, existsSync, writeFileSync, chmodSync} from "node:fs"; +import path from "node:path"; +import {pathToFileURL, fileURLToPath} from "node:url"; +import {createHash} from "node:crypto"; + +const hash = bytes => createHash("sha256").update(bytes).digest("hex"); +const inputs = new Map(); +let total = 0; +const request = JSON.parse(await Bun.stdin.text()); +const bridge = fileURLToPath(new URL("./legacy.mjs", import.meta.url)); + +function bind(file) { + const name = realpathSync(file); + const stat = lstatSync(name, {bigint: true}); + if (!stat.isFile()) throw new Error("Nonregular extension dependency"); + const raw = readFileSync(name); + const item = {path: name, identity: [stat.dev, stat.ino, stat.size, stat.mtimeNs].map(String), sha256: hash(raw)}; + const previous = inputs.get(name); + if (previous && JSON.stringify(previous) !== JSON.stringify(item)) throw new Error("Dependency changed during compilation"); + if (!previous) { + total += raw.length; + if (inputs.size >= 4096 || total > request.maxBytes) throw new Error("Extension dependency budget exceeded"); + inputs.set(name, item); + } + return raw; +} + +function manifests(directory) { + for (let current = directory; ; current = path.dirname(current)) { + const manifest = path.join(current, "package.json"); + if (existsSync(manifest)) bind(manifest); + if (current === path.dirname(current)) break; + } +} + +function sourceText(file) { + const raw=bind(file).toString("utf8"); + if (/\bimport(?:\s|\/\*[\s\S]*?\*\/|\/\/[^\n]*\n)*\(/.test(raw) + || /\brequire\b/.test(raw) || /\btype\s*:\s*["']macro["']/.test(raw)) + throw new Error("Dynamic imports, require and macros need an unsupported dependency boundary"); + const loader=file.endsWith("tsx") ? "tsx" : file.endsWith("jsx") ? "jsx" : /\.[cm]?ts$/.test(file) ? "ts" : "js"; + return new Bun.Transpiler({loader,target:"bun", + define:{"import.meta.url":JSON.stringify(pathToFileURL(file).href)}}).transformSync(raw); +} + +try { + const bundles = []; + let bundleBytes=0; + for (const [index, source] of request.sources.entries()) { + const scanned=new Bun.Transpiler({loader:"js",target:"bun"}).scan(sourceText(source)); + const legacy=!scanned.exports.includes("default"); + const id="mlx-agent.capture."+hash(source+":"+inputs.get(realpathSync(source)).sha256).slice(0,24); + let sdk; + if (legacy && scanned.imports.some(item=>item.path==="@opencode-ai/plugin")) + sdk=Bun.resolveSync("@opencode-ai/plugin",path.dirname(source)); + const entry=legacy ? [ + `import * as module from ${JSON.stringify(source)};`, + `import {legacyPlugin} from ${JSON.stringify(bridge)};`, + sdk ? `import {tool} from ${JSON.stringify(sdk)};` : "", + `export default legacyPlugin({id:${JSON.stringify(id)},module,objectSchema:${sdk ? "tool.schema.object" : "undefined"},toJSONSchema:${sdk ? "tool.schema.toJSONSchema" : "undefined"}});`, + ].join("\n") : `export {default} from ${JSON.stringify(source)};`; + const result = await Bun.build({ + entrypoints: ["capture-entry"], target: "bun", format: "esm", sourcemap: "none", write: false, + plugins: [{name: "capture-local-only", setup(build) { + build.onResolve({filter: /.*/}, args => { + if (args.path==="capture-entry") return {path:args.path,namespace:"capture"}; + if (args.path.startsWith("node:") || args.path === "bun") return {path: args.path, external: true}; + const resolved = Bun.resolveSync(args.path, args.resolveDir || path.dirname(source)); + manifests(path.dirname(resolved)); + return {path: resolved}; + }); + build.onLoad({filter:/.*/,namespace:"capture"},()=>({contents:entry,loader:"js",resolveDir:path.dirname(source)})); + build.onLoad({filter: /.*/}, args => { + if (/\.[cm]?[jt]sx?$/.test(args.path)) return {contents:sourceText(args.path),loader:"js"}; + if (args.path.endsWith(".json")) return {contents:bind(args.path).toString("utf8"),loader:"json"}; + throw new Error("Unsupported local extension dependency format"); + }); + }}], + }); + if (!result.success || result.outputs.length !== 1) throw new Error("Unresolved static imports"); + const bytes = new Uint8Array(await result.outputs[0].arrayBuffer()); + bundleBytes+=bytes.length; + if (bundleBytes > request.maxBytes) throw new Error("Compiled extension budget exceeded"); + const destination = path.join(request.output, `extension-${index}.mjs`); + writeFileSync(destination, bytes, {flag: "wx", mode: 0o600}); + chmodSync(destination, 0o600); + const stat = lstatSync(destination, {bigint: true}); + bundles.push({path: destination, identity: [stat.dev, stat.ino, stat.size, stat.mtimeNs].map(String), + sha256: hash(bytes), bytes: bytes.length, source, legacy, id}); + } + const reply = JSON.stringify({schema: "coding-capture-extensions/1", inputs: [...inputs.values()], bundles}); + if (Buffer.byteLength(reply) > 1024 * 1024) throw new Error("Extension manifest budget exceeded"); + process.stdout.write(reply); +} catch (error) { + process.stderr.write(String(error?.message || "Cannot freeze local extensions").slice(0, 4096)); + process.exitCode = 1; +} diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/legacy.mjs b/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/legacy.mjs new file mode 100644 index 0000000..5eefd6b --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/legacy.mjs @@ -0,0 +1,77 @@ +// Supported legacy hooks are explicit; policy hooks are never silently removed. +import {$} from "bun"; +import {isDeepStrictEqual} from "node:util"; + +const supported = new Set(["config", "tool", "event", "tool.execute.before"]); + +function requireSupported(condition, message) {if (!condition) throw new Error(message)} + +function checkConfiguration(before, after) { + const {provider: beforeProviders, ...beforeRest}=before; + const {provider: afterProviders, ...afterRest}=after; + requireSupported(isDeepStrictEqual(beforeRest,afterRest),"Legacy configuration hook changed unsupported policy or configuration inputs"); + for (const [id, provider] of Object.entries(afterProviders ?? {})) { + if (isDeepStrictEqual(provider,beforeProviders?.[id])) continue; + requireSupported(provider && provider.npm === "@ai-sdk/openai-compatible", + "Legacy configuration hook selected an unsupported provider package"); + for(const model of Object.values(provider.models ?? {})) + requireSupported(!model.provider?.npm,"Legacy hook selected a model-specific package"); + } +} + +export function legacyPlugin({id,module,objectSchema,toJSONSchema}) { + let state; + return { + id, + async bootstrap(configuration, directory) { + requireSupported(!state,"Legacy extension bootstrap was repeated"); + const exports=Object.values(module); + requireSupported(exports.length===1 && typeof exports[0]==="function", + "Legacy extension must export one unambiguous factory"); + const host=new Proxy({$:$.cwd(directory)}, {get(target,key) { + requireSupported(key in target,"Unsupported legacy factory context");return target[key]; + }}); + const hooks=await exports[0](host); + requireSupported(hooks && typeof hooks==="object" && !Array.isArray(hooks),"Invalid legacy hook definition"); + for (const [name,hook] of Object.entries(hooks)) { + requireSupported(supported.has(name),"Unsupported legacy hook: "+name); + requireSupported(name==="tool" ? hook && typeof hook==="object" : typeof hook==="function","Invalid legacy hook: "+name); + } + const tools=Object.entries(hooks.tool ?? {}).map(([name,tool]) => { + requireSupported(typeof tool?.execute==="function" && typeof tool.description==="string" + && typeof objectSchema==="function" && typeof toJSONSchema==="function","Unsupported legacy tool definition"); + const schema=objectSchema(tool.args); + requireSupported(typeof schema.parse==="function","Legacy tool needs an input validator"); + return {name,description:tool.description,input:toJSONSchema(schema),schema,execute:tool.execute}; + }); + const effective=structuredClone(configuration); + if (hooks.config) { + await hooks.config(effective); + checkConfiguration(configuration,effective); + } + state={hooks,tools,directory}; + return effective; + }, + async invoke(method,params={}) { + requireSupported(state,"Owned legacy bootstrap context is missing"); + const {hooks,tools,directory}=state; + if(method==="describe") return {id,directory,hooks:Object.keys(hooks), + tools:tools.map(({name,description,input})=>({name,description,input}))}; + if(method==="tool") { + const tool=tools.find(tool=>tool.name===params.name); + requireSupported(tool,"Unknown legacy tool"); + const content=await tool.execute(tool.schema.parse(params.input),{...params.context,directory}); + requireSupported(typeof content==="string","Unsupported legacy tool result"); + return {content}; + } + if(method==="event" && hooks.event) {await hooks.event({event:params.event});return null} + if(method==="before" && hooks["tool.execute.before"]) { + const event=params.event, legacyTool=event.tool==="shell" ? "bash" : event.tool; + const input={...event,tool:legacyTool},output={args:event.input}; + await hooks["tool.execute.before"](input,output); + return {input:output.args,tool:input.tool===legacyTool ? event.tool : input.tool}; + } + throw new Error("Unsupported legacy operation"); + }, + }; +} diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/proxy.mjs b/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/proxy.mjs new file mode 100644 index 0000000..adfde06 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/proxy.mjs @@ -0,0 +1,47 @@ +import {connection} from "./rpc.mjs"; +const key=Symbol.for("mlx-agent.coding-capture.rpc/1"); +function ownedConnection() { + if(!globalThis[key]) { + const fd=Number(process.env.MLX_AGENT_CAPTURE_EXTENSION_FD); + if(!Number.isInteger(fd) || fd<3) throw new Error("Owned extension channel is missing"); + globalThis[key]=connection(fd); + delete process.env.MLX_AGENT_CAPTURE_EXTENSION_FD; + } + return globalThis[key]; +} + +export function proxyPlugin(id,transport) { + return {id,async setup(ctx) { + const rpc=transport ?? ownedConnection(); + const call=(method,params={})=>rpc.call("extension",{id,method,params}); + const description=await call("describe"); + if(description.id!==id || description.directory!==ctx.location.directory) throw new Error("Extension project changed"); + const registrations=[], controller=new AbortController();let pump=Promise.resolve(); + try { + if(description.tools.length) registrations.push(await ctx.tool.transform(editor=>{ + for(const tool of description.tools) editor.add({...tool,options:{codemode:false},execute:(input,context)=>call("tool",{name:tool.name,input, + context:{sessionID:context.sessionID,agent:context.agent,messageID:context.messageID,id:context.id}})}); + })); + if(description.hooks.includes("tool.execute.before")) registrations.push(await ctx.tool.hook("execute.before",async event=>{ + const result=await call("before",{event});event.input=result.input;event.tool=result.tool; + })); + if(description.hooks.includes("event")) { + // Using filesystem writes here would feed graph updates back into themselves. + registrations.push(await ctx.tool.hook("execute.after",async event=>{ + if(event.status==="completed" && ["edit","write","patch"].includes(event.tool)) + await call("event",{event:{type:"file.edited",properties:{}}}); + })); + pump=(async()=>{ + for await(const event of ctx.event.subscribe({signal:controller.signal})) { + if(controller.signal.aborted) break; + await call("event",{event:{type:event.type,properties:event.data ?? {}}}); + } + })(); + void pump.catch(()=>{}); + } + } catch(error) { + controller.abort();await Promise.allSettled(registrations.map(item=>item.dispose()));throw error; + } + return async()=>{controller.abort();await Promise.all(registrations.map(item=>item.dispose()));await pump}; + }}; +} diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/rpc.mjs b/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/rpc.mjs new file mode 100644 index 0000000..6739a4a --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/rpc.mjs @@ -0,0 +1,54 @@ +// Anonymous inherited sockets are the only transport; there is no listener. +import {createReadStream,createWriteStream,closeSync} from "node:fs"; +import {randomUUID} from "node:crypto"; +const maximum=1024*1024; + +export function connection(fd,handler) { + const reader=createReadStream(null,{fd,autoClose:false}), writer=createWriteStream(null,{fd,autoClose:false}), pending=new Map(); + const prefix=randomUUID();let sequence=0, buffer=Buffer.alloc(0), inFlight=0, closed=false; + function fail(error) { + for(const item of pending.values()) item.reject(error);pending.clear(); + if(!closed) {closed=true;reader.destroy();writer.destroy();try {closeSync(fd)} catch {}} + } + function send(value) { + const raw=Buffer.from(JSON.stringify(value)+"\n"); + if(raw.length>maximum) throw new Error("Extension message budget exceeded"); + writer.write(raw); + } + reader.on("error",fail);writer.on("error",fail); + reader.on("end",()=>fail(new Error("Owned extension host disconnected"))); + reader.on("data",chunk=>{ + buffer=Buffer.concat([buffer,chunk]); + try { + for(let end; (end=buffer.indexOf(10))>=0;) { + if(end>=maximum) throw new Error("Extension message budget exceeded"); + const message=JSON.parse(new TextDecoder("utf-8",{fatal:true}).decode(buffer.subarray(0,end))); + buffer=buffer.subarray(end+1); + if(typeof message.id!=="string" || message.id.length>64) throw new Error("Invalid extension message identity"); + if(message.method!==undefined) { + if(!handler || ++inFlight>16) throw new Error("Unexpected extension request"); + Promise.resolve().then(()=>handler(message.method,message.params)).then( + result=>send({id:message.id,result}),()=>send({id:message.id,error:"Extension hook failed"}) + ).catch(fail).finally(()=>{inFlight--}); + } else { + const request=pending.get(message.id); + if(!request) throw new Error("Unmatched extension reply"); + pending.delete(message.id); + message.error ? request.reject(new Error(message.error)) : request.resolve(message.result); + } + } + if(buffer.length>=maximum) throw new Error("Extension message budget exceeded"); + } catch(error) {fail(error)} + }); + return { + call(method,params) { + return new Promise((resolve,reject)=>{ + if(closed || pending.size>=16) return reject(new Error("Owned extension host is unavailable")); + const id=prefix+":"+(++sequence); + pending.set(id,{resolve,reject}); + try {send({id,method,params})} catch(error) {pending.delete(id);reject(error)} + }); + }, + close:()=>fail(new Error("Owned extension channel closed")), + }; +} diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/worker.mjs b/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/worker.mjs new file mode 100644 index 0000000..0f63f7f --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/coding_capture/worker.mjs @@ -0,0 +1,36 @@ +import {readFileSync,lstatSync} from "node:fs"; +import {pathToFileURL} from "node:url"; +import {createHash} from "node:crypto"; +import {connection} from "./rpc.mjs"; +const hash=raw=>createHash("sha256").update(raw).digest("hex"); +const plan=JSON.parse(readFileSync(process.env.MLX_AGENT_CAPTURE_BOOTSTRAP,"utf8")); +if(plan.schema!=="coding-capture-bootstrap-plan/1") throw new Error("Invalid owned bootstrap plan"); +const instances=new Map();let booted=false; +const descriptor=Number(process.env.MLX_AGENT_CAPTURE_EXTENSION_FD); +delete process.env.MLX_AGENT_CAPTURE_EXTENSION_FD; +delete process.env.MLX_AGENT_CAPTURE_BOOTSTRAP; +delete process.env.BUN_BE_BUN; +connection(descriptor,async(method,params)=>{ + if(method==="bootstrap" && !booted) { + booted=true; + let configuration=JSON.parse(readFileSync(plan.original,"utf8")); + const plugins=[]; + for(const bundle of plan.bundles.filter(item=>item.legacy)) { + if(!lstatSync(bundle.path).isFile() || hash(readFileSync(bundle.path))!==bundle.sha256) + throw new Error("Owned extension bundle changed"); + const plugin=(await import(pathToFileURL(bundle.path).href)).default; + if(plugin.id!==bundle.id || typeof plugin.bootstrap!=="function") throw new Error("Extension identity changed"); + configuration=await plugin.bootstrap(configuration,plan.project); + const description=await plugin.invoke("describe"); + instances.set(plugin.id,plugin); + plugins.push({id:plugin.id,source_sha256:bundle.sha256,description}); + } + return {configuration,plugins}; + } + if(method!=="extension" || !instances.has(params?.id)) throw new Error("Unknown extension operation"); + return instances.get(params.id).invoke(params.method,params.params); +}); +// Keep the PID available to its owner even if the server disconnects while a +// hook has children. The owner terminates the checked group before stdin closes. +process.stdin.resume(); +process.stdin.on("end",()=>process.exit(0)); diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mflux/qwen_image_21/__init__.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mflux/qwen_image_21/__init__.py new file mode 100644 index 0000000..ef96be5 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mflux/qwen_image_21/__init__.py @@ -0,0 +1,5 @@ +"""Qwen-Image 2.1 through mflux, with an optional LoRA merged into the transformer (mlx-agent port).""" + +from .qwen_image_21 import generate, load, merge_lora + +__all__ = ["generate", "load", "merge_lora"] diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mflux/qwen_image_21/convert.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mflux/qwen_image_21/convert.py new file mode 100644 index 0000000..c12136b --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mflux/qwen_image_21/convert.py @@ -0,0 +1,81 @@ +"""Convert Qwen-Image 2.1 (diffusers layout, already in the cache) to an mflux MLX directory. + +``--lora`` merges a PEFT LoRA into the transformer first (``--lora-scale``). +Nothing is downloaded: ``--hf-path`` and ``--lora`` are local paths. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +PIPELINE = "QwenImage21Pipeline" + + +def convert(hf_path: str, mlx_path: str, quantize: bool = False, q_bits: int = 8, lora: str = None, + lora_scale: float = 1.0) -> Path: + import mlx.core as mx + from mlx import nn + from mflux.models.common.config import ModelConfig + from mflux.models.qwen21.qwen21_initializer import Qwen21Initializer + from mflux.models.qwen21.variants.txt2img.qwen_image_21 import QwenImage21 + + from .qwen_image_21 import MFLUX_MODEL, merge_lora + + base, out = Path(hf_path).expanduser(), Path(mlx_path).expanduser() + if not base.is_dir(): + from huggingface_hub import snapshot_download + + base = Path(snapshot_download(hf_path, local_files_only=True)) + if out.exists() and any(out.iterdir()): + raise SystemExit("{0} already exists and is not empty.".format(out)) + try: + index = json.loads((base / "model_index.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + raise SystemExit("{0} is not a diffusers checkpoint (no readable model_index.json).".format(base)) + if index.get("_class_name") != PIPELINE: + raise SystemExit("{0} is a {1}, not a {2}.".format(base, index.get("_class_name"), PIPELINE)) + print("[INFO] Loading {0}".format(base), flush=True) + model = QwenImage21.__new__(QwenImage21) + nn.Module.__init__(model) + Qwen21Initializer._init_config(model, ModelConfig.qwen_image_21()) + weights = Qwen21Initializer._load_weights(str(base)) + if lora: + count = merge_lora(weights.components["transformer"], mx.load(str(Path(lora).expanduser())), lora_scale) + print("[INFO] Merged {0} LoRA layers at scale {1}".format(count, lora_scale), flush=True) + Qwen21Initializer._init_tokenizers(model, str(base)) + Qwen21Initializer._init_models(model) + bits = q_bits if quantize else None + if bits: + print("[INFO] Quantizing to {0} bits".format(bits), flush=True) + Qwen21Initializer._apply_weights(model, weights, bits) + print("[INFO] Writing {0}".format(out), flush=True) + model.save_model(str(out)) + config = { + "model_type": "qwen_image_21", "mflux_model": MFLUX_MODEL, + "components": ["transformer", "text_encoder", "vae"], + "quantization": {"bits": bits} if bits else None, + "lora": {"file": Path(lora).name, "scale": lora_scale} if lora else None, + } + (out / "config.json").write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + print("[INFO] Conversion complete", flush=True) + return out + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--hf-path", required=True, help="diffusers checkpoint directory, or a repo id already in the cache") + parser.add_argument("--mlx-path", required=True, help="destination directory") + parser.add_argument("-q", "--quantize", action="store_true") + parser.add_argument("--q-bits", type=int, default=8, choices=(4, 8)) + parser.add_argument("--lora", default=None, help="local PEFT LoRA .safetensors merged into the transformer") + parser.add_argument("--lora-scale", type=float, default=1.0) + arguments = parser.parse_args(argv) + convert(arguments.hf_path, arguments.mlx_path, arguments.quantize, arguments.q_bits, arguments.lora, arguments.lora_scale) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mflux/qwen_image_21/qwen_image_21.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mflux/qwen_image_21/qwen_image_21.py new file mode 100644 index 0000000..baaace5 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mflux/qwen_image_21/qwen_image_21.py @@ -0,0 +1,73 @@ +"""Qwen-Image 2.1 (Qwen/Qwen-Image-2.1 diffusers layout) through mflux's implementation. + +mflux 0.20 loads no LoRA for this model, so a LoRA is merged into the loaded +transformer weights before quantization: W + scale * B @ A in float32, rounded +to the weight's dtype, which reproduces a checkpoint that ships the merge. +""" + +from __future__ import annotations + +import time +from pathlib import Path + +import mlx.core as mx +import numpy as np + +MFLUX_MODEL = "qwen-image-2.1" + + +def _node(blocks, module: str): + parts = module.split(".") + if len(parts) < 3 or parts[0] != "transformer_blocks" or not parts[1].isdigit(): + raise ValueError("LoRA module {0!r} is not a transformer block layer".format(module)) + node = blocks[int(parts[1])] + for part in parts[2:]: + node = node[int(part)] if part.isdigit() else node[part] + return node + + +def merge_lora(transformer: dict, lora: dict, scale: float) -> int: + """Fold PEFT ``lora_A``/``lora_B`` pairs into ``transformer`` (mflux's loaded weight tree); returns the count.""" + blocks = transformer["transformer_blocks"] + merged = 0 + for name in sorted(lora): + if ".lora_A." not in name: + continue + module, suffix = name.split(".lora_A.") + partner = module + ".lora_B." + suffix + if partner not in lora: + raise ValueError("LoRA {0} has no lora_B".format(module)) + node = _node(blocks, module) + weight = node["weight"] + a, b = lora[name].astype(mx.float32), lora[partner].astype(mx.float32) + if (b.shape[0], a.shape[1]) != tuple(weight.shape): + raise ValueError("LoRA {0} is {1}x{2}, the weight is {3}".format(module, b.shape[0], a.shape[1], tuple(weight.shape))) + node["weight"] = (weight.astype(mx.float32) + scale * (b @ a)).astype(weight.dtype) + merged += 1 + if not merged: + raise ValueError("the LoRA file has no lora_A/lora_B pairs") + return merged + + +def load(path): + from mflux.models.common.config import ModelConfig + from mflux.models.qwen21.variants.txt2img.qwen_image_21 import QwenImage21 + + return QwenImage21(model_path=str(path), model_config=ModelConfig.qwen_image_21()) + + +def generate(model, prompt: str, out: str, width: int = 1024, height: int = 1024, steps: int = 40, seed: int = 42, + guidance: float = 1.0) -> dict: + """Render one image to ``out`` (PNG); returns its dimensions, timing, and pixel spread.""" + started = time.time() + image = model.generate_image( + seed=seed, prompt=prompt, num_inference_steps=steps, width=width, height=height, guidance=guidance, + ) + seconds = time.time() - started + Path(out).parent.mkdir(parents=True, exist_ok=True) + image.save(path=out) + pixels = np.asarray(image.image).astype(np.float32) + return { + "path": str(out), "width": int(pixels.shape[1]), "height": int(pixels.shape[0]), "steps": steps, "seed": seed, + "seconds": round(seconds, 3), "pixel_std": round(float(pixels.std()), 3), + } diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/__init__.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/__init__.py new file mode 100644 index 0000000..1967b6d --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/__init__.py @@ -0,0 +1,4 @@ +from .audio8_asr_infinite import Model, copy_supporting_files, prepare_config +from .config import ModelConfig + +__all__ = ["Model", "ModelConfig", "copy_supporting_files", "prepare_config"] diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/audio.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/audio.py new file mode 100644 index 0000000..0cedd8c --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/audio.py @@ -0,0 +1,96 @@ +"""Streaming audio frontend: the clock windows and log-mel features of the reference decoder. + +Each 80 ms clock step reads its own window (52.5 ms look-back, 2.5 ms +look-ahead) and computes a centered, reflect-padded STFT over that window +alone, so window edges shape the features exactly as in the reference. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass + +import mlx.core as mx +import numpy as np + +from mlx_audio.stt.models.voxtral_realtime.audio import compute_mel_filters + +N_FFT = 400 +HOP_LENGTH = 160 +LOOK_BACK_MS = 52.5 +LOOK_AHEAD_MS = 2.5 + + +def ms_to_samples(milliseconds: float, sampling_rate: int) -> int: + samples = sampling_rate * float(milliseconds) / 1000.0 + if not samples.is_integer(): + raise ValueError("{0} ms is not a whole number of samples at {1} Hz.".format(milliseconds, sampling_rate)) + return int(samples) + + +@dataclass +class StreamLayout: + """The padded stream and its clock windows: window 0 is the prefill, every later window one token.""" + + stream: np.ndarray + samples_per_token: int + prefill_tokens: int + windows: int + look_back: int + look_ahead: int + + @property + def total_tokens(self) -> int: + return self.prefill_tokens + self.windows - 1 + + +def stream_layout(waveform: np.ndarray, *, samples_per_token: int, left_pad_tokens: int, num_delay_tokens: int, + right_pad_text_tokens: int, sampling_rate: int) -> StreamLayout: + prefill_tokens = left_pad_tokens + num_delay_tokens + 1 + right_pad_tokens = num_delay_tokens + 1 + right_pad_text_tokens + stream = np.concatenate([ + np.zeros(left_pad_tokens * samples_per_token, dtype=np.float32), + waveform.astype(np.float32, copy=False), + np.zeros(right_pad_tokens * samples_per_token, dtype=np.float32), + ]) + look_back = ms_to_samples(LOOK_BACK_MS, sampling_rate) + look_ahead = ms_to_samples(LOOK_AHEAD_MS, sampling_rate) + first_end = prefill_tokens * samples_per_token + last_end = stream.shape[0] - look_ahead + windows = 0 if last_end < first_end else (last_end - first_end) // samples_per_token + 1 + return StreamLayout(stream, samples_per_token, prefill_tokens, windows, look_back, look_ahead) + + +def window_batches(layout: StreamLayout): + """Yield (audio [rows, samples], tokens per row): the prefill window, then the one-token windows.""" + step = layout.samples_per_token + first_end = layout.prefill_tokens * step + yield layout.stream[None, : first_end + layout.look_ahead], layout.prefill_tokens + if layout.windows > 1: + starts = first_end + step * np.arange(layout.windows - 1) + offsets = np.arange(-layout.look_back, step + layout.look_ahead) + yield layout.stream[starts[:, None] + offsets[None, :]], 1 + + +class LogMel: + def __init__(self, num_mel_bins: int = 128, sampling_rate: int = 16000, global_log_mel_max: float = 1.5): + self.filters = mx.array(compute_mel_filters(num_mel_bins, N_FFT, sampling_rate), dtype=mx.float32) + n = mx.arange(N_FFT, dtype=mx.float32) + self.window = 0.5 * (1.0 - mx.cos(2.0 * math.pi * n / N_FFT)) + self.floor = global_log_mel_max - 8.0 + + def __call__(self, audio: np.ndarray) -> mx.array: + """[rows, samples] waveform -> [rows, frames, mel] features (torch.stft center=True, last frame dropped).""" + pad = N_FFT // 2 + padded = mx.array(np.pad(audio, ((0, 0), (pad, pad)), mode="reflect"), dtype=mx.float32) + frames = 1 + (padded.shape[1] - N_FFT) // HOP_LENGTH + indices = mx.arange(N_FFT)[None, :] + (mx.arange(frames) * HOP_LENGTH)[:, None] + spectrum = mx.fft.rfft(padded[:, indices] * self.window, n=N_FFT, axis=-1) + power = mx.abs(spectrum[:, :-1, :]) ** 2 + mel = power @ self.filters + log_mel = mx.maximum(mx.log10(mx.maximum(mel, 1e-10)), self.floor) + return (log_mel + 4.0) / 4.0 + + +def as_waveform(samples: np.ndarray) -> np.ndarray: + return np.clip(np.asarray(samples, dtype=np.float32).reshape(-1), -1.0, 1.0) diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/audio8_asr_infinite.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/audio8_asr_infinite.py new file mode 100644 index 0000000..4674547 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/audio8_asr_infinite.py @@ -0,0 +1,494 @@ +"""Audio8 ASR Infinite: Voxtral Realtime audio tower + Qwen2 decoder with delay conditioning. + +Port of the Edge0/Audio8-ASR-Infinite simulated-streaming greedy decoder: +one text token per 80 ms clock step, the audio of each step fused into the +token embedding, the transcription delay applied through a sinusoidal time +embedding plus a frame-length embedding that scales every decoder layer's +MLP input. Weight names are the Hugging Face checkpoint's own. +""" + +from __future__ import annotations + +import glob +import math +import shutil +import time +from pathlib import Path +from typing import List, Optional, Union + +import mlx.core as mx +import mlx.nn as nn +import numpy as np + +from mlx_audio.stt.models.base import STTOutput + +from .audio import LogMel, as_waveform, stream_layout, window_batches +from .config import SAMPLE_RATE, AudioConfig, ModelConfig, TextConfig + +STREAMING_PAD_TOKEN = "[STREAMING_PAD]" +STREAMING_WORD_TOKEN = "[STREAMING_WORD]" +LANGUAGE_TOKENS = {"en": "[LANGUAGE_EN]", "zh": "[LANGUAGE_ZH]"} +DEFAULT_DELAY_MS = 480 +RIGHT_PAD_TEXT_TOKENS = 10 +ROLLING_CONTEXT_TOKENS = 375 +ROLLING_TRIM_INTERVAL_TOKENS = 38 +ROLLING_STABLE_PREFIX_TOKENS = 16 +WINDOW_BATCH = 1024 +SUPPORTING_FILES = ( + "tokenizer.json", + "tokenizer_config.json", + "chat_template.jinja", + "preprocessor_config.json", + "generation_config.json", +) + + +def _rope_inv_freq(head_dim: int, theta: float) -> mx.array: + return 1.0 / (theta ** (mx.arange(0, head_dim, 2, dtype=mx.float32) / head_dim)) + + +def _rotate(x: mx.array, angle: mx.array) -> mx.array: + """Apply the rotate-half RoPE rotation by ``angle`` [..., positions, head_dim / 2].""" + cos = mx.cos(mx.concatenate([angle, angle], axis=-1)).astype(x.dtype) + sin = mx.sin(mx.concatenate([angle, angle], axis=-1)).astype(x.dtype) + half = x.shape[-1] // 2 + rotated = mx.concatenate([-x[..., half:], x[..., :half]], axis=-1) + return x * cos + rotated * sin + + +def prepare_config(config: dict, model_path: Path) -> dict: + """Converter hook: drop ``auto_map`` so the converted directory never points at the repository's code.""" + return {key: value for key, value in config.items() if key != "auto_map"} + + +def copy_supporting_files(source: Path, destination: Path) -> None: + """Converter hook: copy the tokenizer and processor files, never the repository's Python or extra weights.""" + for name in SUPPORTING_FILES: + for found in glob.glob(str(Path(source) / name)): + shutil.copy(found, destination) + + +class CausalConv1d(nn.Conv1d): + def __init__(self, in_channels: int, out_channels: int, kernel_size: int, stride: int = 1): + super().__init__(in_channels, out_channels, kernel_size, stride=stride) + self.left_pad = kernel_size - stride + + def __call__(self, x: mx.array) -> mx.array: + return super().__call__(mx.pad(x, [(0, 0), (self.left_pad, 0), (0, 0)])) + + +class Embedder(nn.Module): + def __init__(self, config: AudioConfig): + super().__init__() + self.conv1 = CausalConv1d(config.num_mel_bins, config.hidden_size, 3) + self.conv2 = CausalConv1d(config.hidden_size, config.hidden_size, 3, stride=2) + + def __call__(self, features: mx.array) -> mx.array: + return nn.gelu(self.conv2(nn.gelu(self.conv1(features)))) + + +class EncoderAttention(nn.Module): + def __init__(self, config: AudioConfig): + super().__init__() + self.heads = config.num_attention_heads + self.kv_heads = config.num_key_value_heads + self.head_dim = config.head_dim + self.theta = config.rope_theta + self.q_proj = nn.Linear(config.hidden_size, self.heads * self.head_dim, bias=True) + self.k_proj = nn.Linear(config.hidden_size, self.kv_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(config.hidden_size, self.kv_heads * self.head_dim, bias=True) + self.o_proj = nn.Linear(self.heads * self.head_dim, config.hidden_size, bias=True) + + def __call__(self, x: mx.array, mask: mx.array, cache: dict) -> mx.array: + """``cache`` holds this layer's un-rotated keys and values of the frames still inside the window.""" + length = x.shape[1] + q = self.q_proj(x).reshape(1, length, self.heads, self.head_dim).transpose(0, 2, 1, 3) + k = self.k_proj(x).reshape(1, length, self.kv_heads, self.head_dim).transpose(0, 2, 1, 3) + v = self.v_proj(x).reshape(1, length, self.kv_heads, self.head_dim).transpose(0, 2, 1, 3) + if cache.get("keys") is not None: + k = mx.concatenate([cache["keys"], k], axis=2) + v = mx.concatenate([cache["values"], v], axis=2) + cache["chunk_keys"], cache["chunk_values"] = k, v + past = k.shape[2] - length + q = mx.fast.rope(q, self.head_dim, traditional=False, base=self.theta, scale=1.0, offset=past) + k = mx.fast.rope(k, self.head_dim, traditional=False, base=self.theta, scale=1.0, offset=0) + out = mx.fast.scaled_dot_product_attention(q, k, v, scale=self.head_dim ** -0.5, mask=mask) + return self.o_proj(out.transpose(0, 2, 1, 3).reshape(1, length, -1)) + + +class EncoderMLP(nn.Module): + def __init__(self, config: AudioConfig): + super().__init__() + self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False) + self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False) + self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=True) + + def __call__(self, x: mx.array) -> mx.array: + return self.down_proj(nn.silu(self.gate_proj(x)) * self.up_proj(x)) + + +class EncoderLayer(nn.Module): + def __init__(self, config: AudioConfig): + super().__init__() + self.self_attn = EncoderAttention(config) + self.self_attn_layer_norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.final_layer_norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.mlp = EncoderMLP(config) + + def __call__(self, x: mx.array, mask: mx.array, cache: dict) -> mx.array: + x = x + self.self_attn(self.self_attn_layer_norm(x), mask, cache) + return x + self.mlp(self.final_layer_norm(x)) + + +class AudioTower(nn.Module): + """Causal Voxtral Realtime encoder with sliding-window attention over the 20 ms frames.""" + + def __init__(self, config: AudioConfig): + super().__init__() + self.config = config + self.embedder = Embedder(config) + self.layers = [EncoderLayer(config) for _ in range(config.num_hidden_layers)] + self.norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def encode(self, frames: mx.array, chunk: Optional[int] = None) -> mx.array: + """[1, T, hidden] conv frames -> [1, T, hidden] hidden states, chunk by chunk. + + Keys are cached un-rotated and re-rotated from position 0 each chunk: + attention depends only on relative offsets, so positions stay bounded + however long the audio is. + """ + window = self.config.sliding_window + chunk = chunk or window + caches = [dict() for _ in self.layers] + outputs = [] + for start in range(0, frames.shape[1], chunk): + x = frames[:, start : start + chunk] + length = x.shape[1] + past = 0 if caches[0].get("keys") is None else caches[0]["keys"].shape[2] + query_positions = mx.arange(past, past + length)[:, None] + key_positions = mx.arange(past + length)[None, :] + offsets = query_positions - key_positions + mask = (offsets >= 0) & (offsets < window) + for layer, cache in zip(self.layers, caches): + x = layer(x, mask, cache) + keep = window - 1 + cache["keys"] = cache.pop("chunk_keys")[:, :, -keep:] + cache["values"] = cache.pop("chunk_values")[:, :, -keep:] + x = self.norm(x) + mx.eval(x, [(cache["keys"], cache["values"]) for cache in caches]) + outputs.append(x) + return mx.concatenate(outputs, axis=1) + + +class Projector(nn.Module): + def __init__(self, config: ModelConfig): + super().__init__() + hidden = config.text_config.hidden_size + self.linear_1 = nn.Linear(config.projection_size, hidden, bias=False) + self.linear_2 = nn.Linear(hidden, hidden, bias=False) + + def __call__(self, x: mx.array) -> mx.array: + return self.linear_2(nn.gelu(self.linear_1(x))) + + +class TextAttention(nn.Module): + def __init__(self, config: TextConfig): + super().__init__() + self.heads = config.num_attention_heads + self.kv_heads = config.num_key_value_heads + self.head_dim = config.head_dim + self.theta = config.rope_theta + self.q_proj = nn.Linear(config.hidden_size, self.heads * self.head_dim, bias=True) + self.k_proj = nn.Linear(config.hidden_size, self.kv_heads * self.head_dim, bias=True) + self.v_proj = nn.Linear(config.hidden_size, self.kv_heads * self.head_dim, bias=True) + self.o_proj = nn.Linear(self.heads * self.head_dim, config.hidden_size, bias=False) + + def __call__(self, x: mx.array, cache: "TextCache", layer: int) -> mx.array: + length = x.shape[1] + offset = cache.length + q = self.q_proj(x).reshape(1, length, self.heads, self.head_dim).transpose(0, 2, 1, 3) + k = self.k_proj(x).reshape(1, length, self.kv_heads, self.head_dim).transpose(0, 2, 1, 3) + v = self.v_proj(x).reshape(1, length, self.kv_heads, self.head_dim).transpose(0, 2, 1, 3) + q = mx.fast.rope(q, self.head_dim, traditional=False, base=self.theta, scale=1.0, offset=offset) + k = mx.fast.rope(k, self.head_dim, traditional=False, base=self.theta, scale=1.0, offset=offset) + k, v = cache.update(layer, k, v) + mask = "causal" if length > 1 else None + out = mx.fast.scaled_dot_product_attention(q, k, v, scale=self.head_dim ** -0.5, mask=mask) + return self.o_proj(out.transpose(0, 2, 1, 3).reshape(1, length, -1)) + + +class TextMLP(nn.Module): + def __init__(self, config: TextConfig): + super().__init__() + self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False) + self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False) + self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False) + + def __call__(self, x: mx.array) -> mx.array: + return self.down_proj(nn.silu(self.gate_proj(x)) * self.up_proj(x)) + + +class AdaRmsNorm(nn.Module): + def __init__(self, config: TextConfig): + super().__init__() + self.linear1 = nn.Linear(config.hidden_size, 32, bias=False) + self.linear2 = nn.Linear(32, config.hidden_size, bias=False) + + def __call__(self, t_cond: mx.array) -> mx.array: + return self.linear2(nn.gelu(self.linear1(t_cond))) + + +class TextLayer(nn.Module): + """Qwen2 decoder layer with the realtime delay scaling before the MLP.""" + + def __init__(self, config: TextConfig): + super().__init__() + self.self_attn = TextAttention(config) + self.mlp = TextMLP(config) + self.input_layernorm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.ada_rms_norm = AdaRmsNorm(config) + + def __call__(self, x: mx.array, scale: mx.array, cache: "TextCache", layer: int) -> mx.array: + x = x + self.self_attn(self.input_layernorm(x), cache, layer) + return x + self.mlp(self.post_attention_layernorm(x) * scale) + + +class TextModel(nn.Module): + def __init__(self, config: TextConfig): + super().__init__() + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size) + self.layers = [TextLayer(config) for _ in range(config.num_hidden_layers)] + self.norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + +class LanguageModel(nn.Module): + def __init__(self, config: TextConfig): + super().__init__() + self.model = TextModel(config) + + def __call__(self, embeds: mx.array, scales: List[mx.array], cache: "TextCache") -> mx.array: + x = embeds + for index, layer in enumerate(self.model.layers): + x = layer(x, scales[index], cache, index) + cache.advance(embeds.shape[1]) + return self.model.embed_tokens.as_linear(self.model.norm(x[:, -1:])) + + +class TextCache: + """Decoder KV cache with the reference rolling window: a stable head plus a re-based tail.""" + + def __init__(self, layers: int, head_dim: int, theta: float, context: int = ROLLING_CONTEXT_TOKENS, + interval: int = ROLLING_TRIM_INTERVAL_TOKENS, stable_prefix: int = ROLLING_STABLE_PREFIX_TOKENS): + self.keys: List[Optional[mx.array]] = [None] * layers + self.values: List[Optional[mx.array]] = [None] * layers + self.length = 0 + self.inv_freq = _rope_inv_freq(head_dim, theta) + self.context = context + self.target = context - (interval - 1) if interval > 1 else context + self.stable_prefix = min(stable_prefix, max(1, context - 1)) if context > 0 else 0 + + def update(self, layer: int, keys: mx.array, values: mx.array): + if self.keys[layer] is not None: + keys = mx.concatenate([self.keys[layer], keys], axis=2) + values = mx.concatenate([self.values[layer], values], axis=2) + self.keys[layer], self.values[layer] = keys, values + return keys, values + + def advance(self, tokens: int) -> None: + self.length += tokens + if self.context <= 0 or self.length <= self.context: + return + trim = self.length - self.target + head = self.stable_prefix + angle = (-float(trim) * self.inv_freq)[None, None, None, :] + for layer, (keys, values) in enumerate(zip(self.keys, self.values)): + tail_keys = _rotate(keys[:, :, head + trim :].astype(mx.float32), angle).astype(keys.dtype) + self.keys[layer] = mx.concatenate([keys[:, :, :head], tail_keys], axis=2) + self.values[layer] = mx.concatenate([values[:, :, :head], values[:, :, head + trim :]], axis=2) + self.length = self.target + + +class Model(nn.Module): + def __init__(self, config: ModelConfig): + super().__init__() + self.config = config + self.audio_tower = AudioTower(config.audio_config) + self.language_model = LanguageModel(config.text_config) + self.multi_modal_projector = Projector(config) + if config.use_frame_len_embedding: + self.frame_len_embedding = nn.Embedding(len(config.supported_frame_lens), config.text_config.hidden_size) + if config.semantic_vad_horizons_seconds: + self.semantic_vad_heads = [ + nn.Linear(config.text_config.hidden_size, config.semantic_vad_num_classes, bias=True) + for _ in config.semantic_vad_horizons_seconds + ] + self._tokenizer = None + self._log_mel = None + + def sanitize(self, weights: dict) -> dict: + sanitized = {} + for key, value in weights.items(): + if key == "language_model.lm_head.weight": + continue + if key.startswith("audio_tower.embedder.conv") and key.endswith(".weight") and value.shape[-1] == 3: + value = value.transpose(0, 2, 1) + sanitized[key] = value + return sanitized + + def model_quant_predicate(self, path: str, module: nn.Module) -> bool: + return path.startswith("language_model.") and "ada_rms_norm" not in path + + @classmethod + def post_load_hook(cls, model: "Model", model_path: Path) -> "Model": + from tokenizers import Tokenizer + + model._tokenizer = Tokenizer.from_file(str(Path(model_path) / "tokenizer.json")) + return model + + def _token_id(self, token: str) -> int: + token_id = self._tokenizer.token_to_id(token) + if token_id is None: + raise ValueError("The tokenizer has no {0} token.".format(token)) + return int(token_id) + + def _frame_len(self, frame_len: Optional[int]) -> int: + frame_len = int(frame_len or self.config.supported_frame_lens[0]) + if frame_len not in self.config.supported_frame_lens: + raise ValueError("frame_len must be one of {0}.".format(self.config.supported_frame_lens)) + return frame_len + + def delay_scales(self, num_delay_tokens: int, frame_len: int) -> List[mx.array]: + """Per-layer MLP input scale ``1 + ada(t_cond)``, computed in the model dtype like the reference.""" + hidden = self.config.text_config.hidden_size + dtype = self.language_model.model.norm.weight.dtype + inv_freq = mx.exp(-math.log(10000.0) * mx.arange(hidden // 2, dtype=mx.float32) / (hidden // 2)) + phase = mx.array(float(num_delay_tokens), dtype=dtype) * inv_freq.astype(dtype) + t_cond = mx.concatenate([mx.cos(phase), mx.sin(phase)]) + if self.config.use_frame_len_embedding: + index = self.config.supported_frame_lens.index(frame_len) + t_cond = t_cond + self.frame_len_embedding.weight[index].astype(dtype) + t_cond = t_cond[None, None, :] + return [1 + layer.ada_rms_norm(t_cond) for layer in self.language_model.model.layers] + + def audio_embeds(self, waveform: np.ndarray, *, frame_len: int, num_delay_tokens: int, left_pad_tokens: int): + """Projected audio for every text position: the prefill window, then one token per clock step.""" + if self._log_mel is None: + self._log_mel = LogMel(self.config.audio_config.num_mel_bins, SAMPLE_RATE) + samples_per_token = SAMPLE_RATE * frame_len * self.config.audio_tower_frame_ms // 1000 + layout = stream_layout( + waveform, samples_per_token=samples_per_token, left_pad_tokens=left_pad_tokens, + num_delay_tokens=num_delay_tokens, right_pad_text_tokens=RIGHT_PAD_TEXT_TOKENS, + sampling_rate=SAMPLE_RATE, + ) + dtype = self.multi_modal_projector.linear_2.weight.dtype + if not mx.issubdtype(dtype, mx.floating): + dtype = mx.bfloat16 + conv_frames = [] + for audio, tokens in window_batches(layout): + for start in range(0, audio.shape[0], WINDOW_BATCH): + features = self._log_mel(audio[start : start + WINDOW_BATCH]).astype(dtype) + frames = self.audio_tower.embedder(features)[:, -tokens * frame_len :] + conv_frames.append(frames.reshape(1, -1, frames.shape[-1])) + mx.eval(conv_frames[-1]) + hidden = self.audio_tower.encode(mx.concatenate(conv_frames, axis=1)) + width = self.config.audio_config.hidden_size + grouped = hidden.reshape(layout.total_tokens, frame_len * width) + if frame_len < self.config.max_frame_len: + grouped = mx.pad(grouped, [(0, 0), (0, (self.config.max_frame_len - frame_len) * width)]) + embeds = self.multi_modal_projector(grouped) + mx.eval(embeds) + return embeds, layout + + def generate( + self, + audio: Union[str, Path, mx.array, np.ndarray, List[mx.array]], + *, + language: str = "en", + transcription_delay_ms: Optional[int] = None, + frame_len: Optional[int] = None, + max_tokens: Optional[int] = None, + verbose: bool = False, + **kwargs, + ) -> STTOutput: + language_token = LANGUAGE_TOKENS.get(str(language).strip().lower()) + if language_token is None: + raise ValueError("Audio8 ASR Infinite transcribes en or zh, got {0!r}.".format(language)) + frame_len = self._frame_len(frame_len) + token_ms = frame_len * self.config.audio_tower_frame_ms + delay_ms = int(transcription_delay_ms or DEFAULT_DELAY_MS) + if delay_ms <= 0 or delay_ms % token_ms: + raise ValueError("transcription_delay_ms must be a positive multiple of {0} ms.".format(token_ms)) + num_delay_tokens = delay_ms // token_ms + left_pad_tokens = self.config.left_pad_tokens(frame_len) + + start_time = time.time() + waveform = as_waveform(self._load_audio(audio)) + embeds, layout = self.audio_embeds( + waveform, frame_len=frame_len, num_delay_tokens=num_delay_tokens, left_pad_tokens=left_pad_tokens, + ) + scales = self.delay_scales(num_delay_tokens, frame_len) + text = self.config.text_config + cache = TextCache(text.num_hidden_layers, text.head_dim, text.rope_theta) + embed_tokens = self.language_model.model.embed_tokens + pad_id = self._token_id(STREAMING_PAD_TOKEN) + eos_id = int(self.config.eos_token_id) + prompt = [int(self.config.bos_token_id), self._token_id(language_token)] + prompt += [pad_id] * (layout.prefill_tokens - 2) + + def step(token_ids: List[int], position: int) -> mx.array: + inputs = embed_tokens(mx.array([token_ids])) + embeds[None, position : position + len(token_ids)] + logits = self.language_model(inputs, scales, cache)[0, -1].astype(mx.float32) + return mx.argmax(mx.where(mx.arange(logits.shape[0]) == eos_id, -mx.inf, logits)) + + limit = layout.windows if max_tokens is None else min(layout.windows, int(max_tokens)) + generated: List[int] = [] + next_token = step(prompt, 0) + decode_start = time.time() + for position in range(layout.prefill_tokens, layout.prefill_tokens + limit - 1): + token = int(next_token.item()) + generated.append(token) + next_token = step([token], position) + mx.async_eval(next_token) + if limit > 0: + generated.append(int(next_token.item())) + end_time = time.time() + + text_value = self._decode(generated) + duration = waveform.shape[0] / SAMPLE_RATE + if verbose: + elapsed = end_time - decode_start + print("Decode: {0} steps in {1:.2f}s ({2:.1f} steps/s); audio {3:.2f}s".format( + len(generated), elapsed, len(generated) / max(elapsed, 1e-9), duration)) + total = end_time - start_time + return STTOutput( + text=text_value, + segments=[{"text": text_value, "start": 0.0, "end": round(duration, 3)}], + language=str(language).strip().lower(), + prompt_tokens=layout.prefill_tokens, + generation_tokens=len(generated), + total_tokens=layout.prefill_tokens + len(generated), + prompt_tps=layout.prefill_tokens / total if total > 0 else 0.0, + generation_tps=len(generated) / (end_time - decode_start) if end_time > decode_start else 0.0, + total_time=total, + ) + + def _decode(self, token_ids: List[int]) -> str: + skipped = { + self._token_id(STREAMING_PAD_TOKEN), + self._token_id(STREAMING_WORD_TOKEN), + int(self.config.bos_token_id), + int(self.config.pad_token_id), + } + visible = [token for token in token_ids if token not in skipped] + return self._tokenizer.decode(visible, skip_special_tokens=True).strip() + + @staticmethod + def _load_audio(audio) -> np.ndarray: + if isinstance(audio, (str, Path)): + from mlx_audio.stt.utils import load_audio + + return np.array(load_audio(str(audio), sr=SAMPLE_RATE)) + if isinstance(audio, list): + audio = audio[0] + return np.array(audio) diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/config.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/config.py new file mode 100644 index 0000000..19bdf2c --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-audio/audio8_asr_infinite/config.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +from mlx_audio.base import BaseModelArgs + +SAMPLE_RATE = 16000 + + +def _rope_theta(params: Optional[dict], default: float) -> float: + if isinstance(params, dict) and "rope_theta" in params: + return float(params["rope_theta"]) + return default + + +@dataclass +class AudioConfig(BaseModelArgs): + hidden_size: int = 1280 + intermediate_size: int = 5120 + num_hidden_layers: int = 32 + num_attention_heads: int = 32 + num_key_value_heads: Optional[int] = None + head_dim: int = 64 + num_mel_bins: int = 128 + rms_norm_eps: float = 1e-5 + rope_parameters: Optional[dict] = None + rope_theta: float = 1_000_000.0 + sliding_window: int = 750 + + def __post_init__(self): + if self.num_key_value_heads is None: + self.num_key_value_heads = self.num_attention_heads + self.rope_theta = _rope_theta(self.rope_parameters, self.rope_theta) + + +@dataclass +class TextConfig(BaseModelArgs): + hidden_size: int = 2048 + intermediate_size: int = 11008 + num_hidden_layers: int = 36 + num_attention_heads: int = 16 + num_key_value_heads: int = 2 + head_dim: Optional[int] = None + rms_norm_eps: float = 1e-6 + rope_parameters: Optional[dict] = None + rope_theta: float = 1_000_000.0 + vocab_size: int = 151936 + tie_word_embeddings: bool = True + model_type: str = "qwen2" + + def __post_init__(self): + if self.model_type != "qwen2": + raise ValueError( + "audio8_asr_infinite supports a qwen2 text decoder, got {0!r}.".format(self.model_type) + ) + if self.head_dim is None: + self.head_dim = self.hidden_size // self.num_attention_heads + self.rope_theta = _rope_theta(self.rope_parameters, self.rope_theta) + + +@dataclass +class ModelConfig(BaseModelArgs): + model_type: str = "audio8_asr_infinite" + audio_config: Optional[AudioConfig] = None + text_config: Optional[TextConfig] = None + supported_frame_lens: List[int] = field(default_factory=lambda: [4, 6, 8]) + audio_tower_frame_ms: int = 20 + use_frame_len_embedding: bool = False + projector_hidden_act: str = "gelu" + semantic_vad_horizons_seconds: Optional[List[float]] = None + semantic_vad_num_classes: int = 8 + streaming_n_left_pad_tokens: int = 18 + streaming_n_left_pad_tokens_by_frame_len: Optional[Dict[str, int]] = None + weight_format_version: int = 2 + bos_token_id: int = 151644 + eos_token_id: int = 151645 + pad_token_id: int = 151643 + + def __post_init__(self): + if self.weight_format_version != 2: + raise ValueError( + "audio8_asr_infinite loads weight_format_version 2, got {0}.".format(self.weight_format_version) + ) + if isinstance(self.audio_config, dict): + self.audio_config = AudioConfig.from_dict(self.audio_config) + if self.audio_config is None: + self.audio_config = AudioConfig() + if isinstance(self.text_config, dict): + self.text_config = TextConfig.from_dict(self.text_config) + if self.text_config is None: + self.text_config = TextConfig() + self.supported_frame_lens = [int(value) for value in self.supported_frame_lens] + self.use_frame_len_embedding = bool(self.use_frame_len_embedding and len(self.supported_frame_lens) > 1) + + @property + def max_frame_len(self) -> int: + return max(self.supported_frame_lens) + + @property + def projection_size(self) -> int: + return self.audio_config.hidden_size * self.max_frame_len + + def left_pad_tokens(self, frame_len: int) -> int: + by_frame_len = self.streaming_n_left_pad_tokens_by_frame_len or {} + return int(by_frame_len.get(str(frame_len), self.streaming_n_left_pad_tokens)) diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-embeddings/laya/__init__.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-embeddings/laya/__init__.py new file mode 100644 index 0000000..4f150d1 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-embeddings/laya/__init__.py @@ -0,0 +1,5 @@ +"""Laya calibrated decision model, ported to MLX by mlx-agent.""" + +from .laya import Model, ModelArgs, load, predict + +__all__ = ["Model", "ModelArgs", "load", "predict"] diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-embeddings/laya/convert.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-embeddings/laya/convert.py new file mode 100644 index 0000000..a3ec4c1 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-embeddings/laya/convert.py @@ -0,0 +1,94 @@ +"""Convert a Laya checkpoint (repository root: encoder config, decision settings, tokenizer) to MLX.""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +from pathlib import Path + +import mlx.core as mx +import mlx.nn as nn +from mlx.utils import tree_flatten + +from .laya import TOKENIZER_FILES, Model, ModelArgs + +SOURCE_FILES = ( + "model.safetensors", + "rl_agent_config.json", + "encoder/config.json", + "tokenizer/tokenizer.json", + "tokenizer/tokenizer_config.json", +) +SETTINGS = ("head_layers", "max_len", "head_max_len", "act_costs", "temperature", "temperature_by_options", "model_name") +QUANTIZE_INCLUDE = ("encoder.layers.", "head.layers.") +GROUP_SIZE = 32 + + +def source_directory(hf_path: str, revision: str = None) -> Path: + path = Path(hf_path).expanduser() + if path.is_dir(): + return path + from huggingface_hub import snapshot_download + + return Path(snapshot_download(hf_path, revision=revision, allow_patterns=list(SOURCE_FILES))) + + +def converted_config(settings: dict, encoder: dict) -> dict: + config = {"model_type": "laya", "encoder": encoder} + config.update({key: settings[key] for key in SETTINGS if key in settings}) + return config + + +def quantizes(path: str, module: nn.Module) -> bool: + """Linear layers of the encoder and decision head; embeddings, norms, and the scorers stay as shipped.""" + return ( + isinstance(module, nn.Linear) + and path.startswith(QUANTIZE_INCLUDE) + and module.weight.shape[-1] % GROUP_SIZE == 0 + ) + + +def convert(hf_path: str, mlx_path: str, quantize: bool = False, q_bits: int = 4, revision: str = None) -> Path: + out = Path(mlx_path).expanduser() + if out.exists() and any(out.iterdir()): + raise SystemExit("{0} already exists and is not empty.".format(out)) + print("[INFO] Loading {0}".format(hf_path), flush=True) + source = source_directory(hf_path, revision) + missing = [name for name in SOURCE_FILES if not (source / name).is_file()] + if missing: + raise SystemExit("{0} is not a Laya checkpoint; missing {1}.".format(hf_path, ", ".join(missing))) + settings = json.loads((source / "rl_agent_config.json").read_text(encoding="utf-8")) + encoder = json.loads((source / "encoder" / "config.json").read_text(encoding="utf-8")) + config = converted_config(settings, encoder) + model = Model(ModelArgs.from_dict(config)) + model.load_weights(list(Model.sanitize(mx.load(str(source / "model.safetensors"))).items()), strict=True) + if quantize: + print("[INFO] Quantizing to {0} bits".format(q_bits), flush=True) + nn.quantize(model, group_size=GROUP_SIZE, bits=q_bits, class_predicate=quantizes) + config["quantization"] = {"group_size": GROUP_SIZE, "bits": q_bits} + out.mkdir(parents=True, exist_ok=True) + print("[INFO] Writing {0}".format(out), flush=True) + mx.save_safetensors(str(out / "model.safetensors"), dict(tree_flatten(model.parameters())), metadata={"format": "mlx"}) + for name in TOKENIZER_FILES: + shutil.copyfile(source / "tokenizer" / name, out / name) + (out / "config.json").write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + print("[INFO] Conversion complete", flush=True) + return out + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--hf-path", required=True, help="Hugging Face repo id or local checkpoint directory") + parser.add_argument("--mlx-path", required=True, help="destination directory") + parser.add_argument("--revision", default=None) + parser.add_argument("-q", "--quantize", action="store_true") + parser.add_argument("--q-bits", type=int, default=4, choices=(4, 8)) + arguments = parser.parse_args(argv) + convert(arguments.hf_path, arguments.mlx_path, arguments.quantize, arguments.q_bits, arguments.revision) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-embeddings/laya/laya.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-embeddings/laya/laya.py new file mode 100644 index 0000000..29acb24 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-embeddings/laya/laya.py @@ -0,0 +1,337 @@ +"""Laya: a ModernBERT encoder with an option-marker decision head (convaiinnovations/laya). + +Each question is rendered as ``[CLS] question: [SEP] [MASK] option ... [SEP] state [SEP]``; +the head scores the hidden state at every option's ``[MASK]`` and a softmax over the question's +options gives its answer distribution. All questions of a request run in one batch. Weight names +are the checkpoint's own, apart from the head's packed attention and ``nn.Sequential`` indices. +""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Optional + +import mlx.core as mx +import mlx.nn as nn + +from mlx_embeddings.models.modernbert import ModelArgs as EncoderArgs +from mlx_embeddings.models.modernbert import ModernBertModel + +QTYPES = {"choice": 0, "score": 1, "noul": 2} +QTYPE_NAMES = {value: key for key, value in QTYPES.items()} +OPTION_TOKENS = 48 +MIN_OPTION_BUDGET = 16 +MASKED_LOGIT = -1e4 +TOKENIZER_FILES = ("tokenizer.json", "tokenizer_config.json") +# transformers' "gelu" is the exact erf form; mlx-embeddings' ModernBERT MLP uses the tanh approximation. +TANH_GELU = {"gelu_new": "tanh", "gelu_pytorch_tanh": "tanh", "gelu_fast": "tanh"} + + +@dataclass +class ModelArgs: + encoder: dict + model_type: str = "laya" + head_layers: int = 2 + max_len: int = 512 + head_max_len: int = 192 + act_costs: Dict[str, float] = field(default_factory=lambda: {"escalate": 0.5}) + temperature: List[float] = field(default_factory=lambda: [1.0, 1.0, 1.0]) + temperature_by_options: Dict[str, float] = field(default_factory=dict) + + @classmethod + def from_dict(cls, params: dict) -> "ModelArgs": + names = cls.__dataclass_fields__ + return cls(**{key: value for key, value in params.items() if key in names}) + + @property + def n_act(self) -> int: + return len(self.act_costs) + 1 + + +def encoder_args(config: dict) -> EncoderArgs: + """ModernBERT arguments from a transformers 4 or 5 encoder config (``rope_parameters`` per layer type).""" + values = dict(config) + rope = values.get("rope_parameters") + if isinstance(rope, dict): + full = rope.get("full_attention") or {} + local = rope.get("sliding_attention") or {} + if "rope_theta" in full: + values["global_rope_theta"] = full["rope_theta"] + if "rope_theta" in local: + values["local_rope_theta"] = local["rope_theta"] + return EncoderArgs.from_dict(values) + + +class HeadAttention(nn.Module): + """``torch.nn.MultiheadAttention`` with a packed input projection and a key padding mask.""" + + def __init__(self, dims: int, heads: int): + super().__init__() + self.heads = heads + self.in_proj = nn.Linear(dims, 3 * dims, bias=True) + self.out_proj = nn.Linear(dims, dims, bias=True) + + def __call__(self, x: mx.array, mask: mx.array) -> mx.array: + batch, length, dims = x.shape + head_dim = dims // self.heads + qkv = self.in_proj(x).reshape(batch, length, 3, self.heads, head_dim).transpose(2, 0, 3, 1, 4) + out = mx.fast.scaled_dot_product_attention(qkv[0], qkv[1], qkv[2], scale=head_dim ** -0.5, mask=mask) + return self.out_proj(out.transpose(0, 2, 1, 3).reshape(batch, length, dims)) + + +class HeadLayer(nn.Module): + """``torch.nn.TransformerEncoderLayer(norm_first=True)``: pre-norm attention, then a ReLU feed-forward.""" + + def __init__(self, dims: int): + super().__init__() + self.self_attn = HeadAttention(dims, max(1, dims // 64)) + self.linear1 = nn.Linear(dims, 4 * dims) + self.linear2 = nn.Linear(4 * dims, dims) + self.norm1 = nn.LayerNorm(dims) + self.norm2 = nn.LayerNorm(dims) + + def __call__(self, x: mx.array, mask: mx.array) -> mx.array: + x = x + self.self_attn(self.norm1(x), mask) + return x + self.linear2(nn.relu(self.linear1(self.norm2(x)))) + + +class Head(nn.Module): + def __init__(self, dims: int, count: int): + super().__init__() + self.layers = [HeadLayer(dims) for _ in range(count)] + + +class Model(nn.Module): + def __init__(self, args: ModelArgs): + super().__init__() + self.args = args + self.encoder = ModernBertModel(encoder_args(args.encoder)) + approx = TANH_GELU.get(args.encoder.get("hidden_activation", "gelu"), "none") + for layer in self.encoder.layers: + layer.mlp.act = nn.GELU(approx=approx) + dims = self.encoder.config.hidden_size + self.head = Head(dims, args.head_layers) + self.type_emb = nn.Embedding(3, dims) + self.scorer = nn.Sequential(nn.LayerNorm(dims), nn.Linear(dims, dims), nn.GELU(), nn.Linear(dims, 1)) + self.act_head = nn.Sequential(nn.Linear(dims + 4, 256), nn.GELU(), nn.Linear(256, args.n_act)) + + def __call__(self, input_ids, attention_mask, marker_pos, marker_mask, qtype): + dtype = self.type_emb.weight.dtype + h = self.encoder(input_ids, attention_mask=attention_mask.astype(dtype))["last_hidden_state"] + h = h + self.type_emb(qtype)[:, None, :] + key_mask = mx.where(attention_mask[:, None, None, :] > 0, 0.0, -mx.inf).astype(h.dtype) + for layer in self.head.layers: + h = layer(h, key_mask) + picked = mx.take_along_axis(h, marker_pos[:, :, None], axis=1) + logits = self.scorer(picked).squeeze(-1).astype(mx.float32) + logits = mx.where(marker_mask, logits, MASKED_LOGIT) + p = mx.softmax(logits, axis=-1) + k = mx.maximum(marker_mask.sum(-1), 2).astype(mx.float32) + entropy = -(p * mx.log(mx.maximum(p, 1e-9))).sum(-1) / mx.log(k) + top = mx.sort(p, axis=-1) + features = mx.stack([top[:, -1], top[:, -1] - top[:, -2], entropy, k / 255.0], axis=-1) + pooled = h[:, 0].astype(mx.float32) + act_logits = self.act_head(mx.concatenate([pooled, features], axis=-1).astype(dtype)) + return logits, act_logits.astype(mx.float32) + + @staticmethod + def sanitize(weights: Dict[str, mx.array]) -> Dict[str, mx.array]: + """Checkpoint names to module names: packed head attention, ``nn.Sequential`` indices, no buffers.""" + result = {} + for name, value in weights.items(): + if name == "temperature": + continue + name = name.replace("self_attn.in_proj_weight", "self_attn.in_proj.weight") + name = name.replace("self_attn.in_proj_bias", "self_attn.in_proj.bias") + for sequential in ("scorer.", "act_head."): + if name.startswith(sequential) and not name.startswith(sequential + "layers."): + name = sequential + "layers." + name[len(sequential):] + result[name] = value + return result + + +# ----------------------------------------------------------------------------- requests + + +class RequestTokenizer: + """The tokenizer calls request rendering needs, over a ``tokenizers`` file (no transformers).""" + + def __init__(self, directory: Path): + from tokenizers import Tokenizer + + self.tokenizer = Tokenizer.from_file(str(Path(directory) / "tokenizer.json")) + settings = json.loads((Path(directory) / "tokenizer_config.json").read_text(encoding="utf-8")) + self.mask_token = settings.get("mask_token", "[MASK]") + ids = {} + for role in ("cls", "sep", "pad", "mask"): + token = settings.get(role + "_token", "[{0}]".format(role.upper())) + value = self.tokenizer.token_to_id(token) + if value is None: + raise ValueError("the tokenizer has no {0} token {1!r}".format(role, token)) + ids[role] = value + self.cls_id, self.sep_id, self.pad_id, self.mask_id = ids["cls"], ids["sep"], ids["pad"], ids["mask"] + + def ids(self, text: str) -> List[int]: + return self.tokenizer.encode(text, add_special_tokens=False).ids + + +def internal_question(definition: dict) -> dict: + """A request question ({type, instructions, criteria}) in the rendering form ({t, ins, crit}).""" + kind = definition.get("type") + if kind not in QTYPES: + raise ValueError("question type must be one of {0}".format(sorted(QTYPES))) + criteria = definition.get("criteria") + if kind == "choice" and isinstance(criteria, list): + criteria = {str(item): None for item in criteria} + if kind == "choice" and (not isinstance(criteria, dict) or len(criteria) < 2): + raise ValueError("a choice question needs at least two criteria") + if kind == "score" and (not isinstance(criteria, list) or len(criteria) < 2): + raise ValueError("a score question needs a list of at least two levels") + if kind == "noul" and criteria is not None and not isinstance(criteria, dict): + raise ValueError("noul criteria are an object with optional true and false texts") + instructions = definition.get("instructions") + if not isinstance(instructions, str): + instructions = json.dumps(instructions) + return {"t": kind, "ins": instructions, "crit": criteria} + + +def option_texts(question: dict) -> List[str]: + """Option texts in label order; noul is always [false, true].""" + kind, criteria = question["t"], question.get("crit") + if kind == "choice": + return [key if not value else "{0}: {1}".format(key, value) for key, value in criteria.items()] + if kind == "score": + return ["level {0}: {1}".format(index, text) for index, text in enumerate(criteria)] + criteria = criteria or {} + return [ + "false: " + (criteria.get("false") or "no, the statement does not hold"), + "true: " + (criteria.get("true") or "yes, the statement holds"), + ] + + +def state_text(state) -> str: + return state if isinstance(state, str) else json.dumps(state, ensure_ascii=False) + + +def build_sequence(tok: RequestTokenizer, state, question: dict, max_len: int, head_max_len: int): + """Token ids for one question and the positions of its option markers.""" + mask = tok.mask_token + head = tok.ids("{0} question: {1}".format(question["t"], str(question["ins"]).replace(mask, " "))) + options = [[tok.mask_id] + tok.ids(" " + text.replace(mask, " "))[:OPTION_TOKENS] for text in option_texts(question)] + budget = head_max_len - sum(len(option) for option in options) + if budget < MIN_OPTION_BUDGET: + per_option = max(4, (head_max_len - MIN_OPTION_BUDGET) // max(1, len(options))) + options = [option[:per_option] for option in options] + budget = head_max_len - sum(len(option) for option in options) + ids = [tok.cls_id] + head[:max(8, budget)] + [tok.sep_id] + markers = [] + for option in options: + markers.append(len(ids)) + ids.extend(option) + ids.append(tok.sep_id) + room = max(0, max_len - len(ids) - 1) + ids = ids + tok.ids(state_text(state).replace(mask, " "))[:room] + [tok.sep_id] + return ids[:max_len], [marker for marker in markers if marker < max_len] + + +def temperature_bucket(qtype: int, k: int) -> str: + size = "2" if k <= 2 else "3-5" if k <= 5 else "6-10" if k <= 10 else "11+" + return "{0}:{1}".format(QTYPE_NAMES[qtype], size) + + +def confidence(p: List[float]) -> float: + """One minus the normalized entropy of the answer distribution.""" + if len(p) < 2: + return 1.0 + entropy = -sum(value * math.log(min(max(value, 1e-12), 1.0)) for value in p) + return 1.0 - entropy / math.log(len(p)) + + +def batch(items: List[dict], pad_id: int): + length = max(len(item["ids"]) for item in items) + width = max(len(item["markers"]) for item in items) + ids, attention, positions, present = [], [], [], [] + for item in items: + padding = length - len(item["ids"]) + ids.append(item["ids"] + [pad_id] * padding) + attention.append([1] * len(item["ids"]) + [0] * padding) + k = len(item["markers"]) + positions.append(item["markers"] + [0] * (width - k)) + present.append([True] * k + [False] * (width - k)) + return ( + mx.array(ids, dtype=mx.int32), mx.array(attention, dtype=mx.int32), + mx.array(positions, dtype=mx.int32), mx.array(present), + mx.array([item["qtype"] for item in items], dtype=mx.int32), + ) + + +def predict(model: Model, tok: RequestTokenizer, state, questions: Dict[str, dict]) -> dict: + """Answer every question about one state in one forward pass (calibrated by the shipped temperatures).""" + if not isinstance(questions, dict) or not questions: + raise ValueError("questions must be a non-empty object of question definitions") + args = model.args + names, parsed, items = list(questions), [], [] + for name in names: + question = internal_question(questions[name]) + ids, markers = build_sequence(tok, state, question, args.max_len, args.head_max_len) + if len(markers) != len(option_texts(question)): + raise ValueError("question {0!r}: options do not fit in {1} tokens".format(name, args.head_max_len)) + parsed.append(question) + items.append({"ids": ids, "markers": markers, "qtype": QTYPES[question["t"]]}) + logits, act = model(*batch(items, tok.pad_id)) + act = mx.softmax(act, axis=-1) + mx.eval(logits, act) + logits, act = logits.tolist(), act.tolist() + answers = {} + for row, (name, question) in enumerate(zip(names, parsed)): + k, qtype = len(items[row]["markers"]), QTYPES[question["t"]] + scale = args.temperature_by_options.get(temperature_bucket(qtype, k), args.temperature[qtype]) + z = [value / scale for value in logits[row][:k]] + top = max(z) + weights = [math.exp(value - top) for value in z] + total = sum(weights) + p = [value / total for value in weights] + extra = {"act_probability": round(act[row][0], 4)} + if question["t"] == "choice": + keys = list(question["crit"]) + answers[name] = { + "type": "choice", "choice": keys[p.index(max(p))], + "probabilities": {key: round(value, 4) for key, value in zip(keys, p)}, + "confidence": round(confidence(p), 4), "laya": extra, + } + elif question["t"] == "score": + answers[name] = { + "type": "score", "score": round(sum(index * value for index, value in enumerate(p)), 4), + "legend": {str(index): text for index, text in enumerate(question["crit"])}, + "probabilities": {str(index): round(value, 4) for index, value in enumerate(p)}, + "confidence": round(confidence(p), 4), "laya": extra, + } + else: + answers[name] = {"type": "noul", "noul": round(p[1], 4), "laya": extra} + tokens = sum(len(item["ids"]) for item in items) + return {"answers": answers, "usage": {"input_tokens": tokens, "output_tokens": 0}} + + +def load(path, dtype: Optional[mx.Dtype] = None): + """A converted Laya directory: (model, tokenizer).""" + directory = Path(path) + config = json.loads((directory / "config.json").read_text(encoding="utf-8")) + model = Model(ModelArgs.from_dict(config)) + weights = {} + for shard in sorted(directory.glob("*.safetensors")): + weights.update(mx.load(str(shard))) + quantization = config.get("quantization") + if quantization: + nn.quantize( + model, group_size=quantization["group_size"], bits=quantization["bits"], + class_predicate=lambda name, module: "{0}.scales".format(name) in weights, + ) + model.load_weights(list(weights.items()), strict=True) + if dtype is not None: + model.set_dtype(dtype) + model.eval() + mx.eval(model.parameters()) + return model, RequestTokenizer(directory) diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-lm/deepseek_v4_dspark/__init__.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-lm/deepseek_v4_dspark/__init__.py new file mode 100644 index 0000000..522de31 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-lm/deepseek_v4_dspark/__init__.py @@ -0,0 +1 @@ +"""DeepSeek-V4 DSpark speculative-decoding drafter, converted from llama.cpp's dflash GGUF by mlx-agent.""" diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-lm/deepseek_v4_dspark/convert.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-lm/deepseek_v4_dspark/convert.py new file mode 100644 index 0000000..9326793 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-lm/deepseek_v4_dspark/convert.py @@ -0,0 +1,311 @@ +"""Convert a DeepSeek-V4 DSpark drafter from llama.cpp's ``dflash`` GGUF to an MLX sidecar. + +The drafter is DeepSeek-V4's DSpark stages: no token embeddings and no output +head (it borrows the target's), fed by the target's hidden states, so it only +runs beside its target model. Tensors keep the names of DeepSeek's checkpoint +with ``mtp.N`` written ``stages.N``. The projections DeepSeek ships in FP8 are +affine-quantized to ``q_bits``; the routed experts, MXFP4 in the source, are +repacked into MLX ``mxfp4`` without rounding; every other tensor keeps its +stored dtype. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np + +ARCHITECTURE = "dflash" +MODEL_TYPE = "deepseek_v4_dspark" +TARGET_MODEL_TYPE = "deepseek_v4" +GROUP_SIZE = 64 +MXFP4 = {"group_size": 32, "bits": 4, "mode": "mxfp4"} +MXFP4_BLOCK_BYTES = 17 +SHARD_BYTES = 5 << 30 +_KEY = ARCHITECTURE + "." + +# llama.cpp's DeepseekV4DSparkModel tensor map (conversion/deepseek.py), reversed. +LAYER_TENSORS = { + "hc_attn_fn.weight": "hc_attn_fn", + "hc_attn_base.weight": "hc_attn_base", + "hc_attn_scale.weight": "hc_attn_scale", + "hc_ffn_fn.weight": "hc_ffn_fn", + "hc_ffn_base.weight": "hc_ffn_base", + "hc_ffn_scale.weight": "hc_ffn_scale", + "attn_sinks.weight": "attn.attn_sink", + "attn_q_a.weight": "attn.wq_a.weight", + "attn_q_b.weight": "attn.wq_b.weight", + "attn_q_a_norm.weight": "attn.q_norm.weight", + "attn_kv.weight": "attn.wkv.weight", + "attn_kv_a_norm.weight": "attn.kv_norm.weight", + "attn_output_a.weight": "attn.wo_a.weight", + "attn_output_b.weight": "attn.wo_b.weight", + "attn_norm.weight": "attn_norm.weight", + "ffn_norm.weight": "ffn_norm.weight", + "ffn_gate_inp.weight": "ffn.gate.weight", + "exp_probs_b.bias": "ffn.gate.bias", + "ffn_gate_shexp.weight": "ffn.shared_experts.w1.weight", + "ffn_down_shexp.weight": "ffn.shared_experts.w2.weight", + "ffn_up_shexp.weight": "ffn.shared_experts.w3.weight", +} +EXPERT_TENSORS = {"ffn_gate_exps.weight": "w1", "ffn_down_exps.weight": "w2", "ffn_up_exps.weight": "w3"} +# llama.cpp writes these without a stage; DeepSeek keeps them on the first and the last stage. +FIRST_STAGE_TENSORS = {"fc.weight": "main_proj.weight", "enc.output_norm.weight": "main_norm.weight"} +LAST_STAGE_TENSORS = { + "output_norm.weight": "norm.weight", + "output_hc_fn.weight": "hc_head_fn", + "output_hc_base.weight": "hc_head_base", + "output_hc_scale.weight": "hc_head_scale", + "markov_w1.weight": "markov_head.markov_w1.weight", + "markov_w2.weight": "markov_head.markov_w2.weight", + "conf_proj.weight": "confidence_head.proj.weight", +} +# DeepSeek ships these in FP8 (llama.cpp stores them Q8_0 or BF16). +QUANTIZED = frozenset(( + "attn.wq_a.weight", "attn.wq_b.weight", "attn.wkv.weight", "attn.wo_a.weight", "attn.wo_b.weight", + "ffn.shared_experts.w1.weight", "ffn.shared_experts.w2.weight", "ffn.shared_experts.w3.weight", + "main_proj.weight", +)) +# llama_expert_gating_func_type values DeepSeek-V4 uses. +SCORING_FUNCS = {4: "sqrtsoftplus"} + + +def read_metadata(reader) -> dict: + """Header fields, without the tokenizer's vocabulary arrays.""" + return { + name: field.contents() for name, field in reader.fields.items() + if not name.startswith("tokenizer.ggml.") or name.endswith("_token_id") + } + + +def _float(value) -> float: + """The shortest decimal that round-trips the stored float32 (1e-06, not 9.99999997e-07).""" + return float(np.format_float_positional(np.float32(value), unique=True, trim="-")) + + +def stage_count(metadata: dict, name: str) -> int: + if metadata.get("general.architecture") != ARCHITECTURE or _KEY + "hyper_connection.count" not in metadata: + raise ValueError("{0} is not a DeepSeek-V4 DSpark drafter (dflash with hyper-connections).".format(name)) + return int(metadata[_KEY + "block_count"]) + + +def tensor_names(stages: int) -> dict: + """GGUF name to MLX name for every tensor but the routed experts.""" + names = {} + for stage in range(stages): + for source, target in LAYER_TENSORS.items(): + names["blk.{0}.{1}".format(stage, source)] = "stages.{0}.{1}".format(stage, target) + for source, target in FIRST_STAGE_TENSORS.items(): + names[source] = "stages.0." + target + for source, target in LAST_STAGE_TENSORS.items(): + names[source] = "stages.{0}.{1}".format(stages - 1, target) + return names + + +def expert_tensors(stages: int) -> dict: + """GGUF name to (stage, projection) for the stacked routed experts.""" + return { + "blk.{0}.{1}".format(stage, source): (stage, projection) + for stage in range(stages) for source, projection in EXPERT_TENSORS.items() + } + + +def check_tensors(present, stages: int, name: str) -> None: + expected = set(tensor_names(stages)) | set(expert_tensors(stages)) + missing, unexpected = sorted(expected - set(present)), sorted(set(present) - expected) + if missing or unexpected: + detail = [] + if missing: + detail.append("missing " + ", ".join(missing[:5])) + if unexpected: + detail.append("unexpected " + ", ".join(unexpected[:5])) + raise ValueError("{0} does not match the DeepSeek-V4 DSpark tensor layout: {1}.".format(name, "; ".join(detail))) + + +def repack_mxfp4(blocks: np.ndarray): + """GGML MXFP4 rows (``n_blocks * 17`` bytes each) as MLX ``mxfp4`` (uint32 codes, uint8 scales). + + Both formats hold E2M1 codes with one E8M0 exponent per 32 values. GGML + stores value ``j`` and ``j + 16`` in the low and high nibble of byte ``j``; + MLX packs eight consecutive codes per uint32, low nibble first. The + exponent byte carries over unchanged, so the repack is exact. + """ + blocks = np.asarray(blocks, dtype=np.uint8) + blocks = blocks.reshape(blocks.shape[:-1] + (-1, MXFP4_BLOCK_BYTES)) + scales = np.ascontiguousarray(blocks[..., 0]) + nibbles = blocks[..., 1:] + codes = np.concatenate((nibbles & 0x0F, nibbles >> 4), axis=-1) + codes = codes.reshape(codes.shape[:-2] + (-1, 8)).astype(np.uint32) + packed = np.zeros(codes.shape[:-1], dtype=np.uint32) + for index in range(8): + packed |= codes[..., index] << np.uint32(4 * index) + return packed, scales + + +def drafter_config(metadata: dict, stages: int, vocab_size: int, markov_rank: int, quantization: dict) -> dict: + """``config.json``: the DSpark fields at the top, the drafter's DeepSeek-V4 parameters in ``text_config``.""" + gating = int(metadata[_KEY + "expert_gating_func"]) + if gating not in SCORING_FUNCS: + raise ValueError("Unexpected expert gating function {0}; DeepSeek-V4 uses sqrtsoftplus (4).".format(gating)) + clamps = {_float(value) for value in metadata[_KEY + "swiglu_clamp_exp"]} + if len(clamps) != 1: + raise ValueError("The SwiGLU clamp differs between stages: {0}.".format(sorted(clamps))) + text = { + "model_type": TARGET_MODEL_TYPE, + "vocab_size": vocab_size, + "hidden_size": int(metadata[_KEY + "embedding_length"]), + "num_attention_heads": int(metadata[_KEY + "attention.head_count"]), + "num_key_value_heads": int(metadata[_KEY + "attention.head_count_kv"]), + "head_dim": int(metadata[_KEY + "attention.key_length"]), + "qk_rope_head_dim": int(metadata[_KEY + "rope.dimension_count"]), + "q_lora_rank": int(metadata[_KEY + "attention.q_lora_rank"]), + "o_lora_rank": int(metadata[_KEY + "attention.output_lora_rank"]), + "o_groups": int(metadata[_KEY + "attention.output_group_count"]), + "n_routed_experts": int(metadata[_KEY + "expert_count"]), + "num_experts_per_tok": int(metadata[_KEY + "expert_used_count"]), + "n_shared_experts": int(metadata[_KEY + "expert_shared_count"]), + "moe_intermediate_size": int(metadata[_KEY + "expert_feed_forward_length"]), + "routed_scaling_factor": _float(metadata[_KEY + "expert_weights_scale"]), + "norm_topk_prob": bool(metadata[_KEY + "expert_weights_norm"]), + "scoring_func": SCORING_FUNCS[gating], + "swiglu_limit": clamps.pop(), + "rms_norm_eps": _float(metadata[_KEY + "attention.layer_norm_rms_epsilon"]), + "rope_theta": _float(metadata[_KEY + "rope.freq_base"]), + "rope_scaling": { + "type": metadata[_KEY + "rope.scaling.type"], + "factor": _float(metadata[_KEY + "rope.scaling.factor"]), + "original_max_position_embeddings": int(metadata[_KEY + "rope.scaling.original_context_length"]), + "beta_fast": _float(metadata[_KEY + "rope.scaling.yarn_beta_fast"]), + "beta_slow": _float(metadata[_KEY + "rope.scaling.yarn_beta_slow"]), + }, + "max_position_embeddings": int(metadata[_KEY + "context_length"]), + "sliding_window": int(metadata[_KEY + "attention.sliding_window"]), + "compress_ratios": [int(value) for value in metadata[_KEY + "attention.compress_ratios"]], + "compress_rope_theta": _float(metadata[_KEY + "attention.compress_rope_freq_base"]), + "index_n_heads": int(metadata[_KEY + "attention.indexer.head_count"]), + "index_head_dim": int(metadata[_KEY + "attention.indexer.key_length"]), + "index_topk": int(metadata[_KEY + "attention.indexer.top_k"]), + "hc_mult": int(metadata[_KEY + "hyper_connection.count"]), + "hc_sinkhorn_iters": int(metadata[_KEY + "hyper_connection.sinkhorn_iterations"]), + "hc_eps": _float(metadata[_KEY + "hyper_connection.epsilon"]), + } + for key, field in (("bos_token_id", "tokenizer.ggml.bos_token_id"), ("eos_token_id", "tokenizer.ggml.eos_token_id")): + if field in metadata: + text[key] = int(metadata[field]) + return { + "model_type": MODEL_TYPE, + "n_mtp_layers": stages, + "dspark_block_size": int(metadata[_KEY + "block_size"]), + "dspark_noise_token_id": int(metadata["tokenizer.ggml.mask_token_id"]), + # llama.cpp stores each id + 1: the input of layer i is layer i - 1's output. + "dspark_target_layer_ids": [int(value) - 1 for value in metadata[_KEY + "target_layers"]], + "dspark_markov_rank": markov_rank, + "dspark_target_name": metadata.get("general.name"), + "quantization": quantization, + "quantization_config": quantization, + "text_config": text, + } + + +class ShardWriter: + """Writes ``model-0000k-of-0000n.safetensors`` shards of at most ``limit`` bytes, then the index.""" + + def __init__(self, directory: Path, limit: int = SHARD_BYTES): + self.directory = Path(directory) + self.limit = limit + self.pending = {} + self.pending_bytes = 0 + self.shards = [] + self.weight_map = {} + self.total = 0 + + def add(self, name: str, array) -> None: + if self.pending and self.pending_bytes + array.nbytes > self.limit: + self._flush() + self.pending[name] = array + self.pending_bytes += array.nbytes + self.total += array.nbytes + + def _flush(self) -> None: + import mlx.core as mx + + path = self.directory / "model-{0:05d}.partial.safetensors".format(len(self.shards) + 1) + mx.save_safetensors(str(path), self.pending, metadata={"format": "mlx"}) + self.shards.append((path, sorted(self.pending))) + self.pending, self.pending_bytes = {}, 0 + + def finish(self) -> int: + if self.pending: + self._flush() + count = len(self.shards) + for index, (path, names) in enumerate(self.shards, start=1): + final = "model-{0:05d}-of-{1:05d}.safetensors".format(index, count) + path.rename(self.directory / final) + self.weight_map.update({name: final for name in names}) + index = {"metadata": {"total_size": self.total}, "weight_map": dict(sorted(self.weight_map.items()))} + (self.directory / "model.safetensors.index.json").write_text(json.dumps(index, indent=2) + "\n", encoding="utf-8") + return count + + +def convert(gguf_path, out_dir, q_bits: int, log=print, shard_bytes: int = SHARD_BYTES) -> dict: + """Write the MLX drafter into ``out_dir`` (which must exist and be empty); ``config.json`` lands last.""" + import gguf + import mlx.core as mx + from gguf import quants + + source = Path(gguf_path) + reader = gguf.GGUFReader(str(source)) + metadata = read_metadata(reader) + stages = stage_count(metadata, source.name) + tensors = {tensor.name: tensor for tensor in reader.tensors} + check_tensors(tensors, stages, source.name) + names, experts = tensor_names(stages), expert_tensors(stages) + dtypes = { + gguf.GGMLQuantizationType.F32: mx.float32, + gguf.GGMLQuantizationType.F16: mx.float16, + gguf.GGMLQuantizationType.BF16: mx.bfloat16, + } + expert_count = int(metadata[_KEY + "expert_count"]) + quantization = {"group_size": GROUP_SIZE, "bits": q_bits, "mode": "affine"} + writer = ShardWriter(out_dir, shard_bytes) + for tensor in reader.tensors: + if tensor.name in experts: + stage, projection = experts[tensor.name] + if tensor.tensor_type != gguf.GGMLQuantizationType.MXFP4 or tensor.data.shape[0] != expert_count: + raise ValueError("{0} must hold {1} MXFP4 experts; found {2} {3}.".format( + tensor.name, expert_count, tensor.data.shape[0], tensor.tensor_type.name)) + log("repacking {0} ({1} experts, mxfp4)".format(tensor.name, expert_count)) + for expert in range(expert_count): + weight, scales = repack_mxfp4(tensor.data[expert]) + base = "stages.{0}.ffn.experts.{1}.{2}".format(stage, expert, projection) + writer.add(base + ".weight", mx.array(weight)) + writer.add(base + ".scales", mx.array(scales)) + quantization[base] = dict(MXFP4) + continue + target = names[tensor.name] + values = quants.dequantize(tensor.data, tensor.tensor_type) + if target.split(".", 2)[2] in QUANTIZED: + log("quantizing {0} -> {1} ({2}-bit)".format(tensor.name, target, q_bits)) + weight, scales, biases = mx.quantize(mx.array(values).astype(mx.bfloat16), group_size=GROUP_SIZE, bits=q_bits) + base = target[: -len(".weight")] + writer.add(base + ".weight", weight) + writer.add(base + ".scales", scales) + writer.add(base + ".biases", biases) + continue + dtype = dtypes.get(tensor.tensor_type) + if dtype is None: + raise ValueError("{0} is stored as {1}; expected F32, F16, or BF16.".format(tensor.name, tensor.tensor_type.name)) + writer.add(target, mx.array(values).astype(dtype)) + shards = writer.finish() + # GGUF shapes list the innermost dimension first: markov_w1 is [rank, vocab]. + markov_rank, vocab_size = (int(size) for size in tensors["markov_w1.weight"].shape) + config = drafter_config(metadata, stages, vocab_size, markov_rank, quantization) + (Path(out_dir) / "config.json").write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + log("wrote {0} tensors in {1} shard(s), {2} bytes".format(len(writer.weight_map), shards, writer.total)) + return { + "port": MODEL_TYPE, + "target": config["dspark_target_name"], + "tensors": len(writer.weight_map), + "bytes": writer.total, + } diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/__init__.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/__init__.py new file mode 100644 index 0000000..10a25e0 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/__init__.py @@ -0,0 +1,6 @@ +"""Wan text-to-video through mlx-video (mlx-agent port): convert a Wan checkpoint, render one prompt to an MP4.""" + +from .t2v import generate, load +from .training_inputs import load_training_session, prepare_training_batch + +__all__ = ["generate", "load", "load_training_session", "prepare_training_batch"] diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/convert.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/convert.py new file mode 100644 index 0000000..739ed97 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/convert.py @@ -0,0 +1,87 @@ +"""Convert a Wan text-to-video checkpoint (original layout, already in the cache) to an mlx-video directory. + +Nothing is downloaded: ``--hf-path`` is a local checkpoint directory or a repo id already in the cache. +The converter writes bfloat16 transformer and T5 weights, a float32 VAE, ``config.json`` (carrying +``model_type`` and, with ``--quantize``, ``quantization``), and the tokenizer files the pipeline loads. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +from pathlib import Path + +MODEL_TYPE = "t2v" +TOKENIZER_SOURCE = ("google", "umt5-xxl") +TOKENIZER_DIR = "tokenizer" +TOKENIZER_FILES = ("special_tokens_map.json", "spiece.model", "tokenizer.json", "tokenizer_config.json") +T5_FILE = "models_t5_umt5-xxl-enc-bf16.pth" +VAE_FILES = ("Wan2.1_VAE.pth", "Wan2.2_VAE.pth") + + +def convert(hf_path: str, mlx_path: str, quantize: bool = False, q_bits: int = 4) -> Path: + base, out = Path(hf_path).expanduser(), Path(mlx_path).expanduser() + if not base.is_dir(): + from huggingface_hub import snapshot_download + + try: + base = Path(snapshot_download(hf_path, local_files_only=True)) + except (OSError, ValueError): + raise SystemExit("{0} is not a checkpoint directory or a repository in the cache.".format(hf_path)) + if out.exists() and any(out.iterdir()): + raise SystemExit("{0} already exists and is not empty.".format(out)) + try: + source = json.loads((base / "config.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + raise SystemExit("{0} is not a Wan checkpoint (no readable config.json).".format(base)) + if not isinstance(source, dict) or source.get("model_type") != MODEL_TYPE: + raise SystemExit("{0} is a {1!r} checkpoint, not a {2!r} text-to-video checkpoint.".format( + base, source.get("model_type") if isinstance(source, dict) else None, MODEL_TYPE)) + missing = [] + if not (base / T5_FILE).is_file(): + missing.append(T5_FILE) + if not any((base / name).is_file() for name in VAE_FILES): + missing.append(VAE_FILES[0]) + if not (any(base.glob("diffusion_pytorch_model*.safetensors")) or (base / "low_noise_model").is_dir()): + missing.append("diffusion_pytorch_model*.safetensors") + if missing: + raise SystemExit("{0} is missing Wan checkpoint files: {1}.".format(base, ", ".join(missing))) + tokenizer = base.joinpath(*TOKENIZER_SOURCE) + absent = [name for name in TOKENIZER_FILES if not (tokenizer / name).is_file()] + if absent: + raise SystemExit("{0} is missing the tokenizer files {1}.".format(tokenizer, ", ".join(absent))) + from mlx_video.convert_wan import convert_wan_checkpoint + + print("[INFO] Converting {0}".format(base), flush=True) + convert_wan_checkpoint( + str(base), str(out), dtype="bfloat16", model_version="auto", quantize=quantize, bits=q_bits, group_size=64, + ) + destination = out / TOKENIZER_DIR + destination.mkdir(parents=True, exist_ok=True) + for name in TOKENIZER_FILES: + shutil.copyfile(tokenizer / name, destination / name) + try: + written = json.loads((out / "config.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + raise SystemExit("the converter wrote no readable config.json in {0}.".format(out)) + if written.get("model_type") != MODEL_TYPE or (quantize and not written.get("quantization")): + raise SystemExit("the converted config.json in {0} lacks model_type {1!r} or its quantization.".format(out, MODEL_TYPE)) + print("[INFO] Conversion complete", flush=True) + return out + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--hf-path", required=True, help="Wan checkpoint directory, or a repo id already in the cache") + parser.add_argument("--mlx-path", required=True, help="destination directory") + parser.add_argument("-q", "--quantize", action="store_true") + parser.add_argument("--q-bits", type=int, default=4, choices=(4, 8)) + arguments = parser.parse_args(argv) + convert(arguments.hf_path, arguments.mlx_path, arguments.quantize, arguments.q_bits) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/t2v.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/t2v.py new file mode 100644 index 0000000..2677512 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/t2v.py @@ -0,0 +1,145 @@ +"""Render one prompt to an MP4 with a converted Wan text-to-video directory, through mlx-video's pipeline. + +The package's ``generate_video`` loads every component itself, so the port times its loaders to split load +time from generation time, redirects its tokenizer to the copy ``convert`` saved beside the weights (no hub +access), and replaces its video writer to measure the frames and encode the MP4 with OpenCV. +""" + +from __future__ import annotations + +import contextlib +import json +import sys +import time +from pathlib import Path +from unittest import mock + +TOKENIZER_REPO = "google/umt5-xxl" +TOKENIZER_DIR = "tokenizer" +LOADERS = ("load_t5_encoder", "load_wan_model", "load_vae_decoder") + + +class Model: + """A converted Wan directory: its path and parsed config.json.""" + + def __init__(self, path: Path, config: dict): + self.path = path + self.config = config + + +def load(path) -> Model: + directory = Path(path).expanduser() + try: + config = json.loads((directory / "config.json").read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise ValueError("no readable config.json in {0}: {1}".format(directory, error)) from error + if not isinstance(config, dict): + raise ValueError("config.json in {0} is not an object".format(directory)) + transformer = ( + ("low_noise_model.safetensors", "high_noise_model.safetensors") if config.get("dual_model") + else ("model.safetensors",) + ) + missing = [name for name in ("t5_encoder.safetensors", "vae.safetensors") + transformer if not (directory / name).is_file()] + if missing: + raise ValueError("{0} is not a complete converted Wan model; missing {1}".format(directory, ", ".join(missing))) + return Model(directory, config) + + +def _encode(frames, out: str, fps: int) -> None: + import cv2 + + height, width = frames.shape[1], frames.shape[2] + for code in ("avc1", "mp4v"): + writer = cv2.VideoWriter(out, cv2.VideoWriter_fourcc(*code), float(fps), (width, height)) + if writer.isOpened(): + for frame in frames: + writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)) + writer.release() + if Path(out).is_file() and Path(out).stat().st_size > 0: + return + else: + writer.release() + if Path(out).exists(): + Path(out).unlink() + raise RuntimeError("OpenCV could not encode an MP4 (tried avc1 and mp4v)") + + +def _pixel_std(frames) -> float: + import numpy as np + + total = squares = 0.0 + for frame in frames: + values = frame.astype(np.float64) + total += float(values.sum()) + squares += float((values * values).sum()) + count = float(frames.size) + mean = total / count + return max(squares / count - mean * mean, 0.0) ** 0.5 + + +def generate(model: Model, prompt: str, out: str, width: int = 832, height: int = 480, frames: int = 81, + fps: int = 16, steps: int = 50, seed: int = 42) -> dict: + """Render one video to ``out`` (MP4); returns its dimensions, timing, memory, and pixel spread.""" + import mlx.core as mx + import mlx_video.generate_wan as wan + from transformers import AutoTokenizer + + spent = {"load": 0.0} + written = {} + original_loaders = {name: getattr(wan, name) for name in LOADERS} + from_pretrained = AutoTokenizer.from_pretrained + local_tokenizer = model.path / TOKENIZER_DIR + + def timed(function): + def wrapper(*arguments, **keywords): + started = time.time() + try: + return function(*arguments, **keywords) + finally: + spent["load"] += time.time() - started + return wrapper + + def redirected(name, *arguments, **keywords): + started = time.time() + try: + if name == TOKENIZER_REPO and local_tokenizer.is_dir(): + name = str(local_tokenizer) + return from_pretrained(name, *arguments, **keywords) + finally: + spent["load"] += time.time() - started + + def capture(video, output_path, fps=None): + written["frames"] = int(video.shape[0]) + written["height"], written["width"] = int(video.shape[1]), int(video.shape[2]) + written["pixel_std"] = _pixel_std(video) + _encode(video, output_path, playback) + + playback = fps + Path(out).parent.mkdir(parents=True, exist_ok=True) + mx.reset_peak_memory() + started = time.time() + patches = [mock.patch.object(wan, name, timed(function)) for name, function in original_loaders.items()] + patches += [ + mock.patch.object(wan, "save_video", capture), + mock.patch.object(AutoTokenizer, "from_pretrained", redirected), + ] + with contextlib.ExitStack() as stack: + for patch in patches: + stack.enter_context(patch) + with contextlib.redirect_stdout(sys.stderr): + wan.generate_video( + model_dir=str(model.path), prompt=prompt, width=width, height=height, num_frames=frames, + steps=steps, seed=seed, output_path=out, + ) + total = time.time() - started + peak_memory_gb = mx.get_peak_memory() / 1e9 + if "frames" not in written: + raise RuntimeError("the pipeline produced no video") + seconds = max(total - spent["load"], 0.0) + return { + "path": str(out), "width": written["width"], "height": written["height"], "frames": written["frames"], + "fps": fps, "duration_seconds": round(written["frames"] / fps, 3), "steps": steps, "seed": seed, + "seconds": round(seconds, 3), "load_seconds": round(spent["load"], 3), + "seconds_per_frame": round(seconds / written["frames"], 3), + "peak_memory_gb": round(peak_memory_gb, 3), "pixel_std": round(written["pixel_std"], 3), + } diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/training.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/training.py new file mode 100644 index 0000000..88ecd18 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/training.py @@ -0,0 +1,292 @@ +"""Native MLX LoRA optimization for a converted single-branch Wan backbone. + +The caller owns dataset preparation, compute admission, and durable artifacts. +This runtime never changes converted model files or writes checkpoint paths. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import math +import zipfile +from dataclasses import asdict + +import mlx.core as mx +import mlx.optimizers as optim +import numpy as np +from mlx import nn +from mlx.utils import tree_flatten, tree_unflatten + +MAX_CHECKPOINT_BYTES = 512 * 1024 * 1024 + + +class TrainableLoRALinear(nn.Module): + def __init__(self, linear, rank, alpha, key): + super().__init__() + out_features, in_features = linear.weight.shape + if isinstance(linear, nn.QuantizedLinear): + in_features = in_features * 32 // linear.bits + self.linear = linear + self.scale = alpha / rank + bound = 1 / math.sqrt(in_features) + self.lora_a = mx.random.uniform( + low=-bound, high=bound, shape=(rank, in_features), key=key + ) + self.lora_b = mx.zeros((out_features, rank)) + + def __call__(self, x): + baseline = self.linear(x) + update = (x.astype(mx.float32) @ self.lora_a.T) @ self.lora_b.T + return baseline + (self.scale * update).astype(baseline.dtype) + + +def _fingerprint(model): + digest = hashlib.sha256() + digest.update(json.dumps(asdict(model.config), sort_keys=True).encode()) + for name, value in sorted(tree_flatten(model.parameters())): + digest.update(name.encode() + b"\0") + digest.update(str(value.dtype).encode() + b"\0") + digest.update(str(value.shape).encode() + b"\0") + material = value.astype(mx.float32) if value.dtype == mx.bfloat16 else value + digest.update(np.asarray(material).tobytes()) + return "sha256:" + digest.hexdigest() + + +class WanMLXTrainingSession: + def __init__(self, model, *, targets, rank, alpha, learning_rate, seed=42): + if not mx.metal.is_available() or mx.default_device() != mx.gpu: + raise ValueError("Wan MLX training requires the Metal GPU.") + if ( + type(rank) is not int + or not 1 <= rank <= 256 + or type(alpha) is not int + or not 1 <= alpha <= 4096 + or type(seed) is not int + or not 0 <= seed < 2**32 + or type(learning_rate) not in (int, float) + or not math.isfinite(learning_rate) + or not 0 < learning_rate <= 1 + ): + raise ValueError("Invalid MLX LoRA configuration.") + if ( + model.config.model_type != "t2v" + or model.config.model_version != "2.1" + or model.config.dual_model + or model.config.in_dim != model.config.out_dim + ): + raise ValueError("Only converted single-branch Wan 2.1 T2V is supported.") + if ( + type(targets) is not tuple + or not targets + or targets != tuple(sorted(set(targets))) + or any(type(name) is not str or not name for name in targets) + ): + raise ValueError("Targets must be explicit, sorted and unique.") + modules = dict(model.named_modules()) + if any( + name not in modules + or not isinstance(modules[name], (nn.Linear, nn.QuantizedLinear)) + for name in targets + ): + raise ValueError("An unsupported LoRA target was requested.") + self.base_identity = _fingerprint(model) + self.model = model + self.targets = targets + self.rank, self.alpha = rank, alpha + self.steps = 0 + self._failed = False + self._metadata = { + "schema": "wan-mlx-training-state/1", + "base": self.base_identity, + "targets": list(targets), + "rank": rank, + "alpha": alpha, + "learning_rate": float(learning_rate), + "seed": seed, + } + keys = mx.random.split(mx.random.key(seed), len(targets)) + replacements = [ + (name, TrainableLoRALinear(modules[name], rank, alpha, keys[index])) + for index, name in enumerate(targets) + ] + model.freeze() + model.update_modules(tree_unflatten(replacements)) + self.optimizer = optim.AdamW(learning_rate=learning_rate, weight_decay=0) + self.optimizer.init(model.trainable_parameters()) + self._require_trainable_boundary() + self._gradient = nn.value_and_grad(model, self._loss) + + def _require_trainable_boundary(self): + expected = { + name + suffix for name in self.targets for suffix in (".lora_a", ".lora_b") + } + actual = dict(tree_flatten(self.model.trainable_parameters())) + if set(actual) != expected: + raise ValueError("The frozen base boundary changed.") + return actual + + def _loss(self, model, clean, noise, context, sigma): + mixture = (1 - sigma[:, None, None, None, None]) * clean + sigma[ + :, None, None, None, None + ] * noise + patch = model.config.patch_size + seq_len = math.prod( + clean.shape[index + 2] // patch[index] for index in range(3) + ) + outputs = model( + list(mixture), + sigma * model.config.num_train_timesteps, + list(context), + seq_len, + ) + prediction = mx.stack(outputs) + return mx.mean(mx.square(prediction.astype(mx.float32) - (noise - clean))) + + def train_step(self, clean, noise, context, sigma): + if self._failed: + raise ValueError( + "Restore a valid checkpoint after a failed optimizer update." + ) + self._require_trainable_boundary() + patch = self.model.config.patch_size + if ( + clean.ndim != 5 + or clean.shape != noise.shape + or clean.shape[0] != 1 + or clean.shape[1] != self.model.config.in_dim + or any( + value <= 0 or value % stride + for value, stride in zip(clean.shape[2:], patch) + ) + or context.ndim != 3 + or context.shape[0] != clean.shape[0] + or not 0 < context.shape[1] <= self.model.config.text_len + or context.shape[2] != self.model.config.text_dim + or sigma.shape != (clean.shape[0],) + ): + raise ValueError("Invalid Wan training batch shape.") + for value in (clean, noise, context, sigma): + if ( + value.dtype not in (mx.float16, mx.bfloat16, mx.float32) + or not mx.all(mx.isfinite(value)).item() + ): + raise ValueError( + "Training inputs must be finite floating-point tensors." + ) + if not mx.all((sigma >= 0) & (sigma <= 1)).item(): + raise ValueError("Flow-matching sigma must be between zero and one.") + loss, gradients = self._gradient(self.model, clean, noise, context, sigma) + flat = dict(tree_flatten(gradients)) + if set(flat) != set(self._require_trainable_boundary()): + raise ValueError("Unexpected optimizer parameters.") + norm = mx.sqrt( + sum(mx.sum(mx.square(value.astype(mx.float32))) for value in flat.values()) + ) + mx.eval(loss, norm, gradients) + if not math.isfinite(loss.item()) or not math.isfinite(norm.item()): + raise ValueError("Non-finite Wan training measurement.") + try: + self.optimizer.update(self.model, gradients) + mx.eval(self.model.trainable_parameters(), self.optimizer.state) + except Exception: + self._failed = True + raise + self.steps += 1 + return {"step": self.steps, "loss": loss.item(), "gradient_norm": norm.item()} + + def adapter_weights(self): + parameters = self._require_trainable_boundary() + result = {} + for name in self.targets: + result[name + ".lora_A.weight"] = parameters[name + ".lora_a"] + result[name + ".lora_B.weight"] = parameters[name + ".lora_b"] + result[name + ".alpha"] = mx.array(self.alpha, dtype=mx.float32) + return result + + def _state(self): + result = { + "adapter." + key: value + for key, value in tree_flatten(self.model.trainable_parameters()) + } + result.update( + { + "optimizer." + key: value + for key, value in tree_flatten(self.optimizer.state) + } + ) + result["completed_steps"] = mx.array(self.steps, dtype=mx.uint64) + return result + + def checkpoint_bytes(self): + if self._failed: + raise ValueError("A failed optimizer update cannot be checkpointed.") + self._require_trainable_boundary() + values = self._state() + metadata = json.dumps(self._metadata, sort_keys=True).encode() + values["metadata"] = mx.array(list(metadata), dtype=mx.uint8) + output = io.BytesIO() + mx.savez(output, **values) + payload = output.getvalue() + if len(payload) > MAX_CHECKPOINT_BYTES: + raise ValueError("MLX checkpoint exceeds its supported size.") + return payload + + def restore_checkpoint(self, payload): + if type(payload) is not bytes or len(payload) > MAX_CHECKPOINT_BYTES: + raise ValueError("Invalid MLX checkpoint.") + expected = self._state() + expected["metadata"] = mx.zeros( + (len(json.dumps(self._metadata, sort_keys=True).encode()),), dtype=mx.uint8 + ) + try: + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + entries = archive.infolist() + names = [entry.filename for entry in entries] + if ( + set(names) != {key + ".npy" for key in expected} + or len(names) != len(expected) + or any( + entry.compress_type != zipfile.ZIP_STORED for entry in entries + ) + or sum(entry.file_size for entry in entries) > MAX_CHECKPOINT_BYTES + ): + raise ValueError + for entry in entries: + reference = expected[entry.filename[:-4]] + if entry.file_size > reference.size * reference.dtype.size + 4096: + raise ValueError + restored = mx.load(io.BytesIO(payload), format="npz") + for key, reference in expected.items(): + value = restored[key] + if value.shape != reference.shape or value.dtype != reference.dtype: + raise ValueError + if ( + value.dtype in (mx.float16, mx.bfloat16, mx.float32) + and not mx.all(mx.isfinite(value)).item() + ): + raise ValueError + metadata = bytes(restored["metadata"].tolist()).decode() + if json.loads(metadata) != self._metadata: + raise ValueError + step = int(restored["completed_steps"].item()) + if step != int(restored["optimizer.step"].item()): + raise ValueError + parameters = [ + (key[len("adapter.") :], value) + for key, value in restored.items() + if key.startswith("adapter.") + ] + optimizer = [ + (key[len("optimizer.") :], value) + for key, value in restored.items() + if key.startswith("optimizer.") + ] + except Exception as error: + raise ValueError("Invalid or incompatible MLX checkpoint.") from error + self.model.update(tree_unflatten(parameters)) + self.optimizer.state = tree_unflatten(optimizer) + self.steps = step + self._failed = False + self._require_trainable_boundary() diff --git a/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/training_inputs.py b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/training_inputs.py new file mode 100644 index 0000000..05eb84d --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/ports/mlx-video/t2v/training_inputs.py @@ -0,0 +1,253 @@ +"""Offline inputs for native MLX training on converted single-branch Wan. + +Callers own compute admission and immutable source snapshots. These helpers +return tensors and content identities; they never download or publish files. +""" +from __future__ import annotations + +import gc +import hashlib +import json +import math +import os +import stat +from dataclasses import dataclass +from pathlib import Path + +MAX_SOURCE_BYTES = 24 * 1024 * 1024 * 1024 +MAX_VIDEO_BYTES = 512 * 1024 * 1024 +MAX_DECODED_PIXELS = 32 * 1024 * 1024 + + +def _digest_regular(path, limit): + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= limit: + raise ValueError("Training sources must be bounded regular files.") + digest = hashlib.sha256() + while chunk := os.read(descriptor, 8 * 1024 * 1024): + digest.update(chunk) + after = os.fstat(descriptor) + if (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) != ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns + ): + raise ValueError("Training source changed while reading.") + return "sha256:" + digest.hexdigest() + finally: + os.close(descriptor) + + +def training_sources(path): + """Validate the supported converted layout and hash every consumed source.""" + root = Path(path).expanduser() + try: + if root.is_symlink() or not root.is_dir(): + raise ValueError("A local converted model directory is required.") + identities = {} + for name in ("config.json", "model.safetensors", "t5_encoder.safetensors", "vae.safetensors"): + identities[name] = _digest_regular(root / name, 65536 if name == "config.json" else MAX_SOURCE_BYTES) + config = json.loads((root / "config.json").read_text()) + if ( + not isinstance(config, dict) + or config.get("model_type") != "t2v" + or config.get("model_version") != "2.1" + or config.get("dual_model", False) is not False + or config.get("in_dim", 16) != 16 + or config.get("out_dim", 16) != 16 + or config.get("vae_z_dim", 16) != 16 + ): + raise ValueError("Only converted single-branch Wan 2.1 T2V is supported.") + tokenizer = root / "tokenizer" + if tokenizer.is_symlink() or not tokenizer.is_dir(): + raise ValueError("A converted local tokenizer is required.") + files = [] + for directory, subdirs, names in os.walk(tokenizer, followlinks=False): + for name in subdirs: + if (Path(directory) / name).is_symlink(): + raise ValueError("Tokenizer links are not supported.") + for name in names: + files.append(Path(directory) / name) + if len(files) > 64: + raise ValueError("Tokenizer exceeds the supported file count.") + if not files: + raise ValueError("A converted local tokenizer is required.") + for file in sorted(files): + identities[file.relative_to(root).as_posix()] = _digest_regular(file, 64 * 1024 * 1024) + if _digest_regular(root / "config.json", 65536) != identities["config.json"]: + raise ValueError("Converted configuration changed while reading.") + return config, identities + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError("Converted training sources are incomplete or unreadable.") from error + + +def _unchanged(root, identities): + if training_sources(root)[1] != identities: + raise ValueError("Converted training sources changed during loading.") + + +def _config(raw): + from mlx_video.models.wan.config import WanModelConfig + return WanModelConfig.from_dict(raw) + + +def load_training_session(path, *, targets, rank, alpha, learning_rate, seed=42): + """Strictly load the converted backbone before freezing it for LoRA.""" + import mlx.core as mx + import mlx.nn as nn + from mlx_video.models.wan.model import WanModel + from .training import WanMLXTrainingSession + + root = Path(path).expanduser() + raw, identities = training_sources(root) + model = WanModel(_config(raw)) + quantization = raw.get("quantization") + if quantization: + from mlx_video.convert_wan import _quantize_predicate + nn.quantize(model, group_size=quantization["group_size"], bits=quantization["bits"], + class_predicate=lambda name, layer: _quantize_predicate(name, layer)) + weights = mx.load(root / "model.safetensors") + model.load_weights(list(weights.items()), strict=True) + mx.eval(model.parameters()) + del weights + _unchanged(root, identities) + return WanMLXTrainingSession(model, targets=targets, rank=rank, alpha=alpha, + learning_rate=learning_rate, seed=seed) + + +def validate_video_shape(shape, config): + """Reject incompatible media without inventing crop, resize or trim rules.""" + stride = config.get("vae_stride", [4, 8, 8]) + patch = config.get("patch_size", [1, 2, 2]) + if ( + len(shape) != 4 or any(type(value) is not int for value in shape) + or not 5 <= shape[0] <= 241 or shape[3] != 3 + or any(not 16 <= value <= 1024 for value in shape[1:3]) + or len(stride) != 3 or len(patch) != 3 + or any(type(value) is not int or value <= 0 for value in (*stride, *patch)) + or (shape[0] - 1) % stride[0] + or ((shape[0] - 1) // stride[0] + 1) % patch[0] + or any(shape[index + 1] % (stride[index + 1] * patch[index + 1]) for index in range(2)) + or math.prod(shape[:3]) > MAX_DECODED_PIXELS + ): + raise ValueError("Training video dimensions or frame count are unsupported.") + + +def _decode_video(path, config): + import cv2 + import numpy as np + + reader = cv2.VideoCapture(str(path)) + frames = [] + try: + if not reader.isOpened(): + raise ValueError("Training video could not be opened.") + metadata = tuple(reader.get(key) for key in ( + cv2.CAP_PROP_FRAME_COUNT, cv2.CAP_PROP_FRAME_HEIGHT, cv2.CAP_PROP_FRAME_WIDTH + )) + if any(not math.isfinite(value) or value != int(value) for value in metadata): + raise ValueError("Training video has invalid dimensions or frame count.") + expected_frames, height, width = map(int, metadata) + validate_video_shape((expected_frames, height, width, 3), config) + while True: + ok, frame = reader.read() + if not ok: + break + height, width, channels = frame.shape + # Reject oversized decoding before retaining another frame. + if channels != 3 or not 16 <= height <= 1024 or not 16 <= width <= 1024: + raise ValueError("Training video dimensions are unsupported.") + if len(frames) >= 241 or (len(frames) + 1) * height * width > MAX_DECODED_PIXELS: + raise ValueError("Training video exceeds the decoded size limit.") + if frames and frame.shape != frames[0].shape: + raise ValueError("Training video dimensions changed.") + frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + if len(frames) != expected_frames: + raise ValueError("Training video could not be decoded completely.") + shape = (len(frames), *frames[0].shape) if frames else (0, 0, 0, 3) + validate_video_shape(shape, config) + return np.stack(frames) + finally: + reader.release() + + +@dataclass(frozen=True) +class PreparedTrainingBatch: + clean: object + noise: object + context: object + sigma: object + source_identities: dict + clip_digest: str + caption_digest: str + + def tensors(self): + return self.clean, self.noise, self.context, self.sigma + + +def prepare_training_batch(path, clip, caption, *, seed=42, sigma=0.5): + """Encode one complete clip and its caption using local converted weights.""" + if ( + type(caption) is not str or not caption.strip() or len(caption) > 2000 + or any(ord(value) < 32 and value not in "\n\t" for value in caption) + or type(seed) is not int or not 0 <= seed < 2**32 + or type(sigma) not in (int, float) or not math.isfinite(sigma) or not 0 <= sigma <= 1 + ): + raise ValueError("Invalid training caption, seed or sigma.") + root, clip = Path(path).expanduser(), Path(clip).expanduser() + raw, identities = training_sources(root) + clip_digest = _digest_regular(clip, MAX_VIDEO_BYTES) + pixels = _decode_video(clip, raw) + + import mlx.core as mx + import numpy as np + from mlx_video.models.wan.loading import encode_text + from mlx_video.models.wan.text_encoder import T5Encoder + from mlx_video.models.wan.vae import WanVAE + from transformers import AutoTokenizer + + if not mx.metal.is_available() or mx.default_device() != mx.gpu: + raise ValueError("Wan MLX training requires the Metal GPU.") + config = _config(raw) + encoder = T5Encoder(vocab_size=config.t5_vocab_size, dim=config.t5_dim, + dim_attn=config.t5_dim_attn, dim_ffn=config.t5_dim_ffn, + num_heads=config.t5_num_heads, num_layers=config.t5_num_layers, + num_buckets=config.t5_num_buckets, shared_pos=False) + weights = mx.load(root / "t5_encoder.safetensors") + encoder.load_weights(list(weights.items()), strict=True) + mx.eval(encoder.parameters()) + del weights + tokenizer = AutoTokenizer.from_pretrained(root / "tokenizer", local_files_only=True, + trust_remote_code=False) + context = encode_text(encoder, tokenizer, caption, config.text_len) + mx.eval(context) + context = mx.array(np.asarray(context.astype(mx.float32)))[None, :, :] + del encoder, tokenizer + gc.collect() + mx.clear_cache() + + vae = WanVAE(z_dim=16, encoder=True) + weights = mx.load(root / "vae.safetensors") + # Converted VAE files contain learned tensors; normalization is model-defined. + for name in ("mean", "std", "inv_std"): + weights.setdefault(name, getattr(vae, name)) + vae.load_weights(list(weights.items()), strict=True) + mx.eval(vae.parameters()) + del weights + video = mx.array(pixels.astype(np.float32) / 127.5 - 1).transpose(3, 0, 1, 2)[None, :, :, :, :] + clean = vae.encode(video) + mx.eval(clean) + clean = mx.array(np.asarray(clean)) + del vae, video, pixels + gc.collect() + mx.clear_cache() + noise = mx.random.normal(clean.shape, key=mx.random.key(seed)) + flow_sigma = mx.array([sigma], dtype=mx.float32) + mx.eval(clean, noise, context, flow_sigma) + if any(not mx.all(mx.isfinite(value)).item() for value in (clean, noise, context)): + raise ValueError("Converted encoders produced non-finite training tensors.") + _unchanged(root, identities) + if _digest_regular(clip, MAX_VIDEO_BYTES) != clip_digest: + raise ValueError("Training media changed during encoding.") + return PreparedTrainingBatch(clean, noise, context, flow_sigma, identities, clip_digest, + "sha256:" + hashlib.sha256(caption.encode()).hexdigest()) diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/resources/references/model-families.md b/packages/codex/mlx-agent/src/mlx_agent/resources/references/model-families.md similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/resources/references/model-families.md rename to packages/codex/mlx-agent/src/mlx_agent/resources/references/model-families.md diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/resources/references/quantization.md b/packages/codex/mlx-agent/src/mlx_agent/resources/references/quantization.md similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/resources/references/quantization.md rename to packages/codex/mlx-agent/src/mlx_agent/resources/references/quantization.md diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/resources/references/troubleshooting.md b/packages/codex/mlx-agent/src/mlx_agent/resources/references/troubleshooting.md similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/resources/references/troubleshooting.md rename to packages/codex/mlx-agent/src/mlx_agent/resources/references/troubleshooting.md diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/resources/serve-recipes.json b/packages/codex/mlx-agent/src/mlx_agent/resources/serve-recipes.json similarity index 58% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/resources/serve-recipes.json rename to packages/codex/mlx-agent/src/mlx_agent/resources/serve-recipes.json index d297d5e..4f9fcb3 100644 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/resources/serve-recipes.json +++ b/packages/codex/mlx-agent/src/mlx_agent/resources/serve-recipes.json @@ -1,17 +1,26 @@ { "schema_version": "1.0", "recipes": { + "strata": { + "executable": "python", + "argv": ["{executable}", "{runtime_adapter}", "--model", "{repo}", "--port", "{port}", "--host", "127.0.0.1", "--max-tokens", "{max_tokens}"], + "adapter_argv": null, + "default_port": 8084, + "readiness": "http://127.0.0.1:{port}/v1/models", + "install_hint": "make install-strata in mlx-workbench" + }, "mlx_lm": { "executable": "mlx_lm.server", "argv": ["{executable}", "--model", "{repo}", "--port", "{port}", "--max-tokens", "{max_tokens}"], "adapter_argv": ["--adapter-path", "{adapter_path}"], + "thinking_off_argv": ["--chat-template-args", "{\"enable_thinking\":false}"], "default_port": 8080, "readiness": "http://127.0.0.1:{port}/v1/models", "install_hint": "pip install mlx-lm" }, "mlx-vlm": { "executable": "mlx_vlm.server", - "argv": ["{executable}", "--model", "{repo}", "--port", "{port}"], + "argv": ["{executable}", "--model", "{repo}", "--port", "{port}", "--host", "127.0.0.1"], "adapter_argv": null, "default_port": 8083, "readiness": "http://127.0.0.1:{port}/v1/models", diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/runtime_preference.py b/packages/codex/mlx-agent/src/mlx_agent/runtime_preference.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/runtime_preference.py rename to packages/codex/mlx-agent/src/mlx_agent/runtime_preference.py diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/scoring.py b/packages/codex/mlx-agent/src/mlx_agent/scoring.py similarity index 100% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/scoring.py rename to packages/codex/mlx-agent/src/mlx_agent/scoring.py diff --git a/packages/codex/mlx-agent/src/mlx_agent/serve.py b/packages/codex/mlx-agent/src/mlx_agent/serve.py new file mode 100644 index 0000000..e4a97cd --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/serve.py @@ -0,0 +1,972 @@ +"""Confirmation-gated launcher for local MLX serving runtimes. + +Serve is the only component that spawns processes, so every mutation goes +through the wire-style preview -> confirm -> receipt flow. Serve never +installs runtimes, never downloads models, binds only to 127.0.0.1, and stops +only processes that it started itself, verified against their receipts. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import secrets +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +from .model_doctor import scan_wired_configs +from .transactions import _atomic_in_directory +from .wiring import require_secret_free_config, validate_health_endpoint + + +SERVE_RECEIPT_SCHEMA_VERSION = "1.0" +SERVE_RECEIPT_KIND = "serve" +MAX_TOKENS_MIN, MAX_TOKENS_MAX, MAX_TOKENS_DEFAULT = 256, 65536, 8192 +PORT_MIN, PORT_MAX = 1, 65535 +READINESS_DEADLINE_DEFAULT = 60.0 +STOP_DEADLINE_SECONDS = 10.0 +MAX_RECEIPTS = 50 + +_RECIPES_PATH = Path(__file__).resolve().parent / "resources" / "serve-recipes.json" + + +class ServeError(RuntimeError): + """Classified serve failure safe to surface in a result envelope.""" + + def __init__(self, code, message, remediation): + super().__init__(message) + self.code = code + self.remediation = remediation + + +def load_recipes(path=None): + location = Path(path) if path is not None else _RECIPES_PATH + value = json.loads(location.read_text(encoding="utf-8")) + recipes = value.get("recipes") + if not isinstance(recipes, dict) or not recipes: + raise ValueError("serve recipe table is missing its recipes") + for name, recipe in recipes.items(): + for key in ("executable", "argv", "default_port", "readiness", "install_hint"): + if key not in recipe: + raise ValueError("serve recipe {0} is missing {1}".format(name, key)) + return recipes + + +def _utc_now(): + return datetime.now(timezone.utc).isoformat() + + +def _preview_hash(plan): + canonical = json.dumps( + {key: plan[key] for key in sorted(plan) if key != "preview_hash"}, + sort_keys=True, separators=(",", ":"), + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def receipts_root(root=None): + base = Path(root) if root is not None else Path.cwd() + return base / ".mlx-agent-receipts" / "serve" + + +def _local_weight_evidence(path): + """Describe only locally declared weight facts; never resolve or fetch a repo.""" + root = Path(path) + try: + location = root / "config.json" + if location.stat().st_size > 1024 * 1024: + return {} + raw = location.read_bytes() + config = json.loads(raw) + if not isinstance(config, dict): + return {} + evidence = {"model_config_fingerprint": hashlib.sha256(raw).hexdigest()} + if any(root.glob("*.safetensors")): + evidence["weight_format"] = "safetensors" + quantization = config.get("quantization") + if isinstance(quantization, dict): + mode = quantization.get("mode", "affine") + if mode not in ("affine", "mxfp4", "mxfp8", "nvfp4"): + mode = "quantized" + if config.get("vq_modules") or config.get("vq_ple"): + description = "VQ + per-tensor " + mode + elif any(isinstance(value, dict) for value in quantization.values()): + description = "per-tensor " + mode + else: + bits = quantization.get("bits") + description = mode + (" {}-bit".format(bits) if type(bits) is int and 1 <= bits <= 16 else "") + evidence["weight_quantization"] = description + return evidence + except (OSError, ValueError): + return {} + + +def resolve_serve_runtime(repo, path, runtime, recipes, hf_cache=None, + manifests=None, registries=None, backend_root=None): + """Select a declared local loader and its isolated executable; never fetch or import model code.""" + from .backends import backend_target, choose_backend, load_manifests, load_registries, lookup + from .convert import cached_snapshot + + manifests = load_manifests() if manifests is None else manifests + registries = load_registries(manifests, root=backend_root) if registries is None else registries + local = Path(path).expanduser() if path else cached_snapshot(repo, hf_cache) if repo else None + config = {} + try: + location = local / "config.json" + if location.stat().st_size > 1024 * 1024: + raise ValueError("config too large") + config = json.loads(location.read_text(encoding="utf-8")) + if not isinstance(config, dict): + raise ValueError("config must be an object") + except (AttributeError, OSError, ValueError, TypeError): + config = {} + if runtime == "auto": + raise ServeError("model_config_unavailable", "Automatic runtime selection needs a readable local config.json.", + "Select a downloaded local model or explicitly review its serving runtime.") from None + if runtime == "strata" or (runtime == "auto" and config.get("serving_runtime") == "strata"): + from .strata_runtime import StrataError, installed_runtime, read_model + try: + if path is None: + raise StrataError("strata_model_required", "Strata serves prepared local GGUF profiles only.") + read_model(path) + return "strata", installed_runtime()["python"] + except StrataError as error: + raise ServeError(error.code, str(error), error.remediation) from error + hits = lookup(config.get("model_type"), manifests, registries) if config.get("model_type") else [] + if runtime == "auto": + backend = choose_backend(hits, "text_llm", "vision_config" in config) + if backend not in ("mlx-lm", "mlx-vlm"): + raise ServeError("unsupported_runtime", "No declared serving backend implements this model architecture.", + "Install a backend with a vetted model adapter; remote runtime code is never executed.") + runtime = "mlx_lm" if backend == "mlx-lm" else backend + else: + backend = "mlx-lm" if runtime == "mlx_lm" else runtime + if config.get("model_type") and not any(hit["backend"] == backend for hit in hits): + raise ServeError("unsupported_runtime", "The selected runtime does not implement this model architecture.", + "Review a fresh serve plan with --runtime auto.") + recipe = recipes.get(runtime) + if recipe is None: + raise ServeError("unsupported_runtime", "No serving recipe exists for this runtime.", "Choose a declared serving runtime.") + manifest = manifests[backend] + if not manifest["builtin"] and not registries.get(backend, {}).get("installed"): + raise ServeError("runtime_not_installed", "The declared {0} backend is not installed.".format(backend), + "Preview and confirm backend install {0}, then review serving again.".format(backend)) + # Optional backends are deliberately outside PATH. Bind the exact reviewed + # executable into both eager and JIT plans instead of changing global PATH. + executable = recipe["executable"] if manifest["builtin"] else str( + backend_target(manifest, backend_root) / "bin" / recipe["executable"]) + return runtime, executable + + +def _adapter_evidence(path): + from .lora import adapter_identity + try: + local = Path(path).expanduser().absolute() + identity = adapter_identity(local) + except (TypeError, ValueError, OSError): + identity = None + if identity is None: + raise ServeError("invalid_adapter", "Adapter files are missing, invalid or linked.", + "Select a completed adapter with adapter_config.json and adapters.safetensors.") + return str(local), identity + + +def _require_adapter_identity(plan): + if plan.get("adapter_path") is None: + return + try: + path, identity = _adapter_evidence(plan["adapter_path"]) + if path == plan["adapter_path"] and identity == plan.get("adapter_identity"): + return + except ServeError: + pass + raise ServeError("adapter_changed", "Adapter files changed after the serving preview.", + "Review a fresh serve plan before starting this adapter.") + + +def _require_training_binding(plan): + binding = plan.get("training_binding") + if binding is None: + return + try: + from .training_artifacts import read_trained_adapter + _, current = read_trained_adapter(binding["training_receipt"]) + if current == binding: + return + except (OSError, ValueError, TypeError, KeyError): + pass + raise ServeError("training_receipt_changed", "The training receipt or its pinned artifacts changed.", + "Review a fresh serving preview for the completed training receipt.") + + +def plan_start(repo, runtime, recipes, port=None, max_tokens=MAX_TOKENS_DEFAULT, + adapter_path=None, path=None, jit=False, receipts_dir=None, hf_cache=None, memory_policy=None, + runtime_executable=None, strata_speculation=False, + strata_expert_memory_gb=None, strata_context_tokens=None, training_receipt=None, thinking="default"): + """Render the start plan and local file identities without side effects.""" + if thinking not in ("default", "off"): + raise ServeError("invalid_arguments", "thinking must be default or off.", "Pass --thinking off or omit it.") + binding = None + if training_receipt is not None: + if repo is not None or path is not None or adapter_path is not None: + raise ServeError("invalid_arguments", "A training receipt selects its own base model and adapter.", + "Use --training-receipt without --repo, --path or --adapter-path.") + if runtime != "mlx_lm": + raise ServeError("unsupported_runtime", "Text training receipts require the MLX-LM serving runtime.", + "Use --runtime auto or mlx_lm.") + try: + from .training_artifacts import read_trained_adapter + _, binding = read_trained_adapter(training_receipt) + except (OSError, ValueError, TypeError, KeyError) as error: + raise ServeError("invalid_training_receipt", str(error), "Select a completed receipt whose base and adapter files are unchanged.") from error + path, adapter_path = binding["base_path"], binding["adapter_path"] + if memory_policy is not None: + from .jit_serve import validate_memory_policy + try: + memory_policy = validate_memory_policy(memory_policy) + except ValueError as error: + raise ServeError("invalid_arguments", str(error), "Correct the memory policy.") from error + if not jit: + raise ServeError("jit_required", "Memory policy requires a JIT endpoint.", "Use --jit.") + has_repo = isinstance(repo, str) and bool(repo.strip()) + has_path = isinstance(path, str) and bool(path.strip()) + if has_repo == has_path: + raise ServeError( + "invalid_arguments", + "serve requires exactly one of a repository identifier or a local path.", + "Pass --repo as publisher/model from the Hugging Face cache, or --path to a local model directory.", + ) + if has_repo and "/" not in repo: + raise ServeError( + "invalid_repo", + "serve requires a publisher/model repository identifier.", + "Pass --repo as publisher/model exactly as it appears in the Hugging Face cache.", + ) + local_path = None + if has_path: + local_path = os.path.abspath(os.path.expanduser(path.strip())) + recipe = recipes.get(runtime) + if recipe is None: + raise ServeError( + "unsupported_runtime", + "serve does not launch the {0} runtime.".format(runtime), + "Ollama and LM Studio manage their own servers; serve supports: {0}.".format( + ", ".join(sorted(recipes)) + ), + ) + if not isinstance(max_tokens, int) or isinstance(max_tokens, bool): + raise ServeError( + "invalid_arguments", "max_tokens must be an integer.", + "Pass --max-tokens between {0} and {1}.".format(MAX_TOKENS_MIN, MAX_TOKENS_MAX), + ) + if not MAX_TOKENS_MIN <= max_tokens <= MAX_TOKENS_MAX: + raise ServeError( + "invalid_arguments", + "max_tokens is outside {0}-{1}.".format(MAX_TOKENS_MIN, MAX_TOKENS_MAX), + "Pass a bounded --max-tokens value.", + ) + selected_port = recipe["default_port"] if port is None else port + if not isinstance(selected_port, int) or isinstance(selected_port, bool): + raise ServeError( + "invalid_arguments", "port must be an integer.", + "Pass --port between {0} and {1}.".format(PORT_MIN, PORT_MAX), + ) + if not PORT_MIN <= selected_port <= PORT_MAX: + raise ServeError( + "invalid_arguments", + "port is outside {0}-{1}.".format(PORT_MIN, PORT_MAX), + "Pass a valid loopback port.", + ) + if adapter_path is not None and not recipe.get("adapter_argv"): + raise ServeError( + "unsupported_runtime", + "The {0} recipe does not support adapter serving.".format(runtime), + "Serve adapters with a runtime whose recipe declares adapter support.", + ) + adapter_identity = None + if adapter_path is not None: + adapter_path, adapter_identity = _adapter_evidence(adapter_path) + model_value = repo.strip() if has_repo else local_path + values = { + "executable": runtime_executable or recipe["executable"], + "repo": model_value, + "port": str(selected_port), + "max_tokens": str(max_tokens), + "runtime_adapter": str(Path(__file__).with_name("strata_runtime.py").resolve()), + } + strata_model = strata_runtime = None + if runtime != "strata" and (strata_expert_memory_gb is not None or strata_context_tokens is not None): + raise ServeError("invalid_arguments", "Strata memory controls require the Strata runtime.", + "Select a prepared local Strata profile.") + if runtime == "strata": + from .strata_runtime import StrataError, installed_runtime, read_model, memory_estimate + try: + if not has_path or adapter_path is not None or type(strata_speculation) is not bool: + raise StrataError("strata_model_required", "Strata requires a local GGUF profile without a LoRA adapter.") + strata_model = read_model(local_path) + strata_context_tokens = 131072 if strata_context_tokens is None else strata_context_tokens + strata_memory_bytes = memory_estimate(strata_model, strata_expert_memory_gb, strata_context_tokens) + strata_runtime = installed_runtime() + if runtime_executable is not None and runtime_executable != strata_runtime["python"]: + raise StrataError("strata_not_installed", "The reviewed Strata executable changed.") + values["executable"] = strata_runtime["python"] + except StrataError as error: + raise ServeError(error.code, str(error), error.remediation) from error + argv = [part.format(**values) for part in recipe["argv"]] + if strata_model: + argv += ["--runtime-root", strata_runtime["root"], "--speculation", "on" if strata_speculation else "off", + "--context-tokens", str(strata_context_tokens), + "--expected-model-fingerprint", strata_model["fingerprint"], + "--expected-runtime-fingerprint", strata_runtime["fingerprint"]] + if strata_expert_memory_gb is not None: + argv += ["--expert-memory-gb", str(strata_expert_memory_gb)] + if adapter_path is not None: + argv.extend( + part.format(adapter_path=str(adapter_path)) for part in recipe["adapter_argv"] + ) + if thinking == "off" and recipe.get("thinking_off_argv"): + # Literal arguments: the chat-template JSON is not a format template. + argv.extend(recipe["thinking_off_argv"]) + plan = { + "repo": repo.strip() if has_repo else None, + "path": local_path, + "runtime": runtime, + "port": selected_port, + "max_tokens": max_tokens, + "adapter_path": str(adapter_path) if adapter_path is not None else None, + "argv": argv, + "readiness": recipe["readiness"].format(port=selected_port), + "bind": "127.0.0.1", + } + if strata_model: + plan.update(strata_model_fingerprint=strata_model["fingerprint"], + strata_runtime_fingerprint=strata_runtime["fingerprint"], + weight_format="gguf", weight_quantization=strata_model["quantization"], weights_preserved=True, + strata_speculation=strata_speculation, strata_expert_memory_gb=strata_expert_memory_gb, + strata_context_tokens=strata_context_tokens, strata_memory_estimate_bytes=strata_memory_bytes, + readiness_deadline=900) + elif has_path: + plan.update(_local_weight_evidence(local_path)) + if adapter_path is not None: + plan["adapter_identity"] = adapter_identity + if thinking == "off": + plan["thinking"] = "off" if recipe.get("thinking_off_argv") else "unsupported" + if binding is not None: + plan["training_binding"] = binding + if jit: + from .convert import cached_snapshot + from .jit_serve import model_fingerprint + local = Path(local_path) if local_path else cached_snapshot(repo.strip(), hf_cache) + if local is None or not local.is_dir(): + raise ServeError("model_not_local", "JIT requires existing local model files.", + "Select an existing local model directory; serving never downloads weights.") + local = local.resolve() + worker = list(argv) + worker[worker.index("--model") + 1] = str(local) + worker[worker.index("--port") + 1] = "{worker_port}" + # Both supported runtimes expose --host. The VLM default binds all + # interfaces, so explicitly fence the private worker to loopback. + if "--host" not in worker: + worker.extend(["--host", "127.0.0.1"]) + root = receipts_root(receipts_dir).absolute() + plan.update(jit=True, local_path=str(local), fingerprint=model_fingerprint(local), + worker_argv=worker, control_config=str(root / "{}.jit-config.json".format(selected_port))) + plan["argv"] = [sys.executable, str(Path(__file__).with_name("jit_serve.py").resolve()), + "--model", model_value, "--port", str(selected_port), + "--config", plan["control_config"]] + if memory_policy is not None: + plan["memory_policy"] = memory_policy + plan["preview_hash"] = _preview_hash(plan) + return plan + + +def _default_which(executable): + return shutil.which(executable) + + +def _default_port_free(port): + import socket + + probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + probe.settimeout(0.5) + return probe.connect_ex(("127.0.0.1", port)) != 0 + finally: + probe.close() + + +def _default_spawn(argv, log_path): + handle = open(log_path, "ab", buffering=0) + process = subprocess.Popen( + argv, + stdin=subprocess.DEVNULL, + stdout=handle, + stderr=subprocess.STDOUT, + start_new_session=True, + env={**os.environ, "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1"}, + ) + return process.pid + + +def _default_readiness(url, deadline_seconds, clock=time.monotonic, sleep=time.sleep, child_pid=None): + validate_health_endpoint(url) + deadline = clock() + deadline_seconds + while clock() < deadline: + if child_pid is not None: + try: + if os.waitpid(child_pid, os.WNOHANG)[0] == child_pid: + return False + except ChildProcessError: + if not _pid_alive(child_pid): + return False + try: + request = urllib.request.Request(url, method="GET") + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + with opener.open(request, timeout=2.0) as response: + if 200 <= response.status < 400: + return True + except (OSError, ValueError, urllib.error.URLError): + pass + sleep(0.5) + return False + + +def _pid_alive(pid): + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + return True + + +def _pid_command(pid): + try: + result = subprocess.run( + ["ps", "-p", str(pid), "-o", "command="], + capture_output=True, text=True, timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + return None + return result.stdout.strip() or None + + +def _default_path_present(path): + return Path(path).is_dir() + + +def start_serve(plan, receipts_dir=None, confirm=False, preview_hash=None, + which=None, model_present=None, path_present=None, port_free=None, + wired_claims=None, spawn=None, readiness=None, + readiness_deadline=READINESS_DEADLINE_DEFAULT, now=_utc_now, + pid_alive=None): + """Execute a reviewed start plan; the only mutating serve entry point.""" + which = which or _default_which + port_free = port_free or _default_port_free + spawn = spawn or _default_spawn + readiness = readiness or (lambda url, deadline: _default_readiness(url, deadline, child_pid=pid if spawn is _default_spawn else None)) + pid_alive = pid_alive or _pid_alive + root = receipts_root(receipts_dir) + + if not confirm: + return {"status": "preview", "plan": plan, "requires_confirmation": True} + if not preview_hash: + raise ServeError( + "preview_hash_required", + "--confirm requires the hash from a reviewed serve preview.", + "Run serve start without --confirm, inspect the plan, then pass --preview-hash.", + ) + if preview_hash != plan["preview_hash"]: + raise ServeError( + "preview_stale", + "The supplied preview hash does not match this start plan.", + "Re-run serve start without --confirm and review the fresh plan.", + ) + _require_training_binding(plan) + _require_adapter_identity(plan) + + if plan["runtime"] == "strata" and not plan.get("jit"): + from .jit_serve import available_memory_bytes + available = available_memory_bytes() + required = plan["strata_memory_estimate_bytes"] + 2_000_000_000 + if type(available) is not int or available < 0: + raise ServeError("memory_unknown", "Available memory could not be measured.", + "Retry after memory reporting is available.") + if available < required: + raise ServeError("insufficient_headroom", "Insufficient memory for the reviewed Strata settings and 2 GB reserve.", + "Reduce the expert budget or context capacity, or free memory.") + + if which(plan["argv"][0]) is None: + recipe_hint = load_recipes()[plan["runtime"]]["install_hint"] + raise ServeError( + "runtime_not_installed", + "The {0} executable is not installed.".format(plan["argv"][0]), + "Install it yourself ({0}); serve never installs runtimes.".format(recipe_hint), + ) + if plan.get("jit"): + from .jit_serve import model_fingerprint + if which(plan["worker_argv"][0]) is None: + raise ServeError("runtime_not_installed", "The model worker executable is not installed.", + load_recipes()[plan["runtime"]]["install_hint"]) + if model_fingerprint(plan["local_path"]) != plan["fingerprint"]: + raise ServeError("preview_stale", "Local model files changed after preview.", + "Review a fresh JIT serve plan.") + if plan["repo"] is not None and model_present is not None and not model_present(plan["repo"]): + raise ServeError( + "model_not_local", + "The model is not present in a local inventory.", + "Download it with the runtime's own pull command first; serve never downloads models.", + ) + if plan["path"] is not None: + present = path_present or _default_path_present + if not present(plan["path"]): + raise ServeError( + "model_not_local", + "The model directory is not present: {0}".format(plan["path"]), + "Point --path at an existing local model directory; serve never downloads models.", + ) + if not port_free(plan["port"]): + raise ServeError( + "port_in_use", + "Port {0} is already bound on 127.0.0.1.".format(plan["port"]), + "Pick a free --port, or stop the process that owns this one.", + ) + if wired_claims is not None: + claimed = wired_claims(plan["port"], plan["runtime"]) + if claimed: + raise ServeError( + "port_in_use", + "A wired config already claims port {0}.".format(plan["port"]), + "Move this server to its own --port.", + ) + receipt_path = root / "{0}.json".format(plan["port"]) + if receipt_path.exists(): + existing = _read_receipt(receipt_path) + if existing is not None and (pid_alive(existing.get("pid", -1)) or _owned_jit_worker(existing, root)): + raise ServeError( + "port_in_use", + "A serve receipt for port {0} is still live.".format(plan["port"]), + "Run serve stop --port {0} first.".format(plan["port"]), + ) + + root.mkdir(parents=True, exist_ok=True) + log_path = root / "{0}.log".format(plan["port"]) + if plan.get("jit"): + config_path = Path(plan["control_config"]) + if config_path.parent != root.absolute() or config_path.is_symlink(): + raise ServeError("invalid_arguments", "Unsafe JIT configuration path.", "Review a fresh serve plan.") + config = {key: plan[key] for key in ("local_path", "fingerprint", "worker_argv", "max_tokens")} + if plan.get("adapter_path") is not None: + config.update(adapter_path=plan["adapter_path"], adapter_identity=plan["adapter_identity"]) + if plan.get("training_binding") is not None: + config["training_binding"] = plan["training_binding"] + if "readiness_deadline" in plan: + config["worker_readiness_seconds"] = plan["readiness_deadline"] + if plan["runtime"] == "strata": + config["runtime"] = "strata" + config["strata_expert_memory_gb"] = plan.get("strata_expert_memory_gb") + config["strata_context_tokens"] = plan.get("strata_context_tokens", 131072) + config.update(model=plan["repo"] or plan["path"], control_token=secrets.token_hex(32), + memory_policy=plan.get("memory_policy"), config_path=str(config_path), + state_path=str(root.absolute() / "{}.jit-state.json".format(plan["port"])), + log_path=str(root.absolute() / "{}.worker.log".format(plan["port"]))) + _atomic_in_directory(root, config_path.name, json.dumps(config).encode(), 0o600) + pid = spawn(plan["argv"], str(log_path)) + if not readiness(plan["readiness"], plan.get("readiness_deadline", readiness_deadline)): + _terminate_pid(pid) + raise ServeError( + "readiness_timeout", + "The server did not answer its readiness endpoint in time.", + "Inspect the serve log at {0}.".format(log_path), + ) + try: + _require_training_binding(plan) + _require_adapter_identity(plan) + except ServeError: + _terminate_pid(pid) + raise + + receipt = { + "schema_version": SERVE_RECEIPT_SCHEMA_VERSION, + "kind": SERVE_RECEIPT_KIND, + "repo": plan["repo"], + "path": plan["path"], + "runtime": plan["runtime"], + "port": plan["port"], + "argv": list(plan["argv"]), + "pid": pid, + "log_path": str(log_path), + "readiness": plan["readiness"], + "started_at": now(), + "preview_hash": plan["preview_hash"], + } + if plan.get("jit"): + receipt["jit"] = True + if plan.get("adapter_path") is not None: + receipt.update(adapter_path=plan["adapter_path"], adapter_identity=plan["adapter_identity"]) + if plan.get("training_binding") is not None: + receipt["training_binding"] = plan["training_binding"] + content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + _atomic_in_directory(root, receipt_path.name, content, 0o600) + return {"status": "started", "receipt": receipt} + + +def _read_receipt(path): + try: + value = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError, json.JSONDecodeError): + return None + if not isinstance(value, dict) or value.get("kind") != SERVE_RECEIPT_KIND: + return None + if not isinstance(value.get("pid"), int) or isinstance(value.get("pid"), bool): + return None + if not isinstance(value.get("argv"), list) or not isinstance(value.get("port"), int): + return None + return value + + +def status_serve(receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command): + """Cross-check serve receipts against live processes; read-only.""" + root = receipts_root(receipts_dir) + entries = [] + if not root.is_dir(): + return entries + for path in sorted(root.glob("*.json")): + if len(entries) >= MAX_RECEIPTS: + break + receipt = _read_receipt(path) + if receipt is None: + continue + alive = pid_alive(receipt["pid"]) + command = pid_command(receipt["pid"]) if alive else None + entries.append({ + "receipt": str(path), + "repo": receipt["repo"], + "path": receipt.get("path"), + "runtime": receipt["runtime"], + "port": receipt["port"], + "pid": receipt["pid"], + "alive": alive, + "argv_match": _argv_matches(receipt, command) if alive else False, + "log_path": receipt.get("log_path"), + "started_at": receipt.get("started_at"), + }) + entries[-1].update(_adapter_metadata(receipt, root)) + if receipt.get("training_binding") is not None: + entries[-1]["training_binding"] = receipt["training_binding"] + if receipt.get("jit"): + entries[-1]["jit"] = True + if alive and entries[-1]["argv_match"]: + try: + entries[-1].update(_jit_control(receipt, root, "status")) + except ServeError: + entries[-1]["model_state"] = "unknown" + else: + entries[-1]["model_state"] = "unknown" + return entries + + +def _adapter_metadata(receipt, root): + if receipt.get("adapter_path") is not None: + return {"adapter_path": receipt["adapter_path"], "adapter_identity": receipt.get("adapter_identity")} + argv = receipt.get("argv", []) + if receipt.get("jit"): + try: + argv = _jit_config(receipt, root)["worker_argv"] + except (ServeError, KeyError): + return {"adapter_status": "unknown"} + if not isinstance(argv, list) or any(not isinstance(value, str) for value in argv): + return {"adapter_status": "unknown"} + if "--adapter-path" in argv: + index = argv.index("--adapter-path") + 1 + if index >= len(argv) or not argv[index]: + return {"adapter_status": "unknown"} + return {"adapter_path": argv[index], "adapter_identity": None} + return {} + + +def _argv_matches(receipt, command, require_port=True): + if not command: + return False + argv = receipt.get("argv") or [] + if not argv: + return False + executable = Path(str(argv[0])).name + # macOS framework Python reports its bundle binary in ps, not the + # interpreter alias used to launch it. Bind a script recipe to its + # absolute entry point instead of accepting any Python process. + if len(argv) > 1 and str(argv[1]).endswith(".py") and Path(str(argv[1])).is_absolute(): + executable = str(argv[1]) + model = receipt.get("repo") or receipt.get("path") + if not model or executable not in command or str(model) not in command: + return False + if receipt.get("kind") == SERVE_RECEIPT_KIND and not receipt.get("jit") and "--adapter-path" in argv: + index = argv.index("--adapter-path") + 1 + if index >= len(argv) or "--adapter-path" not in command or str(argv[index]) not in command: + return False + if receipt.get("jit"): + if "--config" not in argv or str(argv[argv.index("--config") + 1]) not in command: + return False + if require_port: + return "--port {0}".format(receipt.get("port")) in command + return True + + +def _terminate_pid(pid, sig=signal.SIGTERM): + try: + os.kill(pid, sig) + except OSError: + return False + return True + + +def _jit_config(receipt, root): + path = root / "{}.jit-config.json".format(receipt["port"]) + if path.is_symlink(): + raise ServeError("invalid_receipt", "JIT configuration is a symbolic link.", "Inspect the owned receipt directory.") + try: + config = json.loads(path.read_text()) + if not isinstance(config, dict) or not isinstance(config.get("control_token"), str) or len(config["control_token"]) < 16: + raise ValueError("Invalid control token.") + return config + except (OSError, ValueError) as error: + raise ServeError("invalid_receipt", "JIT control configuration is unavailable.", + "Stop and review a fresh endpoint.") from error + + +def _jit_control(receipt, root, action, policy=None): + config = _jit_config(receipt, root) + request = urllib.request.Request("http://127.0.0.1:{}/_mlx/{}".format(receipt["port"], action), + data=json.dumps(policy or {}).encode() if action in ("unload", "policy") else None, + headers={"Authorization": "Bearer " + config["control_token"], "Content-Type": "application/json"}) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + try: + with opener.open(request, timeout=10 if action == "unload" else 2) as response: + state = json.load(response) + if not isinstance(state, dict) or state.get("model_state") not in ("loaded", "unloaded", "loading", "unloading", "failed"): + raise ValueError("Invalid model residency status.") + return state + except urllib.error.HTTPError as error: + try: + detail = json.load(error)["error"] + except (ValueError, KeyError, TypeError): + detail = {"code": "control_failed", "message": "JIT control request failed."} + raise ServeError(detail["code"], detail["message"], "Finish active requests, refresh status and retry.") from error + except (OSError, ValueError) as error: + raise ServeError("control_unavailable", "JIT control status is unavailable.", + "Refresh endpoint status before retrying.") from error + + +def _owned_jit_worker(receipt, root): + if not receipt.get("jit"): + return None + path = root / "{}.jit-state.json".format(receipt["port"]) + if path.is_symlink(): + return None + try: + state = json.loads(path.read_text()) + pid, argv = state["worker_pid"], state["worker_argv"] + if not isinstance(pid, int) or pid <= 0 or state["gateway_pid"] != receipt["pid"]: + return None + config = _jit_config(receipt, root) + expected = {"repo": None, "path": config["local_path"], "argv": argv, + "port": int(argv[argv.index("--port") + 1])} + if _pid_alive(pid) and os.getpgid(pid) == receipt["pid"] and _argv_matches(expected, _pid_command(pid)): + return pid + except (OSError, ValueError, TypeError, KeyError, ServeError): + pass + return None + + +def unload_serve(port, receipts_dir=None, expected_pid=None): + return _configure_owned_jit(port, receipts_dir, expected_pid, "unload") + + +def configure_serve_memory(port, policy, receipts_dir=None, expected_pid=None): + from .jit_serve import validate_memory_policy + try: + policy = validate_memory_policy(policy) + except ValueError as error: + raise ServeError("invalid_arguments", str(error), "Correct the memory policy.") from error + return _configure_owned_jit(port, receipts_dir, expected_pid, "policy", policy) + + +def _configure_owned_jit(port, receipts_dir, expected_pid, action, policy=None): + root = receipts_root(receipts_dir) + receipt = _read_receipt(root / "{}.json".format(port)) + if not receipt or not receipt.get("jit"): + raise ServeError("jit_required", "This endpoint does not support model unload.", + "Enable Load on request for this endpoint first.") + if expected_pid is not None and receipt["pid"] != expected_pid: + raise ServeError("pid_argv_mismatch", "Endpoint identity changed.", "Refresh serving status before unloading.") + if not _pid_alive(receipt["pid"]) or not _argv_matches(receipt, _pid_command(receipt["pid"])): + raise ServeError("pid_argv_mismatch", "Endpoint process no longer matches its receipt.", + "Refresh serving status before unloading.") + state = _jit_control(receipt, root, action, policy) + return dict(state, status="unloaded" if action == "unload" else "configured", port=port, pid=receipt["pid"]) + + +def stop_serve(port, receipts_dir=None, pid_alive=_pid_alive, pid_command=_pid_command, + terminate=_terminate_pid, clock=time.monotonic, sleep=time.sleep): + """Stop exactly the process a serve receipt owns; never kills by port scan.""" + root = receipts_root(receipts_dir) + receipt_path = root / "{0}.json".format(port) + receipt = _read_receipt(receipt_path) if receipt_path.is_file() else None + if receipt is None: + raise ServeError( + "receipt_not_found", + "No serve receipt exists for port {0}.".format(port), + "serve stop only stops processes that serve started; stop foreign processes yourself.", + ) + pid = receipt["pid"] + orphan = _owned_jit_worker(receipt, root) if not pid_alive(pid) else None + if orphan: + pid = orphan + if not pid_alive(pid): + receipt_path.unlink() + return {"status": "already_stopped", "port": port, "pid": pid} + command = pid_command(pid) + if not orphan and not _argv_matches(receipt, command): + raise ServeError( + "pid_argv_mismatch", + "The live pid {0} does not match the serve receipt; refusing to signal it.".format(pid), + "Inspect the process yourself; the receipt is retained at {0}.".format(receipt_path), + ) + if (receipt.get("jit") or receipt.get("runtime") == "strata") and terminate is _terminate_pid and os.getpgid(pid) == receipt["pid"]: + terminate = lambda _pid, sig: os.killpg(receipt["pid"], sig) + terminate(pid, signal.SIGTERM) + deadline = clock() + STOP_DEADLINE_SECONDS + while clock() < deadline: + if not pid_alive(pid): + receipt_path.unlink() + return {"status": "stopped", "port": port, "pid": pid} + sleep(0.2) + terminate(pid, signal.SIGKILL) + if pid_alive(pid): + raise ServeError( + "stop_failed", + "The process did not exit after SIGTERM and SIGKILL.", + "Inspect pid {0} yourself; the receipt is retained at {1}.".format(pid, receipt_path), + ) + receipt_path.unlink() + return {"status": "stopped", "port": port, "pid": pid, "forced": True} + + +def wired_port_claim(roots): + """Build a wired_claims(port, runtime) gate from Wire-managed configs.""" + wired = scan_wired_configs(roots) + claims = {} + for config in wired: + endpoint = config.get("endpoint") + if not endpoint: + continue + try: + parsed = validate_health_endpoint(endpoint) + from urllib.parse import urlsplit + + parts = urlsplit(parsed) + port = parts.port or (443 if parts.scheme == "https" else 80) + except ValueError: + continue + claims.setdefault(port, config["runtime"]) + + def check(port, runtime): + owner = claims.get(port) + return owner is not None and owner != runtime + + return check + + +def default_model_present(hf_cache=None): + from .model_doctor import default_hf_cache, inspect_hf_cache + + cache = Path(hf_cache) if hf_cache is not None else default_hf_cache() + known = {item["id"].casefold() for item in inspect_hf_cache(cache)} + + def check(repo): + return repo.casefold() in known + + return check + + +LAUNCHD_LABEL_PREFIX = "com.mlx-agent.serve." +_LAUNCHD_LABEL = re.compile(r"\A" + LAUNCHD_LABEL_PREFIX.replace(".", r"\.") + r"(\d{1,5})\Z") + + +def launchd_label(port): + return "{0}{1}".format(LAUNCHD_LABEL_PREFIX, port) + + +def render_launchd_plist(plan, log_path=None): + """Render a deterministic launchd plist for a reviewed serve plan.""" + from xml.sax.saxutils import escape + + label = launchd_label(plan["port"]) + log = log_path or str(receipts_root() / "{0}.log".format(plan["port"])) + arguments = "\n".join( + " {0}".format(escape(str(part))) for part in plan["argv"] + ) + return ( + '\n' + '\n' + '\n' + '\n' + ' Label\n' + ' {label}\n' + ' ProgramArguments\n' + ' \n' + '{arguments}\n' + ' \n' + ' RunAtLoad\n' + ' \n' + ' KeepAlive\n' + ' \n' + ' StandardOutPath\n' + ' {log}\n' + ' StandardErrorPath\n' + ' {log}\n' + '\n' + '\n' + ).format(label=escape(label), arguments=arguments, log=escape(str(log))) + + +class LaunchdPlistAdapter: + """Validate the exact launchd plist subset serve renders.""" + + version = "1.0" + runtime = "launchd" + + def __init__(self, path=None): + self.path = Path(path) if path is not None else None + + def validate(self, content): + if not isinstance(content, str): + raise TypeError("plist content must be text") + require_secret_free_config(content) + lines = content.splitlines() + if len(lines) < 8 or not lines[0].startswith("': + raise ValueError("launchd plist must use the managed XML subset") + if lines[-1] != "" or lines[-2] != "": + raise ValueError("launchd plist must close its dict and plist elements") + text = content + for required in ("Label", "ProgramArguments", "RunAtLoad"): + if required not in text: + raise ValueError("launchd plist is missing {0}".format(required)) + label_match = re.search(r"Label\s*\n\s*([^<]+)", text) + if label_match is None or _LAUNCHD_LABEL.fullmatch(label_match.group(1)) is None: + raise ValueError("launchd plist label must use the managed serve prefix") + return True diff --git a/packages/codex/mlx-agent/src/mlx_agent/speak.py b/packages/codex/mlx-agent/src/mlx_agent/speak.py new file mode 100644 index 0000000..dc635fd --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/speak.py @@ -0,0 +1,133 @@ +"""Synthesize one text with a converted text-to-speech model through its backend (writes only the WAV).""" + +from __future__ import annotations + +import json +import math +import re +import subprocess +from pathlib import Path + +from .backends import ( + backend_python, + is_installed, + load_manifests, + load_registries, + lookup, +) +from .inference_runner import InferenceError, RunnerContract, invoke + +SPEAK_SCHEMA = "speak/1" +SPEAK_RUNNER = Path(__file__).resolve().with_name("speak_runner.py") +DEFAULT_TIMEOUT_SECONDS = 600 +MAX_TEXT_CHARS = 2000 +MIN_SPEED, MAX_SPEED = 0.5, 2.0 +_VOICE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}") +_LANG_CODE = re.compile(r"[A-Za-z]{1,8}(?:[-_][A-Za-z0-9]{1,8})?") + + +class SpeakError(InferenceError): + """Classified speech-synthesis failure safe to surface in a result envelope.""" + + +def _invalid(message, remediation): + return SpeakError("invalid_arguments", message, remediation) + + +def _type_candidates(config, model): + """Model types to try: the config's own, or (when it names none) the first token of the directory name.""" + declared = [value for value in (config.get("model_type"), config.get("architecture")) if isinstance(value, str) and value] + if declared: + return declared + name = model.name + if model.parent.name == "snapshots": + name = model.parent.parent.name.split("--")[-1] + token = next((part for part in re.split(r"[^a-z0-9]+", name.lower()) if part), None) + return [token] if token else [] + + +def plan_speak(model_path, text, out, voice=None, speed=1.0, lang_code=None, + manifests=None, registries=None, root=None): + """Pick the speech backend, reading the local config and voice availability.""" + model = Path(model_path).expanduser() + try: + config = json.loads((model / "config.json").read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise SpeakError( + "model_unreadable", "No readable config.json in {0}: {1}".format(model, error), + "Pass the converted model directory.", + ) from error + if not isinstance(config, dict): + raise SpeakError("model_unreadable", "config.json in {0} is not an object.".format(model), "Pass the converted model directory.") + if not isinstance(text, str) or not text.strip() or len(text) > MAX_TEXT_CHARS or any( + ord(character) < 32 and character not in "\n\t" for character in text + ): + raise _invalid("The text must be 1..{0} characters of plain text.".format(MAX_TEXT_CHARS), "Pass --text with a sentence to speak.") + target = Path(out).expanduser() + if not target.is_absolute() or target.suffix.lower() != ".wav" or target.exists(): + raise _invalid("--out must be a new absolute .wav path.", "Pass a path such as /Users/me/Music/speech.wav.") + if not target.parent.is_dir(): + raise _invalid("The directory of --out does not exist.", "Create the directory or pass a path inside an existing one.") + if voice is not None and (not isinstance(voice, str) or not _VOICE.fullmatch(voice)): + raise _invalid("--voice must be a short voice name such as af_heart.", "Pass --voice with a name the model lists.") + if isinstance(speed, bool) or not isinstance(speed, (int, float)) or not math.isfinite(speed) or not MIN_SPEED <= speed <= MAX_SPEED: + raise _invalid("--speed must be {0}..{1}.".format(MIN_SPEED, MAX_SPEED), "Pass --speed 1.0, for example.") + if lang_code is not None and (not isinstance(lang_code, str) or not _LANG_CODE.fullmatch(lang_code)): + raise _invalid("--lang-code must be a short language code such as a or en.", "Pass --lang-code en, for example.") + manifests = load_manifests() if manifests is None else manifests + registries = load_registries(manifests, root) if registries is None else registries + candidates = _type_candidates(config, model) + hits = [] + for candidate in candidates: + hits = sorted( + (hit for hit in lookup(candidate, manifests, registries) if hit["category"] == "text_to_speech"), + key=lambda hit: hit["backend"], + ) + if hits: + break + if not hits: + raise SpeakError( + "not_tts_model", "No text-to-speech backend implements model_type {0!r}.".format(candidates[0] if candidates else None), + "Speech synthesis applies to converted text-to-speech models such as Kokoro.", + ) + manifest = manifests[hits[0]["backend"]] + if not is_installed(manifest, root): + raise SpeakError( + "backend_not_installed", "The {0} backend is not installed.".format(manifest["id"]), + "Install it first: mlx-agent backend install {0}.".format(manifest["id"]), + ) + model_type = hits[0]["module"].rsplit(".", 1)[-1] + argv = [ + str(backend_python(manifest, root)), str(SPEAK_RUNNER), "--model", str(model), "--model-type", model_type, + "--text=" + text, "--out", str(target), "--speed", repr(float(speed)), + ] + selected_voice = voice + if model_type == "kokoro": + name = (voice or "af_heart").removesuffix(".safetensors") + local_voice = model / "voices" / (name + ".safetensors") + if local_voice.is_file(): + selected_voice = str(local_voice.absolute()) + if selected_voice is not None: + argv += ["--voice", selected_voice] + if lang_code is not None: + argv += ["--lang-code", lang_code] + return { + "backend": manifest["id"], "model": str(model), "out": str(target), "voice": voice, + "speed": float(speed), "lang_code": lang_code, "argv": argv, + } + + +CONTRACT = RunnerContract( + verb="speak", error=SpeakError, accept=lambda value, plan: isinstance(value.get("path"), str), + action="synthesize the text", + remediation="Check that the directory is a complete converted text-to-speech model and that --voice and --lang-code suit it.", + activity="Speech synthesis", timeout_remediation="Use shorter text or a larger --timeout.", +) + + +def run_speak(plan, manifests=None, root=None, timeout=DEFAULT_TIMEOUT_SECONDS, runner=subprocess.run): + result = invoke(plan, CONTRACT, manifests, root, timeout, runner) + keys = ("path", "sample_rate", "audio_seconds", "seconds", "load_seconds", "real_time_factor", "peak_memory_gb") + return dict({"schema": SPEAK_SCHEMA, "backend": plan["backend"], "model": plan["model"], + "voice": plan["voice"], "speed": plan["speed"], "lang_code": plan["lang_code"]}, + **{key: result.get(key) for key in keys}) diff --git a/packages/codex/mlx-agent/src/mlx_agent/speak_runner.py b/packages/codex/mlx-agent/src/mlx_agent/speak_runner.py new file mode 100644 index 0000000..0248a1c --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/speak_runner.py @@ -0,0 +1,86 @@ +"""Run inside a speech backend's environment: load one converted TTS model, synthesize one text to a WAV, print JSON.""" + +import argparse +from contextlib import ExitStack +import json +import sys +import time +import wave + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--model-type", required=True) + parser.add_argument("--text", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--speed", type=float, required=True) + parser.add_argument("--voice", default=None) + parser.add_argument("--lang-code", default=None) + arguments = parser.parse_args(argv) + + with ExitStack() as stack: + if arguments.model_type == "kokoro": + if __package__: + from .kokoro_runtime import kokoro_data_directory + else: + from kokoro_runtime import kokoro_data_directory + stack.enter_context(kokoro_data_directory()) + return _synthesize(arguments) + + +def _synthesize(arguments): + + import mlx.core as mx + import numpy as np + from mlx_audio.tts.utils import load + + started = time.time() + model = load(arguments.model, model_type=arguments.model_type) + load_seconds = time.time() - started + + kwargs = {"text": arguments.text, "speed": arguments.speed, "verbose": False} + if arguments.voice is not None: + kwargs["voice"] = arguments.voice + if arguments.lang_code is not None: + kwargs["lang_code"] = arguments.lang_code + + mx.reset_peak_memory() + started = time.time() + chunks = [] + sample_rate = getattr(model, "sample_rate", None) + for result in model.generate(**kwargs): + chunks.append(np.asarray(result.audio, dtype=np.float32).reshape(-1)) + sample_rate = getattr(result, "sample_rate", None) or sample_rate + seconds = time.time() - started + peak_memory_gb = mx.get_peak_memory() / 1e9 + + if not chunks or not sample_rate: + raise RuntimeError("the model produced no audio") + samples = np.concatenate(chunks) + if samples.size == 0: + raise RuntimeError("the model produced no audio") + pcm = (np.clip(samples, -1.0, 1.0) * 32767.0).astype(" 1024 * 1024: + raise ValueError("Unsafe or oversized metadata file") + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("Metadata must be an object") + return value + + +def source_digest(root): + root = Path(root) + digest = hashlib.sha256() + paths = [] + for folder in ("engine/strata_mlx", "http/serve", "http/tools"): + directory = root / folder + if directory.is_symlink(): + raise ValueError("Runtime source directories must not be links") + paths.extend(directory.rglob("*.py")) + paths.append(root / "engine/uv.lock") + paths.append(root / "strata-mlx") + for path in sorted(paths): + if not path.exists(): + continue + if path.is_symlink() or not path.resolve().is_relative_to(root.resolve()): + raise ValueError("Runtime source must remain inside its installation") + digest.update(str(path.relative_to(root)).encode()) + digest.update(path.read_bytes()) + return digest.hexdigest() + + +def installed_runtime(root=None): + root = Path(root) if root is not None else runtime_root() + try: + marker = _json(root / INSTALL_MARKER) + python = root / "engine/.venv/bin/python" + engine = root / "strata-mlx" + required = (root / "engine/strata_mlx/serve.py", root / "http/serve/server.py", root / "http/tools/strata_tokenizer.py") + if (marker.get("engine_revision") != ENGINE_REVISION or marker.get("http_revision") != HTTP_REVISION + or not python.is_file() or not engine.is_file() or engine.is_symlink() or not os.access(engine, os.X_OK) + or engine.read_text() != engine_launcher(root) or not all(p.is_file() for p in required) + or marker.get("source_sha256") != source_digest(root)): + raise ValueError("Missing, modified or incompatible Strata installation") + return {"root": str(root.absolute()), "python": str(python.absolute()), "engine": str(engine.absolute()), "fingerprint": marker["source_sha256"]} + except (OSError, ValueError, TypeError) as error: + raise StrataError("strata_not_installed", str(error)) from error + + +def _local(root, name, directory=False): + if not isinstance(name, str) or not name or Path(name).is_absolute() or any(p in ("", ".", "..") for p in name.split("/")): + raise ValueError("Profile paths must be relative and remain inside the model directory") + path = root / name + for parent in (path, *path.parents): + if parent == root: + break + if parent.is_symlink(): + raise ValueError("Profile paths must not contain symbolic links") + if not path.resolve().is_relative_to(root.resolve()) or not (path.is_dir() if directory else path.is_file()): + raise FileNotFoundError("Missing profile asset: " + name) + return path + + +def file_identity(path): + stat = path.stat() + with path.open("rb") as handle: + header = hashlib.sha256(handle.read(8 << 20)).hexdigest() + return {"bytes": stat.st_size, "mtime_ns": stat.st_mtime_ns, "device": stat.st_dev, "inode": stat.st_ino, "header_sha256": header} + + +def read_model(path): + root = Path(path).expanduser().absolute() + try: + if root.is_symlink(): + raise ValueError("The model profile directory must not be a symbolic link") + config = _json(root / "config.json") + if config.get("serving_runtime") != "strata" or config.get("model_type") != "qwen4_exp": + raise StrataError("strata_model_required", "Strata requires a prepared Qwen3.8-Flash-Next GGUF profile; MLX VQ weights are not supported.") + profile = _json(root / MODEL_MARKER) + if set(profile) != {"schema_version", "gguf", "mtp"} or type(profile["schema_version"]) is not int or profile["schema_version"] != 1: + raise ValueError("Unsupported Strata model profile") + first = _local(root, profile["gguf"]) + if not _FIRST.fullmatch(first.name): + raise ValueError("Only the tested Q2_0 and IQ3_S Qwen3.8-Flash-Next GGUF pairs are supported") + second = _local(root, str(first.relative_to(root)).replace("00001-of-00002", "00002-of-00002")) + from mlx_agent.gguf import read_gguf_header + for index, weight in enumerate((first, second)): + header = read_gguf_header(weight) + architecture = header["metadata"].get("general.architecture") + if architecture != "qwen4exp" and not (index == 1 and architecture is None and header["tensor_count"] == 1): + raise ValueError("GGUF architecture is not Qwen3.8-Flash-Next") + assets = [root / "config.json", root / MODEL_MARKER, first, second] + tokenizer = _local(root, "tokenizer", directory=True) + for name in ("vocab.json", "merges.txt", "token_type.json", "chat_template.jinja"): + assets.append(_local(root, "tokenizer/" + name)) + mtp = None + mtp_bytes = 0 + if profile["mtp"] is not None: + mtp = _local(root, profile["mtp"], directory=True) + manifest_path = _local(root, str(mtp.relative_to(root) / "mtp-manifest.json")) + if manifest_path.stat().st_size > 1024 * 1024: + raise ValueError("Oversized MTP manifest") + tensors = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(tensors, list) or len(tensors) != 31: + raise ValueError("A complete, verified MTP head is required") + assets.append(mtp / "mtp-manifest.json") + for tensor in tensors: + if not isinstance(tensor, dict) or not re.fullmatch(r"[0-9a-f]{64}", str(tensor.get("sha256", ""))): + raise ValueError("MTP tensor verification metadata is missing") + location = _local(root, str(mtp.relative_to(root) / tensor["file"])) + if location.stat().st_size != tensor["bytes"]: + raise ValueError("MTP tensor size changed") + mtp_bytes += location.stat().st_size + assets.append(location) + identities = {str(asset.relative_to(root)): file_identity(asset) for asset in assets} + fingerprint = hashlib.sha256(json.dumps(identities, sort_keys=True).encode()).hexdigest() + # The second shard is the on-disk n-gram lookup table, not resident + # weights. Bound dense expansion by 20%, retain the full MTP file + # sizes, and reserve the pinned engine's 5 GB working allowance plus + # its 30 KB/token float16 cache allowance at the configured capacity. + memory_estimate = math.ceil(first.stat().st_size * 1.2 + mtp_bytes + 5e9 + 131072 * 30000) + return {"root": str(root), "gguf": str(first), "tokenizer": str(tokenizer), "mtp": str(mtp) if mtp else None, + "quantization": "GSQ-RCO " + _FIRST.fullmatch(first.name).group(1), + "mtp_bytes": mtp_bytes, + "fingerprint": fingerprint, "weight_bytes": first.stat().st_size + second.stat().st_size, + "memory_estimate_bytes": memory_estimate} + except StrataError: + raise + except FileNotFoundError as error: + raise StrataError("strata_model_incomplete", str(error)) from error + except (OSError, ValueError, TypeError, KeyError) as error: + raise StrataError("strata_model_invalid", str(error)) from error + + +def memory_estimate(model, expert_memory_gb=None, context_tokens=131072): + """Conservative resident estimate for the reviewed paging budget and context.""" + if type(context_tokens) is not int or not 512 <= context_tokens <= 131072: + raise StrataError("invalid_arguments", "Strata context must be an integer from 512 to 131072 tokens.") + if expert_memory_gb is None: + return model["memory_estimate_bytes"] - 131072 * 30000 + context_tokens * 30000 + if (type(expert_memory_gb) not in (int, float) or not math.isfinite(expert_memory_gb) + or not 0.5 <= expert_memory_gb <= 64): + raise StrataError("invalid_arguments", "Expert memory must be a finite number from 0.5 to 64 GB.") + from mlx_agent.gguf import GGUFError, read_gguf_header + try: + header = read_gguf_header(model["gguf"], include_tensors=True) + experts = dense = floats = 0 + file_bytes = Path(model["gguf"]).stat().st_size + for tensor in header["tensors"]: + block, size = _BLOCKS[tensor["type"]] + count = math.prod(tensor["shape"]) + byte_count = count // block * size + if count % block or header["tensor_data_offset"] + tensor["offset"] + byte_count > file_bytes: + raise ValueError("Tensor data exceeds its GGUF file") + if tensor["name"].endswith("_exps.weight"): + experts += byte_count + else: + dense += byte_count + if tensor["type"] == 14 and tensor["name"] != "output.weight": + floats += count * 2 + if not experts or not dense: + raise ValueError("Expert and dense tensor sizes are required") + mtp_bytes = model.get("mtp_bytes", 0) + # Count dense expansion, Q6_K prompt copies, full draft files, runtime + # working memory and context cache. The n-gram companion remains on disk. + return math.ceil(min(experts, expert_memory_gb * 1e9) + dense * 2 + floats + + mtp_bytes + 5e9 + context_tokens * 30000) + except (GGUFError, OSError, ValueError, KeyError) as error: + raise StrataError("strata_memory_unknown", "Cannot estimate paged Strata memory: " + str(error)) from error + + +def frontend_config(model, runtime, speculation=False, expert_memory_gb=None, context_tokens=131072): + memory_estimate(model, expert_memory_gb, context_tokens) + args = ["--native", model["gguf"], "--spec", "4" if speculation else "1", + "--suffix-draft", "auto" if speculation else "0", "--max-context", str(context_tokens)] + if expert_memory_gb is not None: + args += ["--expert-memory", str(expert_memory_gb)] + if model["mtp"]: + args += ["--mtp", model["mtp"]] + return {"exe": runtime["engine"], "args": args, "cwd": runtime["root"], "tokenizer": model["tokenizer"], + "model_name": model["root"], "host": "127.0.0.1", "open_browser": False, "prevent_sleep": False, + "sampling": {"temperature": 0.0}, "engine_silence_s": 900} + + +def main(): + import argparse + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--runtime-root", required=True) + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--host", choices=["127.0.0.1"], default="127.0.0.1") + parser.add_argument("--max-tokens", type=int, default=8192) + parser.add_argument("--speculation", choices=["on", "off"], default="off") + parser.add_argument("--expert-memory-gb", type=float) + parser.add_argument("--context-tokens", type=int, default=131072) + parser.add_argument("--expected-model-fingerprint", required=True) + parser.add_argument("--expected-runtime-fingerprint", required=True) + args = parser.parse_args() + runtime = installed_runtime(args.runtime_root) + model = read_model(args.model) + if model["fingerprint"] != args.expected_model_fingerprint or runtime["fingerprint"] != args.expected_runtime_fingerprint: + raise StrataError("preview_stale", "The model assets or Strata runtime changed after the serving preview.") + os.environ.update(MLX_AGENT_STRATA_ROOT=runtime["root"], HF_HUB_OFFLINE="1", HF_DATASETS_OFFLINE="1", MLX_ENABLE_TF32="0") + # The only config reaching the frontend is constructed here; model files cannot supply code or commands. + config = frontend_config(model, runtime, args.speculation == "on", args.expert_memory_gb, args.context_tokens) + source = Path(runtime["root"]) / "http" + sys.path[:0] = [str(source), str(source / "tools")] + with tempfile.TemporaryDirectory(prefix="mlx-agent-strata-") as directory: + location = Path(directory) / "server.json" + location.write_text(json.dumps(config), encoding="utf-8") + location.chmod(0o600) + location.with_name("server.shared-settings.json").write_text(json.dumps({"max_tokens": args.max_tokens}), encoding="utf-8") + sys.argv = ["strata-server", "--engine", "strata", "--config", str(location), "--host", "127.0.0.1", "--port", str(args.port)] + engine_log = Path(directory) / "engine.log" + config["log"] = str(engine_log) + location.write_text(json.dumps(config), encoding="utf-8") + try: + runpy.run_path(str(source / "serve/server.py"), run_name="__main__") + finally: + if engine_log.is_file(): + with engine_log.open("rb") as handle: + handle.seek(max(0, engine_log.stat().st_size - 65536)) + print(handle.read().decode("utf-8", errors="replace"), file=sys.stderr) + return 0 + + +if __name__ == "__main__": + # The Strata interpreter gets this package alone, never the installing environment's site-packages. + _package = Path(__file__).resolve().parent + _spec = importlib.util.spec_from_file_location( + "mlx_agent", _package / "__init__.py", submodule_search_locations=[str(_package)]) + sys.modules["mlx_agent"] = importlib.util.module_from_spec(_spec) + _spec.loader.exec_module(sys.modules["mlx_agent"]) + raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/src/mlx_agent/study_recipes.py b/packages/codex/mlx-agent/src/mlx_agent/study_recipes.py new file mode 100644 index 0000000..0b924c5 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/study_recipes.py @@ -0,0 +1,206 @@ +"""Offline recipes for the registered Wan Diffusers study adapter.""" +import hashlib +from importlib import metadata +import json +import os +from pathlib import Path +import platform +import re +import stat + +MODEL_ID = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" +REPOSITORY = "models--Wan-AI--Wan2.1-T2V-1.3B-Diffusers" +ADAPTER_ID = "diffusers.wan2.1-t2v-1.3b.mps.v1" +_REVISION = re.compile(r"[0-9a-f]{40}") +_COMPONENTS = { + "scheduler": ["diffusers", "UniPCMultistepScheduler"], + "text_encoder": ["transformers", "UMT5EncoderModel"], + "tokenizer": ["transformers", "T5TokenizerFast"], + "transformer": ["diffusers", "WanTransformer3DModel"], + "vae": ["diffusers", "AutoencoderKLWan"], +} +_PACKAGES = ("torch", "diffusers", "transformers", "safetensors", "huggingface-hub", + "accelerate", "tokenizers", "sentencepiece") + + +def cache_root(): + # Use exactly the cache resolver used by the generation adapter. + from video_lab.adapters.wan_mps import WanMpsAdapter + return WanMpsAdapter()._effective_cache_root() + + +def runtime_versions(): + versions = {"python": platform.python_version()} + missing = [] + for package in _PACKAGES: + try: + versions[package] = metadata.version(package) + except metadata.PackageNotFoundError: + missing.append(package) + if missing: + raise ValueError("Install video runtime in Studies → Runtime setup. Missing: " + ", ".join(missing)) + from video_lab.runtime_identity import source_tree_digest + versions["video-lab"] = source_tree_digest() + versions["study-recipes"] = "sha256:" + hashlib.sha256(Path(__file__).read_bytes()).hexdigest() + return versions + + +def catalog(root=None): + root = Path(root) if root is not None else cache_root() + snapshots = root / REPOSITORY / "snapshots" + revisions = sorted(path.name for path in snapshots.glob("*") + if _REVISION.fullmatch(path.name) and path.is_dir() and not path.is_symlink()) + reason = None + try: + runtime_versions() + except ValueError as error: + reason = str(error) + if not revisions: + reason = "No cached revision of " + MODEL_ID + ". Add its Diffusers weights before capturing a recipe." + return [{"model_id": MODEL_ID, "adapter_id": ADAPTER_ID, + "display_name": "Wan 2.1 T2V 1.3B (MPS)", "revisions": revisions, + "unavailable_reason": reason}] + + +def validate_settings(value): + fields = {"revision", "prompt", "negative_prompt", "seed", "width", "height", "num_frames", + "fps", "inference_steps", "guidance_scale"} + if not isinstance(value, dict) or set(value) != fields: + raise ValueError("Recipe settings must contain exactly the supported generation fields.") + if not isinstance(value["revision"], str) or not _REVISION.fullmatch(value["revision"]): + raise ValueError("Choose a pinned cached model revision.") + for name in ("prompt", "negative_prompt"): + if not isinstance(value[name], str) or len(value[name]) > 8000 or (name == "prompt" and not value[name].strip()): + raise ValueError("Use a nonempty prompt of at most 8,000 characters and an optional negative prompt.") + for name, low, high, multiple in (("seed", 0, 2**63 - 1, 1), ("width", 16, 4096, 16), + ("height", 16, 4096, 16), ("fps", 1, 120, 1), + ("inference_steps", 1, 1000, 1), ("num_frames", 17, 129, 1)): + item = value[name] + if type(item) is not int or not low <= item <= high or item % multiple: + raise ValueError("Invalid " + name + ": expected " + str(low) + ".." + str(high) + " in multiples of " + str(multiple) + ".") + if (value["num_frames"] - 1) % 16: + raise ValueError("Frame count must equal 16n+1.") + guidance = value["guidance_scale"] + if type(guidance) not in (int, float) or not 0 <= guidance <= 100: + raise ValueError("Guidance must be between 0 and 100.") + return dict(value) + + +def _read(path, repository, limit=None): + target = path.resolve(strict=True) + if not target.is_relative_to(repository): + raise ValueError("A recipe source resolves outside its model cache.") + with target.open("rb") as stream: + before = os.fstat(stream.fileno()) + if not stat.S_ISREG(before.st_mode) or before.st_size <= 0 or (limit is not None and before.st_size > limit): + raise ValueError("Recipe sources must be nonempty regular files within the supported size.") + digest = hashlib.sha256() + content = bytearray() if limit is not None else None + while True: + chunk = stream.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + if content is not None: + content.extend(chunk) + if len(content) > limit: + raise ValueError("Recipe source exceeded the supported size during capture.") + after = os.fstat(stream.fileno()) + current = path.stat() + identity = lambda item: (item.st_dev, item.st_ino, item.st_size, item.st_mtime_ns, item.st_ctime_ns) + if identity(before) != identity(after) or identity(after) != identity(current) or path.resolve(strict=True) != target: + raise ValueError("Recipe source changed during capture.") + return digest.hexdigest(), before.st_size, bytes(content) if content is not None else None + + +def _signature(path): + observed = path.stat() + return path.resolve(strict=True), observed.st_dev, observed.st_ino, observed.st_size, observed.st_mtime_ns, observed.st_ctime_ns + + +def _snapshot(revision, root): + if not isinstance(revision, str) or not _REVISION.fullmatch(revision): + raise ValueError("A pinned cached model revision is required.") + repository = (Path(root) / REPOSITORY).resolve() + folder = repository / "snapshots" / revision + if not folder.is_dir() or folder.is_symlink() or not folder.resolve().is_relative_to(repository): + raise ValueError("The selected cached revision is unavailable.") + paths = [folder / "model_index.json"] + directory_signatures = {folder: _signature(folder)} + for component in _COMPONENTS: + directory = folder / component + if not directory.is_dir() or directory.is_symlink(): + raise ValueError("Cached model is incomplete: missing " + component + ".") + directory_signatures[directory] = _signature(directory) + entries = list(directory.iterdir()) + if any(path.is_dir() for path in entries): + raise ValueError("Unexpected nested model component directory.") + paths.extend(entries) + checksums, sizes, documents, signatures = {}, {}, {}, {} + for path in sorted(paths): + name = path.relative_to(folder).as_posix() + signatures[path] = _signature(path) + digest, size, content = _read(path, repository, 32 * 1024 * 1024 if path.suffix == ".json" else None) + sizes[name] = size + if path.suffix == ".json": + try: + documents[name] = json.loads(content) + except (ValueError, UnicodeError) as error: + raise ValueError("Invalid cached JSON: " + name) from error + checksums[name] = digest + index = documents.get("model_index.json", {}) + if not isinstance(index, dict) or index.get("_class_name") != "WanPipeline" or any(index.get(name) != value for name, value in _COMPONENTS.items()): + raise ValueError("Cached model does not declare the supported Wan pipeline.") + for name in ("transformer/config.json", "vae/config.json", "text_encoder/config.json", + "tokenizer/tokenizer_config.json", "tokenizer/tokenizer.json", "scheduler/scheduler_config.json"): + if not isinstance(documents.get(name), dict): + raise ValueError("Cached model is incomplete: missing " + name + ".") + for component, basename in (("transformer", "diffusion_pytorch_model"), ("vae", "diffusion_pytorch_model"), ("text_encoder", "model")): + index_name = component + "/" + basename + ".safetensors.index.json" + if index_name in documents: + document = documents[index_name] + weight_map = document.get("weight_map") if isinstance(document, dict) else None + if not isinstance(weight_map, dict) or not weight_map: + raise ValueError("Cached weight index is incomplete.") + for name in weight_map.values(): + if not isinstance(name, str) or Path(name).name != name or not name.endswith(".safetensors") or component + "/" + name not in checksums: + raise ValueError("Cached weight shard is missing or invalid.") + elif component + "/" + basename + ".safetensors" not in checksums: + raise ValueError("Cached model is incomplete: missing " + component + " weights.") + weights = hashlib.sha256() + for name in sorted(checksums): + if name.endswith(".safetensors"): + weights.update(name.encode() + b"\0" + checksums[name].encode()) + # Keep the weight digest compatible with WanMpsAdapter's pinned weight check. + assets = {name: digest for name, digest in checksums.items() if not name.endswith(".safetensors")} + for path, signature in {**directory_signatures, **signatures}.items(): + if _signature(path) != signature: + raise ValueError("Cached recipe source changed during capture.") + return "sha256:" + weights.hexdigest(), assets, sizes + + +def prepare(settings, root=None): + from video_lab.domain import Manifest + value = validate_settings(settings) + versions = runtime_versions() + digest, assets, _ = _snapshot(value.pop("revision"), Path(root) if root is not None else cache_root()) + manifest = Manifest(model_id=MODEL_ID, model_revision=settings["revision"], model_checksum=digest, + adapter_id=ADAPTER_ID, adapter_version="1", runtime_versions=versions, source_checksums=assets, + scheduler="unipc", dtype="bfloat16", quantization="none", backend="mps", device="mps", + memory_strategy="resident", hardware_fingerprint=platform.machine(), os_fingerprint=platform.platform(), **value) + return manifest.model_dump(mode="json") + + +def verify(manifest, root=None): + """Recheck a captured recipe before and after model execution.""" + value = manifest.model_dump(mode="json") if hasattr(manifest, "model_dump") else manifest + if value["model_id"] != MODEL_ID or value["adapter_id"] != ADAPTER_ID or value["adapter_version"] != "1": + raise ValueError("Captured recipe adapter identity changed.") + if value["runtime_versions"] != runtime_versions(): + raise ValueError("Captured recipe runtime changed. Capture a new recipe.") + if value["hardware_fingerprint"] != platform.machine() or value["os_fingerprint"] != platform.platform(): + raise ValueError("Captured recipe platform changed. Capture a new recipe.") + digest, assets, sizes = _snapshot(value["model_revision"], Path(root) if root is not None else cache_root()) + if digest != value["model_checksum"] or assets != value["source_checksums"]: + raise ValueError("Captured model weights or assets changed. Capture a new recipe.") + return sizes diff --git a/packages/codex/mlx-agent/src/mlx_agent/task_cli.py b/packages/codex/mlx-agent/src/mlx_agent/task_cli.py new file mode 100644 index 0000000..1bc2ce4 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/task_cli.py @@ -0,0 +1,92 @@ +"""Bounded CLI responses for the engine-owned inference task service.""" + +from pathlib import Path + +from .contracts import ErrorDetail, ResultEnvelope +from .task_contracts import TaskError, capabilities, encoded +from .task_runtime import cancel_task, start_task, task_result, task_status, verify_task_model +from .task_store import TaskStore +from .wiring import redact_secrets + +MAX_RESPONSE_BYTES = 12288 + + +def add_arguments(parser): + commands = parser.add_subparsers(dest="task_command", required=True) + descriptions = { + "capabilities": "describe supported local inference tasks", + "start": "start a durable task from a local JSON request", + "status": "inspect a task without starting inference", + "list": "list recent local tasks", + "result": "retrieve a successful result or its owned artifact reference", + "cancel": "request cancellation of one task", + "verify-model": "check the task's recorded model and runtime identity without running it", + } + for command, description in descriptions.items(): + action = commands.add_parser(command, help=description) + action.add_argument("--state-dir", type=Path, help="absolute task state directory; does not change host compute ownership") + action.add_argument("--json", action="store_true", help="emit the versioned JSON result envelope") + if command == "start": + action.add_argument("--request", type=Path, required=True, help="task-request/1 or task-request/2 JSON file; accepted inputs are retained privately") + elif command in ("status", "result", "cancel", "verify-model"): + action.add_argument("--id", required=True, help="task UUID returned by start") + elif command == "list": + action.add_argument("--limit", type=int, default=20, help="maximum recent records, 1..100 (default: 20)") + + +def summary(record): + value = dict(record) + value["identity"] = {key: record["identity"][key] for key in + ("backend", "backend_version", "engine_version", "python_version", "model_fingerprint") + if key in record["identity"]} + for key in ("supervisor", "execution"): + if value[key] is not None: + value[key] = {"pid": value[key].get("pid")} + if value["error"]: + value["error"] = {"code": str(value["error"].get("code", "task_failed"))[:100], + "message": redact_secrets(str(value["error"].get("message", "Task failed.")))[:1000], + "remediation": redact_secrets(str(value["error"].get("remediation", "Inspect task status.")))[:500]} + return value + + +def _bounded(envelope): + payload = envelope.to_dict() + while len(encoded(payload)) > MAX_RESPONSE_BYTES and payload["data"].get("tasks"): + payload["data"]["tasks"].pop() + payload["data"]["has_more"] = True + if len(encoded(payload)) > MAX_RESPONSE_BYTES and payload["data"].get("inline"): + payload["data"]["inline"] = False + payload["data"]["result"] = None + if len(encoded(payload)) > MAX_RESPONSE_BYTES: + raise TaskError("task_response_too_large", "Task metadata exceeds the response size limit.", "Inspect the private task store and preserve its records.") + return payload + + +def run(arguments): + action = arguments.task_command + operation = "task." + action + try: + store = TaskStore(arguments.state_dir) + if action == "capabilities": + data = capabilities() + elif action == "verify-model": + data = verify_task_model(arguments.id, store.root) + elif action == "list": + data = store.list_page(arguments.limit) + data["tasks"] = [summary(record) for record in data["tasks"]] + elif action == "result": + data = task_result(arguments.id, store.root) + data["task"] = summary(data["task"]) + else: + record = (start_task(arguments.request, store.root) if action == "start" else + cancel_task(arguments.id, store.root) if action == "cancel" else task_status(arguments.id, store.root)) + data = {"task": summary(record)} + envelope = ResultEnvelope.ok(operation, data) + if action == "start" and data["task"]["state"] == "failed": + error = data["task"]["error"] + envelope = ResultEnvelope(operation, "error", data=data, error=ErrorDetail(**error)) + payload = _bounded(envelope) + except TaskError as error: + payload = ResultEnvelope.fail(operation, error.code, redact_secrets(str(error))[:2000], error.remediation[:1000]).to_dict() + print(encoded(payload if arguments.json else payload.get("data") if payload["status"] == "ok" else payload).decode()) + return 0 if payload["status"] == "ok" else 2 diff --git a/packages/codex/mlx-agent/src/mlx_agent/task_contracts.py b/packages/codex/mlx-agent/src/mlx_agent/task_contracts.py new file mode 100644 index 0000000..c114534 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/task_contracts.py @@ -0,0 +1,206 @@ +"""Versioned contracts for durable local inference tasks.""" + +import hashlib +import json +import math +from datetime import datetime +from pathlib import Path +from uuid import UUID + +REQUEST_SCHEMA = "task-request/1" +RUN_SCHEMA = "task-run/1" +MEDIA_REQUEST_SCHEMA = "task-request/2" +MEDIA_RUN_SCHEMA = "task-run/2" +MAX_REQUEST_BYTES = 1024 * 1024 +MEDIA_SUFFIXES = {"audio": (".wav", ".flac", ".mp3"), "image": (".png", ".jpg", ".jpeg", ".webp"), + "video": (".mp4", ".mov", ".m4v")} +STATES = ("starting", "running", "succeeded", "failed", "cancelled") +TERMINAL = ("succeeded", "failed", "cancelled") + + +class TaskError(RuntimeError): + def __init__(self, code, message, remediation="Inspect the task inputs and retry."): + super().__init__(message) + self.code, self.remediation = code, remediation + + def as_dict(self): + return {"code": self.code, "message": str(self), "remediation": self.remediation} + + +def encoded(value): + try: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode("utf-8") + except (ValueError, TypeError, UnicodeError) as error: + raise TaskError("invalid_task", "Task content must be finite UTF-8 JSON.") from error + + +def task_id(value): + try: + if not isinstance(value, str) or str(UUID(value)) != value: + raise ValueError() + except ValueError as error: + raise TaskError("invalid_task_id", "Pass a canonical task UUID.") from error + return value + + +def validate_request(value): + from .task_kinds import find, validate_parameters + if not isinstance(value, dict) or set(value) != {"schema", "kind", "model_path", "parameters"}: + raise TaskError("invalid_request", "A task request needs schema, kind, model_path and parameters.") + kind = find(value["kind"]) + if kind is None or value["schema"] != kind.request_schema: + raise TaskError("unsupported_task", "Unsupported task schema or kind.", "Run mlx-agent task capabilities --json.") + path = value["model_path"] + if not isinstance(path, str) or not Path(path).is_absolute() or any(ord(c) < 32 for c in path): + raise TaskError("invalid_request", "model_path must be an absolute local path.") + parameters = value["parameters"] + if not isinstance(parameters, dict): + raise TaskError("invalid_request", "parameters must be an object.") + parameters = validate_parameters(kind, parameters) + result = dict(value, parameters=parameters) + raw = encoded(result) + if len(raw) > MAX_REQUEST_BYTES: + raise TaskError("request_too_large", "Task requests are limited to 1 MiB.") + return json.loads(raw) + + +def validate_record(value): + if isinstance(value, dict) and value.get("schema") == "coding-capture-run/1": + from .coding_capture_contracts import validate_capture_record + return validate_capture_record(value) + required = {"schema", "id", "kind", "state", "model_path", "created_at", "started_at", "completed_at", + "cancel_requested", "identity", "request", "result", "error", "supervisor", "execution", "execution_unresolved"} + if not isinstance(value, dict): + raise TaskError("invalid_task_record", "Unknown or incomplete task record.", "Preserve the store and use a compatible engine.") + version = value.get("schema") + media = version == MEDIA_RUN_SCHEMA + if media: + required.add("input_artifact") + if set(value) != required or version not in (RUN_SCHEMA, MEDIA_RUN_SCHEMA): + raise TaskError("invalid_task_record", "Unknown or incomplete task record.", "Preserve the store and use a compatible engine.") + task_id(value["id"]) + from .task_kinds import find + kind = find(value["kind"]) + if kind is None or kind.run_schema != version or value["state"] not in STATES: + raise TaskError("invalid_task_record", "Unknown task kind or state.") + if media: + reference = value["input_artifact"] + if kind.input_media: + _artifact_reference(reference) + path = Path(reference["path"]) + if (path.parent.name != value["id"] or path.parent.parent.name != "artifacts" + or path.name != "input" + path.suffix.lower() or path.suffix.lower() not in MEDIA_SUFFIXES[kind.input_media]): + raise TaskError("invalid_task_record", "Input artifact path is outside its task directory.") + elif reference is not None: + raise TaskError("invalid_task_record", "This task kind cannot have an input artifact.") + if not isinstance(value["model_path"], str) or not Path(value["model_path"]).is_absolute(): + raise TaskError("invalid_task_record", "The task model path is invalid.") + if any(type(value[key]) is not bool for key in ("cancel_requested", "execution_unresolved")): + raise TaskError("invalid_task_record", "Task lifecycle flags must be boolean.") + for key in ("created_at", "started_at", "completed_at"): + if value[key] is None and key != "created_at": + continue + try: + stamp = datetime.fromisoformat(value[key]) + if stamp.tzinfo is None: + raise ValueError() + except (ValueError, TypeError) as error: + raise TaskError("invalid_task_record", "Task timestamps require a timezone.") from error + if (value["state"] in TERMINAL) != (value["completed_at"] is not None): + raise TaskError("invalid_task_record", "Task completion timestamp does not match its state.") + for key in ("identity", "request"): + if not isinstance(value[key], dict): + raise TaskError("invalid_task_record", "Task provenance is missing.") + identity = value["identity"] + identity_fields = {"schema", "requested_model_path", "model_path", "model_fingerprint", "backend", + "backend_version", "backend_manifest", "backend_manifest_hash", "requested_executable", + "executable", "executable_identity", "engine_version", "python_version"} + if (set(identity) != identity_fields or identity.get("schema") != "task-identity/1" + or not absolute_path(identity.get("requested_model_path")) + or Path(identity["requested_model_path"]).absolute() != Path(value["model_path"]).absolute() + or not absolute_path(identity.get("model_path")) + or not _sha256(identity.get("model_fingerprint")) + or not _nonempty_string(identity.get("backend")) + or not _nonempty_string(identity.get("backend_version")) + or not isinstance(identity.get("backend_manifest"), dict) + or identity["backend_manifest"].get("id") != identity.get("backend") + or identity["backend_manifest"].get("version") != identity.get("backend_version") + or not _sha256(identity.get("backend_manifest_hash")) + or hashlib.sha256(encoded(identity["backend_manifest"])).hexdigest() != identity["backend_manifest_hash"] + or not absolute_path(identity.get("requested_executable")) + or not absolute_path(identity.get("executable")) + or not _file_identity(identity.get("executable_identity")) + or not _nonempty_string(identity.get("engine_version")) + or not _nonempty_string(identity.get("python_version"))): + raise TaskError("invalid_task_record", "Task model and runtime identity evidence is invalid.") + _artifact_reference(value["request"]) + if value["result"] is not None: + _artifact_reference(value["result"]) + if value["error"] is not None: + error = value["error"] + if (not isinstance(error, dict) or set(error) != {"code", "message", "remediation"} + or any(not _nonempty_string(error.get(key)) for key in ("code", "message", "remediation"))): + raise TaskError("invalid_task_record", "Task error evidence is invalid.") + for key in ("supervisor", "execution"): + process = value[key] + if process is not None and (not isinstance(process, dict) or set(process) != {"pid", "identity"} + or type(process.get("pid")) is not int or process["pid"] <= 0 + or not _nonempty_string(process.get("identity"))): + raise TaskError("invalid_task_record", "Task process identity evidence is invalid.") + if value["execution"] is not None and value["supervisor"] is None: + raise TaskError("invalid_task_record", "Task execution evidence has no supervisor.") + if value["state"] == "succeeded" and value["result"] is None: + raise TaskError("invalid_task_record", "Successful tasks require a durable result.") + if value["state"] == "failed" and value["error"] is None: + raise TaskError("invalid_task_record", "Failed tasks require an actionable error.") + if value["state"] == "running" and (value["started_at"] is None or value["supervisor"] is None or value["execution"] is None): + raise TaskError("invalid_task_record", "Running tasks require start and process evidence.") + if value["state"] == "succeeded" and (value["started_at"] is None or value["supervisor"] is None + or value["execution"] is None or value["error"] is not None or value["execution_unresolved"]): + raise TaskError("invalid_task_record", "Successful task lifecycle evidence is inconsistent.") + if value["state"] != "succeeded" and value["result"] is not None: + raise TaskError("invalid_task_record", "Only successful tasks may contain result evidence.") + if value["state"] == "starting" and value["started_at"] is not None: + raise TaskError("invalid_task_record", "Starting tasks cannot have a start timestamp.") + if value["state"] == "cancelled" and (not value["cancel_requested"] or value["result"] is not None + or value["error"] is not None or value["execution_unresolved"]): + raise TaskError("invalid_task_record", "Cancelled task lifecycle evidence is inconsistent.") + if value["state"] not in ("succeeded", "failed") and value["error"] is not None: + raise TaskError("invalid_task_record", "Only failed tasks may contain error evidence.") + if value["execution_unresolved"] and value["state"] != "failed": + raise TaskError("invalid_task_record", "Only failed tasks may retain unresolved execution evidence.") + encoded(value) + return value + + +def _nonempty_string(value): + return isinstance(value, str) and bool(value.strip()) + + +def absolute_path(value): + return isinstance(value, str) and bool(value) and Path(value).is_absolute() and not any(ord(char) < 32 for char in value) + + +def _sha256(value): + return isinstance(value, str) and len(value) == 64 and all(char in "0123456789abcdef" for char in value) + + +def _file_identity(value): + return isinstance(value, list) and len(value) == 4 and all(type(part) is int and part >= 0 for part in value) + + +def _artifact_reference(value): + if (not isinstance(value, dict) or set(value) != {"path", "sha256", "bytes"} + or not absolute_path(value.get("path")) or not _sha256(value.get("sha256")) + or type(value.get("bytes")) is not int or value["bytes"] < 0): + raise TaskError("invalid_task_record", "Task artifact evidence is invalid.") + + +def capabilities(): + from .task_kinds import KINDS + return {"schema": "task-capabilities/1", "request_schema": REQUEST_SCHEMA, "run_schema": RUN_SCHEMA, + "tasks": [kind.capability() for kind in KINDS.values()]} + + +def finite_number(value): + return not isinstance(value, bool) and isinstance(value, (int, float)) and math.isfinite(value) diff --git a/packages/codex/mlx-agent/src/mlx_agent/task_identity.py b/packages/codex/mlx-agent/src/mlx_agent/task_identity.py new file mode 100644 index 0000000..cc75eaa --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/task_identity.py @@ -0,0 +1,60 @@ +"""Metadata evidence for local inference inputs; weight bytes are not hashed.""" + +import hashlib +import sys +from pathlib import Path + +from . import __version__ +from .task_contracts import TaskError, encoded + + +def _file_identity(path): + info = path.stat() + return [info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns] + + +def _model_fingerprint(model): + entries = [] + if not model.is_dir(): + raise TaskError("model_changed", "The accepted model directory is unavailable.") + for path in sorted(model.rglob("*")): + if path.is_symlink() and not path.exists(): + raise TaskError("model_changed", "The model contains a broken link.") + if path.is_file(): + identity = _file_identity(path) + digest = hashlib.sha256(path.read_bytes()).hexdigest() if path.suffix == ".json" and identity[2] <= 1024 * 1024 else "" + entries.append([str(path.relative_to(model)), *identity, digest]) + return hashlib.sha256(encoded(entries)).hexdigest() + + +def capture_identity(model, manifest, executable): + try: + requested_model = str(Path(model).absolute()) + requested_executable = str(Path(executable).absolute()) + model = Path(model).resolve(strict=True) + executable = Path(executable).resolve(strict=True) + return {"schema": "task-identity/1", "requested_model_path": requested_model, + "model_path": str(model), "model_fingerprint": _model_fingerprint(model), + "backend": manifest["id"], "backend_version": manifest["version"], + "backend_manifest": manifest, "backend_manifest_hash": hashlib.sha256(encoded(manifest)).hexdigest(), + "requested_executable": requested_executable, "executable": str(executable), "executable_identity": _file_identity(executable), + "engine_version": __version__, "python_version": sys.version.split()[0]} + except (OSError, ValueError, KeyError) as error: + raise TaskError("identity_unavailable", "Cannot inspect local model/backend identity: {0}".format(error)) from error + + +def check_identity(expected): + from .backends import BackendError, load_manifests + try: + if expected["schema"] != "task-identity/1": + raise TaskError("identity_changed", "Unsupported task identity algorithm.") + manifest = load_manifests()[expected["backend"]] + current = capture_identity(Path(expected["requested_model_path"]), manifest, Path(expected["requested_executable"])) + for key in ("model_path", "model_fingerprint", "executable", "executable_identity", "backend_manifest_hash"): + if current[key] != expected[key]: + raise TaskError("identity_changed", "Accepted {0} changed.".format(key), "Start a new task with the current local inputs.") + except (KeyError, TypeError) as error: + raise TaskError("identity_changed", "Task identity evidence is invalid.") from error + except (OSError, ValueError, BackendError) as error: + raise TaskError("identity_unavailable", "Cannot read the current backend identity.", + "Repair the backend manifest and retry the identity check.") from error diff --git a/packages/codex/mlx-agent/src/mlx_agent/task_kinds.py b/packages/codex/mlx-agent/src/mlx_agent/task_kinds.py new file mode 100644 index 0000000..e5cdf46 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/task_kinds.py @@ -0,0 +1,306 @@ +"""Durable task kinds: each entry owns its parameters, plan, run, media input and output artifact.""" + +from __future__ import annotations + +import math +import os +import subprocess +import wave +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Dict, Optional + +from .backends import backend_environment +from .decide import plan_decide, run_decide, validate_request as validate_decision +from .describe import plan_describe, run_describe +from .generate import plan_generate, run_generate +from .inference_runner import InferenceError +from .music import plan_music, run_music, validate_parameters as validate_music +from .speak import plan_speak, run_speak +from .task_contracts import (MEDIA_REQUEST_SCHEMA, MEDIA_RUN_SCHEMA, REQUEST_SCHEMA, RUN_SCHEMA, TaskError, + absolute_path, encoded, finite_number) +from .transcribe import plan_transcribe, run_transcribe +from .video import plan_video, run_video + + +@dataclass(frozen=True) +class TaskKind: + """One task kind. `plan` takes (model_path, parameters, output, media, directory).""" + + name: str + result_schema: str + legacy: bool + validate: Callable[[dict], dict] + planner: Callable[..., dict] + runner: Callable[[dict], dict] + input_field: Optional[str] = None + input_media: Optional[str] = None + output: Optional[str] = None + output_check: Optional[Callable[[Path, dict], None]] = None + + @property + def request_schema(self): + return REQUEST_SCHEMA if self.legacy else MEDIA_REQUEST_SCHEMA + + @property + def run_schema(self): + return RUN_SCHEMA if self.legacy else MEDIA_RUN_SCHEMA + + def plan(self, request, directory, input_path=None): + """Plan one run; output lands in `directory`, media comes from `input_path` or the request.""" + parameters = request["parameters"] + media = input_path or (parameters[self.input_field] if self.input_field else None) + output = str(Path(directory) / self.output) if self.output else None + return self.planner(request["model_path"], parameters, output, media, Path(directory)) + + def run(self, plan): + return self.runner(plan) + + def check_output(self, path, plan): + """Fail closed unless the run left a usable output artifact at `path`.""" + if self.output_check is not None: + self.output_check(path, plan) + + def capability(self): + entry = {"kind": self.name, "result_schema": self.result_schema} + if not self.legacy: + entry.update(request_schema=MEDIA_REQUEST_SCHEMA, run_schema=MEDIA_RUN_SCHEMA) + entry.update(cancellation=True, requires_local_model=True, model_verification=True) + return entry + + +def write_json(path, value): + """Write one private JSON file that must not already exist.""" + descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(encoded(value)) + stream.flush() + os.fsync(stream.fileno()) + + +def _options(parameters, keys): + return {key: parameters[key] for key in keys if key in parameters} + + +def _invalid(message): + return TaskError("invalid_request", message) + + +def _require_keys(parameters, required, optional, message): + if set(parameters) - set(required) - set(optional) or not set(required) <= set(parameters): + raise _invalid(message) + + +def _require_string(parameters, *keys): + for key in keys: + if key in parameters and not isinstance(parameters[key], str): + raise _invalid(key + " must be a string.") + + +def _require_integer(parameters, *keys): + for key in keys: + if key in parameters and type(parameters[key]) is not int: + raise _invalid(key + " must be an integer.") + + +def _require_finite(parameters, *keys): + for key in keys: + if key in parameters and not finite_number(parameters[key]): + raise _invalid(key + " must be a finite number.") + + +def _require_absolute(parameters, key): + if not absolute_path(parameters[key]): + raise _invalid(key + " must be an absolute local path.") + + +def _validate_classification(parameters): + if set(parameters) != {"state", "questions"}: + raise _invalid("Classification parameters need state and questions.") + validate_decision(parameters) + return parameters + + +def _validate_music(parameters): + if "caption" not in parameters or set(parameters) - {"caption", "lyrics", "duration", "steps", "seed"}: + raise _invalid("Music parameters need a caption and supported generation settings.") + return validate_music(parameters["caption"], parameters.get("lyrics", "[instrumental]"), + parameters.get("duration", 15.0), parameters.get("steps", 30), parameters.get("seed", 42)) + + +def _validate_transcribe(parameters): + _require_keys(parameters, ("audio_path",), ("language",), "Transcription parameters need audio_path and optional language.") + _require_absolute(parameters, "audio_path") + _require_string(parameters, "language") + return parameters + + +def _validate_speak(parameters): + _require_keys(parameters, ("text",), ("voice", "speed", "lang_code"), "Speech parameters need text and supported synthesis settings.") + _require_string(parameters, "text", "voice") + if "speed" in parameters and (isinstance(parameters["speed"], bool) or not isinstance(parameters["speed"], (int, float)) + or not math.isfinite(parameters["speed"])): + raise _invalid("speed must be a finite number.") + _require_string(parameters, "lang_code") + return parameters + + +def _validate_image_generation(parameters): + _require_keys(parameters, ("prompt",), ("width", "height", "steps", "seed"), + "Image-generation parameters need prompt and optional size, steps and seed.") + _require_string(parameters, "prompt") + _require_integer(parameters, "width", "height", "steps", "seed") + return parameters + + +def _validate_image_understanding(parameters): + _require_keys(parameters, ("image_path", "prompt"), ("max_tokens", "temperature", "max_pixels"), + "Image-understanding parameters need image_path, prompt and supported options.") + _require_absolute(parameters, "image_path") + _require_string(parameters, "prompt") + _require_integer(parameters, "max_tokens", "max_pixels") + _require_finite(parameters, "temperature") + return parameters + + +def _validate_video_generation(parameters): + _require_keys(parameters, ("prompt",), ("width", "height", "frames", "steps", "seed", "fps"), + "Video-generation parameters need prompt and supported options.") + _require_string(parameters, "prompt") + _require_integer(parameters, "width", "height", "frames", "steps", "seed") + _require_finite(parameters, "fps") + return parameters + + +def _validate_video_understanding(parameters): + _require_keys(parameters, ("video_path", "prompt"), ("fps", "temperature", "max_tokens", "max_pixels"), + "Video-understanding parameters need video_path, prompt and supported options.") + _require_absolute(parameters, "video_path") + _require_string(parameters, "prompt") + _require_integer(parameters, "max_tokens", "max_pixels") + _require_finite(parameters, "fps", "temperature") + return parameters + + +def _plan_classification(model, parameters, output, media, directory): + path = directory / "decision.json" + write_json(path, parameters) + return plan_decide(model, str(path)) + + +def _plan_music(model, parameters, output, media, directory): + return plan_music(model, parameters["caption"], parameters["lyrics"], output, + duration=parameters["duration"], steps=parameters["steps"], seed=parameters["seed"]) + + +def _plan_transcribe(model, parameters, output, media, directory): + return plan_transcribe(model, media, **_options(parameters, ("language",))) + + +def _plan_speak(model, parameters, output, media, directory): + return plan_speak(model, parameters["text"], output, **_options(parameters, ("voice", "speed", "lang_code"))) + + +def _plan_image_generation(model, parameters, output, media, directory): + return plan_generate(model, parameters["prompt"], output, **_options(parameters, ("width", "height", "steps", "seed"))) + + +def _plan_image_understanding(model, parameters, output, media, directory): + return plan_describe(model, parameters["prompt"], image=media, + **_options(parameters, ("max_tokens", "temperature", "max_pixels"))) + + +def _plan_video_generation(model, parameters, output, media, directory): + return plan_video(model, parameters["prompt"], output, + **_options(parameters, ("width", "height", "frames", "fps", "steps", "seed"))) + + +def _plan_video_understanding(model, parameters, output, media, directory): + return plan_describe(model, parameters["prompt"], video=media, + **_options(parameters, ("fps", "max_tokens", "temperature", "max_pixels"))) + + +def _wav_check(activity): + def check(path, plan): + try: + with wave.open(str(path), "rb") as audio: + frames = audio.getnframes() + channels = audio.getnchannels() + sample_width = audio.getsampwidth() + if frames <= 0 or channels <= 0 or sample_width <= 0 or audio.getframerate() <= 0: + raise TaskError("invalid_audio_output", "{0} did not produce usable WAV audio.".format(activity)) + frame_bytes = channels * sample_width + chunk_frames = max(1, 65536 // frame_bytes) + received = 0 + remaining = frames + while remaining: + count = min(remaining, chunk_frames) + received += len(audio.readframes(count)) + remaining -= count + if received != frames * frame_bytes: + raise TaskError("invalid_audio_output", "{0} produced a truncated WAV payload.".format(activity)) + except (OSError, EOFError, wave.Error) as error: + raise TaskError("invalid_audio_output", "{0} did not produce a readable WAV file.".format(activity)) from error + return check + + +def _backend_validator(script, code, unchecked, undecodable, remediation): + def check(path, plan, runner=subprocess.run): + validator = Path(__file__).with_name(script) + try: + completed = runner([plan["argv"][0], str(validator), str(path)], stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30, + env=backend_environment(), check=False) + except (OSError, subprocess.SubprocessError) as error: + raise TaskError(code, unchecked, "Check the installed {0} and retry the task.".format(remediation)) from error + if completed.returncode != 0: + raise TaskError(code, undecodable, "Check the {0} output and retry the task.".format(remediation)) + return check + + +check_png = _backend_validator("task_png_validator.py", "invalid_image_output", + "Image generation could not validate its PNG output.", + "Image generation did not produce a fully decodable PNG.", "image backend") +check_mp4 = _backend_validator("task_mp4_validator.py", "invalid_video_output", + "Video generation could not validate its MP4 output.", + "Video generation did not produce a decodable MP4 video stream.", "video backend") + + +KINDS: Dict[str, TaskKind] = {kind.name: kind for kind in ( + TaskKind("classification", "decide/1", True, _validate_classification, _plan_classification, + lambda plan: run_decide(plan)), + TaskKind("music_generation", "music/1", True, _validate_music, _plan_music, lambda plan: run_music(plan), + output="output.wav", output_check=_wav_check("Music generation")), + TaskKind("transcribe", "transcribe/1", False, _validate_transcribe, _plan_transcribe, + lambda plan: run_transcribe(plan), input_field="audio_path", input_media="audio"), + TaskKind("speak", "speak/1", False, _validate_speak, _plan_speak, lambda plan: run_speak(plan), + output="output.wav", output_check=_wav_check("Speech synthesis")), + TaskKind("image_generation", "generate/1", False, _validate_image_generation, _plan_image_generation, + lambda plan: run_generate(plan), output="output.png", output_check=lambda path, plan: check_png(path, plan)), + TaskKind("image_understanding", "describe/1", False, _validate_image_understanding, _plan_image_understanding, + lambda plan: run_describe(plan), input_field="image_path", input_media="image"), + TaskKind("video_generation", "video/1", False, _validate_video_generation, _plan_video_generation, + lambda plan: run_video(plan), output="output.mp4", output_check=lambda path, plan: check_mp4(path, plan)), + TaskKind("video_understanding", "describe/1", False, _validate_video_understanding, _plan_video_understanding, + lambda plan: run_describe(plan), input_field="video_path", input_media="video"), +)} + + +def find(name): + """The kind named `name`, or None for anything that is not a known kind name.""" + return KINDS.get(name) if isinstance(name, str) else None + + +def get(name): + kind = find(name) + if kind is None: + raise TaskError("unsupported_task", "Unsupported task schema or kind.", "Run mlx-agent task capabilities --json.") + return kind + + +def validate_parameters(kind, parameters): + """Normalized parameters for `kind`; classified inference failures become task errors.""" + try: + return kind.validate(parameters) + except InferenceError as error: + raise TaskError(error.code, str(error), error.remediation) from error diff --git a/packages/codex/mlx-agent/src/mlx_agent/task_mp4_validator.py b/packages/codex/mlx-agent/src/mlx_agent/task_mp4_validator.py new file mode 100644 index 0000000..356f826 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/task_mp4_validator.py @@ -0,0 +1,46 @@ +"""Decode-check a generated MP4 in the selected video backend environment.""" + +from __future__ import annotations + +import math +import sys +from pathlib import Path + + +def valid_mp4(path: Path) -> bool: + capture = None + try: + import cv2 + + if path.suffix.lower() != ".mp4" or not path.is_file(): + return False + capture = cv2.VideoCapture(str(path)) + if not capture.isOpened(): + return False + fps = float(capture.get(cv2.CAP_PROP_FPS)) + frame_count = float(capture.get(cv2.CAP_PROP_FRAME_COUNT)) + width = float(capture.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = float(capture.get(cv2.CAP_PROP_FRAME_HEIGHT)) + if (not all(math.isfinite(value) for value in (fps, frame_count, width, height)) + or fps <= 0 or frame_count <= 0 or not frame_count.is_integer() + or width <= 0 or height <= 0 + or frame_count / fps <= 0): + return False + decoded = 0 + while True: + ok, frame = capture.read() + if not ok: + break + if frame is None or getattr(frame, "size", 0) <= 0: + return False + decoded += 1 + return decoded == int(frame_count) + except Exception: + return False + finally: + if capture is not None: + capture.release() + + +if __name__ == "__main__": + raise SystemExit(0 if len(sys.argv) == 2 and valid_mp4(Path(sys.argv[1])) else 1) diff --git a/packages/codex/mlx-agent/src/mlx_agent/task_png_validator.py b/packages/codex/mlx-agent/src/mlx_agent/task_png_validator.py new file mode 100644 index 0000000..af91248 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/task_png_validator.py @@ -0,0 +1,25 @@ +"""Decode-check a generated PNG in the selected image backend environment.""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +def valid_png(path: Path) -> bool: + try: + from PIL import Image + + with Image.open(path) as image: + if image.format != "PNG" or image.width <= 0 or image.height <= 0: + return False + image.verify() + with Image.open(path) as image: + image.load() + return True + except (ImportError, OSError, ValueError, SyntaxError): + return False + + +if __name__ == "__main__": + raise SystemExit(0 if len(sys.argv) == 2 and valid_png(Path(sys.argv[1])) else 1) diff --git a/packages/codex/mlx-agent/src/mlx_agent/task_runtime.py b/packages/codex/mlx-agent/src/mlx_agent/task_runtime.py new file mode 100644 index 0000000..908e52d --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/task_runtime.py @@ -0,0 +1,375 @@ +"""Durable local inference supervised independently of the calling client.""" + +import hashlib +import json +import os +import signal +import subprocess +import sys +import tempfile +import time +import wave +from datetime import datetime, timezone +from pathlib import Path + +from . import task_kinds +from .backends import BackendError, load_manifests +from .compute import ComputeBusy, ComputeLease, group_members, process_alive, process_identity +from .inference_runner import InferenceError +from .task_contracts import MAX_REQUEST_BYTES, TERMINAL, TaskError, validate_request +from .task_identity import capture_identity, check_identity +from .task_kinds import write_json +from .task_store import TaskStore +from .wiring import redact_secrets + +POLL_SECONDS = .1 +STARTUP_SECONDS = 15 +STOP_SECONDS = 5 + + +def _error(error): + return {"code": getattr(error, "code", "task_execution_failed"), + "message": redact_secrets(str(error))[-2000:], + "remediation": getattr(error, "remediation", "Inspect this task's local log and retry after resolving the failure.")} + + +def _spawn(role, identifier, store, launcher=subprocess.Popen): + path = store.artifact_path(identifier, role + ".log") + descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0), 0o600) + environment = dict(os.environ) + source = str(Path(__file__).resolve().parents[1]) + environment["PYTHONPATH"] = source + (os.pathsep + environment["PYTHONPATH"] if environment.get("PYTHONPATH") else "") + try: + with os.fdopen(descriptor, "wb") as output: + return launcher([sys.executable, "-m", "mlx_agent.task_worker", "--role", role, "--id", identifier, + "--state-dir", str(store.root)], stdin=subprocess.DEVNULL, stdout=output, + stderr=subprocess.STDOUT, start_new_session=True, env=environment, umask=0o077) + except OSError as error: + raise TaskError("task_launch_failed", "Cannot launch the task worker: {0}".format(error)) from error + + +def start_task(request_path, state_dir=None, launcher=subprocess.Popen): + store = TaskStore(state_dir) + staged_input = None + try: + path = Path(request_path) + if path.is_symlink() or not path.is_file(): + raise TaskError("invalid_request", "The task request must be a readable regular file.") + with path.open("rb") as stream: + raw = stream.read(MAX_REQUEST_BYTES + 1) + if len(raw) > MAX_REQUEST_BYTES: + raise TaskError("request_too_large", "Task requests are limited to 1 MiB.") + request = validate_request(json.loads(raw)) + kind = task_kinds.get(request["kind"]) + if kind.input_media: + staged_input = store.stage_input(request["parameters"][kind.input_field], media_kind=kind.input_media) + with tempfile.TemporaryDirectory(prefix="mlx-task-plan-") as directory: + plan = kind.plan(request, Path(directory), staged_input["staging_path"] if staged_input else None) + identity = capture_identity(Path(request["model_path"]), load_manifests()[plan["backend"]], Path(plan["argv"][0])) + except (OSError, ValueError, InferenceError, BackendError) as error: + if staged_input is not None: + store.discard_staged_input(staged_input) + raise TaskError(getattr(error, "code", "invalid_request"), str(error), getattr(error, "remediation", "Check the local model, request and installed backend.")) from error + except TaskError: + if staged_input is not None: + store.discard_staged_input(staged_input) + raise + try: + record = store.create(request, identity, input_artifact=staged_input["reference"] if staged_input else None, + staged_input=staged_input) + except (TaskError, OSError, ValueError): + if staged_input is not None: + store.discard_staged_input(staged_input) + raise + try: + _spawn("supervisor", record["id"], store, launcher) + except TaskError as error: + return store.transition(record["id"], ("starting",), {"state": "failed", "error": _error(error)}) + return store.get(record["id"]) + + +def _unresolved(record): + execution = record["execution"] + if not execution: + return False + try: + return bool(group_members(execution["pid"])) + except (OSError, ValueError, KeyError, subprocess.SubprocessError): + return True + + +def task_status(identifier, state_dir=None): + store = TaskStore(state_dir) + record = store.get(identifier) + if record["state"] in TERMINAL: + return record + owner = record["supervisor"] + if owner: + try: + observed = process_identity(owner["pid"]) + alive = process_alive(owner["pid"]) + except (OSError, TypeError, ValueError, subprocess.SubprocessError) as error: + raise TaskError("task_process_inspection_failed", "The task process could not be inspected.", + "Retry status when the local process table is available; no process was signaled.") from error + if alive and observed == owner["identity"]: + return record + if alive and observed is None: + raise TaskError("task_identity_unavailable", "The live supervisor identity could not be inspected.", + "Retry status when the local process table is available; no process was signaled.") + else: + age = (datetime.now(timezone.utc) - datetime.fromisoformat(record["created_at"])).total_seconds() + if age < STARTUP_SECONDS: + return record + try: + return store.transition(identifier, ("starting", "running"), {"state": "failed", "execution_unresolved": _unresolved(record), + "error": {"code": "task_interrupted", "message": "The task supervisor stopped before recording a result.", + "remediation": "Inspect the task log and any unresolved execution before starting new work."}}) + except TaskError as error: + if error.code == "task_state_changed": + return store.get(identifier) + raise + + +def cancel_task(identifier, state_dir=None): + return TaskStore(state_dir).request_cancel(identifier) + + +def verify_task_model(identifier, state_dir=None): + """Check recorded model/runtime identity without changing historical state.""" + record = TaskStore(state_dir).get(identifier) + check_identity(record["identity"]) + return {"id": record["id"], "model_path": record["model_path"], + "model_fingerprint": record["identity"]["model_fingerprint"], "matches": True} + + +def task_result(identifier, state_dir=None): + store = TaskStore(state_dir) + record = task_status(identifier, state_dir) + if record["state"] != "succeeded": + raise TaskError("task_not_succeeded", "This task has no successful result.", "Inspect its status and error before retrying.") + path = store.artifact_path(identifier, "result.json") + reference = record["result"] + try: + digest = hashlib.sha256() + size = 0 + with path.open("rb") as source: + for chunk in iter(lambda: source.read(65536), b""): + size += len(chunk) + digest.update(chunk) + if str(path) != reference.get("path") or size != reference.get("bytes") or digest.hexdigest() != reference.get("sha256"): + raise TaskError("result_changed", "The saved task result changed.") + inline = size <= 8192 + result = json.loads(path.read_bytes()) if inline else None + kind = task_kinds.get(record["kind"]) + if kind.input_media or kind.output: + # The result names its media artifacts; each one is checked against the task directory. + result = result if result is not None else json.loads(path.read_bytes()) + if not isinstance(result, dict): + raise TaskError("result_changed", "The task result is not a JSON object.") + if kind.input_media: + store.verify_input_artifact(identifier, record["input_artifact"], media_kind=kind.input_media) + if result.get("input_artifact") != record["input_artifact"]: + raise TaskError("result_changed", "The media input reference changed.") + if kind.output: + store.verify_artifact(identifier, result.get("output_artifact"), kind.output, code="output_artifact") + return {"task": record, "artifact": reference, "inline": inline, "result": result if inline else None} + except (OSError, ValueError) as error: + raise TaskError("result_unavailable", "The saved task result is unavailable.") from error + + +def _artifact_reference(path): + digest = hashlib.sha256() + size = 0 + with path.open("rb") as source: + for chunk in iter(lambda: source.read(65536), b""): + digest.update(chunk) + size += len(chunk) + return {"path": str(path), "sha256": digest.hexdigest(), "bytes": size} + + +def _known_members(process, identity, previous): + members = group_members(process.pid) + if process_identity(process.pid) == identity: + return {pid: observed for pid in members if (observed := process_identity(pid))} + return {pid: value for pid, value in previous.items() if pid in members and process_identity(pid) == value} + + +def _wait_for_execution_ready(process, store, identifier): + path = store.artifact_path(identifier, "execute.ready") + deadline = time.monotonic() + STARTUP_SECONDS + while time.monotonic() < deadline: + if path.exists(): + return + if process.poll() is not None: + raise TaskError("task_worker_start_failed", "The execution worker exited before completing startup.") + time.sleep(POLL_SECONDS) + raise TaskError("task_worker_start_timeout", "The execution worker did not complete startup before the deadline.") + + +def _stop(process, identity, known, grace=STOP_SECONDS): + for sig in (signal.SIGTERM, signal.SIGKILL): + members = group_members(process.pid) + if not members: + process.poll() + return True + known = _known_members(process, identity, known) + if not known: + return False + try: + os.killpg(process.pid, sig) + except ProcessLookupError: + return True + except OSError: + if not group_members(process.pid): + return True + raise + deadline = time.monotonic() + grace + while time.monotonic() < deadline: + process.poll() + if not group_members(process.pid): + return True + time.sleep(POLL_SECONDS) + return not group_members(process.pid) + + +def _finish(store, identifier, state, **changes): + try: + return store.transition(identifier, ("starting", "running"), dict(changes, state=state)) + except TaskError as error: + if error.code == "task_cancel_requested" and state == "succeeded": + return store.transition(identifier, ("starting", "running"), {"state": "cancelled", "error": None}) + if error.code != "task_state_changed": + raise + return store.get(identifier) + + +def supervise(identifier, state_dir, lease_factory=ComputeLease): + store = TaskStore(state_dir) + process, identity, known = None, None, {} + claimed = False + try: + own_identity = process_identity(os.getpid()) + if not own_identity: + raise TaskError("task_identity_unavailable", "Cannot establish the supervisor identity.") + record = store.get(identifier) + if record["state"] != "starting" or record["supervisor"] is not None: + return 1 + store.transition(identifier, ("starting",), {"supervisor": {"pid": os.getpid(), "identity": own_identity}}) + claimed = True + with lease_factory("task:" + identifier) as lease: + record = store.get(identifier) + if record["cancel_requested"]: + _finish(store, identifier, "cancelled") + return 0 + check_identity(record["identity"]) + store.read_request(identifier) + process = _spawn("execute", identifier, store) + identity = process_identity(process.pid) + if not identity: + raise TaskError("task_identity_unavailable", "Cannot establish the execution worker identity.") + known = {process.pid: identity} + lease.bind_process_group(process.pid) + store.transition(identifier, ("starting",), {"execution": {"pid": process.pid, "identity": identity}}) + _wait_for_execution_ready(process, store, identifier) + identity = process_identity(process.pid) + if not identity: + raise TaskError("task_identity_unavailable", "Cannot establish the ready execution worker identity.") + known = {process.pid: identity} + lease.bind_process_group(process.pid) + store.transition(identifier, ("starting",), {"state": "running", "execution": {"pid": process.pid, "identity": identity}}) + while process.poll() is None: + known = _known_members(process, identity, known) + record = store.get(identifier) + if record["cancel_requested"]: + stopped = _stop(process, identity, known) + _finish(store, identifier, "cancelled" if stopped else "failed", execution_unresolved=not stopped, + error=None if stopped else _error(TaskError("task_stop_unconfirmed", "The task execution group could not be stopped safely."))) + return 0 if stopped else 1 + time.sleep(POLL_SECONDS) + if group_members(process.pid) and not _stop(process, identity, known): + raise TaskError("task_stop_unconfirmed", "The execution worker exited but its group remains active.") + record = store.get(identifier) + if record["cancel_requested"]: + _finish(store, identifier, "cancelled") + return 0 + outcome_path = store.artifact_path(identifier, "execution-result.json") + try: + outcome = json.loads(outcome_path.read_bytes()) + except (OSError, ValueError) as error: + raise TaskError("task_execution_failed", "The execution worker did not produce a readable result.") from error + if process.returncode != 0 or outcome.get("error"): + error = outcome.get("error") or _error(TaskError("task_execution_failed", "The execution worker failed.")) + _finish(store, identifier, "failed", error=error) + return 1 + check_identity(record["identity"]) + store.read_request(identifier) + result = store.write_result(identifier, outcome["result"]) + # Re-read cancellation at the last publication boundary. + if store.get(identifier)["cancel_requested"]: + _finish(store, identifier, "cancelled") + else: + _finish(store, identifier, "succeeded", result=result, error=None) + return 0 + except (TaskError, ComputeBusy, OSError, ValueError, subprocess.SubprocessError) as error: + if not claimed: + print(str(error), file=sys.stderr) + return 1 + unresolved = False + if process is not None: + try: + unresolved = not _stop(process, identity, known) + except (OSError, ValueError, subprocess.SubprocessError): + unresolved = True + if isinstance(error, ComputeBusy): + error = TaskError("compute_busy", str(error), "Wait for the current host model operation to stop, then start a new task.") + try: + if not unresolved and store.get(identifier)["cancel_requested"]: + _finish(store, identifier, "cancelled", error=None) + else: + _finish(store, identifier, "failed", error=_error(error), execution_unresolved=unresolved) + except TaskError as storage_error: + print(str(storage_error), file=sys.stderr) + return 1 + + +def execute(identifier, state_dir): + store = TaskStore(state_dir) + try: + ready_path = store.artifact_path(identifier, "execute.ready") + descriptor = os.open(ready_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0), 0o600) + os.close(descriptor) + # Never run until the supervisor has durably bound the group to its lease. + deadline = time.monotonic() + STARTUP_SECONDS + while True: + record = store.get(identifier) + if record["state"] in TERMINAL or record["cancel_requested"]: + return 1 + if record["execution"] and record["execution"]["pid"] == os.getpid() and record["state"] == "running": + break + if time.monotonic() >= deadline: + return 1 + time.sleep(POLL_SECONDS) + check_identity(record["identity"]) + request = store.read_request(identifier) + record = store.get(identifier) + directory = store.artifact_path(identifier, "execution-result.json").parent + kind = task_kinds.get(request["kind"]) + input_path = None + if kind.input_media: + input_path = str(store.verify_input_artifact(identifier, record["input_artifact"], media_kind=kind.input_media)) + plan = kind.plan(request, directory, input_path) + result = kind.run(plan) + if kind.input_media: + result["input_artifact"] = record["input_artifact"] + if kind.output: + output = store.artifact_path(identifier, kind.output) + kind.check_output(output, plan) + result["output_artifact"] = _artifact_reference(output) + write_json(store.artifact_path(identifier, "execution-result.json"), {"result": result}) + return 0 + except (TaskError, InferenceError, BackendError, OSError, ValueError, wave.Error) as error: + try: + write_json(store.artifact_path(identifier, "execution-result.json"), {"error": _error(error)}) + except (TaskError, OSError): + pass + return 1 diff --git a/packages/codex/mlx-agent/src/mlx_agent/task_store.py b/packages/codex/mlx-agent/src/mlx_agent/task_store.py new file mode 100644 index 0000000..ac7c421 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/task_store.py @@ -0,0 +1,481 @@ +"""Private, transactional task history and immutable local artifacts.""" + +import hashlib +import json +import os +import sqlite3 +import stat +import sys +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 +from typing import Dict, Optional, Sequence + +from . import task_kinds +from .task_contracts import (MAX_REQUEST_BYTES, MEDIA_RUN_SCHEMA, MEDIA_SUFFIXES, RUN_SCHEMA, TERMINAL, TaskError, encoded, + _artifact_reference, task_id, validate_record, validate_request) + + +def default_root(): + if sys.platform == "darwin": + return Path.home() / "Library/Application Support/mlx-agent/tasks" + return Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local/state")) / "mlx-agent/tasks" + + +def utc_now(): + return datetime.now(timezone.utc).isoformat() + + +def ensure_private(path, directory=True): + for item in (path, *path.parents): + if item.is_symlink(): + raise TaskError("unsafe_task_storage", "Task storage cannot traverse symbolic links.") + info = path.stat() + expected_type = stat.S_ISDIR if directory else stat.S_ISREG + if not expected_type(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077: + raise TaskError("unsafe_task_storage", "Task storage must be private and owned by the current user.") + + +class TaskStore: + def __init__(self, root=None): + self.root = Path(root) if root is not None else default_root() + if not self.root.is_absolute(): + raise TaskError("invalid_state_dir", "Task state directory must be absolute.") + self.database = self.root / "tasks.sqlite3" + + @contextmanager + def _connection(self, write=False): + connection = None + try: + if not self.root.exists() and not self.root.is_symlink() and not write: + yield None + return + if write: + # Check parents before creating anything through a caller-supplied path. + for item in (self.root, *self.root.parents): + if item.is_symlink(): + raise TaskError("unsafe_task_storage", "Task storage cannot traverse symbolic links.") + self.root.mkdir(parents=True, mode=0o700, exist_ok=True) + ensure_private(self.root) + if not self.database.exists() and not self.database.is_symlink(): + if not write: + yield None + return + try: + descriptor = os.open(self.database, os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0), 0o600) + os.close(descriptor) + except FileExistsError: + pass + ensure_private(self.database, directory=False) + uri = self.database.as_uri() + ("?mode=rw" if write else "?mode=ro") + connection = sqlite3.connect(uri, uri=True, timeout=5) + if write: + connection.execute("BEGIN IMMEDIATE") + version = connection.execute("PRAGMA user_version").fetchone()[0] + if version == 0 and write: + tables = connection.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() + if tables: + raise TaskError("invalid_task_store", "An unversioned database cannot be adopted as task storage.") + connection.execute("CREATE TABLE tasks (id TEXT PRIMARY KEY, created_at TEXT NOT NULL, record TEXT NOT NULL)") + connection.execute("PRAGMA user_version = 1") + # Initialization is independent of a later failed task write. + connection.commit() + connection.execute("BEGIN IMMEDIATE") + elif version != 1: + raise TaskError("unsupported_task_store", "Unsupported task database version.", "Preserve the database and use a compatible engine.") + yield connection + if write: + connection.commit() + except TaskError: + raise + except (OSError, sqlite3.Error, ValueError) as error: + raise TaskError("task_storage_failed", "Cannot access task storage: {0}".format(error), "Check the task directory, free space and permissions; preserve existing records.") from error + finally: + if connection is not None: + connection.close() + + def _load(self, connection, identifier): + task_id(identifier) + row = connection.execute("SELECT record FROM tasks WHERE id = ?", (identifier,)).fetchone() if connection else None + if row is None: + raise TaskError("task_not_found", "No task with that ID.", "Use mlx-agent task list with the same --state-dir.") + try: + value = validate_record(json.loads(row[0])) + if value["id"] != identifier: + raise TaskError("invalid_task_record", "Task row identity does not match its record.") + return value + except (ValueError, TypeError) as error: + raise TaskError("invalid_task_record", "The stored task record is not readable JSON.") from error + + def _save(self, connection, record): + validate_record(record) + connection.execute("UPDATE tasks SET record = ? WHERE id = ?", (encoded(record).decode(), record["id"])) + + @contextmanager + def updating(self, identifier): + """Hold the write transaction over one record; changes to the yielded record are saved if the block succeeds.""" + with self._connection(write=True) as connection: + record = self._load(connection, identifier) + yield record + self._save(connection, record) + + def _ensure_artifact_root(self): + for item in (self.root, *self.root.parents): + if item.is_symlink(): + raise TaskError("unsafe_task_storage", "Task storage cannot traverse symbolic links.") + self.root.mkdir(parents=True, mode=0o700, exist_ok=True) + ensure_private(self.root) + artifacts = self.root / "artifacts" + artifacts.mkdir(mode=0o700, exist_ok=True) + ensure_private(artifacts) + return artifacts + + @staticmethod + def fsync_directory(path): + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + def stage_input(self, source, media_kind="audio"): + """Copy a supported media input into a private, unaccepted staging directory.""" + source = Path(source).expanduser() + suffix = source.suffix.lower() + if media_kind not in MEDIA_SUFFIXES or not source.is_absolute() or suffix not in MEDIA_SUFFIXES[media_kind]: + description = {"audio": "WAV, FLAC or MP3", "image": "PNG, JPEG or WebP", + "video": "MP4, MOV or M4V"}.get(media_kind, "supported media") + raise TaskError("invalid_input", "Media input must be an absolute {0} path.".format(description)) + identifier = str(uuid4()) + artifacts = self._ensure_artifact_root() + staging = artifacts / (".staging-" + identifier) + try: + staging.mkdir(mode=0o700) + ensure_private(staging) + if source.is_symlink(): + raise TaskError("invalid_input", "Media input must be a regular file, not a symbolic link.") + source_descriptor = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + output_path = staging / ("input" + suffix) + output_descriptor = None + try: + before = os.fstat(source_descriptor) + if not stat.S_ISREG(before.st_mode): + raise TaskError("invalid_input", "Media input must be a readable regular file.") + output_descriptor = os.open(output_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | + getattr(os, "O_NOFOLLOW", 0), 0o600) + digest, size = hashlib.sha256(), 0 + with os.fdopen(source_descriptor, "rb") as input_stream, os.fdopen(output_descriptor, "wb") as output_stream: + source_descriptor = output_descriptor = None + while True: + chunk = input_stream.read(1024 * 1024) + if not chunk: + break + output_stream.write(chunk) + digest.update(chunk) + size += len(chunk) + output_stream.flush() + os.fsync(output_stream.fileno()) + after = os.fstat(input_stream.fileno()) + current = os.stat(source, follow_symlinks=False) + identity = lambda value: (value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns, value.st_ctime_ns) + if (identity(before) != identity(after) or identity(before) != identity(current) + or size != before.st_size): + raise TaskError("input_changed", "Transcription input changed while it was being copied.") + ensure_private(output_path, directory=False) + self.fsync_directory(staging) + final = artifacts / identifier / output_path.name + return {"id": identifier, "staging_path": str(output_path), "staging_directory": str(staging), + "reference": {"path": str(final), "sha256": digest.hexdigest(), "bytes": size}} + finally: + if source_descriptor is not None: + os.close(source_descriptor) + if output_descriptor is not None: + os.close(output_descriptor) + except (OSError, TaskError) as error: + self.discard_staged_input({"id": identifier, "staging_directory": str(staging)}) + if isinstance(error, TaskError): + raise + raise TaskError("input_copy_failed", "Cannot safely copy the media input: {0}".format(error), + "Check that the source is readable and the task directory has free space.") from error + + def discard_staged_input(self, staged_input): + """Remove only the private staging directory allocated for this submission.""" + identifier = task_id(staged_input.get("id")) + expected = self.root / "artifacts" / (".staging-" + identifier) + if Path(staged_input.get("staging_directory", "")) != expected or expected.is_symlink(): + raise TaskError("invalid_input", "The staged input does not belong to this submission.") + if expected.exists(): + ensure_private(expected) + for child in expected.iterdir(): + if child.is_symlink() or not child.is_file(): + raise TaskError("unsafe_task_storage", "Staged input contains unexpected filesystem entries.") + child.unlink() + expected.rmdir() + + def get(self, identifier): + with self._connection() as connection: + return self._load(connection, identifier) + + def list(self, limit=20): + return self.list_page(limit)["tasks"] + + def list_page(self, limit: int = 20, kinds: Optional[Sequence[str]] = None) -> Dict: + if type(limit) is not int or not 1 <= limit <= 100: + raise TaskError("invalid_limit", "Task history limit must be between 1 and 100.") + if kinds is not None and (not isinstance(kinds, (tuple, list)) or not kinds + or any(not isinstance(kind, str) or not kind for kind in kinds)): + raise TaskError("invalid_kind", "Task history kinds must be a nonempty list of names.") + with self._connection() as connection: + if connection is None: + return {"tasks": [], "has_more": False} + cursor = connection.execute("SELECT id, record FROM tasks ORDER BY created_at DESC, id DESC") + tasks = [] + for identifier, raw in cursor: + try: + value = json.loads(raw) + except (ValueError, TypeError) as error: + raise TaskError("invalid_task_record", "The stored task record is not readable JSON.") from error + if kinds is not None and isinstance(value, dict) and value.get("kind") not in kinds: + continue + record = validate_record(value) + if record["id"] != identifier: + raise TaskError("invalid_task_record", "Task row identity does not match its record.") + tasks.append(record) + if len(tasks) > limit: + break + return {"tasks": tasks[:limit], "has_more": len(tasks) > limit} + + def create_capture(self, task_id: str, request_ref: Dict, identity: Dict, preview_hash: str) -> Dict: + """Index an accepted in-volume request without copying its body outside the volume.""" + from .coding_capture_contracts import KIND, RUN_SCHEMA as CAPTURE_RUN_SCHEMA + record = {"schema": CAPTURE_RUN_SCHEMA, "id": task_id, "kind": KIND, "state": "starting", + "created_at": utc_now(), "started_at": None, "completed_at": None, "cancel_requested": False, + "request": request_ref, "preview_hash": preview_hash, "context": identity, "processes": {}, + "client_outcome": "pending", "attribution": "pending", "result": None, "error": None, + "execution_unresolved": False, "artifacts_removed_at": None} + record = json.loads(encoded(validate_record(record))) + with self._connection(write=True) as connection: + try: + connection.execute("INSERT INTO tasks VALUES (?, ?, ?)", (task_id, record["created_at"], encoded(record).decode())) + except sqlite3.IntegrityError as error: + raise TaskError("task_exists", "A task with that ID already exists.") from error + return record + + def artifact_path(self, identifier, name): + task_id(identifier) + if not isinstance(name, str) or not name or name.startswith(".") or Path(name).name != name or "/" in name or "\\" in name: + raise TaskError("invalid_artifact", "Artifact names must be plain file names.") + try: + for directory in (self.root, self.root / "artifacts", self.root / "artifacts" / identifier): + ensure_private(directory) + path = self.root / "artifacts" / identifier / name + if path.is_symlink(): + raise TaskError("unsafe_task_storage", "Task artifacts cannot be symbolic links.") + if path.exists(): + ensure_private(path, directory=False) + return path + except OSError as error: + raise TaskError("task_storage_failed", "Cannot access the task artifact directory.") from error + + def _write_artifact(self, identifier, name, value): + target = self.artifact_path(identifier, name) + if target.exists(): + raise TaskError("artifact_exists", "Task evidence cannot be overwritten.") + raw = encoded(value) + temporary = target.parent / (".write-" + str(uuid4())) + try: + descriptor = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(raw) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, target) + descriptor = os.open(target.parent, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + except OSError as error: + raise TaskError("task_storage_failed", "Cannot save task evidence: {0}".format(error)) from error + finally: + temporary.unlink(missing_ok=True) + return {"path": str(target), "sha256": hashlib.sha256(raw).hexdigest(), "bytes": len(raw)} + + def create(self, request, identity, input_artifact=None, staged_input=None): + request = validate_request(request) + media = request["schema"] == "task-request/2" + if not media and (staged_input is not None or input_artifact is not None): + raise TaskError("invalid_task", "Legacy tasks cannot carry a media input snapshot.") + takes_input = task_kinds.get(request["kind"]).input_media is not None + if takes_input and (staged_input is None or input_artifact is None): + raise TaskError("invalid_task", "This task kind requires a retained input snapshot.") + if not takes_input and (staged_input is not None or input_artifact is not None): + raise TaskError("invalid_task", "This task kind does not accept an input snapshot.") + identifier = staged_input["id"] if staged_input is not None else str(uuid4()) + if staged_input is not None: + staged_directory = self.root / "artifacts" / (".staging-" + identifier) + staged_path = Path(staged_input.get("staging_path", "")) + if (Path(staged_input.get("staging_directory", "")) != staged_directory + or staged_path.parent != staged_directory or staged_path.is_symlink()): + raise TaskError("invalid_input", "The staged input does not match this task submission.") + ensure_private(staged_directory) + ensure_private(staged_path, directory=False) + digest, size = hashlib.sha256(), 0 + try: + with staged_path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + size += len(chunk) + except OSError as error: + self.discard_staged_input(staged_input) + raise TaskError("input_copy_failed", "The staged media input is unavailable.") from error + if digest.hexdigest() != input_artifact.get("sha256") or size != input_artifact.get("bytes"): + self.discard_staged_input(staged_input) + raise TaskError("input_changed", "The staged transcription input changed before acceptance.") + artifacts = self._ensure_artifact_root() + final_directory = artifacts / identifier + moved = False + try: + with self._connection(write=True) as connection: + if staged_input is not None: + expected_stage = artifacts / (".staging-" + identifier) + if (Path(staged_input.get("staging_directory", "")) != expected_stage + or Path(input_artifact.get("path", "")).parent != final_directory + or not expected_stage.is_dir() or final_directory.exists()): + raise TaskError("invalid_input", "The staged input does not match this task submission.") + os.replace(expected_stage, final_directory) + moved = True + self.fsync_directory(artifacts) + ensure_private(final_directory) + else: + final_directory.mkdir(mode=0o700, exist_ok=True) + ensure_private(final_directory) + reference = self._write_artifact(identifier, "request.json", request) + record = {"schema": MEDIA_RUN_SCHEMA if media else RUN_SCHEMA, "id": identifier, + "kind": request["kind"], "state": "starting", "model_path": request["model_path"], + "created_at": utc_now(), "started_at": None, "completed_at": None, + "cancel_requested": False, "identity": identity, "request": reference, "result": None, + "error": None, "supervisor": None, "execution": None, "execution_unresolved": False} + if media: + record["input_artifact"] = input_artifact + validate_record(record) + connection.execute("INSERT INTO tasks VALUES (?, ?, ?)", (identifier, record["created_at"], encoded(record).decode())) + return record + except (TaskError, OSError, sqlite3.Error, ValueError): + if staged_input is not None: + try: + self.discard_staged_input(staged_input) + except TaskError: + pass + if moved: + try: + self.get(identifier) + except TaskError as error: + if error.code == "task_not_found": + self._remove_unaccepted_directory(final_directory) + raise + + @staticmethod + def _remove_unaccepted_directory(directory): + if not directory.exists() or directory.is_symlink(): + return + ensure_private(directory) + for child in directory.iterdir(): + if child.is_symlink() or not child.is_file(): + raise TaskError("unsafe_task_storage", "Unaccepted task directory contains unexpected entries.") + child.unlink() + directory.rmdir() + + def verify_input_artifact(self, identifier, reference, media_kind="audio"): + if media_kind not in MEDIA_SUFFIXES: + raise TaskError("invalid_input", "The retained input media kind is unsupported.") + names = tuple("input" + suffix for suffix in MEDIA_SUFFIXES[media_kind]) + return self.verify_artifact(identifier, reference, allowed_names=names, + code="input_artifact") + + def verify_artifact(self, identifier, reference, expected_name=None, allowed_names=(), code="artifact"): + task_id(identifier) + try: + _artifact_reference(reference) + except TaskError as error: + raise TaskError(code + "_changed", "The task artifact reference is invalid.") from error + path = Path(reference["path"]) + names = set(allowed_names) + if expected_name is not None: + names.add(expected_name) + if (path.parent.name != identifier or path.parent.parent != self.root / "artifacts" + or path.name not in names): + raise TaskError(code + "_changed", "The task artifact is outside its owned directory.") + owned = self.artifact_path(identifier, path.name) + digest, size = hashlib.sha256(), 0 + try: + with owned.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + size += len(chunk) + except OSError as error: + raise TaskError(code + "_unavailable", "The retained task artifact is unavailable.") from error + if str(owned) != reference["path"] or digest.hexdigest() != reference["sha256"] or size != reference["bytes"]: + raise TaskError(code + "_changed", "The retained task artifact changed.") + return owned + + def transition(self, identifier, expected, changes): + with self._connection(write=True) as connection: + record = self._load(connection, identifier) + capture = record["kind"] == "coding_capture" + mutable = {"state", "started_at", "completed_at", "result", "error", "execution_unresolved"} + mutable.update(("processes", "client_outcome", "attribution") if capture else ("supervisor", "execution")) + if not isinstance(changes, dict) or set(changes) - mutable: + raise TaskError("invalid_transition", "Task identity and accepted inputs are immutable.") + if record["state"] in TERMINAL or record["state"] not in expected: + raise TaskError("task_state_changed", "The task state has already changed.", "Read its current status.") + if "supervisor" in changes and record["supervisor"] is not None: + raise TaskError("task_state_changed", "The task already has a supervisor.") + if capture and "processes" in changes and isinstance(changes["processes"], dict): + if any(changes["processes"].get(role) != identity for role, identity in record["processes"].items()): + raise TaskError("task_state_changed", "Accepted capture process identities cannot be replaced.") + next_state = changes.get("state", record["state"]) + if next_state == "succeeded" and record["cancel_requested"]: + raise TaskError("task_cancel_requested", "Cancellation was requested before result publication.") + if record["state"] == "running" and next_state == "starting": + raise TaskError("invalid_transition", "Running tasks cannot restart under the same identity.") + record.update(changes) + if next_state == "running" and record["started_at"] is None: + record["started_at"] = utc_now() + if next_state in TERMINAL: + record["completed_at"] = utc_now() + self._save(connection, record) + return record + + def request_cancel(self, identifier): + with self._connection(write=True) as connection: + record = self._load(connection, identifier) + if record["state"] not in TERMINAL: + record["cancel_requested"] = True + self._save(connection, record) + return record + + def read_request(self, identifier): + record = self.get(identifier) + if record["kind"] == "coding_capture": + raise TaskError("capture_storage_required", "Read capture requests through their bounded owned volume.") + try: + path = self.artifact_path(identifier, "request.json") + if path.stat().st_size > MAX_REQUEST_BYTES: + raise TaskError("request_changed", "The saved request exceeds its size limit.") + raw = path.read_bytes() + if hashlib.sha256(raw).hexdigest() != record["request"].get("sha256"): + raise TaskError("request_changed", "The accepted request snapshot changed.") + return validate_request(json.loads(raw)) + except (OSError, ValueError) as error: + raise TaskError("request_changed", "The accepted request snapshot is unavailable.") from error + + def write_result(self, identifier, value): + with self._connection(write=True) as connection: + record = self._load(connection, identifier) + if record["kind"] == "coding_capture": + raise TaskError("capture_storage_required", "Publish capture results inside their bounded owned volume.") + if record["state"] in TERMINAL: + raise TaskError("task_state_changed", "A terminal task cannot acquire new result evidence.") + return self._write_artifact(identifier, "result.json", value) diff --git a/packages/codex/mlx-agent/src/mlx_agent/task_worker.py b/packages/codex/mlx-agent/src/mlx_agent/task_worker.py new file mode 100644 index 0000000..c1c0430 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/task_worker.py @@ -0,0 +1,19 @@ +"""Internal process entry points for the engine-owned inference lifecycle.""" + +import argparse +from pathlib import Path + +from .task_runtime import execute, supervise + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("--role", choices=("supervisor", "execute"), required=True) + parser.add_argument("--id", required=True) + parser.add_argument("--state-dir", type=Path, required=True) + args = parser.parse_args(argv) + return (supervise if args.role == "supervisor" else execute)(args.id, args.state_dir) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/src/mlx_agent/taxonomy.py b/packages/codex/mlx-agent/src/mlx_agent/taxonomy.py new file mode 100644 index 0000000..af69ad4 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/taxonomy.py @@ -0,0 +1,266 @@ +"""Deterministic task type and use-case labels for models (no model calls).""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +from .backends import ( + TYPE_PREFERENCE, BackendError, choose_backend, is_vision_type, load_manifests, load_registries, lookup, + lookup_squashed, squash_type, +) +from .gguf import DRAFT_ARCHITECTURES, MAX_CONFIG_BYTES +from .modality import detect_facets, detect_modalities + +TASK_TYPES = ( + "text_llm", "vision_language", "speech_to_text", "text_to_speech", + "embedding", "classification", "image_generation", "video_generation", "music_generation", "speculative_draft", "other", +) +# Ordered specific-first; the last entry is the type's default use case. +USE_CASES = { + "text_llm": ("coding", "reasoning", "general_chat"), + "vision_language": ("ocr_documents", "video", "vision"), + "speech_to_text": ("realtime_transcription", "diarization", "speech_translation", "transcription"), + "text_to_speech": ("voice_cloning", "narration"), + "embedding": ("reranking", "retrieval"), + "classification": ("moderation", "routing", "classification"), + "image_generation": ("image_generation",), + "video_generation": ("video_generation",), + "music_generation": ("music_generation",), + "speculative_draft": ("speculative_decoding",), + "other": (), +} +PIPELINE_TYPES = { + "text-generation": "text_llm", + "text2text-generation": "text_llm", + "conversational": "text_llm", + "image-text-to-text": "vision_language", + "visual-question-answering": "vision_language", + "image-to-text": "vision_language", + "video-text-to-text": "vision_language", + "document-question-answering": "vision_language", + "any-to-any": "vision_language", + "automatic-speech-recognition": "speech_to_text", + "text-to-speech": "text_to_speech", + "text-to-audio": "text_to_speech", + "feature-extraction": "embedding", + "sentence-similarity": "embedding", + "text-ranking": "embedding", + "text-classification": "classification", + "zero-shot-classification": "classification", + "text-to-image": "image_generation", + "image-to-image": "image_generation", + "text-to-video": "video_generation", +} +_SPECIFIC_TOKENS = ( + ("coding", ("code", "coder", "coding", "starcoder", "codestral", "devstral")), + ("reasoning", ("reasoning", "reasoner", "thinking", "r1", "qwq")), + ("ocr_documents", ("ocr", "document", "docvqa")), + ("video", ("video",)), + ("realtime_transcription", ("realtime", "real-time", "streaming")), + ("diarization", ("diariz",)), + ("speech_translation", ("translat",)), + ("voice_cloning", ("clone", "cloning", "zero-shot")), + ("reranking", ("rerank",)), + ("moderation", ("moderation", "guardrail", "toxic")), + ("routing", ("routing", "router")), +) +# llama.cpp multimodal projectors (mmproj files) are model parts, not models. +_GGUF_PROJECTOR_ARCHITECTURES = ("clip", "mmproj") +# A converted DSpark drafter's config.json (mlx-agent's port, mlx-community's sidecars). +_DRAFT_CONFIG_KEYS = ("dspark_target_layer_ids",) +_VISION_KEYS = ("vision_config", "vision_tower", "mm_vision_tower", "visual", "vision_encoder") +_EMBEDDING_NAME = re.compile(r"(? MAX_CONFIG_BYTES: + return None, () + value = json.loads(location.read_text(encoding="utf-8")) + except (OSError, ValueError, TypeError): + return None, () + if not isinstance(value, dict): + return None, () + model_type = value.get("model_type") if isinstance(value.get("model_type"), str) else None + return model_type, tuple(sorted(key for key in value if isinstance(key, str))) + + +def annotate_inventory(report, manifests=None, registries=None): + """Add a "task" object to every scanned GGUF model and MLX output.""" + if manifests is None: + try: + manifests = load_manifests() + except (BackendError, OSError, ValueError): + manifests = {} + if registries is None: + registries = load_registries(manifests) + for item in report.get("models", []): + item["task"] = classify( + item.get("name"), gguf_architecture=item.get("architecture"), + manifests=manifests, registries=registries, local=True, + ) + for output in report.get("outputs", []): + model_type, keys = read_config_type(output.get("path", "")) + name = _output_name(output) + if model_type is None: + model_type = name_model_type(name, manifests, registries) + output["task"] = classify( + name, model_type=model_type, config_keys=keys, + manifests=manifests, registries=registries, local=True, + ) + if output.get("runtime") is None: + backend = output_backend(model_type, keys, output["task"]["type"], manifests, registries) + if backend is not None: + output["backend"] = backend + return report + + +def _output_name(output): + """The repository name for a Hugging Face cache snapshot (its folder is a revision hash), else the folder name.""" + parts = Path(str(output.get("path", ""))).parts + if len(parts) >= 3 and parts[-2] == "snapshots" and parts[-3].startswith("models--"): + return parts[-3].split("--")[-1] + return output.get("name") + + +def name_model_type(name, manifests, registries): + """A backend model module named by the folder's leading word, for configs without a model_type.""" + match = re.match(r"[a-z]+", str(name or "").lower()) + if match and lookup(match.group(0), manifests, registries): + return match.group(0) + return None + + +def output_backend(model_type, config_keys, task_type, manifests, registries): + """The backend that runs a local MLX output for its task, and whether it is installed.""" + hits = lookup(model_type, manifests, registries) if model_type else [] + backend = choose_backend(hits, task_type, "vision_config" in config_keys) + if backend is None and not hits: + # Configs without a model_type (Kokoro) fall back to the task's only backend. + preference = TYPE_PREFERENCE.get(task_type, ()) + backend = preference[0] if len(preference) == 1 else None + if backend is None or backend not in manifests: + return None + return {"id": backend, "installed": bool(registries.get(backend, {}).get("installed"))} diff --git a/packages/codex/mlx-agent/src/mlx_agent/training_artifacts.py b/packages/codex/mlx-agent/src/mlx_agent/training_artifacts.py new file mode 100644 index 0000000..84221d9 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/training_artifacts.py @@ -0,0 +1,43 @@ +"""Read a completed training receipt and recheck its local artifacts.""" +import hashlib +import json +from pathlib import Path + + +def read_trained_adapter(receipt_path): + from .jit_serve import model_fingerprint + from .lora import adapter_identity + from .wiring import _MODEL + + path = Path(receipt_path).expanduser() + if not path.is_absolute() or path.is_symlink() or not path.is_file(): + raise ValueError("Choose a bounded local training receipt.") + with path.open("rb") as stream: + payload = stream.read(65537) + if len(payload) > 65536 or path.is_symlink(): + raise ValueError("Choose a bounded local training receipt.") + receipt = json.loads(payload) + if (not isinstance(receipt, dict) or receipt.get("kind") != "lora" + or receipt.get("exit_status") != "done" or type(receipt.get("pid")) is not int + or not isinstance(receipt.get("argv"), list)): + raise ValueError("A completed training receipt with verified adapter output is required.") + repo = receipt.get("repo") + if not isinstance(repo, str) or not _MODEL.fullmatch(repo): + raise ValueError("The training receipt has no valid base model identifier.") + output = receipt.get("out") + if not isinstance(output, str) or not output: + raise ValueError("The training receipt has no adapter output path.") + adapter = Path(output).expanduser().absolute() + identity = adapter_identity(adapter) + if not identity or identity != receipt.get("adapter_identity"): + raise ValueError("Adapter content no longer matches the completed training receipt.") + base = receipt.get("base_identity") + if not isinstance(base, dict) or not isinstance(base.get("path"), str) or not base["path"]: + raise ValueError("This historical receipt has no pinned base identity; run a newly reviewed training job.") + model = Path(base["path"]).expanduser().absolute() + if not model.is_dir() or model_fingerprint(model) != base.get("fingerprint"): + raise ValueError("The trained base-model file identity changed.") + binding = {"training_receipt": str(path), "training_receipt_sha256": hashlib.sha256(payload).hexdigest(), + "training_run_id": receipt.get("run_id"), "base_repo": repo, "base_path": str(model), + "base_fingerprint": base["fingerprint"], "adapter_path": str(adapter), "adapter_identity": identity} + return receipt, binding diff --git a/packages/codex/mlx-agent/src/mlx_agent/training_eval_runtime.py b/packages/codex/mlx-agent/src/mlx_agent/training_eval_runtime.py new file mode 100644 index 0000000..4c59073 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/training_eval_runtime.py @@ -0,0 +1,31 @@ +"""Optional MLX inference child; invoked only for reviewed local evaluations.""" +import json +from pathlib import Path +import sys + +def generate_outputs(plan, variant, load, generate, make_sampler, seed): + if variant not in ("baseline", "adapter"): + raise ValueError("Unknown evaluation variant.") + model, tokenizer = load(plan["base_path"], adapter_path=plan["adapter_path"] if variant == "adapter" else None) + prompts = plan["prompt_suite"]["prompts"] if "prompt_suite" in plan else [ + {"id": "prompt", "text": plan["prompt"], "max_tokens": plan["max_tokens"]}] + samples = [] + for entry in prompts: + seed(plan["seed"]) + text = generate(model, tokenizer, prompt=entry["text"], max_tokens=entry["max_tokens"], + sampler=make_sampler(temp=plan["temperature"]), verbose=False) + samples.append({"id": entry["id"], "text": text}) + return {"samples": samples} if "prompt_suite" in plan else {"text": samples[0]["text"]} + + +def main(): + plan = json.loads(Path(sys.argv[1]).read_text()) + variant = sys.argv[2] + import mlx.core as mx + from mlx_lm import load, generate + from mlx_lm.sample_utils import make_sampler + outputs = generate_outputs(plan, variant, load, generate, make_sampler, mx.random.seed) + Path(sys.argv[3]).write_text(json.dumps(outputs)) + +if __name__ == "__main__": + main() diff --git a/packages/codex/mlx-agent/src/mlx_agent/training_evaluation.py b/packages/codex/mlx-agent/src/mlx_agent/training_evaluation.py new file mode 100644 index 0000000..e221f6f --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/training_evaluation.py @@ -0,0 +1,235 @@ +"""Link a cached-base/adapter comparison to its existing training receipt.""" +from __future__ import annotations +import hashlib +import json +import os +import copy +import re +from pathlib import Path +import sys +import tempfile +import uuid + +from mlx_agent.compute import ComputeLease +from mlx_agent.lab import _run_child + + +def _digest(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _prompt(text, limit): + if not isinstance(text, str) or not text.strip() or len(text) > 8000: + raise ValueError("Evaluation prompt must contain 1..8000 characters.") + if type(limit) is not int or not 1 <= limit <= 512: + raise ValueError("Evaluation token limit must be 1..512.") + + +def validate_suite(value): + if not isinstance(value, dict) or set(value) != {"id", "name", "prompts"}: + raise ValueError("An evaluation suite needs id, name and prompts only.") + if not isinstance(value["id"], str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", value["id"]): + raise ValueError("Invalid evaluation suite id.") + name = value["name"] + if not isinstance(name, str) or not name.strip() or len(name) > 200 or any(ord(c) < 32 for c in name): + raise ValueError("Evaluation suite name must contain 1..200 characters without controls.") + prompts = value["prompts"] + if not isinstance(prompts, list) or not 1 <= len(prompts) <= 16: + raise ValueError("An evaluation suite must contain 1..16 prompts.") + identities = set() + for entry in prompts: + if not isinstance(entry, dict) or set(entry) != {"id", "text", "max_tokens"}: + raise ValueError("Evaluation entries support id, text and max_tokens only.") + identity = entry["id"] + if not isinstance(identity, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", identity) or identity in identities: + raise ValueError("Evaluation prompt ids must be valid and unique.") + identities.add(identity) + _prompt(entry["text"], entry["max_tokens"]) + return copy.deepcopy(value) + + +def preview(receipt_path, prompt=None, max_tokens=128, prompt_suite=None): + if prompt_suite is not None: + if prompt is not None or max_tokens != 128: + raise ValueError("Choose either a prompt suite or a single prompt and token limit.") + prompt_suite = validate_suite(prompt_suite) + else: + _prompt(prompt, max_tokens) + from mlx_agent.training_artifacts import read_trained_adapter + receipt, binding = read_trained_adapter(receipt_path) + path = Path(binding["training_receipt"]) + model = Path(binding["base_path"]) + identity = binding["adapter_identity"] + files = [p for p in model.rglob("*") if p.is_file()] + if not files or len(files) > 256: + raise ValueError("Base-model inventory is unavailable or exceeds its bound.") + inventory = {str(p.relative_to(model)): {"sha256": _digest(p), "bytes": p.stat().st_size} + for p in sorted(files)} + plan = {"schema_version": "text-training-evaluation/1", "training_receipt": str(path), + "training_receipt_sha256": binding["training_receipt_sha256"], "training_run_id": receipt.get("run_id"), + "base_repo": receipt["repo"], "base_path": str(model), "base_revision": model.name, + "base_inventory": inventory, "dataset": receipt.get("dataset"), "adapter_path": binding["adapter_path"], + "adapter_identity": identity, "prompt": prompt, "max_tokens": max_tokens, + "temperature": 0.0, "seed": 42} + if prompt_suite is not None: + plan.update(schema_version="text-training-evaluation/2", prompt_suite=prompt_suite) + del plan["prompt"], plan["max_tokens"] + plan["preview_hash"] = hashlib.sha256(json.dumps(plan, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + return plan + + +def history(database): + with database.connect() as connection: + present = connection.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='text_training_evaluations'").fetchone() + if not present: + return [] + return [dict(row) for row in connection.execute( + "SELECT * FROM text_training_evaluations ORDER BY created_at DESC, rowid DESC LIMIT 200")] + + +def latest_reviews(database, evaluation_id, digest): + with database.connect() as connection: + present = connection.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='text_training_evaluation_reviews'").fetchone() + if not present: + return [] + return [dict(row) for row in connection.execute("""SELECT r.* FROM text_training_evaluation_reviews r + WHERE evaluation_id=? AND artifact_sha256=? AND NOT EXISTS ( + SELECT 1 FROM text_training_evaluation_reviews newer WHERE newer.evaluation_id=r.evaluation_id + AND newer.prompt_id=r.prompt_id AND newer.artifact_sha256=r.artifact_sha256 + AND newer.rowid>r.rowid) ORDER BY r.rowid""", (evaluation_id, digest))] + + +def report_samples(report): + if report.get("schema_version") == "text-training-evaluation/2": + return report["samples"] + if report.get("schema_version") == "text-training-evaluation/1": + return [{"id": "prompt", "prompt": report["prompt"], "max_tokens": report["max_tokens"], "outputs": report["outputs"]}] + raise ValueError("Unsupported retained evaluation schema.") + + +def inspect(database, artifacts, evaluation_id): + with database.connect() as connection: + present = connection.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='text_training_evaluations'").fetchone() + row = connection.execute("SELECT * FROM text_training_evaluations WHERE id=?", (evaluation_id,)).fetchone() if present else None + record = dict(row) if row else None + if record is None: + raise ValueError("Evaluation was not found in this workspace.") + artifact = artifacts.get(record["artifact_id"]) + if artifact.sha256 != record["artifact_sha256"] or artifact.byte_size > 1024 * 1024: + raise ValueError("Evaluation artifact does not match its retained record.") + with database.connect() as connection: + with artifacts.open_verified_with_connection(connection, artifact.id, + sha256=artifact.sha256, byte_size=artifact.byte_size, + media_type="application/json") as handle: + content = os.read(handle.descriptor, artifact.byte_size + 1) + artifacts.reverify_open_handle(connection, handle) + report = json.loads(content) + return {"evaluation_id": evaluation_id, "artifact_sha256": artifact.sha256, "report": report, + "reviews": latest_reviews(database, evaluation_id, artifact.sha256)} + + +def run(database, artifacts, request, generate=None): + with ComputeLease("text-training-evaluation") as lease: + plan = preview(request["receipt"], request.get("prompt"), request.get("max_tokens", 128), request.get("prompt_suite")) + if request.get("preview_hash") != plan["preview_hash"]: + raise ValueError("The evaluation plan changed; review a fresh preview.") + with tempfile.TemporaryDirectory(prefix="mlx-training-evaluation-") as directory: + folder = Path(directory) + file = folder / "plan.json" + file.write_text(json.dumps(plan)) + outputs = {} + for variant in ("baseline", "adapter"): + if generate is not None: + outputs[variant] = generate(plan, variant) + continue + destination = folder / (variant + ".json") + environment = {**os.environ, "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1"} + result = _run_child([sys.executable, "-m", "mlx_agent.training_eval_runtime", + str(file), variant, str(destination)], environment, timeout=900, + pass_fds=(lease.descriptor,)) + if result.returncode != 0: + raise ValueError("Local evaluation failed: " + result.stderr[-2048:]) + if not destination.is_file() or destination.stat().st_size > 1024 * 1024: + raise ValueError("Evaluation produced no bounded output.") + output = json.loads(destination.read_text()) + outputs[variant] = output["samples"] if "prompt_suite" in plan else output["text"] + if preview(request["receipt"], request.get("prompt"), request.get("max_tokens", 128), request.get("prompt_suite")) != plan: + raise ValueError("Evaluation inputs changed during generation; no result was accepted.") + if "prompt_suite" in plan: + prompts = plan["prompt_suite"]["prompts"] + for variant in outputs: + values = outputs[variant] + if not isinstance(values, list) or len(values) != len(prompts) or any( + not isinstance(value, dict) or value.get("id") != entry["id"] + or not isinstance(value.get("text"), str) or not value["text"].strip() + for entry, value in zip(prompts, values)): + raise ValueError("Evaluation did not return every paired prompt in order.") + samples = [{"id": entry["id"], "prompt": entry["text"], "max_tokens": entry["max_tokens"], + "outputs": {variant: outputs[variant][index]["text"] for variant in outputs}} + for index, entry in enumerate(prompts)] + report = {**plan, "samples": samples, "judgment": None} + else: + if any(not isinstance(text, str) or not text.strip() for text in outputs.values()): + raise ValueError("Evaluation returned an empty response.") + report = {**plan, "outputs": outputs, "judgment": None} + encoded = json.dumps(report, sort_keys=True).encode() + if len(encoded) > 1024 * 1024: + raise ValueError("Evaluation report exceeds its retained size bound.") + staged = artifacts.stage_bytes(encoded, "application/json") + artifact = artifacts.commit(staged, None) + identity = "text_eval_" + uuid.uuid4().hex + with database.connect() as connection: + connection.execute("""CREATE TABLE IF NOT EXISTS text_training_evaluations ( + id TEXT PRIMARY KEY, training_receipt TEXT NOT NULL, training_run_id TEXT, + artifact_id TEXT NOT NULL REFERENCES artifacts(id), artifact_sha256 TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')))""") + columns = {row[1] for row in connection.execute("PRAGMA table_info(text_training_evaluations)")} + for name, kind in (("prompt_count", "INTEGER"), ("suite_name", "TEXT")): + if name not in columns: + connection.execute("ALTER TABLE text_training_evaluations ADD COLUMN " + name + " " + kind) + connection.execute("INSERT INTO text_training_evaluations(id,training_receipt,training_run_id,artifact_id,artifact_sha256,prompt_count,suite_name) VALUES (?,?,?,?,?,?,?)", + (identity, plan["training_receipt"], plan["training_run_id"], artifact.id, artifact.sha256, + len(report_samples(report)), plan.get("prompt_suite", {}).get("name"))) + return {"evaluation_id": identity, "artifact_id": artifact.id, "artifact_sha256": artifact.sha256, + "report": report, "reviews": [], "path": str(artifacts.resolve(artifact.id))} + + +def review(database, artifacts, reviewer, request): + identity, digest, prompt_id = request["evaluation_id"], request["artifact_sha256"], request["prompt_id"] + preference, notes = request["preference"], request.get("notes", "") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ValueError("A retained evaluation SHA-256 is required.") + if preference not in ("baseline", "adapter", "tie", "neither") or not isinstance(notes, str) or len(notes) > 2000: + raise ValueError("Choose a supported preference and notes of at most 2000 characters.") + if not isinstance(reviewer, str) or not reviewer.strip() or len(reviewer) > 256: + raise ValueError("A bounded reviewer identity is required.") + with database.connect() as connection: + connection.execute("BEGIN IMMEDIATE") + if not connection.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='text_training_evaluations'").fetchone(): + raise ValueError("Evaluation was not found in this workspace.") + record = connection.execute("SELECT * FROM text_training_evaluations WHERE id=?", (identity,)).fetchone() + if record is None or record["artifact_sha256"] != digest: + raise ValueError("The retained evaluation identity changed; inspect it again.") + artifact = artifacts.get(record["artifact_id"]) + if artifact.sha256 != digest or artifact.byte_size > 1024 * 1024: + raise ValueError("The retained evaluation artifact changed.") + with artifacts.open_verified_with_connection(connection, artifact.id, sha256=digest, + byte_size=artifact.byte_size, media_type="application/json") as handle: + report = json.loads(os.read(handle.descriptor, artifact.byte_size + 1)) + if prompt_id not in {sample["id"] for sample in report_samples(report)}: + raise ValueError("The prompt is not part of this retained evaluation.") + connection.execute("""CREATE TABLE IF NOT EXISTS text_training_evaluation_reviews ( + id TEXT PRIMARY KEY, evaluation_id TEXT NOT NULL REFERENCES text_training_evaluations(id), + artifact_sha256 TEXT NOT NULL, prompt_id TEXT NOT NULL, reviewer_id TEXT NOT NULL, + preference TEXT NOT NULL, notes TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')))""") + review_id = "text_review_" + uuid.uuid4().hex + connection.execute("INSERT INTO text_training_evaluation_reviews(id,evaluation_id,artifact_sha256,prompt_id,reviewer_id,preference,notes) VALUES (?,?,?,?,?,?,?)", + (review_id, identity, digest, prompt_id, reviewer, preference, notes)) + artifacts.reverify_open_handle(connection, handle) + return {"evaluation_id": identity, "artifact_sha256": digest, "review_id": review_id, + "reviews": latest_reviews(database, identity, digest)} diff --git a/packages/codex/mlx-agent/src/mlx_agent/training_runner.py b/packages/codex/mlx-agent/src/mlx_agent/training_runner.py new file mode 100644 index 0000000..4362f85 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/training_runner.py @@ -0,0 +1,30 @@ +"""Record the actual trainer exit without inferring success from a folder.""" +import argparse +import json +from pathlib import Path +import subprocess +import sys + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from mlx_agent.atomicio import write_text_atomic + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--result", type=Path, required=True) + parser.add_argument("command", nargs=argparse.REMAINDER) + arguments = parser.parse_args() + command = arguments.command[1:] if arguments.command[:1] == ["--"] else arguments.command + if not command: + parser.error("training command is required") + try: + code = subprocess.run(command).returncode + result = {"schema_version": "training-exit/1", "exit_code": code} + except OSError as error: + code = 127 + result = {"schema_version": "training-exit/1", "exit_code": code, "error": str(error)} + write_text_atomic(arguments.result, json.dumps(result)) + return code + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/transactions.py b/packages/codex/mlx-agent/src/mlx_agent/transactions.py similarity index 86% rename from packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/transactions.py rename to packages/codex/mlx-agent/src/mlx_agent/transactions.py index ee06116..bf94c78 100644 --- a/packages/codex/mlx-agent/skills/mlx-scout/src/mlx_agent/transactions.py +++ b/packages/codex/mlx-agent/src/mlx_agent/transactions.py @@ -33,6 +33,8 @@ RECEIPT_SCHEMA_VERSION = "2.1" +# Per-target progress of a pending transaction; receipt.json is rewritten only at its start and end. +STEP_JOURNAL_NAME = "steps.jsonl" MAX_PREVIEW_CHARS = 12000 _STATUSES = {"pending", "applied", "failed", "rolled_back", "rollback_failed", "recovery_required"} _HASH = __import__("re").compile(r"^[0-9a-f]{64}$") @@ -151,17 +153,25 @@ def _filesystem_parent_digest(directory_fd): def _legacy_parent_scopes(targets, create=False): """Open and deduplicate physical target parents without following aliases.""" scopes = {} + walked = {} try: for target in sorted({_physical_absolute(item) for item in targets}, key=str): + if target.parent in walked: + # The parent was opened for an earlier target; a missing one stays skipped. + if walked[target.parent] is not None: + scopes[walked[target.parent]]["targets"].append(target) + continue try: _parent, directory_fd = _walk_directory(target.parent, create=create) except ValueError as error: if str(error).startswith("path component does not exist:"): + walked[target.parent] = None continue raise try: item = os.fstat(directory_fd) identity = (item.st_dev, item.st_ino) + walked[target.parent] = identity if identity in scopes: os.close(directory_fd) scopes[identity]["targets"].append(target) @@ -550,18 +560,21 @@ def _target_locks(targets, create_parents=False, lock_root=None): identities = {} held = [] try: - for target in canonical: - _parent, target_parent_fd = _walk_directory( - target.parent, create=create_parents - ) - digest = _filesystem_target_lock_digest(target, target_parent_fd) - if digest in identities: - os.close(target_parent_fd) - continue - identities[digest] = { - "target": target, - "target_parent_fd": target_parent_fd, - } + # Targets sharing a parent are bound to one walk of that parent. + parents = {} + try: + for target in canonical: + if target.parent not in parents: + parents[target.parent] = _walk_directory(target.parent, create=create_parents)[1] + digest = _filesystem_target_lock_digest(target, parents[target.parent]) + if digest not in identities: + identities[digest] = { + "target": target, + "target_parent_fd": os.dup(parents[target.parent]), + } + finally: + for parent_fd in parents.values(): + os.close(parent_fd) for digest in sorted(identities): identity = identities[digest] target = identity["target"] @@ -612,8 +625,41 @@ def _target_locks(targets, create_parents=False, lock_root=None): def _assert_safe_target(path): + """Require a link-free parent and an absent or regular leaf, without opening or reading the leaf.""" value = _absolute(path) - _read_target(value) + _parent, directory = _walk_directory(value.parent) + try: + return _check_safe_leaf(value, directory) + finally: + os.close(directory) + + +def _assert_safe_targets(paths): + """Check many targets as _assert_safe_target does, walking each distinct parent once.""" + checked, parents = {}, {} + try: + for name in paths: + value = _absolute(name) + if value.parent not in parents: + parents[value.parent] = _walk_directory(value.parent)[1] + checked[name] = _check_safe_leaf(value, parents[value.parent]) + finally: + for directory in parents.values(): + os.close(directory) + return checked + + +def _check_safe_leaf(value, directory): + try: + item = os.stat(value.name, dir_fd=directory, follow_symlinks=False) + except FileNotFoundError: + return value + except OSError as error: + raise ValueError("refusing unsafe target leaf {0}: {1}".format(value, error)) + if stat.S_ISLNK(item.st_mode): + raise ValueError("refusing unsafe target leaf {0}: symbolic link".format(value)) + if not stat.S_ISREG(item.st_mode): + raise ValueError("target is not a regular file: {0}".format(value)) return value @@ -640,8 +686,7 @@ def _write_all(descriptor, content): def _atomic_in_directory(directory, name, content, mode, validator=None): """Write and fsync a stage file before atomically replacing ``name``.""" - directory = _assert_safe_directory(directory) - dir_fd = _open_directory(directory) + _directory, dir_fd = _walk_directory(directory) stage_name = ".mlx-agent-stage-{0}".format(uuid.uuid4().hex) stage_fd = None try: @@ -788,8 +833,9 @@ def from_dict(cls, value, receipt_path=""): raise ValueError("receipt targets are malformed") if any(set(targets) != set(item) for item in maps): raise ValueError("receipt target maps do not match") + safe_targets = _assert_safe_targets(targets) for index, target_name in enumerate(targets): - target = _assert_safe_target(target_name) + target = safe_targets[target_name] if str(target) != target_name or value["target_roots"][target_name] != str(target.parent): raise ValueError("receipt target is not rooted safely") mode = value["target_modes"][target_name] @@ -845,20 +891,22 @@ def __init__(self, receipts_dir=None, health_checker=None, fault_injector=None, self._preview = "" self._preview_hash = "" self._expected_current = None + self._journal_fd = None def preview(self, changes, expected_current=None): if not isinstance(changes, (list, tuple)) or not changes: raise ValueError("changes must be a non-empty list") prepared, diffs, binding = [], [], [] - seen = set() + seen, diff_chars = set(), 0 for change in changes: - if not isinstance(change, dict) or "path" not in change or "content" not in change: - raise ValueError("each change requires path and content") + remove = isinstance(change, dict) and change.get("remove") is True + if not isinstance(change, dict) or "path" not in change or (remove == ("content" in change)): + raise ValueError("each change requires path and content, or path and remove") path = _absolute(change["path"]) if str(path) in seen: raise ValueError("each target may appear only once") seen.add(str(path)) - content = change["content"] + content = "" if remove else change["content"] if not isinstance(content, str): raise TypeError("change content must be text") adapter = change.get("adapter") or ConfigAdapter.detect(path, runtime=change.get("runtime")) @@ -868,6 +916,8 @@ def preview(self, changes, expected_current=None): if not self.create_target_parents or not str(error).startswith("path component does not exist:"): raise before, existed, mode = b"", False, None + if remove and not existed: + raise ValueError("a removal requires an existing target") before_text = before.decode("utf-8") if not getattr(adapter, "secret_scan_exempt", False): require_secret_free_config(before_text) @@ -875,14 +925,21 @@ def preview(self, changes, expected_current=None): endpoint = change.get("endpoint") if endpoint: validate_health_endpoint(endpoint) - adapter.validate(content) + if not remove: + adapter.validate(content) after = content.encode("utf-8") - diffs.append("".join(difflib.unified_diff( - redact_secrets(before_text).splitlines(True), redact_secrets(content).splitlines(True), - fromfile=str(path), tofile=str(path), lineterm="", - ))) - binding.append({"path": str(path), "before": _sha256(before), "after": _sha256(after), "exists": existed, "mode": mode, "endpoint": _sha256(str(endpoint or "").encode("utf-8"))}) - prepared.append({"path": path, "content": content, "adapter": adapter, "endpoint": endpoint, "before_hash": _sha256(before), "existed": existed, "mode": mode}) + # Diffs past the preview bound are cut by _bounded; they are not built at all. + if diff_chars <= MAX_PREVIEW_CHARS: + diffs.append("".join(difflib.unified_diff( + redact_secrets(before_text).splitlines(True), redact_secrets(content).splitlines(True), + fromfile=str(path), tofile=str(path), lineterm="", + ))) + diff_chars += len(diffs[-1]) + (len(diffs) > 1) + entry = {"path": str(path), "before": _sha256(before), "after": _sha256(after), "exists": existed, "mode": mode, "endpoint": _sha256(str(endpoint or "").encode("utf-8"))} + if remove: + entry["remove"] = True + binding.append(entry) + prepared.append({"path": path, "content": content, "adapter": adapter, "endpoint": endpoint, "before_hash": _sha256(before), "existed": existed, "mode": mode, "remove": remove}) self._changes = prepared self._expected_current = self._normalize_expected_current(expected_current) self._preview = self._bounded("\n".join(diffs)) @@ -949,22 +1006,33 @@ def _apply_locked(self): if _sha256(current) != receipt.before_hashes[str(path)] or exists != change["existed"] or mode != receipt.target_modes[str(path)]: raise ValueError("preview is stale; target changed before replacement") mutation_started = True + if change["remove"]: + _remove_target(path) + receipt.after_hashes[str(path)] = _sha256(b"") + receipt.after_modes[str(path)] = None + self._record_step(receipt, path) + self._fault("after_replace:{0}".format(index)) + if _read_target(path)[1]: + raise ValueError("removal left the target in place") + receipt.validations[str(path)] = {"pre": True, "post": True, "passed": True} + self._record_step(receipt, path) + continue self._replace_target(path, change["content"].encode("utf-8"), change["adapter"], receipt.target_modes[str(path)]) receipt.after_hashes[str(path)] = _sha256(change["content"].encode("utf-8")) receipt.after_modes[str(path)] = ( receipt.target_modes[str(path)] if receipt.target_modes[str(path)] is not None else 0o600 ) - self._persist(receipt) + self._record_step(receipt, path) self._fault("after_replace:{0}".format(index)) - change["adapter"].validate(_read_regular(path).decode("utf-8")) after, after_exists, after_mode = _read_target(path) + change["adapter"].validate(after.decode("utf-8")) if not after_exists or (receipt.target_modes[str(path)] is not None and after_mode != receipt.target_modes[str(path)]): raise ValueError("replacement mode does not match the reviewed target mode") receipt.after_hashes[str(path)] = _sha256(after) receipt.after_modes[str(path)] = after_mode receipt.validations[str(path)] = {"pre": True, "post": True, "passed": True} - self._persist(receipt) + self._record_step(receipt, path) receipt.validations["health_check"] = self._run_health_checks() if receipt.validations["health_check"]["passed"]: receipt.status = "applied" @@ -977,6 +1045,10 @@ def _apply_locked(self): receipt.status = "failed" return receipt self._finish_restore(receipt) + finally: + if self._journal_fd is not None: + os.close(self._journal_fd) + self._journal_fd = None return receipt def _normalize_expected_current(self, expected_current): @@ -1068,7 +1140,20 @@ def _persist(self, receipt): if self.receipt_writer is not None: self.receipt_writer(Path(receipt.receipt_path), receipt.to_dict()) return - _atomic_in_directory(root, "receipt.json", (json.dumps(receipt.to_dict(), indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600) + _write_receipt(root, receipt) + + def _record_step(self, receipt, path): + """Journal one target's progress without rewriting the whole receipt.""" + if self.receipt_writer is not None: + self._persist(receipt) + return + target = str(path) + step = {"target": target, "after_hash": receipt.after_hashes[target], "after_mode": receipt.after_modes[target]} + if target in receipt.validations: + step["validation"] = receipt.validations[target] + if self._journal_fd is None: + self._journal_fd = _open_directory(receipt.transaction_root) + _append_step(self._journal_fd, step) def _finish_restore(self, receipt): if _restore_receipt(receipt, expected_current=_receipt_after_state(receipt)): @@ -1166,6 +1251,83 @@ def _current_state(target_name): return {"hash": _sha256(content), "exists": exists, "mode": mode} +def _write_receipt(root, receipt): + """Replace receipt.json; a finished receipt supersedes its step journal.""" + _atomic_in_directory(root, "receipt.json", (json.dumps(receipt.to_dict(), indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600) + if receipt.status == "pending": + return + directory = _open_directory(root) + try: + os.unlink(STEP_JOURNAL_NAME, dir_fd=directory) + os.fsync(directory) + except FileNotFoundError: + pass + finally: + os.close(directory) + + +def _append_step(directory, step): + """Append one fsynced record to a pending transaction's step journal through its open root.""" + flags = os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(STEP_JOURNAL_NAME, flags | os.O_EXCL, 0o600, dir_fd=directory) + created = True + except FileExistsError: + descriptor = os.open(STEP_JOURNAL_NAME, flags, 0o600, dir_fd=directory) + created = False + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise ValueError("receipt step journal is not a regular file") + _write_all(descriptor, (json.dumps(step, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")) + os.fsync(descriptor) + finally: + os.close(descriptor) + if created: + os.fsync(directory) + + +def replay_step_journal(value, root): + """Apply the journaled steps of a pending transaction to its decoded receipt.""" + if not isinstance(value, dict) or value.get("status") != "pending": + return value + content, exists, _mode = _read_target(Path(root) / STEP_JOURNAL_NAME) + if not exists: + return value + targets = value.get("targets") + maps = [value.get(name) for name in ("after_hashes", "after_modes", "validations")] + if not isinstance(targets, list) or not all(isinstance(item, dict) for item in maps): + raise ValueError("receipt fields are malformed or untrusted") + known = set(targets) + # The last segment is empty, or a record an interrupted append never finished. + for line in content.split(b"\n")[:-1]: + try: + step = json.loads(line.decode("utf-8")) + except ValueError: + raise ValueError("receipt step journal is malformed") + if ( + not isinstance(step, dict) + or not {"target", "after_hash", "after_mode"} <= set(step) <= {"target", "after_hash", "after_mode", "validation"} + or step["target"] not in known + or not isinstance(step["after_hash"], str) or len(step["after_hash"]) != 64 + or any(character not in "0123456789abcdef" for character in step["after_hash"]) + or not (step["after_mode"] is None or type(step["after_mode"]) is int) + or not isinstance(step.get("validation", {}), dict) + ): + raise ValueError("receipt step journal is malformed") + maps[0][step["target"]] = step["after_hash"] + maps[1][step["target"]] = step["after_mode"] + if "validation" in step: + maps[2][step["target"]] = step["validation"] + return value + + +def load_receipt(receipt_path): + """Read a receipt, replaying the step journal of a transaction that did not finish.""" + location = _assert_safe_target(receipt_path) + value = json.loads(_read_regular(location).decode("utf-8")) + return Receipt.from_dict(replay_step_journal(value, location.parent), str(location)) + + def _receipt_after_state(receipt): return { target: { @@ -1265,18 +1427,14 @@ def _rollback_preview_locked(receipt, location): def preview_rollback(receipt_path): """Return a hash-bound, non-secret snapshot of the current rollback targets.""" location = _assert_safe_target(receipt_path) - initial = Receipt.from_dict( - json.loads(_read_regular(location).decode("utf-8")), str(location) - ) + initial = load_receipt(location) lock_context = ( _migrate_legacy_target_locks(initial.targets, initial.lock_root) if initial.lock_root is not None else nullcontext() ) with lock_context: with _target_locks(initial.targets, lock_root=initial.lock_root): - receipt = Receipt.from_dict( - json.loads(_read_regular(location).decode("utf-8")), str(location) - ) + receipt = load_receipt(location) return _rollback_preview_locked(receipt, location) @@ -1288,14 +1446,14 @@ def rollback( ): """Restore a receipt through a user preview hash or installer-internal CAS.""" location = _assert_safe_target(receipt_path) - initial = Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) + initial = load_receipt(location) lock_context = ( _migrate_legacy_target_locks(initial.targets, initial.lock_root) if initial.lock_root is not None else nullcontext() ) with lock_context: with _target_locks(initial.targets, lock_root=initial.lock_root): - receipt = Receipt.from_dict(json.loads(_read_regular(location).decode("utf-8")), str(location)) + receipt = load_receipt(location) if ( expected_receipt_hash is not None and _rollback_receipt_hash(receipt) != expected_receipt_hash @@ -1306,7 +1464,8 @@ def rollback( raise ValueError("reviewed rollback hashes do not match receipt targets") for target_name in receipt.targets: current, exists, _mode = _read_target(target_name) - if not exists or _sha256(current) != expected_after_hashes[target_name]: + expected_exists = receipt.after_modes[target_name] is not None + if exists != expected_exists or _sha256(current) != expected_after_hashes[target_name]: raise ValueError("receipt target changed after reviewed rollback preview") else: if preview_hash is None: @@ -1336,7 +1495,7 @@ def rollback( else: receipt.status = "rollback_failed" try: - _atomic_in_directory(Path(receipt.transaction_root), "receipt.json", (json.dumps(receipt.to_dict(), indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600) + _write_receipt(Path(receipt.transaction_root), receipt) except Exception as error: receipt.status = "recovery_required" receipt.validations["receipt_write"] = {"passed": False, "message": redact_secrets(str(error))} diff --git a/packages/codex/mlx-agent/src/mlx_agent/transcribe.py b/packages/codex/mlx-agent/src/mlx_agent/transcribe.py new file mode 100644 index 0000000..8acfe32 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/transcribe.py @@ -0,0 +1,78 @@ +"""Transcribe one audio file with a converted speech model through its backend (read-only).""" + +from __future__ import annotations + +import json +import re +import subprocess +from pathlib import Path + +from .backends import ( + backend_python, + is_installed, + load_manifests, + load_registries, + lookup, +) +from .inference_runner import InferenceError, RunnerContract, invoke + +TRANSCRIBE_SCHEMA = "transcribe/1" +TRANSCRIBE_RUNNER = Path(__file__).resolve().with_name("transcribe_runner.py") +DEFAULT_TIMEOUT_SECONDS = 300 +_LANGUAGE = re.compile(r"[a-z]{2,8}") + + +class TranscribeError(InferenceError): + """Classified transcription failure safe to surface in a result envelope.""" + + +def plan_transcribe(model_path, audio_path, language=None, manifests=None, registries=None, root=None): + """Pick the speech backend for a converted model directory; pure apart from reading config.json.""" + model = Path(model_path).expanduser().resolve() + try: + config = json.loads((model / "config.json").read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise TranscribeError( + "model_unreadable", "No readable config.json in {0}: {1}".format(model, error), + "Pass the converted model directory.", + ) from error + model_type = config.get("model_type") if isinstance(config, dict) else None + audio = Path(audio_path).expanduser().resolve() + if not audio.is_file(): + raise TranscribeError("audio_not_found", "No audio file at {0}.".format(audio), "Pass a WAV, FLAC, or MP3 file.") + if language is not None and not _LANGUAGE.fullmatch(language): + raise TranscribeError("invalid_arguments", "language must be a short lowercase code.", "Pass --language en, for example.") + manifests = load_manifests() if manifests is None else manifests + registries = load_registries(manifests, root) if registries is None else registries + backends = sorted({hit["backend"] for hit in lookup(model_type, manifests, registries) if hit["category"] == "speech_to_text"}) + if not backends: + raise TranscribeError( + "not_speech_model", "No speech-to-text backend implements model_type {0!r}.".format(model_type), + "Transcription applies to converted speech-to-text models.", + ) + manifest = manifests[backends[0]] + if not is_installed(manifest, root): + raise TranscribeError( + "backend_not_installed", "The {0} backend is not installed.".format(manifest["id"]), + "Install it first: mlx-agent backend install {0}.".format(manifest["id"]), + ) + argv = [str(backend_python(manifest, root)), str(TRANSCRIBE_RUNNER), "--model", str(model), "--audio", str(audio)] + if language: + argv += ["--language", language] + return {"backend": manifest["id"], "model": str(model), "audio": str(audio), "language": language, "argv": argv} + + +CONTRACT = RunnerContract( + verb="transcribe", error=TranscribeError, accept=lambda value, plan: isinstance(value.get("text"), str), + action="transcribe the clip", remediation="Check that the directory is a complete converted speech model.", + activity="Transcription", timeout_remediation="Use a shorter clip or a larger --timeout.", +) + + +def run_transcribe(plan, manifests=None, root=None, timeout=DEFAULT_TIMEOUT_SECONDS, runner=subprocess.run): + result = invoke(plan, CONTRACT, manifests, root, timeout, runner) + return { + "schema": TRANSCRIBE_SCHEMA, "backend": plan["backend"], "model": plan["model"], "audio": plan["audio"], + "language": plan["language"], "text": result["text"], + "seconds": result.get("seconds"), "audio_seconds": result.get("audio_seconds"), + } diff --git a/packages/codex/mlx-agent/src/mlx_agent/transcribe_runner.py b/packages/codex/mlx-agent/src/mlx_agent/transcribe_runner.py new file mode 100644 index 0000000..194e9ef --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/transcribe_runner.py @@ -0,0 +1,66 @@ +"""Run inside a speech backend's environment: load one converted model, transcribe one clip, print JSON.""" + +import argparse +import inspect +import json +import sys +import time + + +def _restore_whisper_end_token(model): + """Give a Whisper tokenizer the no-speech token its vocabulary has. + + mlx-audio looks up `<|nospeech|>`, which only large-v3 vocabularies carry; earlier ones call it + `<|nocaptions|>`. The failed lookup returns the unknown token, `<|endoftext|>`, which decoding then + suppresses as the no-speech token, so every transcript runs to the token limit. + """ + get_tokenizer = getattr(model, "get_tokenizer", None) + if not callable(get_tokenizer): + return + try: + probe = get_tokenizer() + vocabulary = probe.hf_tokenizer.get_vocab() + if probe.no_speech != probe.eot: + return + except Exception: + return + no_speech = vocabulary.get("<|nospeech|>", vocabulary.get("<|nocaptions|>")) + tokenizer_type = type(probe) + restored = type(tokenizer_type.__name__, (tokenizer_type,), {"no_speech": property(lambda self: no_speech)}) + + def restored_tokenizer(*args, **kwargs): + tokenizer = get_tokenizer(*args, **kwargs) + tokenizer.__class__ = restored + return tokenizer + + model.get_tokenizer = restored_tokenizer + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--audio", required=True) + parser.add_argument("--language", default=None) + arguments = parser.parse_args(argv) + + from mlx_audio.stt.utils import load_audio, load_model + + started = time.time() + model = load_model(arguments.model) + _restore_whisper_end_token(model) + samples = load_audio(arguments.audio) + kwargs = {} + if arguments.language and "language" in inspect.signature(model.generate).parameters: + kwargs["language"] = arguments.language + output = model.generate(arguments.audio, **kwargs) + text = getattr(output, "text", output if isinstance(output, str) else "") + print(json.dumps({ + "text": str(text).strip(), + "seconds": round(time.time() - started, 3), + "audio_seconds": round(int(samples.shape[0]) / 16000, 3), + }), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/verification.py b/packages/codex/mlx-agent/src/mlx_agent/verification.py similarity index 96% rename from packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/verification.py rename to packages/codex/mlx-agent/src/mlx_agent/verification.py index 65b5e75..122480a 100644 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/verification.py +++ b/packages/codex/mlx-agent/src/mlx_agent/verification.py @@ -7,6 +7,9 @@ import ipaddress import json import math +import subprocess +import sys +import tempfile import threading import time from copy import deepcopy @@ -74,6 +77,23 @@ "globals", "locals", "compile", "breakpoint", "exit", "quit", "getattr", "setattr", "delattr", "vars", "help", "print", ) +CODING_PROBE_TIMEOUT_SECONDS = 5 +_CODING_PROBE_CHECK = ( + "import json, sys\n" + "namespace = {'__builtins__': {}}\n" + "try:\n" + " exec(compile(sys.stdin.read(), '', 'exec'), namespace)\n" + " add = namespace.get('add')\n" + " if not callable(add):\n" + " reason = 'missing_function'\n" + " elif add(2, 3) != 5 or add(-1, 1) != 0:\n" + " reason = 'wrong_answer'\n" + " else:\n" + " reason = 'valid'\n" + "except Exception:\n" + " reason = 'wrong_answer'\n" + "sys.stdout.write(json.dumps(reason))\n" +) REASONING_PROBE_PROMPT = ( "A farmer has 17 sheep. All but 9 run away. " "How many sheep does the farmer have left? Reply with only the number." @@ -97,7 +117,7 @@ def _role_probe_outcome(reason): return {"valid": reason == "valid", "reason": reason} -def validate_coding_response(response): +def validate_coding_response(response, runner=subprocess.run): """Validate the fixed coding probe without retaining raw model output.""" try: content, _hidden = _generation_text(response) @@ -119,21 +139,31 @@ def validate_coding_response(response): ] if not function_defs: return _role_probe_outcome("missing_function") - namespace = {"__builtins__": {}} - try: - exec(compile(tree, "", "exec"), namespace) - add = namespace.get("add") - if not callable(add): - return _role_probe_outcome("missing_function") - if add(2, 3) != 5 or add(-1, 1) != 0: - return _role_probe_outcome("wrong_answer") - except Exception: - return _role_probe_outcome("wrong_answer") - return _role_probe_outcome("valid") + return _role_probe_outcome(_check_probe_code(code, runner)) except Exception: return _role_probe_outcome("invalid_response") +def _check_probe_code(code, runner): + """Run the answer in an isolated interpreter with no environment and a wall-clock limit.""" + with tempfile.TemporaryDirectory() as directory: + try: + completed = runner( + [sys.executable, "-I", "-S", "-c", _CODING_PROBE_CHECK], + input=code, capture_output=True, text=True, cwd=directory, env={}, + timeout=CODING_PROBE_TIMEOUT_SECONDS, start_new_session=True, + ) + except subprocess.TimeoutExpired: + return "wrong_answer" + try: + reason = json.loads(completed.stdout) + except ValueError: + return "wrong_answer" + if completed.returncode != 0 or reason not in ("valid", "wrong_answer", "missing_function"): + return "wrong_answer" + return reason + + def _extract_probe_code(content): if not isinstance(content, str) or not content.strip(): return None diff --git a/packages/codex/mlx-agent/src/mlx_agent/video.py b/packages/codex/mlx-agent/src/mlx_agent/video.py new file mode 100644 index 0000000..b4bf0a3 --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/video.py @@ -0,0 +1,148 @@ +"""Render one prompt with a converted text-to-video model through its backend (writes only the MP4).""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +from .backends import ( + backend_python, + is_installed, + load_manifests, + load_registries, + lookup, +) +from .inference_runner import InferenceError, RunnerContract, invoke + +VIDEO_SCHEMA = "video/1" +VIDEO_RUNNER = Path(__file__).resolve().with_name("video_runner.py") +DEFAULT_TIMEOUT_SECONDS = 3600 +MAX_PROMPT_CHARS = 2000 +MAX_SIDE, MIN_SIDE = 1920, 64 +MAX_FRAMES = 241 +MAX_FPS = 60 +DEFAULT_PATCH, DEFAULT_SPATIAL_STRIDE, DEFAULT_TEMPORAL_STRIDE = 2, 8, 4 +DEFAULT_STEPS, DEFAULT_FPS = 50, 16 +DEFAULT_WIDTH, DEFAULT_HEIGHT, DEFAULT_FRAMES = 832, 480, 81 + + +class VideoError(InferenceError): + """Classified video-generation failure safe to surface in a result envelope.""" + + +def _invalid(message, remediation): + return VideoError("invalid_arguments", message, remediation) + + +def _is_int(value): + return isinstance(value, int) and not isinstance(value, bool) + + +def _stride(config, key, index, default): + value = config.get(key) + if isinstance(value, (list, tuple)) and len(value) == 3 and _is_int(value[index]) and value[index] > 0: + return value[index] + return default + + +def _config_int(config, key, default): + value = config.get(key) + return value if _is_int(value) and value > 0 else default + + +def plan_video(model_path, prompt, out, width=DEFAULT_WIDTH, height=DEFAULT_HEIGHT, frames=DEFAULT_FRAMES, + fps=None, steps=None, seed=42, manifests=None, registries=None, root=None): + """Pick the video backend for a converted model directory; pure apart from reading config.json. + + The size alignment, the frame rule, the area cap, and the step and rate defaults come from the + model's own config.json (its VAE and patch strides), so a model with other strides is held to its own. + """ + model = Path(model_path).expanduser() + try: + config = json.loads((model / "config.json").read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise VideoError( + "model_unreadable", "No readable config.json in {0}: {1}".format(model, error), + "Pass the converted model directory.", + ) from error + if not isinstance(config, dict): + raise VideoError("model_unreadable", "config.json in {0} is not an object.".format(model), "Pass the converted model directory.") + if not isinstance(prompt, str) or not prompt.strip() or len(prompt) > MAX_PROMPT_CHARS or any( + ord(character) < 32 and character not in "\n\t" for character in prompt + ): + raise _invalid("The prompt must be 1..{0} characters of text.".format(MAX_PROMPT_CHARS), "Pass --prompt with plain text.") + target = Path(out).expanduser() + if not target.is_absolute() or target.suffix.lower() != ".mp4" or target.exists(): + raise _invalid("--out must be a new absolute .mp4 path.", "Pass a path such as /Users/me/Movies/clip.mp4.") + if not target.parent.is_dir(): + raise _invalid("The directory of --out does not exist.", "Create the directory or pass a path inside an existing one.") + alignment = _stride(config, "patch_size", 1, DEFAULT_PATCH) * _stride(config, "vae_stride", 1, DEFAULT_SPATIAL_STRIDE) + temporal = _stride(config, "vae_stride", 0, DEFAULT_TEMPORAL_STRIDE) + area_cap = config.get("max_area") if _is_int(config.get("max_area")) and config.get("max_area") > 0 else None + for name, value in (("width", width), ("height", height)): + if not _is_int(value) or not MIN_SIDE <= value <= MAX_SIDE or value % alignment: + raise _invalid( + "--{0} must be a multiple of {1} from {2} to {3}.".format(name, alignment, MIN_SIDE, MAX_SIDE), + "Pass --{0} {1}, for example.".format(name, 480 if name == "height" else 832), + ) + if area_cap and width * height > area_cap: + raise _invalid("This model renders at most {0} pixels per frame.".format(area_cap), "Pass a smaller --width and --height.") + if not _is_int(frames) or not 1 + temporal <= frames <= MAX_FRAMES or (frames - 1) % temporal: + raise _invalid( + "--frames must be {0}n+1 from {1} to {2}.".format(temporal, 1 + temporal, MAX_FRAMES), + "Pass --frames 17, for example.", + ) + fps = _config_int(config, "sample_fps", DEFAULT_FPS) if fps is None else fps + if not _is_int(fps) or not 1 <= fps <= MAX_FPS: + raise _invalid("--fps must be 1..{0}.".format(MAX_FPS), "Pass --fps 16, for example.") + steps = _config_int(config, "sample_steps", DEFAULT_STEPS) if steps is None else steps + if not _is_int(steps) or not 1 <= steps <= 100: + raise _invalid("--steps must be 1..100.", "Pass --steps 30, for example.") + if not _is_int(seed) or not 0 <= seed < 2 ** 32: + raise _invalid("--seed must be 0..4294967295.", "Pass --seed 42, for example.") + model_type = config.get("model_type") + manifests = load_manifests() if manifests is None else manifests + registries = load_registries(manifests, root) if registries is None else registries + hits = sorted( + (hit for hit in lookup(model_type, manifests, registries) if hit["category"] == "video_generation"), + key=lambda hit: hit["backend"], + ) + if not hits: + raise VideoError( + "not_video_model", "No video-generation backend implements model_type {0!r}.".format(model_type), + "Video generation applies to converted text-to-video models such as Wan2.1 T2V.", + ) + manifest = manifests[hits[0]["backend"]] + if not is_installed(manifest, root): + raise VideoError( + "backend_not_installed", "The {0} backend is not installed.".format(manifest["id"]), + "Install it first: mlx-agent backend install {0}.".format(manifest["id"]), + ) + argv = [ + str(backend_python(manifest, root)), str(VIDEO_RUNNER), "--module", hits[0]["module"], "--model", str(model), + "--prompt=" + prompt, "--out", str(target), "--width", str(width), "--height", str(height), + "--frames", str(frames), "--fps", str(fps), "--steps", str(steps), "--seed", str(seed), + ] + return { + "backend": manifest["id"], "model": str(model), "out": str(target), + "width": width, "height": height, "frames": frames, "fps": fps, "steps": steps, "seed": seed, "argv": argv, + } + + +CONTRACT = RunnerContract( + verb="video", error=VideoError, accept=lambda value, plan: isinstance(value.get("path"), str), + action="render the prompt", + remediation="Check that the directory is a complete converted text-to-video model, or use fewer frames, steps, or a smaller size.", + activity="Video generation", timeout_remediation="Use fewer frames or steps, or a larger --timeout.", +) + + +def run_video(plan, manifests=None, root=None, timeout=DEFAULT_TIMEOUT_SECONDS, runner=subprocess.run): + result = invoke(plan, CONTRACT, manifests, root, timeout, runner) + keys = ( + "path", "width", "height", "frames", "fps", "duration_seconds", "steps", "seed", "seconds", "load_seconds", + "seconds_per_frame", "peak_memory_gb", "pixel_std", + ) + return dict({"schema": VIDEO_SCHEMA, "backend": plan["backend"], "model": plan["model"]}, + **{key: result.get(key) for key in keys}) diff --git a/packages/codex/mlx-agent/src/mlx_agent/video_runner.py b/packages/codex/mlx-agent/src/mlx_agent/video_runner.py new file mode 100644 index 0000000..72e6caa --- /dev/null +++ b/packages/codex/mlx-agent/src/mlx_agent/video_runner.py @@ -0,0 +1,38 @@ +"""Run inside a video backend's environment: render one prompt with a converted model to an MP4, print JSON.""" + +import argparse +import importlib +import json +import sys +import time + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--module", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--prompt", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--width", type=int, required=True) + parser.add_argument("--height", type=int, required=True) + parser.add_argument("--frames", type=int, required=True) + parser.add_argument("--fps", type=int, required=True) + parser.add_argument("--steps", type=int, required=True) + parser.add_argument("--seed", type=int, required=True) + arguments = parser.parse_args(argv) + + port = importlib.import_module(arguments.module) + started = time.time() + model = port.load(arguments.model) + open_seconds = time.time() - started + result = port.generate( + model, arguments.prompt, arguments.out, width=arguments.width, height=arguments.height, + frames=arguments.frames, fps=arguments.fps, steps=arguments.steps, seed=arguments.seed, + ) + result["load_seconds"] = round(float(result.get("load_seconds", 0.0)) + open_seconds, 3) + print(json.dumps(result), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/watch.py b/packages/codex/mlx-agent/src/mlx_agent/watch.py similarity index 97% rename from packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/watch.py rename to packages/codex/mlx-agent/src/mlx_agent/watch.py index 63519d1..f89a362 100644 --- a/packages/codex/mlx-agent/skills/mlx-bench/src/mlx_agent/watch.py +++ b/packages/codex/mlx-agent/src/mlx_agent/watch.py @@ -180,7 +180,8 @@ def diff_snapshots(baseline, current): "base": base, "detail": "weight bytes changed from {0} to {1}".format(old_bytes, new_bytes), }) - if previous.get("gated") != info.get("gated"): + # "unknown" (or a missing value) means the Hub did not report access; that is not a change. + if previous.get("gated") != info.get("gated") and not {previous.get("gated"), info.get("gated")} & {None, "unknown"}: findings.append({ "code": "gated_changed", "repo": repo, diff --git a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/wiring.py b/packages/codex/mlx-agent/src/mlx_agent/wiring.py similarity index 96% rename from packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/wiring.py rename to packages/codex/mlx-agent/src/mlx_agent/wiring.py index a0e753d..953edd2 100644 --- a/packages/codex/mlx-agent/skills/mlx-adopt/src/mlx_agent/wiring.py +++ b/packages/codex/mlx-agent/src/mlx_agent/wiring.py @@ -22,9 +22,14 @@ r"(?im)(^|[\s,{&?;])([\"']?(?:" r"[a-z0-9_-]*(?:token|secret|passw(?:or)?d|authorization|credentials?|auth)|" r"[a-z0-9_-]*(?:api|access|private)[_-]?key" - r")[\"']?\s*[:=]\s*)" + r")[\"']?\s*[:=]\s*(?:(?:bearer|basic|digest|token)\s+(?=[^\s,}\]\r\n]))?)" r"([^\s,}\]\r\n]+|\"(?:\\.|[^\"])*\"|'(?:\\.|[^'])*')" ) +# Provider tokens recognizable without a key: Hugging Face, GitHub, OpenAI-style and Slack. +_PREFIXED_TOKEN = re.compile( + r"\b(?:hf_[A-Za-z0-9]{30,}|gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{30,}" + r"|sk-[A-Za-z0-9_-]{20,}|xox[abprs]-[A-Za-z0-9-]{10,})" +) _URL_USERINFO = re.compile(r"(https?://)([^/@\s]+@)", re.I) _OLLAMA = re.compile(r"\A# MLX_AGENT_WIRE v1\nFROM (" + _MODEL.pattern[1:-1] + r")\n\Z") _ENV_REFERENCE = re.compile( @@ -120,10 +125,21 @@ def _redact_json(value): if isinstance(value, list): return [_redact_json(item) for item in value] if isinstance(value, str): - return re.sub(r"https?://[^\s\"']+", lambda match: redact_endpoint(match.group(0)), value) + return _redact_text(value) return value +def _redact_text(text): + text = _URL_USERINFO.sub(r"\1@", text) + def redact_assignment(match): + value = match.group(3) + replacement = value if "" in value else "" + return match.group(1) + match.group(2) + replacement + text = _ASSIGNMENT_SECRET.sub(redact_assignment, text) + text = _PREFIXED_TOKEN.sub("", text) + return re.sub(r"https?://[^\s\"']+", lambda match: redact_endpoint(match.group(0)), text) + + def redact_endpoint(endpoint): """Return only a non-credential origin/path representation of an endpoint.""" try: @@ -150,16 +166,7 @@ def redact_secrets(content): parsed = None if parsed is not None: return json.dumps(_redact_json(parsed), sort_keys=True, separators=(",", ": ")) - text = _URL_USERINFO.sub(r"\1@", text) - def redact_assignment(match): - value = match.group(3) - replacement = value if "" in value else "" - return match.group(1) + match.group(2) + replacement - text = _ASSIGNMENT_SECRET.sub(redact_assignment, text) - def redact_url(match): - return redact_endpoint(match.group(0)) - text = re.sub(r"https?://[^\s\"']+", redact_url, text) - return text + return _redact_text(text) def validate_health_endpoint(endpoint): diff --git a/scripts/sync-codex-package.mjs b/scripts/sync-codex-package.mjs new file mode 100644 index 0000000..6d1802c --- /dev/null +++ b/scripts/sync-codex-package.mjs @@ -0,0 +1,73 @@ +#!/usr/bin/env node +import { copyFile, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { CODEX_PACKAGES, packageTreeHash } from "./validate-catalog.mjs"; + +async function copyTree(source, target, relative, excluded) { + const entry = await lstat(source); + if (entry.isSymbolicLink()) throw new Error(`source symlink rejected: ${relative}`); + if (entry.isFile()) { + if (excluded.has(relative)) return; + await mkdir(path.dirname(target), { recursive: true }); + await copyFile(source, target); + return; + } + if (!entry.isDirectory()) throw new Error(`source entry is not a file or directory: ${relative}`); + for (const name of (await readdir(source)).sort()) { + await copyTree(path.join(source, name), path.join(target, name), path.posix.join(relative, name), excluded); + } +} + +export async function syncCodexPackage({ root, name, source, commit }) { + const config = CODEX_PACKAGES[name]; + if (!config) throw new Error(`no pinned Codex package: ${name}`); + if (typeof commit !== "string" || !/^[0-9a-f]{40}$/.test(commit)) throw new Error("commit must be a full lowercase SHA"); + if (typeof source !== "string" || !source) throw new Error("source checkout is required"); + const sourceRoot = path.join(source, config.sourcePath); + const codexRoot = path.join(root, "packages/codex"); + const packageRoot = path.join(codexRoot, name); + const staging = await mkdtemp(path.join(codexRoot, `.${name}-`)); + try { + const excluded = new Set(config.excluded ?? []); + for (const base of config.included) await copyTree(path.join(sourceRoot, base), path.join(staging, base), base, excluded); + const skills = (await readdir(path.join(staging, "skills"), { withFileTypes: true })).filter((item) => item.isDirectory()).map((item) => item.name).sort(); + const tree = await packageTreeHash(staging, config.included); + const provenance = { + repository: `cavi-ai/${name}`, + source_commit: commit, + source_path: config.sourcePath, + included: config.included, + ...(config.excluded ? { excluded: config.excluded, exclusion_reason: config.exclusionReason } : {}), + integrity: { algorithm: "sha256", tree } + }; + await writeFile(path.join(staging, "provenance.json"), `${JSON.stringify(provenance, null, 2)}\n`); + const registryPath = path.join(codexRoot, "expected-integrity.json"); + const registry = JSON.parse(await readFile(registryPath, "utf8")); + registry[name] = { repository: `cavi-ai/${name}`, source_commit: commit, source_path: config.sourcePath, tree, skills }; + await rm(packageRoot, { recursive: true, force: true }); + await rename(staging, packageRoot); + await writeFile(registryPath, `${JSON.stringify(registry, null, 2)}\n`); + return { name, commit, tree, skills }; + } finally { + await rm(staging, { recursive: true, force: true }); + } +} + +function option(argv, flag) { + const index = argv.indexOf(flag); + return index >= 0 ? argv[index + 1] : undefined; +} + +const invoked = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invoked) { + const argv = process.argv.slice(2); + try { + const result = await syncCodexPackage({ root: process.cwd(), name: option(argv, "--name"), source: option(argv, "--source"), commit: option(argv, "--commit") }); + console.log(JSON.stringify(result)); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/scripts/sync-codex-package.test.mjs b/scripts/sync-codex-package.test.mjs new file mode 100644 index 0000000..bae027a --- /dev/null +++ b/scripts/sync-codex-package.test.mjs @@ -0,0 +1,80 @@ +import assert from "node:assert/strict"; +import { cp, mkdir, mkdtemp, readFile, readdir, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { syncCodexPackage } from "./sync-codex-package.mjs"; +import { validateCatalog } from "./validate-catalog.mjs"; + +const repositoryRoot = path.resolve(new URL("..", import.meta.url).pathname); +const commit = "a".repeat(40); + +async function catalogCopy() { + const root = await mkdtemp(path.join(tmpdir(), "plugins-sync-")); + for (const item of [".agents", ".claude-plugin", "catalog.json", "catalog.schema.json", "packages", "providers"]) { + await cp(path.join(repositoryRoot, item), path.join(root, item), { recursive: true }); + } + return root; +} + +async function mlxAgentSource(skills = ["mlx-adopt", "mlx-scout"]) { + const source = await mkdtemp(path.join(tmpdir(), "mlx-agent-source-")); + const provider = path.join(source, "providers/codex"); + await mkdir(path.join(provider, ".codex-plugin"), { recursive: true }); + await writeFile(path.join(provider, ".codex-plugin/plugin.json"), JSON.stringify({ name: "mlx-agent", version: "9.9.9", skills: "./skills/" })); + await writeFile(path.join(provider, ".mlx-agent-generated-files.json"), "{}\n"); + for (const skill of skills) { + await mkdir(path.join(provider, "skills", skill), { recursive: true }); + await writeFile(path.join(provider, "skills", skill, "SKILL.md"), `---\nname: ${skill}\ndescription: source\n---\n`); + } + await mkdir(path.join(provider, "scripts"), { recursive: true }); + await writeFile(path.join(provider, "scripts/mlx-agent"), "#!/usr/bin/env python3\n"); + await writeFile(path.join(provider, "scripts/mlx-agent-mcp"), "#!/usr/bin/env python3\n"); + await mkdir(path.join(provider, "src/mlx_agent"), { recursive: true }); + await writeFile(path.join(provider, "src/mlx_agent/__init__.py"), "__version__ = \"9.9.9\"\n"); + await writeFile(path.join(provider, "src/mlx_agent/mcp_server.py"), "transport\n"); + await writeFile(path.join(provider, "requirements-lab.txt"), "lab==1\n"); + await writeFile(path.join(provider, "requirements-lab-inference.txt"), "inference==1\n"); + return source; +} + +test("vendors a tagged source tree that passes catalog validation", async () => { + const root = await catalogCopy(); + await writeFile(path.join(root, "packages/codex/mlx-agent/stale.txt"), "stale\n"); + const result = await syncCodexPackage({ root, name: "mlx-agent", source: await mlxAgentSource(), commit }); + assert.deepEqual(result.skills, ["mlx-adopt", "mlx-scout"]); + assert.deepEqual(await validateCatalog(root), []); + const packageRoot = path.join(root, "packages/codex/mlx-agent"); + assert.deepEqual((await readdir(packageRoot)).sort(), [".codex-plugin", "provenance.json", "requirements-lab-inference.txt", "requirements-lab.txt", "scripts", "skills", "src"]); + assert.deepEqual(await readdir(path.join(packageRoot, "scripts")), ["mlx-agent"]); + assert.deepEqual(await readdir(path.join(packageRoot, "src/mlx_agent")), ["__init__.py"]); + const registry = JSON.parse(await readFile(path.join(root, "packages/codex/expected-integrity.json"), "utf8")); + assert.equal(registry["mlx-agent"].source_commit, commit); + assert.equal(registry["mlx-agent"].tree, JSON.parse(await readFile(path.join(packageRoot, "provenance.json"), "utf8")).integrity.tree); + assert.equal(registry["obsidian-agent"].source_commit, JSON.parse(await readFile(path.join(repositoryRoot, "packages/codex/expected-integrity.json"), "utf8"))["obsidian-agent"].source_commit); +}); + +test("records the vendored skill inventory so later drift is rejected", async () => { + const root = await catalogCopy(); + await syncCodexPackage({ root, name: "mlx-agent", source: await mlxAgentSource(["mlx-adopt", "mlx-bench", "mlx-scout"]), commit }); + assert.deepEqual(await validateCatalog(root), []); + await mkdir(path.join(root, "packages/codex/mlx-agent/skills/mlx-extra"), { recursive: true }); + await writeFile(path.join(root, "packages/codex/mlx-agent/skills/mlx-extra/SKILL.md"), "---\nname: mlx-extra\n---\n"); + const errors = await validateCatalog(root); + assert(errors.some((error) => error.includes("codex package skill inventory mismatch: mlx-agent"))); + assert(errors.some((error) => error.includes("codex package upstream digest mismatch: mlx-agent"))); +}); + +test("refuses symlinks, unknown packages and abbreviated commits without touching the catalog", async () => { + const root = await catalogCopy(); + const before = await readFile(path.join(root, "packages/codex/expected-integrity.json"), "utf8"); + const source = await mlxAgentSource(); + await symlink("/etc/hosts", path.join(source, "providers/codex/skills/mlx-adopt/hosts")); + await assert.rejects(syncCodexPackage({ root, name: "mlx-agent", source, commit }), /source symlink rejected: skills\/mlx-adopt\/hosts/); + await assert.rejects(syncCodexPackage({ root, name: "loop-killer", source, commit }), /no pinned Codex package/); + await assert.rejects(syncCodexPackage({ root, name: "mlx-agent", source, commit: "abc1234" }), /full lowercase SHA/); + assert.equal(await readFile(path.join(root, "packages/codex/expected-integrity.json"), "utf8"), before); + assert.deepEqual(await validateCatalog(root), []); + assert.deepEqual((await readdir(path.join(root, "packages/codex"))).filter((item) => item.startsWith(".")), []); +}); diff --git a/scripts/validate-catalog.mjs b/scripts/validate-catalog.mjs index 68eeca3..8a66397 100644 --- a/scripts/validate-catalog.mjs +++ b/scripts/validate-catalog.mjs @@ -10,15 +10,16 @@ export const MARKETPLACE = "cavi-ai"; // Every plugin the catalog may list. REQUIRED is the subset that must be present and ships a pinned Codex package. const CANONICAL = ["mlx-agent", "obsidian-agent", "loop-killer"]; const REQUIRED = ["mlx-agent", "obsidian-agent"]; -const INCLUDED_ROOTS = [".codex-plugin", "skills"]; -const SKILL_INVENTORIES = { - "mlx-agent": ["mlx-adopt", "mlx-bench", "mlx-scout", "mlx-wire"], - "obsidian-agent": ["build-retrospective", "connection-finder", "consistent-tagging", "daily-rollup", "dedup-merge", "frontmatter-normalizer", "manifest-content", "manifest-core", "manifest-feature", "manifest-infra", "manifest-pm", "manifest-research", "manifest-risk", "manifest-vault", "meeting-cleanup", "moc-builder", "note-splitter", "outline-to-draft", "plan-to-spec", "source-digest", "summarize-and-link", "task-harvester", "tracker-driver", "vault-grounding", "vault-synthesis", "wikilink-weaver"] +// What each pinned Codex package mirrors from its source repository. The pinned commit, tree and skills live in packages/codex/expected-integrity.json. +export const CODEX_PACKAGES = { + "mlx-agent": { + sourcePath: "providers/codex", + included: [".codex-plugin", "requirements-lab-inference.txt", "requirements-lab.txt", "scripts", "skills", "src"], + excluded: ["scripts/mlx-agent-mcp", "src/mlx_agent/mcp_server.py"], + exclusionReason: "Codex skills use the packaged CLI entrypoint; MCP transport artifacts are not required or distributed by this catalog." + }, + "obsidian-agent": { sourcePath: ".", included: [".codex-plugin", "skills"] } }; -const SOURCE_COMMITS = { "mlx-agent": "923627988cf98e984fa36e0b940f86e7263e2958", "obsidian-agent": "7efecb058fce8185056cbe7868d6de1af11879a9" }; -const SOURCE_PATHS = { "mlx-agent": "providers/codex", "obsidian-agent": "." }; -const MLX_EXCLUDED = [".mlx-agent-generated-files.json", "skills/*/scripts/mlx-agent-mcp", "skills/*/src/mlx_agent/mcp_server.py"]; -const MLX_EXCLUSION_REASON = "Codex skills use the packaged CLI entrypoint; MCP transport artifacts are not required or distributed by this catalog."; const DISCOVERY_COMMANDS = { agy: { "mlx-agent": "git clone https://github.com/cavi-ai/mlx-agent.git && agy plugin install ./mlx-agent/providers/agy" @@ -75,10 +76,11 @@ async function filesUnder(directory, symlinks = [], nonregular = []) { return nested.flat(); } -async function packageTreeHash(packageRoot, included) { +export async function packageTreeHash(packageRoot, included) { const hash = createHash("sha256"); for (const base of included) { - const files = (await filesUnder(path.join(packageRoot, base))).sort(); + const target = path.join(packageRoot, base); + const files = (await lstat(target)).isFile() ? [target] : (await filesUnder(target)).sort(); for (const file of files) { hash.update(path.relative(packageRoot, file)); hash.update("\0"); @@ -147,11 +149,13 @@ function validateTrustedIntegrity(registry, errors) { if (!Object.hasOwn(registry, name)) { errors.push(`trusted integrity registry missing package: ${name}`); continue; } const record = registry[name]; if (!record || typeof record !== "object" || Array.isArray(record)) { errors.push(`trusted integrity ${name} must be an object`); continue; } - for (const key of unknownKeys(record, ["repository", "source_commit", "source_path", "tree"])) errors.push(`trusted integrity ${name} unexpected property: ${key}`); + for (const key of unknownKeys(record, ["repository", "source_commit", "source_path", "tree", "skills"])) errors.push(`trusted integrity ${name} unexpected property: ${key}`); if (record.repository !== `cavi-ai/${name}`) errors.push(`trusted integrity ${name} repository mismatch`); - if (record.source_commit !== SOURCE_COMMITS[name]) errors.push(`trusted integrity ${name} source_commit mismatch`); - if (record.source_path !== SOURCE_PATHS[name]) errors.push(`trusted integrity ${name} source_path mismatch`); + if (typeof record.source_commit !== "string" || !/^[0-9a-f]{40}$/.test(record.source_commit)) errors.push(`trusted integrity ${name} source_commit must be a full lowercase commit`); + if (record.source_path !== CODEX_PACKAGES[name].sourcePath) errors.push(`trusted integrity ${name} source_path mismatch`); if (typeof record.tree !== "string" || !/^[0-9a-f]{64}$/.test(record.tree)) errors.push(`trusted integrity ${name} tree must be 64 lowercase hexadecimal characters`); + const skills = record.skills; + if (!Array.isArray(skills) || !skills.length || skills.some((skill) => typeof skill !== "string" || !/^[a-z0-9-]+$/.test(skill)) || JSON.stringify(skills) !== JSON.stringify([...new Set(skills)].sort())) errors.push(`trusted integrity ${name} skills must be sorted unique skill names`); } } @@ -205,25 +209,27 @@ async function validateCodexPackage(root, raw, canonical, trustedIntegrity, erro const provenance = JSON.parse(await readFile(path.join(packageRoot, "provenance.json"), "utf8")); const manifest = JSON.parse(await readFile(path.join(packageRoot, ".codex-plugin/plugin.json"), "utf8")); if (manifest.name !== raw.name) errors.push(`codex package manifest mismatch: ${raw.name}`); + const config = CODEX_PACKAGES[raw.name]; + if (!config) { errors.push(`codex package has no pinned configuration: ${raw.name}`); return; } if (provenance.repository !== canonical.repository) errors.push(`codex package provenance repository mismatch: ${raw.name}`); - if (provenance.source_commit !== SOURCE_COMMITS[raw.name]) errors.push(`codex package source commit mismatch: ${raw.name}`); - if (provenance.source_path !== SOURCE_PATHS[raw.name]) errors.push(`codex package source path mismatch: ${raw.name}`); - const provenanceKeys = raw.name === "mlx-agent" ? ["repository", "source_commit", "source_path", "included", "excluded", "exclusion_reason", "integrity"] : ["repository", "source_commit", "source_path", "included", "integrity"]; + if (provenance.source_commit !== trustedIntegrity?.source_commit) errors.push(`codex package source commit mismatch: ${raw.name}`); + if (provenance.source_path !== config.sourcePath) errors.push(`codex package source path mismatch: ${raw.name}`); + const provenanceKeys = config.excluded ? ["repository", "source_commit", "source_path", "included", "excluded", "exclusion_reason", "integrity"] : ["repository", "source_commit", "source_path", "included", "integrity"]; if (unknownKeys(provenance, provenanceKeys).length || unknownKeys(provenance.integrity, ["algorithm", "tree"]).length) errors.push(`codex package provenance shape invalid: ${raw.name}`); - if (raw.name === "mlx-agent" && (JSON.stringify(provenance.excluded) !== JSON.stringify(MLX_EXCLUDED) || provenance.exclusion_reason !== MLX_EXCLUSION_REASON)) errors.push("codex package exclusions mismatch: mlx-agent"); - if (JSON.stringify(provenance.included) !== JSON.stringify(INCLUDED_ROOTS)) errors.push(`codex package included roots mismatch: ${raw.name}`); - if (!trustedIntegrity || trustedIntegrity.repository !== canonical.repository || trustedIntegrity.source_commit !== SOURCE_COMMITS[raw.name] || trustedIntegrity.source_path !== SOURCE_PATHS[raw.name] || typeof trustedIntegrity.tree !== "string") errors.push(`codex package trusted integrity invalid: ${raw.name}`); + if (config.excluded && (JSON.stringify(provenance.excluded) !== JSON.stringify(config.excluded) || provenance.exclusion_reason !== config.exclusionReason)) errors.push(`codex package exclusions mismatch: ${raw.name}`); + if (JSON.stringify(provenance.included) !== JSON.stringify(config.included)) errors.push(`codex package included roots mismatch: ${raw.name}`); + if (!trustedIntegrity || trustedIntegrity.repository !== canonical.repository || trustedIntegrity.source_path !== config.sourcePath || typeof trustedIntegrity.tree !== "string") errors.push(`codex package trusted integrity invalid: ${raw.name}`); if (provenance.integrity?.algorithm !== "sha256" || !Array.isArray(provenance.included)) { errors.push(`codex package integrity metadata invalid: ${raw.name}`); } else if (symlinks.length === 0 && nonregular.length === 0) { - const actualTree = await packageTreeHash(packageRoot, INCLUDED_ROOTS); + const actualTree = await packageTreeHash(packageRoot, config.included); if (actualTree !== provenance.integrity.tree) errors.push(`codex package integrity mismatch: ${raw.name}`); if (actualTree !== trustedIntegrity?.tree) errors.push(`codex package upstream digest mismatch: ${raw.name}`); } - const allowedTop = new Set([".codex-plugin", "skills", "provenance.json"]); + const allowedTop = new Set([...config.included, "provenance.json"]); for (const item of await readdir(packageRoot)) if (!allowedTop.has(item)) errors.push(`codex package unexpected root entry: ${raw.name}: ${item}`); const skills = (await readdir(path.join(packageRoot, "skills"), { withFileTypes: true })).filter((item) => item.isDirectory()).map((item) => item.name).sort(); - if (JSON.stringify(skills) !== JSON.stringify(SKILL_INVENTORIES[raw.name])) errors.push(`codex package skill inventory mismatch: ${raw.name}`); + if (JSON.stringify(skills) !== JSON.stringify(trustedIntegrity?.skills)) errors.push(`codex package skill inventory mismatch: ${raw.name}`); if (!packageFiles.some((file) => /\/skills\/[^/]+\/SKILL\.md$/.test(file))) errors.push(`codex package has no skills: ${raw.name}`); if (packageFiles.some((file) => path.basename(file) === ".mcp.json" || path.basename(file) === "mlx-agent-mcp" || path.basename(file) === "mcp_server.py")) { errors.push(`codex package contains MCP transport: ${raw.name}`); diff --git a/scripts/validate-catalog.test.mjs b/scripts/validate-catalog.test.mjs index 9401900..84eda98 100644 --- a/scripts/validate-catalog.test.mjs +++ b/scripts/validate-catalog.test.mjs @@ -29,10 +29,15 @@ function plugin(name, repository = `cavi-ai/${name}`, supportedHosts = pluginHos }; } -async function fixtureTreeHash(packageRoot) { - const walk = async (directory) => (await Promise.all((await readdir(directory, { withFileTypes: true })).map(async (item) => item.isDirectory() ? walk(path.join(directory, item.name)) : [path.join(directory, item.name)]))).flat(); +const includedRoots = { + "mlx-agent": [".codex-plugin", "requirements-lab-inference.txt", "requirements-lab.txt", "scripts", "skills", "src"], + "obsidian-agent": [".codex-plugin", "skills"] +}; + +async function fixtureTreeHash(packageRoot, roots = includedRoots[path.basename(packageRoot)]) { + const walk = async (target) => (await stat(target)).isFile() ? [target] : (await Promise.all((await readdir(target, { withFileTypes: true })).map(async (item) => item.isDirectory() ? walk(path.join(target, item.name)) : [path.join(target, item.name)]))).flat(); const hash = createHash("sha256"); - for (const base of [".codex-plugin", "skills"]) for (const file of (await walk(path.join(packageRoot, base))).sort()) { + for (const base of roots) for (const file of (await walk(path.join(packageRoot, base))).sort()) { hash.update(path.relative(packageRoot, file)); hash.update("\0"); hash.update(await readFile(file)); hash.update("\0"); } return hash.digest("hex"); @@ -46,7 +51,15 @@ async function createFixturePackage(root, name) { await mkdir(path.join(packageRoot, "skills", skill), { recursive: true }); await writeFile(path.join(packageRoot, "skills", skill, "SKILL.md"), `---\nname: ${skill}\ndescription: fixture\n---\n`); } - await writeFile(path.join(packageRoot, "provenance.json"), JSON.stringify({ repository: `cavi-ai/${name}`, source_commit: sourceCommits[name], source_path: name === "mlx-agent" ? "providers/codex" : ".", included: [".codex-plugin", "skills"], ...(name === "mlx-agent" ? { excluded: [".mlx-agent-generated-files.json", "skills/*/scripts/mlx-agent-mcp", "skills/*/src/mlx_agent/mcp_server.py"], exclusion_reason: "Codex skills use the packaged CLI entrypoint; MCP transport artifacts are not required or distributed by this catalog." } : {}), integrity: { algorithm: "sha256", tree: await fixtureTreeHash(packageRoot) } })); + if (name === "mlx-agent") { + await mkdir(path.join(packageRoot, "scripts"), { recursive: true }); + await mkdir(path.join(packageRoot, "src/mlx_agent"), { recursive: true }); + await writeFile(path.join(packageRoot, "scripts/mlx-agent"), "#!/usr/bin/env python3\n"); + await writeFile(path.join(packageRoot, "src/mlx_agent/__init__.py"), "__version__ = \"1.0.0\"\n"); + await writeFile(path.join(packageRoot, "requirements-lab.txt"), "fixture==1\n"); + await writeFile(path.join(packageRoot, "requirements-lab-inference.txt"), "fixture==1\n"); + } + await writeFile(path.join(packageRoot, "provenance.json"), JSON.stringify({ repository: `cavi-ai/${name}`, source_commit: sourceCommits[name], source_path: name === "mlx-agent" ? "providers/codex" : ".", included: includedRoots[name], ...(name === "mlx-agent" ? { excluded: ["scripts/mlx-agent-mcp", "src/mlx_agent/mcp_server.py"], exclusion_reason: "Codex skills use the packaged CLI entrypoint; MCP transport artifacts are not required or distributed by this catalog." } : {}), integrity: { algorithm: "sha256", tree: await fixtureTreeHash(packageRoot) } })); } async function fixture({ catalog, claude = [], codex = [], agy = [], opencode = [], openclaw = [], hermes = [] }) { @@ -68,7 +81,7 @@ async function fixture({ catalog, claude = [], codex = [], agy = [], opencode = const expectedIntegrity = {}; for (const name of Object.keys(inventories)) { const provenance = JSON.parse(await readFile(path.join(root, `packages/codex/${name}/provenance.json`), "utf8")); - expectedIntegrity[name] = { repository: provenance.repository, source_commit: provenance.source_commit, source_path: provenance.source_path, tree: provenance.integrity.tree }; + expectedIntegrity[name] = { repository: provenance.repository, source_commit: provenance.source_commit, source_path: provenance.source_path, tree: provenance.integrity.tree, skills: inventories[name] }; } await writeFile(path.join(root, "packages/codex/expected-integrity.json"), JSON.stringify(expectedIntegrity)); return root; @@ -395,7 +408,10 @@ test("requires a closed and fully typed trusted-integrity registry", async () => [(registry) => { registry["mlx-agent"].extra = true; }, "trusted integrity mlx-agent unexpected property: extra"], [(registry) => { delete registry["mlx-agent"].source_path; }, "trusted integrity mlx-agent source_path mismatch"], [(registry) => { registry["obsidian-agent"].tree = "A".repeat(64); }, "trusted integrity obsidian-agent tree must be 64 lowercase hexadecimal characters"], - [(registry) => { registry["obsidian-agent"].tree = "0".repeat(63); }, "trusted integrity obsidian-agent tree must be 64 lowercase hexadecimal characters"] + [(registry) => { registry["obsidian-agent"].tree = "0".repeat(63); }, "trusted integrity obsidian-agent tree must be 64 lowercase hexadecimal characters"], + [(registry) => { registry["mlx-agent"].source_commit = "main"; }, "trusted integrity mlx-agent source_commit must be a full lowercase commit"], + [(registry) => { registry["mlx-agent"].skills = ["mlx-wire", "mlx-adopt"]; }, "trusted integrity mlx-agent skills must be sorted unique skill names"], + [(registry) => { delete registry["obsidian-agent"].skills; }, "trusted integrity obsidian-agent skills must be sorted unique skill names"] ]; for (const [mutateOrValue, expected] of cases) { const root = await fixture({ catalog: canonical }); From e2ba0fac6b11cd04580705215f2ca7ca6d202369 Mon Sep 17 00:00:00 2001 From: Sasan Sotoodehfar Date: Thu, 22 Oct 2026 16:16:39 -0400 Subject: [PATCH 2/2] feat: pin every host to the released tag and commit --- .claude-plugin/marketplace.json | 28 ++++- ...ex-package.yml => sync-plugin-release.yml} | 28 ++--- README.md | 36 +++--- packages/codex/expected-integrity.json | 9 +- packages/codex/obsidian-agent/provenance.json | 9 +- .../skills/cloud-reply/SKILL.md | 59 +++++++++ .../skills/note-to-artifact/SKILL.md | 92 ++++++++++++++ .../skills/research-workbench/SKILL.md | 56 +++++++++ .../skills/session-to-note/SKILL.md | 53 ++++++++ .../scripts/parse-transcript.mjs | 61 ++++++++++ .../scripts/parse-transcript.test.mjs | 40 ++++++ .../skills/vault-routines/SKILL.md | 55 +++++++++ providers/agy/catalog.json | 4 +- providers/hermes/catalog.json | 4 +- providers/openclaw/catalog.json | 4 +- providers/opencode/catalog.json | 8 +- releases.json | 10 ++ scripts/sync-codex-package.mjs | 73 ----------- scripts/sync-release.mjs | 115 ++++++++++++++++++ ...package.test.mjs => sync-release.test.mjs} | 61 +++++++--- scripts/validate-catalog.mjs | 61 +++++++--- scripts/validate-catalog.test.mjs | 51 +++++++- 22 files changed, 760 insertions(+), 157 deletions(-) rename .github/workflows/{sync-codex-package.yml => sync-plugin-release.yml} (73%) create mode 100644 packages/codex/obsidian-agent/skills/cloud-reply/SKILL.md create mode 100644 packages/codex/obsidian-agent/skills/note-to-artifact/SKILL.md create mode 100644 packages/codex/obsidian-agent/skills/research-workbench/SKILL.md create mode 100644 packages/codex/obsidian-agent/skills/session-to-note/SKILL.md create mode 100644 packages/codex/obsidian-agent/skills/session-to-note/scripts/parse-transcript.mjs create mode 100644 packages/codex/obsidian-agent/skills/session-to-note/scripts/parse-transcript.test.mjs create mode 100644 packages/codex/obsidian-agent/skills/vault-routines/SKILL.md create mode 100644 releases.json delete mode 100644 scripts/sync-codex-package.mjs create mode 100644 scripts/sync-release.mjs rename scripts/{sync-codex-package.test.mjs => sync-release.test.mjs} (50%) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 4184a39..a23f864 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,21 +1,39 @@ { "name": "cavi-ai", - "owner": { "name": "Cavi AI", "url": "https://github.com/cavi-ai" }, - "metadata": { "description": "Cavi AI plugins for agent hosts." }, + "owner": { + "name": "Cavi AI", + "url": "https://github.com/cavi-ai" + }, + "metadata": { + "description": "Cavi AI plugins for agent hosts." + }, "plugins": [ { "name": "mlx-agent", - "source": { "source": "github", "repo": "cavi-ai/mlx-agent" }, + "source": { + "source": "github", + "repo": "cavi-ai/mlx-agent", + "ref": "v0.10.0", + "sha": "6e6effd57bc4fce90760d938a939aef1f18e1f49" + }, "description": "Discover, verify, and wire local MLX-optimized models on Apple Silicon." }, { "name": "obsidian-agent", - "source": { "source": "github", "repo": "cavi-ai/obsidian-agent" }, + "source": { + "source": "github", + "repo": "cavi-ai/obsidian-agent", + "ref": "v0.1.0", + "sha": "b0b0cf03658b4f74c632db54834cb9add337c40a" + }, "description": "Portable Obsidian vault workflows powered by the official Obsidian CLI." }, { "name": "loop-killer", - "source": { "source": "github", "repo": "cavi-ai/loop-killer" }, + "source": { + "source": "github", + "repo": "cavi-ai/loop-killer" + }, "description": "Ends a stop-hook re-prompt loop: after repeated re-prompt turns with no tool work, it clears the active goal and says so." } ] diff --git a/.github/workflows/sync-codex-package.yml b/.github/workflows/sync-plugin-release.yml similarity index 73% rename from .github/workflows/sync-codex-package.yml rename to .github/workflows/sync-plugin-release.yml index 21285e2..6d21c0a 100644 --- a/.github/workflows/sync-codex-package.yml +++ b/.github/workflows/sync-plugin-release.yml @@ -1,4 +1,4 @@ -name: Sync Codex package +name: Sync plugin release on: repository_dispatch: @@ -6,24 +6,24 @@ on: workflow_dispatch: inputs: name: - description: Plugin whose Codex package to vendor + description: Catalog plugin that released required: true type: choice - options: [mlx-agent, obsidian-agent] + options: [mlx-agent, obsidian-agent, loop-killer] tag: - description: Release tag to vendor, for example v0.10.0 + description: Release tag, for example v0.10.0 required: true permissions: contents: read concurrency: - group: sync-codex-package-${{ github.event.client_payload.slug || inputs.name }} + group: sync-plugin-release-${{ github.event.client_payload.slug || inputs.name }} cancel-in-progress: false jobs: sync: - if: github.event_name == 'workflow_dispatch' || contains(fromJSON('["mlx-agent","obsidian-agent"]'), github.event.client_payload.slug) + if: github.event_name == 'workflow_dispatch' || contains(fromJSON('["mlx-agent","obsidian-agent","loop-killer"]'), github.event.client_payload.slug) runs-on: ubuntu-latest permissions: contents: write @@ -54,12 +54,12 @@ jobs: with: node-version: 22 - - name: Vendor the tagged package + - name: Point every host at the release run: | set -euo pipefail resolved="$(git -C .upstream rev-parse HEAD)" if [ -n "$COMMIT" ] && [ "$resolved" != "$COMMIT" ]; then echo "$TAG resolves to $resolved, not $COMMIT"; exit 1; fi - node scripts/sync-codex-package.mjs --name "$NAME" --source .upstream --commit "$resolved" + node scripts/sync-release.mjs --name "$NAME" --source .upstream --tag "$TAG" --commit "$resolved" rm -rf .upstream node --test node scripts/validate-catalog.mjs @@ -71,24 +71,24 @@ jobs: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | set -euo pipefail - if [ -z "$(git status --porcelain -- packages/codex)" ]; then echo "packages/codex/$NAME already matches $TAG"; exit 0; fi + if [ -z "$(git status --porcelain)" ]; then echo "The catalog already points $NAME at $TAG"; exit 0; fi branch="sync/$NAME-$TAG" if git ls-remote --exit-code --heads origin "$branch" > /dev/null; then echo "Branch $branch already exists"; exit 1; fi git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git switch -c "$branch" - git add -A -- packages/codex - git commit -m "chore: vendor $NAME $TAG Codex package" + git add -A -- releases.json .claude-plugin providers packages + git commit -m "chore: sync $NAME $TAG" git push origin "$branch" body="$(mktemp)" { echo "## Changes" echo - echo "- \`packages/codex/$NAME\` mirrors \`cavi-ai/$NAME\` \`$TAG\` (\`$RESOLVED\`)." - echo "- \`packages/codex/expected-integrity.json\` pins that commit, its tree hash and skill inventory." + echo "- \`releases.json\` pins \`cavi-ai/$NAME\` to \`$TAG\` (\`$RESOLVED\`)." + echo "- Every host projection of $NAME installs that release: the Claude marketplace source, each discovery install command and, when the plugin ships one, the vendored Codex package." echo echo "## Verification" echo echo "- \`node --test\` and \`node scripts/validate-catalog.mjs\` passed in $RUN_URL." } > "$body" - gh pr create --base main --head "$branch" --title "chore: vendor $NAME $TAG" --body-file "$body" + gh pr create --base main --head "$branch" --title "chore: sync $NAME $TAG" --body-file "$body" diff --git a/README.md b/README.md index ef11260..26daab3 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,17 @@ The machine-readable source of truth is [`catalog.json`](catalog.json). Claude and Codex consume native marketplace projections. Agy, OpenCode, OpenClaw, and Hermes use the discovery records under `providers/`, which link to each source repository's tested installer; those files do not claim a native marketplace protocol. Portable AgentSkills packages remain available from each plugin repository. +## Releases + +[`releases.json`](releases.json) pins each released plugin to one tag and commit, and every host installs that release: the Claude marketplace source carries the tag as `ref` and the commit as `sha`, each discovery install command clones the tag, and the Codex package is copied from it. `loop-killer` has no release yet, so its Claude source follows the default branch. + +A plugin's release workflow sends a `cavi-oss-release` dispatch. [Sync plugin release](.github/workflows/sync-plugin-release.yml) checks out the tag, confirms it resolves to the released commit, points every host at it with `scripts/sync-release.mjs`, runs the catalog gate and opens a pull request. The same workflow runs by hand from the Actions tab with a plugin name and tag. Locally: + +```sh +node scripts/sync-release.mjs --name mlx-agent --source --tag --commit +node --test && node scripts/validate-catalog.mjs +``` + ## Claude Code Add the marketplace, then install a plugin: @@ -33,25 +44,14 @@ codex plugin add mlx-agent@cavi-ai codex plugin add obsidian-agent@cavi-ai ``` -The Codex projection at [`.agents/plugins/marketplace.json`](.agents/plugins/marketplace.json) uses Codex's documented marketplace-local source layout. `packages/codex/` contains minimal, self-contained projections copied from exact upstream commits. Each package includes provenance and a deterministic tree hash; catalog validation rejects local drift. - -These package projections are distribution artifacts, not independent workflow sources. Product development remains in the linked plugin repository. - -A plugin release refreshes its package. The plugin's release workflow sends a `cavi-oss-release` dispatch, and [Sync Codex package](.github/workflows/sync-codex-package.yml) checks out the tag, confirms it resolves to the released commit, vendors it with `scripts/sync-codex-package.mjs`, runs the catalog gate and opens a pull request. The same workflow runs by hand from the Actions tab with a plugin name and tag. Locally: - -```sh -node scripts/sync-codex-package.mjs --name mlx-agent --source --commit -node --test && node scripts/validate-catalog.mjs -``` - -`packages/codex/expected-integrity.json` pins each package's source commit, tree hash and skill inventory; `provenance.json` inside the package must agree with it. +The Codex projection at [`.agents/plugins/marketplace.json`](.agents/plugins/marketplace.json) uses Codex's documented marketplace-local source layout, so `packages/codex/` holds a self-contained copy of each release's Codex package. These copies are distribution artifacts, not independent workflow sources; product development remains in the linked plugin repository. `packages/codex/expected-integrity.json` pins each copy's release commit, tree hash and skill inventory, `provenance.json` inside the package must agree with it, and validation rejects local drift. ## Agy -Agy discovery metadata is in [`providers/agy/catalog.json`](providers/agy/catalog.json). Each entry records the source repository's native installation command: +Agy discovery metadata is in [`providers/agy/catalog.json`](providers/agy/catalog.json). Each entry records the source repository's native installation command for the pinned release: ```sh -git clone https://github.com/cavi-ai/mlx-agent.git +git clone --branch --depth 1 https://github.com/cavi-ai/mlx-agent.git agy plugin install ./mlx-agent/providers/agy ``` @@ -62,10 +62,10 @@ The discovery catalog itself does not install anything. `obsidian-agent` does no OpenCode discovery metadata is in [`providers/opencode/catalog.json`](providers/opencode/catalog.json). Both source repositories provide preview-first installers: ```sh -git clone https://github.com/cavi-ai/mlx-agent.git +git clone --branch --depth 1 https://github.com/cavi-ai/mlx-agent.git python3 mlx-agent/scripts/mlx-agent install opencode --scope user --dry-run --json -git clone https://github.com/cavi-ai/obsidian-agent.git +git clone --branch --depth 1 https://github.com/cavi-ai/obsidian-agent.git node obsidian-agent/scripts/install.mjs --host opencode --scope user --dry-run ``` @@ -76,7 +76,7 @@ Review the destination paths and use the exact preview hash when confirming. OpenClaw and Hermes discovery metadata is in [`providers/openclaw/catalog.json`](providers/openclaw/catalog.json) and [`providers/hermes/catalog.json`](providers/hermes/catalog.json). The `mlx-agent` installer places its skills in `$OPENCLAW_STATE_DIR/skills` or `$HERMES_HOME/skills`: ```sh -git clone https://github.com/cavi-ai/mlx-agent.git +git clone --branch --depth 1 https://github.com/cavi-ai/mlx-agent.git python3 mlx-agent/scripts/mlx-agent install openclaw --scope user --dry-run --json python3 mlx-agent/scripts/mlx-agent install hermes --scope user --dry-run --json ``` @@ -96,4 +96,4 @@ node --test node scripts/validate-catalog.mjs ``` -Validation rejects identity drift, unknown hosts, packages for undeclared hosts, duplicate entries, non-plugin products, and projections that disagree with the canonical host list, repository, or package path. +Validation rejects identity drift, unknown hosts, packages for undeclared hosts, duplicate entries, non-plugin products, projections that disagree with the canonical host list, repository, or package path, and any host projection that does not install the plugin's pinned release. diff --git a/packages/codex/expected-integrity.json b/packages/codex/expected-integrity.json index 115bc19..21cab16 100644 --- a/packages/codex/expected-integrity.json +++ b/packages/codex/expected-integrity.json @@ -16,11 +16,12 @@ }, "obsidian-agent": { "repository": "cavi-ai/obsidian-agent", - "source_commit": "7efecb058fce8185056cbe7868d6de1af11879a9", + "source_commit": "b0b0cf03658b4f74c632db54834cb9add337c40a", "source_path": ".", - "tree": "e4b9ecdc9d7d76011d955784f64d2b2f2074e5c776b5411d9fc332dd07734df2", + "tree": "b1cc6f52489f22b76d6b1700060302e11ce335ee841fa8af8ce38dcbf96c9c7a", "skills": [ "build-retrospective", + "cloud-reply", "connection-finder", "consistent-tagging", "daily-rollup", @@ -37,13 +38,17 @@ "meeting-cleanup", "moc-builder", "note-splitter", + "note-to-artifact", "outline-to-draft", "plan-to-spec", + "research-workbench", + "session-to-note", "source-digest", "summarize-and-link", "task-harvester", "tracker-driver", "vault-grounding", + "vault-routines", "vault-synthesis", "wikilink-weaver" ] diff --git a/packages/codex/obsidian-agent/provenance.json b/packages/codex/obsidian-agent/provenance.json index b3be3be..978e62e 100644 --- a/packages/codex/obsidian-agent/provenance.json +++ b/packages/codex/obsidian-agent/provenance.json @@ -1,10 +1,13 @@ { "repository": "cavi-ai/obsidian-agent", - "source_commit": "7efecb058fce8185056cbe7868d6de1af11879a9", + "source_commit": "b0b0cf03658b4f74c632db54834cb9add337c40a", "source_path": ".", - "included": [".codex-plugin", "skills"], + "included": [ + ".codex-plugin", + "skills" + ], "integrity": { "algorithm": "sha256", - "tree": "e4b9ecdc9d7d76011d955784f64d2b2f2074e5c776b5411d9fc332dd07734df2" + "tree": "b1cc6f52489f22b76d6b1700060302e11ce335ee841fa8af8ce38dcbf96c9c7a" } } diff --git a/packages/codex/obsidian-agent/skills/cloud-reply/SKILL.md b/packages/codex/obsidian-agent/skills/cloud-reply/SKILL.md new file mode 100644 index 0000000..1bea254 --- /dev/null +++ b/packages/codex/obsidian-agent/skills/cloud-reply/SKILL.md @@ -0,0 +1,59 @@ +--- +name: cloud-reply +description: Use when running as a dispatched cloud Claude Code session against an Obsidian vault repo — do the task, write the result back as a reply note, and open a PR so Companion can pull it into the vault on any device. +portable: false +--- + +# Cloud reply + +> **Claude adapter only.** This workflow depends on Claude cloud sessions, +> Companion reply polling, and GitHub pull requests. The official Obsidian CLI +> cannot provide that transport. Keep it out of universal capability lists and +> move it into the dedicated Claude adapter when that provider package lands. + +You are a cloud Claude Code session fired against an Obsidian vault that is this +Git repo (e.g. via a Companion "Send to cloud session" routine). The user is +usually on their phone and can't see your terminal — your **reply note + PR is +the only way your work reaches them**, so land it where Companion looks. + +**REQUIRED SUB-SKILL:** obsidian-agent:vault-grounding + +## The discipline + +1. **Ground the work in the repo.** The task arrives as free text in the run + context. Read the actual notes before you assert anything (cite their paths); + the vault is the source of truth, not memory. +2. **Write one reply note into the configured reply folder.** Companion polls a + folder the user configures (`cloudReplyFolder`); its default is + `Claude/Replies/`, but do not assume it. Look in the repo first — if a folder + already holds reply notes, use that one. Only fall back to `Claude/Replies/` + when the repo has none, and say in the PR body which folder you chose. A reply + written to the wrong folder is invisible to the user. + Name it in kebab-case from the task (e.g. `weekly-decisions-rollup.md`), with + YAML frontmatter: `title`, `created` (YYYY-MM-DD), `source: claude-cloud`, and + `tags` (reuse the vault's existing tags). Keep the body skimmable: what you + did, key findings (with `[[wikilinks]]` to the notes), and any follow-ups. +3. **Keep all changes in one commit on a reply branch.** If the task also + created or edited other notes, include them. Commit to a new branch + `claude/reply-` — never push to `main`. +4. **Open a pull request** titled after the task. The PR is the review gate: the + user merges it, the reply note lands on `main`, and Companion's "Pull cloud + session replies" fetches it onto their device. +5. **Blocked is honest.** If you can't finish, still write the reply note saying + what's blocked and why, and open the PR — a blocked run must not go silent. + +## Hard requirements + +- The reply note lives in the repo's existing reply folder (default + `Claude/Replies/`) with valid frontmatter, and the PR body names the folder used. +- Work reaches the user as a **PR**, not a direct push to `main`. +- Every claim about the vault traces to a note you actually read. + +## Common mistakes + +- Doing great work but leaving no reply note — the user sees nothing. +- Pushing to `main` instead of a `claude/reply-` branch + PR. +- Inventing vault facts from memory instead of reading the notes. +- A reply note with no frontmatter (it won't index cleanly in the vault). +- Hardcoding `Claude/Replies/` when the user configured a different reply folder — + the reply lands where Companion isn't looking and the user sees nothing. diff --git a/packages/codex/obsidian-agent/skills/note-to-artifact/SKILL.md b/packages/codex/obsidian-agent/skills/note-to-artifact/SKILL.md new file mode 100644 index 0000000..9e71659 --- /dev/null +++ b/packages/codex/obsidian-agent/skills/note-to-artifact/SKILL.md @@ -0,0 +1,92 @@ +--- +name: note-to-artifact +description: Use when turning notes, data, or a plan into a beautiful, self-contained HTML artifact (report, dashboard, plan, table, or diagram) in the Anthropic-flavored "HTML is all you need" style — especially for Obsidian notes rendered in a ```claude-html``` block. +portable: false +--- + +# Note → artifact + +> **Claude adapter only.** This workflow requires Anthropic-specific artifact +> conventions and Companion's `claude-html` renderer. Portable workflows return +> Markdown and Mermaid instead. Keep this skill out of universal capability +> lists and move it into the dedicated Claude adapter when that provider +> package lands. + +Turn source material into **one self-contained HTML document** that is +beautiful, information-dense, and renders with no external resources. + +This style is inspired by Thariq Shihipar's "HTML is all you need to make +effective reports/dashboards" (see the repo NOTICE/README for attribution). + +## Design system + +**Colors** +- Background `#FAF9F5` (warm ivory); surfaces `#FFFFFF` cards on `#F0EEE6` + panels; borders `#D1CFC5` +- Text `#141413` (near-black), `#3D3D3A` (muted), `#87867F` (subtle) +- Accent `#D97757` (clay/terracotta); secondary `#788C5D` (olive), `#6A9BCC` (blue) + +**Typography** +- Serif for headings (`ui-serif, Georgia`), sans (`system-ui`) for body, mono + (`"SF Mono", Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace`) for code/labels +- Line-height ~1.6; large headings; letter-spacing on small eyebrow labels + +**Layout** +- Max-width ~960px, centered, 32–48px padding +- Card-based; rounded corners (8–16px); subtle shadows +- CSS grid for stat rows and multi-column sections + +**Charts** +- Inline SVG only — no external libraries. Use the clay/olive/blue palette. + +## Format selection + +Pick the format that fits the content: **plan**, **report**, **table**, +**diagram/flow**, or **dashboard**. Lead with the single most important number +or takeaway. + +## Hard requirements + +- Output a SINGLE self-contained HTML document — no external CSS/JS/font/image + requests. Inline everything. +- Escape any user/content text interpolated into HTML. +- When saving into Obsidian, wrap the document in a ```` ```claude-html ```` + fenced block so the Companion for Claude plugin renders it inline. +- Output the **complete** document — close every tag and the ` +``` + +## Extending the style + +Project-specific variations live in `design-system/extensions/` (additive +overrides layered on top of this base — never edit the base set). If an +extension applies to the current vault, prefer its tokens over the defaults. diff --git a/packages/codex/obsidian-agent/skills/research-workbench/SKILL.md b/packages/codex/obsidian-agent/skills/research-workbench/SKILL.md new file mode 100644 index 0000000..07a14d8 --- /dev/null +++ b/packages/codex/obsidian-agent/skills/research-workbench/SKILL.md @@ -0,0 +1,56 @@ +--- +name: research-workbench +description: Use when running a formal research project on typed research records — creating a project, importing sources, capturing evidence with locators, building supported claims, auditing provenance, and generating an evidence-backed outline via the research_* tools. +portable: false +--- + +# Research workbench + +> **Claude adapter only.** The typed `research_*` record API has no equivalent +> in the official Obsidian CLI. Keep this skill out of universal capability +> lists and move it with its owning research tools into the dedicated Claude +> adapter. Use `obsidian-agent:source-digest` for portable plain-note research. + +Preserve continuity from research question through an evidence-backed outline. + +**REQUIRED SUB-SKILL:** obsidian-agent:vault-grounding + +## Process + +1. **Project** — identify the active `research-project`; if none exists, offer + `research_project_create` with a focused question. +2. **Source** — import source material with `research_source_import`; preserve + readable content and its computed fingerprint. +3. **Evidence** — capture an exact excerpt and locator with + `research_evidence_capture`, then use `research_evidence_review` before + treating it as trusted. +4. **Claim** — use `research_claim_create` to keep supporting, challenging, and + contextual relations distinct. +5. **Link** — attach further evidence to an existing claim with + `research_claim_link`, choosing `supports`, `challenges`, or `contextualizes` + deliberately; the relation is not decoration, it is what the audit and the + outline read. +6. **Audit** — run `research_audit`; surface stale fingerprints, missing + locators, broken references, unreviewed evidence, and unsupported claims. +7. **Outline** — use `research_outline_generate`; include only reviewed, + locatable, non-stale evidence as trusted support. + +## Hard requirements + +- Use research tools instead of hand-editing canonical research records. +- Never present proposed, rejected, stale, disconnected, or unsupported material + as trusted support. +- Ask before material vault writes and report the records created or updated. +- Run only the stages needed for the user's request, then name the next useful + stage. +- If a required tool is unavailable, identify it and stop rather than fabricate + state. + +## Common mistakes + +- Hand-editing a research record instead of using the `research_*` tool that owns it. +- Treating captured evidence as trusted before `research_evidence_review`. +- Generating an outline without running `research_audit` first, so stale + fingerprints and broken references reach the draft. +- Recording every relation as `supports` because it is the default-looking option. +- Inventing project state when a `research_*` tool is unavailable instead of stopping. diff --git a/packages/codex/obsidian-agent/skills/session-to-note/SKILL.md b/packages/codex/obsidian-agent/skills/session-to-note/SKILL.md new file mode 100644 index 0000000..3bd07e6 --- /dev/null +++ b/packages/codex/obsidian-agent/skills/session-to-note/SKILL.md @@ -0,0 +1,53 @@ +--- +name: session-to-note +description: Use when distilling or saving a Claude session into the Obsidian vault, capturing what was decided or learned in a session, or turning session memory into a persistent, linked knowledge note. +portable: false +--- + +# Session → note + +> **Claude adapter only.** Transcript discovery and parsing depend on Claude's +> private session layout and plugin-root environment. The official Obsidian CLI +> does not expose host transcripts. Keep this skill out of universal capability +> lists and move it into the dedicated Claude adapter. + +Distill the current Claude session into ONE consolidated vault note — the ideas, +decisions, and conclusions you'd remember walking out of a meeting, not a +keystroke log. + +**REQUIRED SUB-SKILL:** obsidian-agent:vault-grounding + +## Process + +1. **Get clean turns.** Locate the active session transcript (`.jsonl`, under + `~/.claude/projects//`) and run the bundled parser to strip + thinking, tool calls/results, and harness noise: + `node "${CLAUDE_PLUGIN_ROOT}/skills/session-to-note/scripts/parse-transcript.mjs" ` +2. **Distill, don't transcribe.** From the clean turns, extract: the goal, key + decisions, what was built/changed, open questions, and the few durable ideas + worth keeping. Drop chatter. Aim for a note a reader understands in a minute. +3. **Tag it.** Apply tags via obsidian-agent:consistent-tagging (reuse the + vault's taxonomy). Always include a `session` tag. +4. **Write the note.** `note_create` with a clear title (topic + date) and a + body that leads with the goal and the decisions. `note_create` writes + `title`, `created` (date), `source`, and `tags` frontmatter automatically. +5. **Stamp the type.** `note_create` cannot set arbitrary frontmatter, so call + `update_frontmatter` on the new note with `fields: { type: "session" }` — + this is what makes it index as a session in search/Dataview/the tag pane. +6. **Link into the graph.** Use obsidian-agent:wikilink-weaver to connect the + new note to related existing notes, so the session becomes connected + knowledge, not an island. + +## Hard requirements + +- The note is a distillation, NOT the raw transcript. No tool calls, no + thinking, no harness noise. +- Date/time + `type: session` + a `session` tag, so it indexes for search, + Dataview, and the tag pane. +- At least attempt to link the note to related notes via wikilink-weaver. + +## Common mistakes + +- Pasting the raw conversation instead of distilling it. +- Missing frontmatter/date → the note doesn't index. +- Leaving the note orphaned (no links in or out). diff --git a/packages/codex/obsidian-agent/skills/session-to-note/scripts/parse-transcript.mjs b/packages/codex/obsidian-agent/skills/session-to-note/scripts/parse-transcript.mjs new file mode 100644 index 0000000..95dead2 --- /dev/null +++ b/packages/codex/obsidian-agent/skills/session-to-note/scripts/parse-transcript.mjs @@ -0,0 +1,61 @@ +#!/usr/bin/env node +// Read a Claude Code session transcript (.jsonl) and emit clean conversational +// turns — user/assistant prose only, with thinking, tool calls/results, and +// harness noise stripped. Dependency-free (Node stdlib). + +import { readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +const NOISE_PREFIXES = [" b && b.type === "text" && typeof b.text === "string") + .map((b) => b.text) + .join("\n"); +} + +function isNoise(text) { + const t = text.trim(); + if (t === "" || t === "[Request interrupted by user]") return true; + return NOISE_PREFIXES.some((p) => t.startsWith(p)); +} + +/** Filter raw .jsonl lines to [{ role, text }] conversational turns. */ +export function filterTranscript(lines) { + const turns = []; + for (const line of lines) { + let rec; + try { + rec = JSON.parse(line); + } catch { + continue; + } + if (rec.type !== "user" && rec.type !== "assistant") continue; + const msg = rec.message ?? {}; + const text = blockText(rec.type, msg.content).trim(); + if (!text || isNoise(text)) continue; + turns.push({ role: rec.type, text }); + } + return turns; +} + +function main() { + const path = process.argv[2]; + if (!path) { + console.error("usage: parse-transcript.mjs "); + process.exit(1); + } + const lines = readFileSync(path, "utf8").split("\n").filter(Boolean); + const turns = filterTranscript(lines); + for (const t of turns) { + console.log(`## ${t.role === "user" ? "User" : "Claude"}\n\n${t.text}\n`); + } +} + +// Run main only when invoked directly, not when imported by the test. +// pathToFileURL handles paths with spaces/symlinks (e.g. "Application Support"). +if (import.meta.url === pathToFileURL(process.argv[1]).href) main(); diff --git a/packages/codex/obsidian-agent/skills/session-to-note/scripts/parse-transcript.test.mjs b/packages/codex/obsidian-agent/skills/session-to-note/scripts/parse-transcript.test.mjs new file mode 100644 index 0000000..abfc61d --- /dev/null +++ b/packages/codex/obsidian-agent/skills/session-to-note/scripts/parse-transcript.test.mjs @@ -0,0 +1,40 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { filterTranscript } from "./parse-transcript.mjs"; + +const lines = [ + JSON.stringify({ type: "queue-operation", operation: "x" }), + JSON.stringify({ type: "attachment", attachment: {} }), + JSON.stringify({ type: "user", message: { role: "user", content: "What's the plan for Atlas?" } }), + JSON.stringify({ type: "assistant", message: { role: "assistant", content: [ + { type: "thinking", thinking: "secret reasoning" }, + { type: "text", text: "Atlas ships in two phases." }, + { type: "tool_use", name: "Read", input: {} }, + ] } }), + JSON.stringify({ type: "user", message: { role: "user", content: [ + { type: "tool_result", content: "file contents", tool_use_id: "t1" }, + ] } }), + JSON.stringify({ type: "user", message: { role: "user", content: [{ type: "text", text: "[Request interrupted by user]" }] } }), + JSON.stringify({ type: "user", message: { role: "user", content: [{ type: "text", text: "/clear" }] } }), + "not json", +]; + +test("keeps only user/assistant text turns, drops noise", () => { + const turns = filterTranscript(lines); + assert.deepEqual(turns, [ + { role: "user", text: "What's the plan for Atlas?" }, + { role: "assistant", text: "Atlas ships in two phases." }, + ]); +}); + +test("joins multiple text blocks and drops interleaved tool/system records", () => { + const turns = filterTranscript([ + JSON.stringify({ type: "system", content: "system event" }), + JSON.stringify({ type: "assistant", message: { role: "assistant", content: [ + { type: "text", text: "First point." }, + { type: "server_tool_use", name: "advisor", input: {} }, + { type: "text", text: "Second point." }, + ] } }), + ]); + assert.deepEqual(turns, [{ role: "assistant", text: "First point.\nSecond point." }]); +}); diff --git a/packages/codex/obsidian-agent/skills/vault-routines/SKILL.md b/packages/codex/obsidian-agent/skills/vault-routines/SKILL.md new file mode 100644 index 0000000..16140da --- /dev/null +++ b/packages/codex/obsidian-agent/skills/vault-routines/SKILL.md @@ -0,0 +1,55 @@ +--- +name: vault-routines +description: Use when a vault result has recurring value and the user wants it repeated on a schedule. Registers a scheduled cloud agent in Claude Code; in Companion it can only fire a routine the user already created. +portable: false +--- + +# Vault routines + +> **Claude adapter only.** Scheduling and routine-fire URLs belong to the +> Claude/Companion host integration; the official Obsidian CLI has no scheduler. +> Keep this skill out of universal capability lists and move it into the +> dedicated Claude adapter. + +Turn a one-off vault result into an editable, scheduled routine. You leverage +existing scheduling machinery; you never build scheduling yourself. + +## When a result has recurring value + +After producing something the user will want again on a cadence (a weekly +review, a vault health sweep, a Monday task pull, a new-papers digest), offer a +routine. Phrase it as a concrete, editable proposal — never schedule silently. + +> "This is useful on a cadence. Want me to set it up as a routine — e.g. every +> Monday 8am? It runs as a cloud session and you can edit or cancel it anytime." + +## Rules + +1. **Consent first.** Never create a schedule without the user agreeing to it. +2. **Concrete cadence.** Propose a specific time/frequency tied to the task, not + "regularly." +3. **Editable + cancellable.** Tell the user the routine is editable and how it + runs (cloud session), so it never feels like a black box. +4. **Scope the routine.** A routine re-runs the same skill against the live + vault. State exactly what it will do each run and where output lands. + +## Wiring it — branch on where you are running + +You are in exactly one of two runtimes. Establish which before promising anything. + +**Claude Code (desktop CLI).** Use the `/schedule` machinery to register a +scheduled cloud agent against the command or skill invocation that produced the +result. Confirm back the cadence, the next run time, and how to edit or cancel it. + +**Companion (in Obsidian).** Companion cannot create a routine. It can only +*fire* a routine the user already created in the Claude Code web UI and whose +"fire" URL they pasted into Companion's settings. If none is configured, say so +plainly and give the two steps — create the routine in the web UI, paste its fire +URL into Companion settings — instead of implying a schedule was set. + +## What you cannot do + +- You cannot create a schedule from inside Companion. +- You cannot schedule anything without the user agreeing to a specific cadence. +- Never report a routine as scheduled unless you registered it and can state its + next run. diff --git a/providers/agy/catalog.json b/providers/agy/catalog.json index 02003c8..a0bc626 100644 --- a/providers/agy/catalog.json +++ b/providers/agy/catalog.json @@ -7,7 +7,9 @@ "name": "mlx-agent", "repository": "cavi-ai/mlx-agent", "package": "providers/agy", - "install": { "command": "git clone https://github.com/cavi-ai/mlx-agent.git && agy plugin install ./mlx-agent/providers/agy" } + "install": { + "command": "git clone --branch v0.10.0 --depth 1 https://github.com/cavi-ai/mlx-agent.git && agy plugin install ./mlx-agent/providers/agy" + } } ] } diff --git a/providers/hermes/catalog.json b/providers/hermes/catalog.json index e8c3f33..6c7021a 100644 --- a/providers/hermes/catalog.json +++ b/providers/hermes/catalog.json @@ -7,7 +7,9 @@ "name": "mlx-agent", "repository": "cavi-ai/mlx-agent", "package": "providers/agentskills", - "install": { "command": "git clone https://github.com/cavi-ai/mlx-agent.git && python3 mlx-agent/scripts/mlx-agent install hermes --scope user --dry-run --json" } + "install": { + "command": "git clone --branch v0.10.0 --depth 1 https://github.com/cavi-ai/mlx-agent.git && python3 mlx-agent/scripts/mlx-agent install hermes --scope user --dry-run --json" + } } ] } diff --git a/providers/openclaw/catalog.json b/providers/openclaw/catalog.json index 42e40d8..6889739 100644 --- a/providers/openclaw/catalog.json +++ b/providers/openclaw/catalog.json @@ -7,7 +7,9 @@ "name": "mlx-agent", "repository": "cavi-ai/mlx-agent", "package": "providers/agentskills", - "install": { "command": "git clone https://github.com/cavi-ai/mlx-agent.git && python3 mlx-agent/scripts/mlx-agent install openclaw --scope user --dry-run --json" } + "install": { + "command": "git clone --branch v0.10.0 --depth 1 https://github.com/cavi-ai/mlx-agent.git && python3 mlx-agent/scripts/mlx-agent install openclaw --scope user --dry-run --json" + } } ] } diff --git a/providers/opencode/catalog.json b/providers/opencode/catalog.json index 47eb7b2..842dad3 100644 --- a/providers/opencode/catalog.json +++ b/providers/opencode/catalog.json @@ -7,13 +7,17 @@ "name": "mlx-agent", "repository": "cavi-ai/mlx-agent", "package": "providers/opencode", - "install": { "command": "git clone https://github.com/cavi-ai/mlx-agent.git && python3 mlx-agent/scripts/mlx-agent install opencode --scope user --dry-run --json" } + "install": { + "command": "git clone --branch v0.10.0 --depth 1 https://github.com/cavi-ai/mlx-agent.git && python3 mlx-agent/scripts/mlx-agent install opencode --scope user --dry-run --json" + } }, { "name": "obsidian-agent", "repository": "cavi-ai/obsidian-agent", "package": "providers/opencode", - "install": { "command": "git clone https://github.com/cavi-ai/obsidian-agent.git && node obsidian-agent/scripts/install.mjs --host opencode --scope user --dry-run" } + "install": { + "command": "git clone --branch v0.1.0 --depth 1 https://github.com/cavi-ai/obsidian-agent.git && node obsidian-agent/scripts/install.mjs --host opencode --scope user --dry-run" + } } ] } diff --git a/releases.json b/releases.json new file mode 100644 index 0000000..dc0ac10 --- /dev/null +++ b/releases.json @@ -0,0 +1,10 @@ +{ + "mlx-agent": { + "tag": "v0.10.0", + "commit": "6e6effd57bc4fce90760d938a939aef1f18e1f49" + }, + "obsidian-agent": { + "tag": "v0.1.0", + "commit": "b0b0cf03658b4f74c632db54834cb9add337c40a" + } +} diff --git a/scripts/sync-codex-package.mjs b/scripts/sync-codex-package.mjs deleted file mode 100644 index 6d1802c..0000000 --- a/scripts/sync-codex-package.mjs +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env node -import { copyFile, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -import { CODEX_PACKAGES, packageTreeHash } from "./validate-catalog.mjs"; - -async function copyTree(source, target, relative, excluded) { - const entry = await lstat(source); - if (entry.isSymbolicLink()) throw new Error(`source symlink rejected: ${relative}`); - if (entry.isFile()) { - if (excluded.has(relative)) return; - await mkdir(path.dirname(target), { recursive: true }); - await copyFile(source, target); - return; - } - if (!entry.isDirectory()) throw new Error(`source entry is not a file or directory: ${relative}`); - for (const name of (await readdir(source)).sort()) { - await copyTree(path.join(source, name), path.join(target, name), path.posix.join(relative, name), excluded); - } -} - -export async function syncCodexPackage({ root, name, source, commit }) { - const config = CODEX_PACKAGES[name]; - if (!config) throw new Error(`no pinned Codex package: ${name}`); - if (typeof commit !== "string" || !/^[0-9a-f]{40}$/.test(commit)) throw new Error("commit must be a full lowercase SHA"); - if (typeof source !== "string" || !source) throw new Error("source checkout is required"); - const sourceRoot = path.join(source, config.sourcePath); - const codexRoot = path.join(root, "packages/codex"); - const packageRoot = path.join(codexRoot, name); - const staging = await mkdtemp(path.join(codexRoot, `.${name}-`)); - try { - const excluded = new Set(config.excluded ?? []); - for (const base of config.included) await copyTree(path.join(sourceRoot, base), path.join(staging, base), base, excluded); - const skills = (await readdir(path.join(staging, "skills"), { withFileTypes: true })).filter((item) => item.isDirectory()).map((item) => item.name).sort(); - const tree = await packageTreeHash(staging, config.included); - const provenance = { - repository: `cavi-ai/${name}`, - source_commit: commit, - source_path: config.sourcePath, - included: config.included, - ...(config.excluded ? { excluded: config.excluded, exclusion_reason: config.exclusionReason } : {}), - integrity: { algorithm: "sha256", tree } - }; - await writeFile(path.join(staging, "provenance.json"), `${JSON.stringify(provenance, null, 2)}\n`); - const registryPath = path.join(codexRoot, "expected-integrity.json"); - const registry = JSON.parse(await readFile(registryPath, "utf8")); - registry[name] = { repository: `cavi-ai/${name}`, source_commit: commit, source_path: config.sourcePath, tree, skills }; - await rm(packageRoot, { recursive: true, force: true }); - await rename(staging, packageRoot); - await writeFile(registryPath, `${JSON.stringify(registry, null, 2)}\n`); - return { name, commit, tree, skills }; - } finally { - await rm(staging, { recursive: true, force: true }); - } -} - -function option(argv, flag) { - const index = argv.indexOf(flag); - return index >= 0 ? argv[index + 1] : undefined; -} - -const invoked = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); -if (invoked) { - const argv = process.argv.slice(2); - try { - const result = await syncCodexPackage({ root: process.cwd(), name: option(argv, "--name"), source: option(argv, "--source"), commit: option(argv, "--commit") }); - console.log(JSON.stringify(result)); - } catch (error) { - console.error(error.message); - process.exitCode = 1; - } -} diff --git a/scripts/sync-release.mjs b/scripts/sync-release.mjs new file mode 100644 index 0000000..29ba449 --- /dev/null +++ b/scripts/sync-release.mjs @@ -0,0 +1,115 @@ +#!/usr/bin/env node +import { copyFile, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { CANONICAL, CODEX_PACKAGES, DISCOVERY_HOSTS, RELEASES, claudeSource, discoveryInstallCommand, packageTreeHash } from "./validate-catalog.mjs"; + +const CLAUDE_MARKETPLACE = ".claude-plugin/marketplace.json"; +const CODEX_INTEGRITY = "packages/codex/expected-integrity.json"; + +async function readJson(file) { + return JSON.parse(await readFile(file, "utf8")); +} + +async function writeJson(file, value) { + await writeFile(file, `${JSON.stringify(value, null, 2)}\n`); +} + +async function copyTree(source, target, relative, excluded) { + const entry = await lstat(source); + if (entry.isSymbolicLink()) throw new Error(`source symlink rejected: ${relative}`); + if (entry.isFile()) { + if (excluded.has(relative)) return; + await mkdir(path.dirname(target), { recursive: true }); + await copyFile(source, target); + return; + } + if (!entry.isDirectory()) throw new Error(`source entry is not a file or directory: ${relative}`); + for (const name of (await readdir(source)).sort()) { + await copyTree(path.join(source, name), path.join(target, name), path.posix.join(relative, name), excluded); + } +} + +async function stageCodexPackage({ root, name, source, commit }) { + const config = CODEX_PACKAGES[name]; + if (typeof source !== "string" || !source) throw new Error(`${name} ships a Codex package, so a source checkout is required`); + const staging = await mkdtemp(path.join(root, "packages/codex", `.${name}-`)); + try { + const excluded = new Set(config.excluded ?? []); + for (const base of config.included) await copyTree(path.join(source, config.sourcePath, base), path.join(staging, base), base, excluded); + const skills = (await readdir(path.join(staging, "skills"), { withFileTypes: true })).filter((item) => item.isDirectory()).map((item) => item.name).sort(); + const tree = await packageTreeHash(staging, config.included); + await writeJson(path.join(staging, "provenance.json"), { + repository: `cavi-ai/${name}`, + source_commit: commit, + source_path: config.sourcePath, + included: config.included, + ...(config.excluded ? { excluded: config.excluded, exclusion_reason: config.exclusionReason } : {}), + integrity: { algorithm: "sha256", tree } + }); + return { staging, record: { repository: `cavi-ai/${name}`, source_commit: commit, source_path: config.sourcePath, tree, skills } }; + } catch (error) { + await rm(staging, { recursive: true, force: true }); + throw error; + } +} + +// Point every host projection of one plugin at a release: the release pin, the Claude marketplace +// source, each discovery install command and, when the plugin ships one, the vendored Codex package. +export async function syncRelease({ root, name, source, tag, commit }) { + if (!CANONICAL.includes(name)) throw new Error(`not a catalog plugin: ${name}`); + if (typeof tag !== "string" || !/^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/.test(tag)) throw new Error("tag must be vX.Y.Z"); + if (typeof commit !== "string" || !/^[0-9a-f]{40}$/.test(commit)) throw new Error("commit must be a full lowercase SHA"); + const release = { tag, commit }; + + const releases = await readJson(path.join(root, RELEASES)); + const pinned = Object.fromEntries(CANONICAL.filter((plugin) => plugin === name || Object.hasOwn(releases, plugin)).map((plugin) => [plugin, plugin === name ? release : releases[plugin]])); + const hosts = []; + const claude = await readJson(path.join(root, CLAUDE_MARKETPLACE)); + for (const entry of claude.plugins) if (entry.name === name) { entry.source = claudeSource(name, release); hosts.push("claude"); } + const discovery = {}; + for (const host of DISCOVERY_HOSTS) { + discovery[host] = await readJson(path.join(root, `providers/${host}/catalog.json`)); + for (const entry of discovery[host].plugins) { + if (entry.name !== name) continue; + entry.install = { ...entry.install, command: discoveryInstallCommand(host, name, release) }; + hosts.push(host); + } + } + const staged = CODEX_PACKAGES[name] ? await stageCodexPackage({ root, name, source, commit }) : null; + try { + await writeJson(path.join(root, RELEASES), pinned); + await writeJson(path.join(root, CLAUDE_MARKETPLACE), claude); + for (const host of DISCOVERY_HOSTS) await writeJson(path.join(root, `providers/${host}/catalog.json`), discovery[host]); + if (staged) { + const integrity = await readJson(path.join(root, CODEX_INTEGRITY)); + integrity[name] = staged.record; + const packageRoot = path.join(root, "packages/codex", name); + await rm(packageRoot, { recursive: true, force: true }); + await rename(staged.staging, packageRoot); + await writeJson(path.join(root, CODEX_INTEGRITY), integrity); + hosts.push("codex"); + } + } finally { + if (staged) await rm(staged.staging, { recursive: true, force: true }); + } + return { name, tag, commit, hosts: hosts.sort(), ...(staged ? { codex: { tree: staged.record.tree, skills: staged.record.skills } } : {}) }; +} + +function option(argv, flag) { + const index = argv.indexOf(flag); + return index >= 0 ? argv[index + 1] : undefined; +} + +const invoked = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invoked) { + const argv = process.argv.slice(2); + try { + const result = await syncRelease({ root: process.cwd(), name: option(argv, "--name"), source: option(argv, "--source"), tag: option(argv, "--tag"), commit: option(argv, "--commit") }); + console.log(JSON.stringify(result)); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/scripts/sync-codex-package.test.mjs b/scripts/sync-release.test.mjs similarity index 50% rename from scripts/sync-codex-package.test.mjs rename to scripts/sync-release.test.mjs index bae027a..de327d8 100644 --- a/scripts/sync-codex-package.test.mjs +++ b/scripts/sync-release.test.mjs @@ -4,20 +4,23 @@ import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; -import { syncCodexPackage } from "./sync-codex-package.mjs"; +import { syncRelease } from "./sync-release.mjs"; import { validateCatalog } from "./validate-catalog.mjs"; const repositoryRoot = path.resolve(new URL("..", import.meta.url).pathname); const commit = "a".repeat(40); +const tag = "v9.9.9"; async function catalogCopy() { const root = await mkdtemp(path.join(tmpdir(), "plugins-sync-")); - for (const item of [".agents", ".claude-plugin", "catalog.json", "catalog.schema.json", "packages", "providers"]) { + for (const item of [".agents", ".claude-plugin", "catalog.json", "catalog.schema.json", "packages", "providers", "releases.json"]) { await cp(path.join(repositoryRoot, item), path.join(root, item), { recursive: true }); } return root; } +const readJson = async (root, relative) => JSON.parse(await readFile(path.join(root, relative), "utf8")); + async function mlxAgentSource(skills = ["mlx-adopt", "mlx-scout"]) { const source = await mkdtemp(path.join(tmpdir(), "mlx-agent-source-")); const provider = path.join(source, "providers/codex"); @@ -39,42 +42,64 @@ async function mlxAgentSource(skills = ["mlx-adopt", "mlx-scout"]) { return source; } -test("vendors a tagged source tree that passes catalog validation", async () => { +test("points every host of a released plugin at its tag and commit", async () => { const root = await catalogCopy(); await writeFile(path.join(root, "packages/codex/mlx-agent/stale.txt"), "stale\n"); - const result = await syncCodexPackage({ root, name: "mlx-agent", source: await mlxAgentSource(), commit }); - assert.deepEqual(result.skills, ["mlx-adopt", "mlx-scout"]); + const result = await syncRelease({ root, name: "mlx-agent", source: await mlxAgentSource(), tag, commit }); + assert.deepEqual(result.hosts, ["agy", "claude", "codex", "hermes", "openclaw", "opencode"]); assert.deepEqual(await validateCatalog(root), []); + + assert.deepEqual((await readJson(root, "releases.json"))["mlx-agent"], { tag, commit }); + const claude = (await readJson(root, ".claude-plugin/marketplace.json")).plugins.find((entry) => entry.name === "mlx-agent"); + assert.deepEqual(claude.source, { source: "github", repo: "cavi-ai/mlx-agent", ref: tag, sha: commit }); + for (const host of ["agy", "opencode", "openclaw", "hermes"]) { + const entry = (await readJson(root, `providers/${host}/catalog.json`)).plugins.find((item) => item.name === "mlx-agent"); + assert.match(entry.install.command, /^git clone --branch v9\.9\.9 --depth 1 https:\/\/github\.com\/cavi-ai\/mlx-agent\.git && /); + } const packageRoot = path.join(root, "packages/codex/mlx-agent"); assert.deepEqual((await readdir(packageRoot)).sort(), [".codex-plugin", "provenance.json", "requirements-lab-inference.txt", "requirements-lab.txt", "scripts", "skills", "src"]); assert.deepEqual(await readdir(path.join(packageRoot, "scripts")), ["mlx-agent"]); assert.deepEqual(await readdir(path.join(packageRoot, "src/mlx_agent")), ["__init__.py"]); - const registry = JSON.parse(await readFile(path.join(root, "packages/codex/expected-integrity.json"), "utf8")); - assert.equal(registry["mlx-agent"].source_commit, commit); - assert.equal(registry["mlx-agent"].tree, JSON.parse(await readFile(path.join(packageRoot, "provenance.json"), "utf8")).integrity.tree); - assert.equal(registry["obsidian-agent"].source_commit, JSON.parse(await readFile(path.join(repositoryRoot, "packages/codex/expected-integrity.json"), "utf8"))["obsidian-agent"].source_commit); + const integrity = await readJson(root, "packages/codex/expected-integrity.json"); + assert.equal(integrity["mlx-agent"].source_commit, commit); + assert.deepEqual(integrity["mlx-agent"].skills, ["mlx-adopt", "mlx-scout"]); + + const pins = await readJson(root, "releases.json"); + const obsidian = (await readJson(root, ".claude-plugin/marketplace.json")).plugins.find((entry) => entry.name === "obsidian-agent"); + assert.deepEqual(obsidian.source, { source: "github", repo: "cavi-ai/obsidian-agent", ref: pins["obsidian-agent"].tag, sha: pins["obsidian-agent"].commit }); +}); + +test("syncs a plugin without a Codex package from its tag alone", async () => { + const root = await catalogCopy(); + const result = await syncRelease({ root, name: "loop-killer", tag: "v0.1.0", commit }); + assert.deepEqual(result.hosts, ["claude"]); + assert.deepEqual((await readJson(root, "releases.json"))["loop-killer"], { tag: "v0.1.0", commit }); + assert.deepEqual(await validateCatalog(root), []); }); test("records the vendored skill inventory so later drift is rejected", async () => { const root = await catalogCopy(); - await syncCodexPackage({ root, name: "mlx-agent", source: await mlxAgentSource(["mlx-adopt", "mlx-bench", "mlx-scout"]), commit }); + await syncRelease({ root, name: "mlx-agent", source: await mlxAgentSource(["mlx-adopt", "mlx-bench", "mlx-scout"]), tag, commit }); assert.deepEqual(await validateCatalog(root), []); await mkdir(path.join(root, "packages/codex/mlx-agent/skills/mlx-extra"), { recursive: true }); await writeFile(path.join(root, "packages/codex/mlx-agent/skills/mlx-extra/SKILL.md"), "---\nname: mlx-extra\n---\n"); const errors = await validateCatalog(root); - assert(errors.some((error) => error.includes("codex package skill inventory mismatch: mlx-agent"))); - assert(errors.some((error) => error.includes("codex package upstream digest mismatch: mlx-agent"))); + assert(errors.includes("codex package skill inventory mismatch: mlx-agent")); + assert(errors.includes("codex package upstream digest mismatch: mlx-agent")); }); -test("refuses symlinks, unknown packages and abbreviated commits without touching the catalog", async () => { +test("refuses bad input before writing anything", async () => { const root = await catalogCopy(); - const before = await readFile(path.join(root, "packages/codex/expected-integrity.json"), "utf8"); + const files = ["releases.json", ".claude-plugin/marketplace.json", "providers/agy/catalog.json", "packages/codex/expected-integrity.json"]; + const before = await Promise.all(files.map((file) => readFile(path.join(root, file), "utf8"))); const source = await mlxAgentSource(); await symlink("/etc/hosts", path.join(source, "providers/codex/skills/mlx-adopt/hosts")); - await assert.rejects(syncCodexPackage({ root, name: "mlx-agent", source, commit }), /source symlink rejected: skills\/mlx-adopt\/hosts/); - await assert.rejects(syncCodexPackage({ root, name: "loop-killer", source, commit }), /no pinned Codex package/); - await assert.rejects(syncCodexPackage({ root, name: "mlx-agent", source, commit: "abc1234" }), /full lowercase SHA/); - assert.equal(await readFile(path.join(root, "packages/codex/expected-integrity.json"), "utf8"), before); + await assert.rejects(syncRelease({ root, name: "mlx-agent", source, tag, commit }), /source symlink rejected: skills\/mlx-adopt\/hosts/); + await assert.rejects(syncRelease({ root, name: "mlx-agent", tag, commit }), /source checkout is required/); + await assert.rejects(syncRelease({ root, name: "ghost", tag, commit }), /not a catalog plugin/); + await assert.rejects(syncRelease({ root, name: "loop-killer", tag: "0.1.0", commit }), /tag must be vX\.Y\.Z/); + await assert.rejects(syncRelease({ root, name: "loop-killer", tag, commit: "abc1234" }), /full lowercase SHA/); + assert.deepEqual(await Promise.all(files.map((file) => readFile(path.join(root, file), "utf8"))), before); assert.deepEqual(await validateCatalog(root), []); assert.deepEqual((await readdir(path.join(root, "packages/codex"))).filter((item) => item.startsWith(".")), []); }); diff --git a/scripts/validate-catalog.mjs b/scripts/validate-catalog.mjs index 8a66397..7620566 100644 --- a/scripts/validate-catalog.mjs +++ b/scripts/validate-catalog.mjs @@ -8,8 +8,10 @@ export const HOSTS = ["claude", "codex", "agy", "opencode", "openclaw", "hermes" // Marketplace identity: what users type after @ in `plugin install @`. export const MARKETPLACE = "cavi-ai"; // Every plugin the catalog may list. REQUIRED is the subset that must be present and ships a pinned Codex package. -const CANONICAL = ["mlx-agent", "obsidian-agent", "loop-killer"]; +export const CANONICAL = ["mlx-agent", "obsidian-agent", "loop-killer"]; const REQUIRED = ["mlx-agent", "obsidian-agent"]; +// releases.json pins a released plugin to one tag and commit; every host projection of that plugin installs it. +export const RELEASES = "releases.json"; // What each pinned Codex package mirrors from its source repository. The pinned commit, tree and skills live in packages/codex/expected-integrity.json. export const CODEX_PACKAGES = { "mlx-agent": { @@ -20,22 +22,34 @@ export const CODEX_PACKAGES = { }, "obsidian-agent": { sourcePath: ".", included: [".codex-plugin", "skills"] } }; -const DISCOVERY_COMMANDS = { +// The install step each discovery host runs inside a fresh clone of the plugin repository. +const DISCOVERY_INSTALLS = { agy: { - "mlx-agent": "git clone https://github.com/cavi-ai/mlx-agent.git && agy plugin install ./mlx-agent/providers/agy" + "mlx-agent": "agy plugin install ./mlx-agent/providers/agy" }, opencode: { - "mlx-agent": "git clone https://github.com/cavi-ai/mlx-agent.git && python3 mlx-agent/scripts/mlx-agent install opencode --scope user --dry-run --json", - "obsidian-agent": "git clone https://github.com/cavi-ai/obsidian-agent.git && node obsidian-agent/scripts/install.mjs --host opencode --scope user --dry-run" + "mlx-agent": "python3 mlx-agent/scripts/mlx-agent install opencode --scope user --dry-run --json", + "obsidian-agent": "node obsidian-agent/scripts/install.mjs --host opencode --scope user --dry-run" }, openclaw: { - "mlx-agent": "git clone https://github.com/cavi-ai/mlx-agent.git && python3 mlx-agent/scripts/mlx-agent install openclaw --scope user --dry-run --json" + "mlx-agent": "python3 mlx-agent/scripts/mlx-agent install openclaw --scope user --dry-run --json" }, hermes: { - "mlx-agent": "git clone https://github.com/cavi-ai/mlx-agent.git && python3 mlx-agent/scripts/mlx-agent install hermes --scope user --dry-run --json" + "mlx-agent": "python3 mlx-agent/scripts/mlx-agent install hermes --scope user --dry-run --json" } }; -const DISCOVERY_HOSTS = Object.keys(DISCOVERY_COMMANDS); +export const DISCOVERY_HOSTS = Object.keys(DISCOVERY_INSTALLS); + +export function discoveryInstallCommand(host, name, release) { + const install = DISCOVERY_INSTALLS[host]?.[name]; + if (!install) return undefined; + const clone = release ? `git clone --branch ${release.tag} --depth 1` : "git clone"; + return `${clone} https://github.com/cavi-ai/${name}.git && ${install}`; +} + +export function claudeSource(name, release) { + return release ? { source: "github", repo: `cavi-ai/${name}`, ref: release.tag, sha: release.commit } : { source: "github", repo: `cavi-ai/${name}` }; +} const LEGACY = new Set(["claude-plugins", "claude-obsidian", "claude-obsidian-plugin"]); const PROJECTIONS = { claude: ".claude-plugin/marketplace.json", @@ -159,7 +173,21 @@ function validateTrustedIntegrity(registry, errors) { } } -function validateNativeProjection(host, projection, errors) { +function validateReleases(releases, errors) { + if (!releases || typeof releases !== "object" || Array.isArray(releases)) { + errors.push("releases must be an object"); + return; + } + for (const name of Object.keys(releases)) if (!CANONICAL.includes(name)) errors.push(`release pin for unknown plugin: ${name}`); + for (const name of REQUIRED) if (!Object.hasOwn(releases, name)) errors.push(`release pin missing: ${name}`); + for (const [name, release] of Object.entries(releases)) { + if (!release || typeof release !== "object" || Array.isArray(release) || unknownKeys(release, ["tag", "commit"]).length) { errors.push(`release pin ${name} must have only tag and commit`); continue; } + if (typeof release.tag !== "string" || !/^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/.test(release.tag)) errors.push(`release pin ${name} tag must be vX.Y.Z`); + if (typeof release.commit !== "string" || !/^[0-9a-f]{40}$/.test(release.commit)) errors.push(`release pin ${name} commit must be a full lowercase commit`); + } +} + +function validateNativeProjection(host, projection, releases, errors) { const rootAllowed = host === "claude" ? ["name", "owner", "metadata", "plugins"] : ["name", "interface", "plugins"]; for (const key of unknownKeys(projection, rootAllowed)) errors.push(`${host} native root property invalid: ${key}`); if (host === "claude") { @@ -168,7 +196,8 @@ function validateNativeProjection(host, projection, errors) { } else if (typeof projection?.interface?.displayName !== "string" || unknownKeys(projection?.interface, ["displayName"]).length) errors.push("codex native interface invalid"); for (const entry of projection.plugins ?? []) { if (host === "claude") { - if (unknownKeys(entry, ["name", "source", "description"]).length || entry?.source?.source !== "github" || typeof entry?.source?.repo !== "string" || unknownKeys(entry.source, ["source", "repo"]).length) errors.push(`claude native source invalid: ${entry?.name}`); + if (unknownKeys(entry, ["name", "source", "description"]).length || entry?.source?.source !== "github" || typeof entry?.source?.repo !== "string" || unknownKeys(entry.source, ["source", "repo", "ref", "sha"]).length) errors.push(`claude native source invalid: ${entry?.name}`); + else if (JSON.stringify(entry.source) !== JSON.stringify(claudeSource(entry.name, releases?.[entry.name]))) errors.push(`claude source does not match its release pin: ${entry?.name}`); } else { if (unknownKeys(entry, ["name", "source", "policy", "category"]).length || entry?.source?.source !== "local" || typeof entry?.source?.path !== "string" || unknownKeys(entry.source, ["source", "path"]).length) errors.push(`codex native source invalid: ${entry?.name}`); if (entry?.policy?.installation !== "AVAILABLE" || entry?.policy?.authentication !== "ON_INSTALL" || unknownKeys(entry?.policy, ["installation", "authentication"]).length) errors.push(`codex native policy invalid: ${entry?.name}`); @@ -177,11 +206,12 @@ function validateNativeProjection(host, projection, errors) { } } -async function validateCodexPackage(root, raw, canonical, trustedIntegrity, errors) { +async function validateCodexPackage(root, raw, canonical, trustedIntegrity, release, errors) { if (raw?.source?.source !== "local") { errors.push(`codex source must be marketplace-local: ${raw?.name}`); return; } + if (trustedIntegrity && trustedIntegrity.source_commit !== release?.commit) errors.push(`codex package does not match its release pin: ${raw.name}`); const codexPath = canonical.packages?.codex?.path; if (typeof codexPath !== "string" || !codexPath) { errors.push(`codex package path invalid: ${raw?.name}`); @@ -247,6 +277,9 @@ export async function validateCatalog(root = process.cwd()) { const trustedIntegrity = await json(root, "packages/codex/expected-integrity.json", errors); if (trustedIntegrity !== JSON_FAILURE) validateTrustedIntegrity(trustedIntegrity, errors); const trustedRegistry = trustedIntegrity === JSON_FAILURE ? null : trustedIntegrity; + const releaseFile = await json(root, RELEASES, errors); + if (releaseFile !== JSON_FAILURE) validateReleases(releaseFile, errors); + const releases = releaseFile === JSON_FAILURE ? null : releaseFile; if (catalog === JSON_FAILURE) return errors.sort(); validateCanonicalSchema(schema === JSON_FAILURE ? null : schema, catalog, errors); if (!catalog || typeof catalog !== "object" || Array.isArray(catalog)) return errors.sort(); @@ -293,7 +326,7 @@ export async function validateCatalog(root = process.cwd()) { if (projection === JSON_FAILURE) continue; if (!projection || typeof projection !== "object" || Array.isArray(projection)) { errors.push(`${host} projection must be an object`); continue; } if (projection.name !== MARKETPLACE) errors.push(`${host} projection name must be ${MARKETPLACE}`); - if (host === "claude" || host === "codex") validateNativeProjection(host, projection, errors); + if (host === "claude" || host === "codex") validateNativeProjection(host, projection, releases, errors); if (DISCOVERY_HOSTS.includes(host)) { if (projection.host !== host) errors.push(`${host} projection host mismatch`); if (projection.protocol !== "discovery") errors.push(`${host} projection must use discovery protocol`); @@ -308,7 +341,7 @@ export async function validateCatalog(root = process.cwd()) { if (DISCOVERY_HOSTS.includes(host) && (typeof raw?.install?.command !== "string" || !raw.install.command.trim())) { errors.push(`${host} install command missing: ${entry.name}`); } - if (DISCOVERY_HOSTS.includes(host) && DISCOVERY_COMMANDS[host]?.[entry.name] !== raw?.install?.command) errors.push(`${host} install command mismatch: ${entry.name}`); + if (DISCOVERY_HOSTS.includes(host) && discoveryInstallCommand(host, entry.name, releases?.[entry.name]) !== raw?.install?.command) errors.push(`${host} install command mismatch: ${entry.name}`); if (projected.has(entry.name)) errors.push(`${host} duplicate projection: ${entry.name}`); projected.add(entry.name); const canonical = byName.get(entry.name); @@ -317,7 +350,7 @@ export async function validateCatalog(root = process.cwd()) { continue; } if (host === "codex" && raw?.source) { - await validateCodexPackage(root, raw, canonical, trustedRegistry?.[raw.name], errors); + await validateCodexPackage(root, raw, canonical, trustedRegistry?.[raw.name], releases?.[raw.name], errors); entry.repository = canonical.repository; entry.package = raw.source.path?.replace(/^\.\//, ""); } diff --git a/scripts/validate-catalog.test.mjs b/scripts/validate-catalog.test.mjs index 84eda98..826459a 100644 --- a/scripts/validate-catalog.test.mjs +++ b/scripts/validate-catalog.test.mjs @@ -5,7 +5,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; -import { MARKETPLACE, validateCatalog } from "./validate-catalog.mjs"; +import { MARKETPLACE, claudeSource, discoveryInstallCommand, validateCatalog } from "./validate-catalog.mjs"; const hosts = ["claude", "codex", "agy", "opencode", "openclaw", "hermes", "agentskills"]; const inventories = { @@ -13,6 +13,7 @@ const inventories = { "obsidian-agent": ["build-retrospective", "connection-finder", "consistent-tagging", "daily-rollup", "dedup-merge", "frontmatter-normalizer", "manifest-content", "manifest-core", "manifest-feature", "manifest-infra", "manifest-pm", "manifest-research", "manifest-risk", "manifest-vault", "meeting-cleanup", "moc-builder", "note-splitter", "outline-to-draft", "plan-to-spec", "source-digest", "summarize-and-link", "task-harvester", "tracker-driver", "vault-grounding", "vault-synthesis", "wikilink-weaver"] }; const sourceCommits = { "mlx-agent": "923627988cf98e984fa36e0b940f86e7263e2958", "obsidian-agent": "7efecb058fce8185056cbe7868d6de1af11879a9" }; +const releases = { "mlx-agent": { tag: "v1.0.0", commit: sourceCommits["mlx-agent"] }, "obsidian-agent": { tag: "v1.0.0", commit: sourceCommits["obsidian-agent"] } }; const pluginHosts = { "mlx-agent": ["claude", "codex", "agy", "opencode", "openclaw", "hermes", "agentskills"], "obsidian-agent": ["claude", "codex", "opencode", "agentskills"] @@ -69,7 +70,8 @@ async function fixture({ catalog, claude = [], codex = [], agy = [], opencode = for (const host of ["agy", "opencode", "openclaw", "hermes"]) await mkdir(path.join(root, "providers", host), { recursive: true }); await cp(path.resolve(new URL("../catalog.schema.json", import.meta.url).pathname), path.join(root, "catalog.schema.json")); await writeFile(path.join(root, "catalog.json"), JSON.stringify(catalog)); - const nativeClaude = claude.map((item) => item.source ? item : { name: item.name, source: { source: "github", repo: item.repository }, description: `${item.name} fixture` }); + await writeFile(path.join(root, "releases.json"), JSON.stringify(releases)); + const nativeClaude = claude.map((item) => item.source ? item : { name: item.name, source: { ...claudeSource(item.name, releases[item.name]), repo: item.repository }, description: `${item.name} fixture` }); const nativeCodex = codex.map((item) => item.source ? item : { name: item.name, source: { source: "local", path: `./packages/codex/${item.name}` }, policy: { installation: "AVAILABLE", authentication: "ON_INSTALL" }, category: "Productivity" }); await writeFile(path.join(root, ".claude-plugin/marketplace.json"), JSON.stringify({ name: MARKETPLACE, owner: { name: "Cavi AI", url: "https://github.com/cavi-ai" }, metadata: { description: "fixture" }, plugins: nativeClaude })); await writeFile(path.join(root, ".agents/plugins/marketplace.json"), JSON.stringify({ name: MARKETPLACE, interface: { displayName: "Cavi AI" }, plugins: nativeCodex })); @@ -97,9 +99,48 @@ const entry = (name, host) => ({ name, repository: `cavi-ai/${name}`, package: canonical.plugins.find((item) => item.name === name).packages[host].path, - ...(host === "agy" ? { install: { command: "git clone https://github.com/cavi-ai/mlx-agent.git && agy plugin install ./mlx-agent/providers/agy" } } : {}), - ...(host === "opencode" ? { install: { command: name === "mlx-agent" ? "git clone https://github.com/cavi-ai/mlx-agent.git && python3 mlx-agent/scripts/mlx-agent install opencode --scope user --dry-run --json" : "git clone https://github.com/cavi-ai/obsidian-agent.git && node obsidian-agent/scripts/install.mjs --host opencode --scope user --dry-run" } } : {}), - ...(host === "openclaw" || host === "hermes" ? { install: { command: `git clone https://github.com/cavi-ai/mlx-agent.git && python3 mlx-agent/scripts/mlx-agent install ${host} --scope user --dry-run --json` } } : {}) + ...(["agy", "opencode", "openclaw", "hermes"].includes(host) ? { install: { command: discoveryInstallCommand(host, name, releases[name]) } } : {}) +}); + +test("pins discovery installs and the Claude source to the released tag and commit", () => { + const release = { tag: "v1.2.3", commit: "b".repeat(40) }; + assert.equal(discoveryInstallCommand("agy", "mlx-agent", release), "git clone --branch v1.2.3 --depth 1 https://github.com/cavi-ai/mlx-agent.git && agy plugin install ./mlx-agent/providers/agy"); + assert.equal(discoveryInstallCommand("opencode", "obsidian-agent", release), "git clone --branch v1.2.3 --depth 1 https://github.com/cavi-ai/obsidian-agent.git && node obsidian-agent/scripts/install.mjs --host opencode --scope user --dry-run"); + assert.equal(discoveryInstallCommand("hermes", "mlx-agent", undefined), "git clone https://github.com/cavi-ai/mlx-agent.git && python3 mlx-agent/scripts/mlx-agent install hermes --scope user --dry-run --json"); + assert.deepEqual(claudeSource("mlx-agent", release), { source: "github", repo: "cavi-ai/mlx-agent", ref: "v1.2.3", sha: "b".repeat(40) }); + assert.deepEqual(claudeSource("loop-killer", undefined), { source: "github", repo: "cavi-ai/loop-killer" }); +}); + +test("rejects host projections and packages that drift from the release pin", async () => { + const root = await fixture({ catalog: canonical, claude: [{ name: "mlx-agent", source: { source: "github", repo: "cavi-ai/mlx-agent" }, description: "unpinned" }, entry("obsidian-agent", "claude")], codex: [entry("mlx-agent", "codex"), entry("obsidian-agent", "codex")], agy: [{ ...entry("mlx-agent", "agy"), install: { command: discoveryInstallCommand("agy", "mlx-agent", undefined) } }], opencode: [entry("mlx-agent", "opencode"), entry("obsidian-agent", "opencode")], openclaw: [entry("mlx-agent", "openclaw")], hermes: [entry("mlx-agent", "hermes")] }); + const pinsPath = path.join(root, "releases.json"); + const pins = JSON.parse(await readFile(pinsPath, "utf8")); + pins["obsidian-agent"].commit = "c".repeat(40); + await writeFile(pinsPath, JSON.stringify(pins)); + const errors = await validateCatalog(root); + assert(errors.includes("claude source does not match its release pin: mlx-agent"), errors.join("\n")); + assert(errors.includes("agy install command mismatch: mlx-agent"), errors.join("\n")); + assert(errors.includes("codex package does not match its release pin: obsidian-agent"), errors.join("\n")); + assert(errors.includes("claude source does not match its release pin: obsidian-agent"), errors.join("\n")); +}); + +test("requires well-formed release pins for every plugin with a Codex package", async () => { + const cases = [ + [(pins) => { delete pins["mlx-agent"]; }, "release pin missing: mlx-agent"], + [(pins) => { pins.ghost = { tag: "v1.0.0", commit: "a".repeat(40) }; }, "release pin for unknown plugin: ghost"], + [(pins) => { pins["mlx-agent"].tag = "1.0.0"; }, "release pin mlx-agent tag must be vX.Y.Z"], + [(pins) => { pins["mlx-agent"].commit = "abc1234"; }, "release pin mlx-agent commit must be a full lowercase commit"], + [(pins) => { pins["mlx-agent"].branch = "main"; }, "release pin mlx-agent must have only tag and commit"] + ]; + for (const [mutate, expected] of cases) { + const root = await fixture({ catalog: canonical }); + const pinsPath = path.join(root, "releases.json"); + const pins = JSON.parse(await readFile(pinsPath, "utf8")); + mutate(pins); + await writeFile(pinsPath, JSON.stringify(pins)); + const errors = await validateCatalog(root); + assert(errors.includes(expected), `missing ${expected}\n${errors.join("\n")}`); + } }); test("accepts the canonical plugins and truthful host projections", async () => {